updater & find_redundant: rewrite in ruby
This commit is contained in:
Felix Yan
parent
a11f555167
commit
b3e5feb64b
|
|
@ -32,7 +32,7 @@ ENV.fetch("JOBS", "1").to_i.times.each do
|
|||
|
||||
if v.check_domain_verbose(domain, enable_cdnlist: false, show_green: true)
|
||||
Filelock '.git.lock' do
|
||||
puts `./updater.py -a #{domain}`
|
||||
puts `./updater.rb -a #{domain}`
|
||||
puts `git commit -S -am "accelerated-domains: add #{domain}"` if $?.success?
|
||||
puts `./update-local` if $?.success?
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,77 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
''' Find redundant items in accelerated-domains.china.conf.
|
||||
e.g. 'bar.foo.com' is redundant for 'foo.com'.
|
||||
'''
|
||||
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
def load(conf_file):
|
||||
''' Parse conf file & Prepare data structure
|
||||
Returns: [ ['abc', 'com'],
|
||||
['bar', 'foo', 'com'],
|
||||
... ]
|
||||
'''
|
||||
|
||||
results = []
|
||||
if isinstance(conf_file, str):
|
||||
lines = open(conf_file, 'r').readlines()
|
||||
elif isinstance(conf_file, Iterable):
|
||||
lines = iter(conf_file)
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line == '' or line.startswith('#'):
|
||||
continue
|
||||
# A domain name is case-insensitive and
|
||||
# consists of several labels, separated by a full stop
|
||||
domain_name = line.split('/')[1]
|
||||
domain_name = domain_name.lower()
|
||||
domain_labels = domain_name.split('.')
|
||||
results.append(domain_labels)
|
||||
|
||||
# Sort results by domain labels' length
|
||||
results.sort(key=len)
|
||||
return results
|
||||
|
||||
|
||||
def find(labelses):
|
||||
''' Find redundant items by a tree of top-level domain label to sub-level.
|
||||
`tree` is like { 'com': { 'foo: { 'bar': LEAF },
|
||||
'abc': LEAF },
|
||||
'org': ... }
|
||||
'''
|
||||
redundant = []
|
||||
tree = {}
|
||||
LEAF = 1
|
||||
for labels in labelses:
|
||||
domain = '.'.join(labels)
|
||||
# Init root node as current node
|
||||
node = tree
|
||||
while len(labels) > 0:
|
||||
label = labels.pop()
|
||||
if label in node:
|
||||
# If child node is a LEAF node,
|
||||
# current domain must be an existed domain or a subdomain of an existed.
|
||||
if node[label] == LEAF:
|
||||
print(f"Redundant found: {domain} at {'.'.join(labels)}")
|
||||
redundant.append(domain)
|
||||
break
|
||||
else:
|
||||
# Create a leaf node if current label is last one
|
||||
if len(labels) == 0:
|
||||
node[label] = LEAF
|
||||
# Create a branch node
|
||||
else:
|
||||
node[label] = {}
|
||||
# Iterate to child node
|
||||
node = node[label]
|
||||
return redundant
|
||||
|
||||
|
||||
def find_redundant(conf_file):
|
||||
return find(load(conf_file))
|
||||
|
||||
if __name__ == '__main__':
|
||||
find_redundant('accelerated-domains.china.conf')
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
""" Find redundant items in accelerated-domains.china.conf.
|
||||
e.g. 'bar.foo.com' is redundant for 'foo.com'.
|
||||
"""
|
||||
|
||||
def load(conf_file)
|
||||
""" Parse conf file & Prepare data structure
|
||||
Returns: [ ['abc', 'com'],
|
||||
['bar', 'foo', 'com'],
|
||||
... ]
|
||||
"""
|
||||
|
||||
results = []
|
||||
if conf_file.is_a? String
|
||||
lines = File.readlines(conf_file)
|
||||
elsif conf_file.is_a? Array
|
||||
lines = conf_file
|
||||
end
|
||||
lines.map do |line|
|
||||
line = line.chomp
|
||||
next if line.empty? or line.start_with?('#')
|
||||
# A domain name is case-insensitive and
|
||||
# consists of several labels, separated by a full stop
|
||||
domain_name = line.split('/')[1]
|
||||
domain_name = domain_name.downcase
|
||||
domain_labels = domain_name.split('.')
|
||||
results << domain_labels
|
||||
end
|
||||
|
||||
# Sort results by domain labels' length
|
||||
results.sort_by(&:length)
|
||||
end
|
||||
|
||||
|
||||
LEAF = 1
|
||||
def find(labelses)
|
||||
""" Find redundant items by a tree of top-level domain label to sub-level.
|
||||
`tree` is like { 'com': { 'foo: { 'bar': LEAF },
|
||||
'abc': LEAF },
|
||||
'org': ... }
|
||||
"""
|
||||
redundant = []
|
||||
tree = {}
|
||||
labelses.each do |labels|
|
||||
domain = labels.join('.')
|
||||
# Init root node as current node
|
||||
node = tree
|
||||
until labels.empty?
|
||||
label = labels.pop
|
||||
if node.include? label
|
||||
# If child node is a LEAF node,
|
||||
# current domain must be an existed domain or a subdomain of an existed.
|
||||
if node[label] == LEAF
|
||||
puts "Redundant found: #{domain} at #{labels.join('.')}"
|
||||
redundant << domain
|
||||
break
|
||||
end
|
||||
else
|
||||
# Create a leaf node if current label is last one
|
||||
if labels.empty?
|
||||
node[label] = LEAF
|
||||
# Create a branch node
|
||||
else
|
||||
node[label] = {}
|
||||
end
|
||||
end
|
||||
# Iterate to child node
|
||||
node = node[label]
|
||||
end
|
||||
end
|
||||
redundant
|
||||
end
|
||||
|
||||
def find_redundant(conf_file)
|
||||
return find(load(conf_file))
|
||||
end
|
||||
|
||||
if __FILE__ == $0
|
||||
find_redundant('accelerated-domains.china.conf')
|
||||
end
|
||||
83
updater.py
83
updater.py
|
|
@ -1,83 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import unicode_literals
|
||||
from argparse import ArgumentParser
|
||||
import idna
|
||||
import sys
|
||||
import find_redundant
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser(description="dnsmasq-china-list updater")
|
||||
parser.add_argument(
|
||||
'-a', '--add',
|
||||
metavar="DOMAIN",
|
||||
nargs="+",
|
||||
help='Add one or more new domain(s) (implies -s)',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-d', '--delete',
|
||||
metavar="DOMAIN",
|
||||
nargs="+",
|
||||
default=[],
|
||||
help='Remove one or more old domain(s) (implies -s)',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-s', '--sort',
|
||||
action='store_true',
|
||||
default=True,
|
||||
help='Sort the list (default action)',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-f', '--file',
|
||||
nargs=1,
|
||||
default=["accelerated-domains.china.conf"],
|
||||
help="Specify the file to update (accelerated-domains.china.conf by default)",
|
||||
)
|
||||
|
||||
options = parser.parse_args()
|
||||
|
||||
with open(options.file[0]) as f:
|
||||
lines = list(f)
|
||||
|
||||
changed = False
|
||||
|
||||
if options.add:
|
||||
options.sort = True
|
||||
|
||||
for domain in options.add:
|
||||
encoded_domain = idna.encode(domain).decode()
|
||||
new_line = f"server=/{encoded_domain}/114.114.114.114\n"
|
||||
disabled_line = f"#server=/{encoded_domain}/114.114.114.114"
|
||||
if new_line in lines:
|
||||
print(f"Domain already exists: {domain}")
|
||||
else:
|
||||
for line in lines:
|
||||
if line.startswith(disabled_line):
|
||||
print(f"Domain already disabled: {domain}")
|
||||
break
|
||||
else:
|
||||
print(f"New domain added: {domain}")
|
||||
lines.append(new_line)
|
||||
changed = True
|
||||
|
||||
options.delete += find_redundant.find_redundant(lines)
|
||||
|
||||
if options.delete:
|
||||
options.sort = True
|
||||
|
||||
for domain in options.delete:
|
||||
target_line = f"server=/{idna.encode(domain).decode()}/114.114.114.114\n"
|
||||
if target_line not in lines:
|
||||
print(f"Failed to remove domain {domain}: not found.")
|
||||
else:
|
||||
print(f"Domain removed: {domain}")
|
||||
lines.remove(target_line)
|
||||
changed = True
|
||||
|
||||
if (options.add or options.delete) and not changed:
|
||||
sys.exit(1)
|
||||
|
||||
if options.sort:
|
||||
lines.sort(key=lambda x: x.lstrip("#"))
|
||||
|
||||
with open(options.file[0], "w") as f:
|
||||
f.write(''.join(filter(lambda line: line.strip(), lines)))
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
#!/usr/bin/ruby
|
||||
require 'domain_name'
|
||||
require 'optparse'
|
||||
require 'ostruct'
|
||||
|
||||
options = OpenStruct.new
|
||||
options.sort = true
|
||||
options.file = "accelerated-domains.china.conf"
|
||||
options.add = []
|
||||
options.delete = []
|
||||
OptionParser.new do |opts|
|
||||
opts.banner = "dnsmasq-china-list updater"
|
||||
|
||||
opts.on("-s", "--[no-]sort", "Sort the list (default action)") do |s|
|
||||
options.sort = s
|
||||
end
|
||||
|
||||
opts.on("-f", "--file FILE", "Specify the file to update (accelerated-domains.china.conf by default)") do |f|
|
||||
options.file = f
|
||||
end
|
||||
|
||||
opts.on("-a", "--add domain1,domain2", Array, "Add domain(s) to the list (implies -s)") do |a|
|
||||
options.add = a
|
||||
options.sort = true
|
||||
end
|
||||
|
||||
opts.on("-d", "--delete domain1,domain2", Array, "Remove domain(s) from the list (implies -s)") do |d|
|
||||
options.delete = d
|
||||
options.sort = true
|
||||
end
|
||||
end.parse!
|
||||
|
||||
lines = File.readlines(options.file).filter { |line| !line.empty? }
|
||||
disabled_lines = lines.filter { |line| line.start_with?("#") }
|
||||
|
||||
changed = false
|
||||
|
||||
options.add.each do |domain|
|
||||
domain = DomainName.normalize(domain)
|
||||
new_line = "server=/#{domain}/114.114.114.114\n"
|
||||
disabled_line = "#server=/#{domain}/114.114.114.114"
|
||||
if lines.include? new_line
|
||||
puts "Domain already exists: #{domain}"
|
||||
else
|
||||
if disabled_lines.any? { |line| line.start_with? disabled_line }
|
||||
puts "Domain already disabled: #{domain}"
|
||||
else
|
||||
# Check for duplicates
|
||||
test_domain = domain
|
||||
while test_domain.include? '.'
|
||||
test_domain = test_domain.partition('.').last
|
||||
_new_line = "server=/#{test_domain}/114.114.114.114\n"
|
||||
_disabled_line = "#server=/#{test_domain}/114.114.114.114"
|
||||
if lines.include? _new_line
|
||||
puts "Redundant domain already exists: #{test_domain}"
|
||||
break
|
||||
elsif disabled_lines.any? { |line| line.start_with? _disabled_line }
|
||||
puts "Redundant domain already disabled: #{test_domain}"
|
||||
break
|
||||
end
|
||||
end
|
||||
next if test_domain.include? '.'
|
||||
|
||||
puts "New domain added: #{domain}"
|
||||
lines << new_line
|
||||
changed = true
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
options.delete.each do |domain|
|
||||
domain = DomainName.normalize(domain)
|
||||
target_line = "server=/#{domain}/114.114.114.114\n"
|
||||
unless lines.include? target_line
|
||||
puts "Failed to remove domain #{domain}: not found."
|
||||
else
|
||||
puts "Domain removed: #{domain}"
|
||||
lines.delete(target_line)
|
||||
changed = true
|
||||
end
|
||||
end
|
||||
|
||||
fail "No changes made." if (options.add.length || options.delete.length) && !changed
|
||||
|
||||
lines.sort_by! { |x| x.delete_prefix("#") } if options.sort
|
||||
|
||||
File.write(options.file, lines.join)
|
||||
Loading…
Reference in New Issue