hacamer
/
dnsmasq-china-list
mirror of https://github.com/felixonmars/dnsmasq-china-list.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

verify: refactor and improve

This commit is contained in:
Felix Yan 2018-12-14 01:41:07 +08:00
parent f7f0dae638
commit 79540ae6d6
2 changed files with 149 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cdn-testlist.txt
View File

@ -1,4 +1,5 @@
www.azure.cn.mschcdn.com www.azure.cn.mschcdn.com
www.gov.cn www.gov.cn
a1.cdn-hotels.com a1.cdn-hotels.com
cdn.jsdelivr.net cdn.jsdelivr.net
edge.yunjiasu.com

213
verify.py
View File

@ -6,85 +6,166 @@ import random
import ipaddress import ipaddress
import tldextract import tldextract
with open("ns-whitelist.txt") as f:
whitelist = list([l.rstrip('\n') for l in f if l])
with open("ns-blacklist.txt") as f: class ChnroutesNotAvailable(Exception):
blacklist = list([l.rstrip('\n') for l in f if l]) pass
with open("cdn-testlist.txt") as f: class NSNotAvailable(Exception):
cdnlist = list([l.rstrip('\n') for l in f if l]) pass
try: # OK
with open("/usr/share/chnroutes2/chnroutes.txt") as f: class OK(Exception):
chnroutes = list([l.rstrip('\n') for l in f if l and not l.startswith("#")]) pass
except:
print(colored("Failed to load chnroutes, CDN check disabled"), "red") class WhitelistMatched(OK):
chnroutes = None pass
class CDNListVerified(OK):
pass
class NSVerified(OK):
pass
# Not OK
class NotOK(Exception):
pass
class NXDOMAIN(NotOK):
pass
class BlacklistMatched(NotOK):
pass
class CDNListNotVerified(NotOK):
pass
class NSNotVerified(NotOK):
pass
with open("accelerated-domains.china.raw.txt") as f: class ChinaListVerify(object):
domains = random.sample([line.rstrip('\n') for line in f], 100) whitelist_file = "ns-whitelist.txt"
# domains = [line.rstrip('\n') for line in f][13820:13830] blacklist_file = "ns-blacklist.txt"
cdnlist_file = "cdn-testlist.txt"
chnroutes_file = "/usr/share/chnroutes2/chnroutes.txt"
def __init__(self):
self.whitelist = self.load_list(self.whitelist_file)
self.blacklist = self.load_list(self.blacklist_file)
self.cdnlist = self.load_list(self.cdnlist_file)
def cn_ip_test(domain): try:
answers = dns.resolver.query(domain, 'A') self.chnroutes = self.load_list(self.chnroutes_file)
answer = answers[0].to_text() except FileNotFoundError:
print(colored("Failed to load chnroutes, CDN check disabled", "red"))
return any(ipaddress.IPv4Address(answer) in ipaddress.IPv4Network(n) for n in chnroutes) self.chnroutes = None
def load_list(self, filename):
with open(filename) as f:
return list([l.rstrip('\n') for l in f if l and not l.startswith("#")])
for domain in domains: def test_cn_ip(self, domain):
if domain: if self.chnroutes is None:
nameserver = None raise ChnroutesNotAvailable
nameserver_text = ""
ns_failed = False answers = dns.resolver.query(domain, 'A')
answer = answers[0].to_text()
return any(ipaddress.IPv4Address(answer) in ipaddress.IPv4Network(n) for n in self.chnroutes)
def check_whitelist(self, nameservers):
if any(i in " ".join(nameservers) for i in self.whitelist):
raise WhitelistMatched
def check_blacklist(self, nameservers):
if any(i in " ".join(nameservers) for i in self.blacklist):
raise BlacklistMatched
def check_cdnlist(self, domain):
if self.test_cn_ip(domain):
raise CDNListVerified
else:
raise CDNListNotVerified
def check_domain(self, domain):
nameservers = []
try: try:
answers = dns.resolver.query(domain, 'NS') answers = dns.resolver.query(domain, 'NS')
except dns.resolver.NXDOMAIN: except dns.resolver.NXDOMAIN:
print(colored("NXDOMAIN found in domain: " + domain, "white", "on_red")) raise NXDOMAIN
continue except:
except Exception: pass
ns_failed = True else:
else:
for rdata in answers: for rdata in answers:
if nameserver is None: nameserver = rdata.to_text()
nameserver = rdata.to_text() if tldextract.extract(nameserver).registered_domain:
nameserver_text += rdata.to_text() nameservers.append(nameserver)
testdomain = None self.check_whitelist(nameservers)
if any(i in nameserver_text for i in whitelist):
print(colored("NS Whitelist matched for domain: " + domain, "green")) # Assuming CDNList for non-TLDs
elif domain.count(".") > 1 and tldextract.extract(domain).registered_domain != domain or any(testdomain.endswith(domain) for testdomain in cdnlist): if domain.count(".") > 1 and tldextract.extract(domain).registered_domain != domain:
for testdomain in cdnlist: self.check_cdnlist(domain)
if testdomain.endswith(domain):
break for testdomain in self.cdnlist:
else: if testdomain.endswith(domain):
testdomain = domain self.check_cdnlist(testdomain)
if chnroutes:
try: self.check_blacklist(nameservers)
if cn_ip_test(testdomain):
print(colored("CDNList matched and verified for domain: " + domain, "green")) for nameserver in nameservers:
else: try:
print(colored("CDNList matched but failed to verify for domain: " + domain, "red")) if self.test_cn_ip(nameserver):
except: raise NSVerified
print("Failed to find A for cdnlist domain:", testdomain) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
continue pass
else:
print(colored("CDNList matched (but verification is not available) for domain: " + domain)) if nameservers:
elif any(i in nameserver_text for i in blacklist): raise NSNotVerified
print(colored("NS Blacklist matched for domain: " + domain, "red"))
else: else:
if ns_failed: raise NSNotAvailable
print("Failed to find NS for domain: " + domain)
elif chnroutes: def check_domain_quiet(self, domain):
try: try:
if cn_ip_test(nameserver): self.check_domain(domain)
print(colored("NS verified for domain: " + domain, "green")) except OK:
else: return True
print(colored("NS failed to verify for domain: " + domain, "red")) except NotOK:
except: return False
print("Failed to find A for NS domain:", nameserver, "domain:", domain) except:
return None
else:
return None
def check_domain_list(self, domain_list, sample=100):
domains = self.load_list(domain_list)
if sample:
domains = random.sample(domains, sample)
for domain in domains:
try:
self.check_domain(domain)
except NXDOMAIN:
print(colored("NXDOMAIN found in domain: " + domain, "white", "on_red"))
except WhitelistMatched:
print(colored("NS Whitelist matched for domain: " + domain, "green"))
except CDNListVerified:
print(colored("CDNList matched and verified for domain: " + domain, "green"))
except CDNListNotVerified:
print(colored("CDNList matched but failed to verify for domain: " + domain, "red"))
except BlacklistMatched:
print(colored("NS Blacklist matched for domain: " + domain, "red"))
except NSVerified:
print(colored("NS verified for domain: " + domain, "green"))
except NSNotVerified:
print(colored("NS failed to verify for domain: " + domain, "red"))
except ChnroutesNotAvailable:
print("Additional Check disabled due to missing chnroutes. domain:", domain)
except NSNotAvailable:
print("Failed to get correct name server for domain:", domain)
else: else:
print("Neutral domain:", domain) raise NotImplementedError
if __name__ == "__main__":
v = ChinaListVerify()
v.check_domain_list("accelerated-domains.china.raw.txt")