verify.rb

   1 #!/usr/bin/ruby
   2
   3 require 'colorize'
   4 require 'concurrent'
   5 require 'ipaddr'
   6 require 'public_suffix'
   7 require 'resolv'
   8
   9 class ChinaListVerify
  10     def initialize(
  11         dns=nil,
  12         whitelist_file: "ns-whitelist.txt",
  13         blacklist_file: "ns-blacklist.txt",
  14         cdnlist_file: "cdn-testlist.txt",
  15         chnroutes_file: "/usr/share/china_ip_list.txt"
  16     )
  17         @dns = dns
  18         @whitelist = load_list whitelist_file
  19         @blacklist = load_list blacklist_file
  20         @cdnlist = load_list cdnlist_file
  21         @tld_ns = Concurrent::Hash.new
  22
  23         begin
  24             @chnroutes = load_list(chnroutes_file).map { |line| IPAddr.new line }
  25         rescue Errno::ENOENT
  26             puts "Failed to load chnroutes, CDN check disabled".red
  27             @chnroutes = nil
  28         end
  29     end
  30
  31     def load_list(filename)
  32         File.readlines(filename).each do |line|
  33             line if !line.chomp!.empty? and !line.start_with?("#")
  34         end
  35     end
  36
  37     def test_cn_ip(domain, response: nil)
  38         if @chnroutes == nil
  39             raise "chnroutes not loaded"
  40         end
  41
  42         answers = nil
  43         if response != nil && !response.empty?
  44             answers = response.filter_map { |n, r| r if n.to_s == domain && r.class == Resolv::DNS::Resource::IN::A }
  45         end
  46
  47         if answers == nil || answers.empty?
  48             answers = resolve(domain, 'A')
  49         end
  50
  51         answers.each do |answer|
  52             answer = IPAddr.new answer.address.to_s
  53             if @chnroutes.any? { |range| range.include? answer }
  54                 return true
  55             end
  56         end
  57
  58         return false
  59     end
  60
  61     def resolve(domain, rdtype="A", server: nil, with_glue: false)
  62         rdtype = Kernel.const_get("Resolv::DNS::Resource::IN::#{rdtype}")
  63         if !server
  64             if !@dns
  65                 resolver = Resolv::DNS.new
  66             else
  67                 resolver = Resolv::DNS.new(nameserver: @dns)
  68             end
  69         else
  70             server = [server] unless server.is_a? Array
  71             resolver = Resolv::DNS.new(nameserver: server)
  72         end
  73         if !with_glue
  74             resolver.getresources(domain, rdtype)
  75         else
  76             # Workaround for https://github.com/ruby/resolv/issues/27
  77             result = []
  78             glue = []
  79             n0 = Resolv::DNS::Name.create domain
  80             resolver.fetch_resource(domain, rdtype) {|reply, reply_name|
  81                 reply.each_resource {|n, ttl, data|
  82                     if n0 == n && data.is_a?(rdtype)
  83                         result << data
  84                     else
  85                         glue << [n, data]
  86                     end
  87                 }
  88             }
  89             return result, glue
  90         end
  91     end
  92
  93     def get_ns_for_tld(tld)
  94         if !@tld_ns.has_key? tld
  95             answers = resolve(tld + ".", "NS")
  96             results = []
  97             answers.each do |answer|
  98                 ips = resolve answer.name.to_s
  99                 ips.each do |ip|
 100                     results << ip.address.to_s
 101                 end
 102             end
 103             @tld_ns[tld] = results
 104         end
 105
 106         @tld_ns[tld]
 107     end
 108
 109     def check_whitelist(nameservers)
 110         @whitelist.each { |pattern| nameservers.each {|ns| return pattern if ns.end_with? pattern }}
 111         nil
 112     end
 113
 114     def check_blacklist(nameservers)
 115         @blacklist.each { |pattern| nameservers.each {|ns| return pattern if ns.end_with? pattern }}
 116         nil
 117     end
 118
 119     def check_cdnlist(domain)
 120         test_cn_ip domain
 121     end
 122
 123     def check_domain(domain, enable_cdnlist: true)
 124         nameservers = []
 125         nxdomain = false
 126         begin
 127             tld_ns = get_ns_for_tld(PublicSuffix.parse(domain, ignore_private: true).tld)
 128         rescue PublicSuffix::DomainNotAllowed, PublicSuffix::DomainInvalid
 129             yield nil, "Invalid domain #{domain}"
 130             return nil
 131         end
 132         response, glue = self.resolve(
 133             domain + ".",
 134             'NS',
 135             server: tld_ns,
 136             with_glue: true
 137         )
 138         response.each do |rdata|
 139             begin
 140                 nameserver = rdata.name.to_s
 141                 if PublicSuffix.valid?(nameserver, ignore_private: true)
 142                     nameservers << nameserver
 143                 end
 144
 145                 if result = check_whitelist(nameservers)
 146                     yield true, "NS Whitelist #{result} matched for domain #{domain}" if block_given?
 147                     return true
 148                 end
 149             rescue NoMethodError => e
 150                 puts "Ignoring error: #{e}"
 151             end
 152         end
 153
 154         if enable_cdnlist
 155             @cdnlist.each do |testdomain|
 156                 if testdomain == domain or testdomain.end_with? "." + domain
 157                     if result = check_cdnlist(testdomain)
 158                         yield true, "CDN List matched (#{testdomain}) and verified #{result} for domain #{domain}" if block_given?
 159                         return true
 160                     end
 161                 end
 162             end
 163
 164             # Assuming CDNList for non-TLDs
 165             if domain.count(".") > 1 and PublicSuffix.domain(domain, ignore_private: true) != domain
 166                 if result = check_cdnlist(domain)
 167                     yield true, "CDN List matched and verified #{result} for domain #{domain}" if block_given?
 168                     return true if result
 169                 end
 170             end
 171         end
 172
 173         if result = check_blacklist(nameservers)
 174             yield false, "NS Blacklist #{result} matched for domain #{domain}" if block_given?
 175             return false
 176         end
 177
 178         nameservers.each do |nameserver|
 179             if result = test_cn_ip(nameserver, response: glue)
 180                 yield true, "NS #{nameserver} verified #{result} for domain #{domain}" if block_given?
 181                 return true
 182             end
 183         end
 184
 185         if !nameservers.empty?
 186             yield false, "NS #{nameservers[0]} not verified for domain #{domain}" if block_given?
 187             return false
 188         else
 189             yield nil, "Failed to get correct name server for domain #{domain}" if block_given?
 190             return nil
 191         end
 192     end
 193
 194     def check_domain_verbose(domain, show_green: false, **kwargs)
 195         check_domain(domain, **kwargs) do |result, message|
 196             if result == true
 197                 puts message.green if show_green
 198             elsif result == false
 199                 puts message.red
 200             else
 201                 puts message.yellow
 202             end
 203         end
 204     end
 205
 206     def check_domain_list(domain_list, sample: 30, show_green: false, jobs: Concurrent.processor_count)
 207         domains = load_list domain_list
 208         if sample > 0
 209             domains = domains.sample(sample)
 210         else
 211             domains.shuffle!
 212         end
 213         pool = Concurrent::FixedThreadPool.new(jobs)
 214         domains.each do |domain|
 215             pool.post do
 216                 if check_domain_verbose(domain, show_green: show_green)
 217                     yield domain if block_given?
 218                 end
 219             end
 220         end
 221         pool.shutdown
 222         pool.wait_for_termination
 223     end
 224 end
 225
 226 # Operates on the raw file to preserve commented out lines
 227 def CheckRedundant(lines, disabled_lines, domain)
 228     new_line = "server=/#{domain}/114.114.114.114\n"
 229     disabled_line = "#server=/#{domain}/114.114.114.114"
 230     if lines.include? new_line
 231         puts "Domain already exists: #{domain}"
 232         return false
 233     elsif disabled_lines.any? { |line| line.start_with? disabled_line }
 234         puts "Domain already disabled: #{domain}"
 235         return false
 236     else
 237         # Check for duplicates
 238         test_domain = domain
 239         while test_domain.include? '.'
 240             test_domain = test_domain.partition('.').last
 241             _new_line = "server=/#{test_domain}/114.114.114.114\n"
 242             _disabled_line = "#server=/#{test_domain}/114.114.114.114"
 243             if lines.include? _new_line
 244                 puts "Redundant domain already exists: #{test_domain}"
 245                 return false
 246             elsif disabled_lines.any? { |line| line.start_with? _disabled_line }
 247                 puts "Redundant domain already disabled: #{test_domain}"
 248                 return false
 249             end
 250         end
 251     end
 252     return new_line
 253 end
 254
 255 if __FILE__ == $0
 256     require 'optparse'
 257     require 'ostruct'
 258
 259     options = OpenStruct.new
 260     options.file = "accelerated-domains.china.raw.txt"
 261     options.sample = 0
 262     options.verbose = false
 263     options.domain = nil
 264     options.dns = nil
 265     OptionParser.new do |opts|
 266         opts.banner = 'A simple verify library for dnsmasq-china-list'
 267
 268         opts.on("-f", "--file FILE", "File to check") do |f|
 269             options.file = f
 270         end
 271
 272         opts.on("-s", "--sample SAMPLE", Integer, "Verify only a limited sample. Pass 0 to example all entries") do |s|
 273             options.sample = s
 274         end
 275
 276         opts.on("-v", "--[no-]verbose", "Show green results") do |v|
 277             options.verbose = v
 278         end
 279
 280         opts.on("-d", "--domain DOMAIN", "Verify a domain instead of checking a list. Will ignore the other list options.") do |d|
 281             options.domain = d
 282         end
 283
 284         opts.on("-D", "--dns DNS", "Specify a DNS server to use instead of the system default one.") do |d|
 285             options.dns = d
 286         end
 287
 288         opts.on_tail("-h", "--help", "Show this message") do
 289             puts opts
 290             exit
 291         end
 292     end.parse!
 293
 294     v = ChinaListVerify.new options.dns
 295
 296     if options.domain
 297         exit v.check_domain_verbose(options.domain, show_green: options.verbose) == true
 298     else
 299         v.check_domain_list(options.file, sample: options.sample, show_green: options.verbose)
 300     end
 301 end