etc/agfw/autoproxy-gfwlist/checkRules.py

   1 #!/usr/bin/python -u
   2 # vim: ai et fileencoding=utf-8 ts=4 sw=4:
   3
   4 '''
   5     This script will read `list.txt', and write both stdout and `list.err'.
   6
   7     Rules which are comments or regexs will be ignored.
   8
   9     * For `.example.com', if you got anything other than 56, then unless the
  10       page contains DPI keyword, consider the rule invalid.
  11
  12     * For `||example.com', if you got anything other than 28, then consider
  13       the rule invalid. (For issue 117, see below.)
  14
  15     * For `|https://*.example.com', if you got anything other than 35, check
  16       it manually before considering it invalid.
  17
  18     In addition, if you got 6 or 7, check the rule again manually. If in
  19     doubt, check the rule again manually. Also, please remember: `Garbage
  20     in, garbage out.'
  21
  22     XXX: As a workaround for issue 117, `|http://example.com/' will be
  23     tested as `.example.com/'.
  24 '''
  25
  26 from urllib import unquote
  27 import re
  28 import subprocess
  29 import sys
  30
  31 testurl = "http://%s/?"
  32 iplist = ["66.249.89.19", "66.249.89.32", "66.249.89.33", "66.249.89.34", "66.249.89.35", "66.249.89.37", "66.249.89.38", "66.249.89.39", "66.249.89.40", "66.249.89.44", "66.249.89.45", "66.249.89.46", "66.249.89.47", "66.249.89.49", "66.249.89.50", "66.249.89.51", "66.249.89.52", "66.249.89.53", "66.249.89.54", "66.249.89.56", "66.249.89.57", "66.249.89.58", "66.249.89.59", "66.249.89.60", "66.249.89.61", "66.249.89.62", "66.249.89.63", "66.249.89.64", "66.249.89.65", "66.249.89.66", "66.249.89.68", "66.249.89.69", "66.249.89.72", "66.249.89.73", "66.249.89.74", "66.249.89.76", "66.249.89.77", "66.249.89.78", "66.249.89.79", "66.249.89.81", "66.249.89.82", "66.249.89.83", "66.249.89.84", "66.249.89.91", "66.249.89.93", "66.249.89.95", "66.249.89.96", "66.249.89.98", "66.249.89.99", "66.249.89.100", "66.249.89.101", "66.249.89.104", "66.249.89.112", "66.249.89.115", "66.249.89.116", "66.249.89.118", "66.249.89.120", "66.249.89.123", "66.249.89.128", "66.249.89.132", "66.249.89.136", "66.249.89.137", "66.249.89.141", "66.249.89.142", "66.249.89.143", "66.249.89.144", "66.249.89.145", "66.249.89.146", "66.249.89.148", "66.249.89.149", "66.249.89.152", "66.249.89.154", "66.249.89.155", "66.249.89.161", "66.249.89.162", "66.249.89.163", "66.249.89.164", "66.249.89.165", "66.249.89.184", "66.249.89.189", "66.249.89.190", "66.249.89.193", "66.249.89.210",
  33           "66.102.13.17", "66.102.13.18", "66.102.13.19", "66.102.13.24", "66.102.13.32", "66.102.13.33", "66.102.13.34", "66.102.13.35", "66.102.13.36", "66.102.13.37", "66.102.13.38", "66.102.13.39", "66.102.13.40", "66.102.13.41", "66.102.13.42", "66.102.13.43", "66.102.13.44", "66.102.13.45", "66.102.13.46", "66.102.13.47", "66.102.13.48", "66.102.13.49", "66.102.13.51", "66.102.13.52", "66.102.13.53", "66.102.13.54", "66.102.13.56", "66.102.13.57", "66.102.13.58", "66.102.13.59", "66.102.13.60", "66.102.13.61", "66.102.13.62", "66.102.13.63", "66.102.13.64", "66.102.13.65", "66.102.13.66", "66.102.13.68", "66.102.13.69", "66.102.13.72", "66.102.13.73", "66.102.13.74", "66.102.13.75", "66.102.13.76", "66.102.13.77", "66.102.13.78", "66.102.13.79", "66.102.13.81", "66.102.13.82", "66.102.13.83", "66.102.13.84", "66.102.13.91", "66.102.13.93", "66.102.13.95", "66.102.13.96", "66.102.13.98", "66.102.13.99", "66.102.13.100", "66.102.13.101", "66.102.13.102", "66.102.13.103", "66.102.13.104", "66.102.13.105", "66.102.13.106", "66.102.13.112", "66.102.13.113", "66.102.13.115", "66.102.13.118", "66.102.13.120", "66.102.13.123", "66.102.13.128", "66.102.13.132", "66.102.13.136", "66.102.13.137", "66.102.13.138", "66.102.13.139", "66.102.13.141", "66.102.13.142", "66.102.13.143", "66.102.13.144", "66.102.13.145", "66.102.13.146", "66.102.13.147", "66.102.13.148", "66.102.13.149", "66.102.13.152", "66.102.13.154", "66.102.13.155", "66.102.13.156", "66.102.13.157", "66.102.13.161", "66.102.13.162", "66.102.13.163", "66.102.13.164", "66.102.13.165", "66.102.13.166", "66.102.13.167", "66.102.13.178", "66.102.13.184", "66.102.13.190", "66.102.13.191", "66.102.13.193", "66.102.13.210"]
  34 offset = 0
  35
  36 INVALID = 0
  37 IP = 1
  38 TLS = 2
  39 URL = 4
  40
  41 expect = {IP: 28, TLS: 35, URL: 56, INVALID: 0}
  42
  43 def getUrl(rule):
  44     global offset
  45
  46     if rule.startswith('||'): return ('http://' + rule[2:], IP)
  47     if rule.startswith('|https'): return (rule[1:], TLS)
  48     if rule.startswith('|http://'):
  49         rule = '.' + rule[8:] # XXX: issue 117
  50     offset += 1
  51     if offset >= len(iplist): offset = 0
  52     return (testurl % iplist[offset] + rule, URL)
  53
  54 def main():
  55     fin = open('list.txt', 'r')
  56     ferr = open('list.err', 'w')
  57     line = 0
  58     if len(sys.argv) > 1:
  59         startfrom = int(sys.argv[1])
  60     else:
  61         startfrom = 0
  62     for rule in fin:
  63         line += 1
  64         rule = rule.strip()
  65         if line < startfrom: continue
  66         if not rule: continue
  67         if rule.startswith('[AutoProxy'): continue
  68         if rule.startswith('!'): continue
  69         if rule.startswith('@@'): continue
  70         if rule.startswith('/') and rule.endswith('/'): continue
  71         (test, t) = getUrl(rule)
  72         val1 = subprocess.call(['/usr/bin/curl', '-4', '-I', '-m', '5', test], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  73         val2 = subprocess.call(['/usr/bin/curl', '-4', '-I', '-m', '5', test], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  74         if (t==IP and val1==28 and val2==28) or (t==TLS and val1==35 or val2==35) or (t==URL and val1==56 or val2==56):
  75             pass
  76         elif t==IP and val1==56 or val2==56:
  77             print line, '\t', test, '=>', '\033[31mexpecting %d, got %d %d\033[0m' % (expect[t], val1, val2)
  78         else:
  79             print line, '\t', test, '=>', '\033[1;31mexpecting %d, got %d %d\033[0m' % (expect[t], val1, val2)
  80             ferr.write(str(line) + ': "' + rule + '", expecting %d, got %d %d' % (expect[t], val1, val2) + '\n')
  81
  82
  83 if __name__ == '__main__':
  84     main()