netwerk/dns/prepare_tlds.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import codecs
   6 import encodings.idna
   7 import re
   8 import sys
   9
  10 from make_dafsa import words_to_bin, words_to_cxx
  11
  12 """
  13 Processes a file containing effective TLD data.  See the following URL for a
  14 description of effective TLDs and of the file format that this script
  15 processes (although for the latter you're better off just reading this file's
  16 short source code).
  17
  18 http://wiki.mozilla.org/Gecko:Effective_TLD_Service
  19 """
  20
  21
  22 def getEffectiveTLDs(path):
  23     file = codecs.open(path, "r", "UTF-8")
  24     domains = set()
  25     for line in file:
  26         # line always contains a line terminator unless the file is empty
  27         if len(line) == 0:
  28             raise StopIteration
  29         line = line.rstrip()
  30         # comment, empty, or superfluous line for explicitness purposes
  31         if line.startswith("//") or not line.strip():
  32             continue
  33         line = re.split(r"[ \t\n]", line, 1)[0]
  34         entry = EffectiveTLDEntry(line)
  35         domain = entry.domain()
  36         assert domain not in domains, "repeating domain %s makes no sense" % domain
  37         domains.add(domain)
  38         yield entry
  39
  40
  41 def _normalizeHostname(domain):
  42     """
  43     Normalizes the given domain, component by component.  ASCII components are
  44     lowercased, while non-ASCII components are processed using the ToASCII
  45     algorithm.
  46     """
  47
  48     def convertLabel(label):
  49         if _isASCII(label):
  50             return label.lower()
  51         return encodings.idna.ToASCII(label).decode("utf-8")
  52
  53     return ".".join(map(convertLabel, domain.split(".")))
  54
  55
  56 def _isASCII(s):
  57     "True if s consists entirely of ASCII characters, false otherwise."
  58     for c in s:
  59         if ord(c) > 127:
  60             return False
  61     return True
  62
  63
  64 class EffectiveTLDEntry:
  65     """
  66     Stores an entry in an effective-TLD name file.
  67     """
  68
  69     _exception = False
  70     _wild = False
  71
  72     def __init__(self, line):
  73         """
  74         Creates a TLD entry from a line of data, which must have been stripped of
  75         the line ending.
  76         """
  77         if line.startswith("!"):
  78             self._exception = True
  79             domain = line[1:]
  80         elif line.startswith("*."):
  81             self._wild = True
  82             domain = line[2:]
  83         else:
  84             domain = line
  85         self._domain = _normalizeHostname(domain)
  86
  87     def domain(self):
  88         "The domain this represents."
  89         return self._domain
  90
  91     def exception(self):
  92         "True if this entry's domain denotes does not denote an effective TLD."
  93         return self._exception
  94
  95     def wild(self):
  96         "True if this entry represents a class of effective TLDs."
  97         return self._wild
  98
  99
 100 #################
 101 # DO EVERYTHING #
 102 #################
 103
 104
 105 def main(output, effective_tld_filename, output_format="cxx"):
 106     """
 107     effective_tld_filename is the effective TLD file to parse.
 108     based on the output format, either a C++ array of a binary representation
 109     of a DAFSA representing the eTLD file is then printed to standard output
 110     or a binary file is written to disk.
 111     """
 112
 113     def typeEnum(etld):
 114         """
 115         Maps the flags to the DAFSA's enum types.
 116         """
 117         if etld.exception():
 118             return 1
 119         elif etld.wild():
 120             return 2
 121         else:
 122             return 0
 123
 124     def dafsa_words():
 125         """
 126         make_dafsa expects lines of the form "<domain_name><enum_value>"
 127         """
 128         for etld in getEffectiveTLDs(effective_tld_filename):
 129             yield "%s%d" % (etld.domain(), typeEnum(etld))
 130
 131     """ words_to_bin() returns a bytes while words_to_cxx() returns string """
 132     if output_format == "bin":
 133         output.write(words_to_bin(dafsa_words()))
 134     else:
 135         output.write(words_to_cxx(dafsa_words()))
 136
 137
 138 if __name__ == "__main__":
 139     """
 140     This program can output the DAFSA in two formats:
 141     as C++ code that will be included and compiled at build time
 142     or as a binary file that will be published in Remote Settings.
 143
 144     Flags for format options:
 145     "cxx" -> C++ array [default]
 146     "bin" -> Binary file
 147     """
 148
 149     output_format = "bin" if "--bin" in sys.argv else "cxx"
 150     main(sys.stdout, sys.argv[1], output_format=output_format)