tools/make-services.py

   1 #!/usr/bin/env python3
   2 #
   3 # Parses the CSV version of the IANA Service Name and Transport Protocol Port Number Registry
   4 # and generates a services(5) file.
   5 #
   6 # Wireshark - Network traffic analyzer
   7 # By Gerald Combs <gerald@wireshark.org>
   8 # Copyright 2013 Gerald Combs
   9 #
  10 # SPDX-License-Identifier: GPL-2.0-or-later
  11
  12 import sys
  13 import getopt
  14 import csv
  15 import re
  16 import collections
  17 import urllib.request, urllib.error, urllib.parse
  18 import codecs
  19
  20 iana_svc_url = 'https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv'
  21
  22 __doc__ = '''\
  23 Usage: make-services.py [url]
  24
  25 url defaults to
  26     %s
  27 ''' % (iana_svc_url)
  28
  29
  30 services_file = 'epan/services-data.c'
  31
  32 exclude_services = [
  33     '^spr-itunes',
  34     '^spl-itunes',
  35     '^shilp',
  36     ]
  37
  38 min_source_lines = 14000 # Size was ~ 14800 on 2017-07-20
  39
  40 def parse_port(port_str):
  41
  42     p = port_str.split('-')
  43     try:
  44         if len(p) == 1:
  45             return tuple([int(p[0])])
  46         if len(p) == 2:
  47             return tuple(range(int(p[0]), int(p[1]) + 1))
  48     except ValueError:
  49         pass
  50     return ()
  51
  52 def port_to_str(port):
  53     if len(port) == 2:
  54         return str(port[0]) + '-' + str(port[1])
  55     return str(port[0])
  56
  57 def parse_rows(svc_fd):
  58     port_reader = csv.reader(svc_fd)
  59     count = 0
  60
  61     # Header positions as of 2013-08-06
  62     headers = next(port_reader)
  63
  64     try:
  65         sn_pos = headers.index('Service Name')
  66     except Exception:
  67         sn_pos = 0
  68     try:
  69         pn_pos = headers.index('Port Number')
  70     except Exception:
  71         pn_pos = 1
  72     try:
  73         tp_pos = headers.index('Transport Protocol')
  74     except Exception:
  75         tp_pos = 2
  76     try:
  77         desc_pos = headers.index('Description')
  78     except Exception:
  79         desc_pos = 3
  80
  81     services_map = {}
  82
  83     for row in port_reader:
  84         service = row[sn_pos]
  85         port = parse_port(row[pn_pos])
  86         proto = row[tp_pos]
  87         description = row[desc_pos]
  88         count += 1
  89
  90         if len(service) < 1 or not port or len(proto) < 1:
  91             continue
  92
  93         if re.search('|'.join(exclude_services), service):
  94             continue
  95
  96         # max 15 chars
  97         service = service[:15].rstrip()
  98
  99         # replace blanks (for some non-standard long names)
 100         service = service.replace(" ", "-")
 101
 102         description = description.replace("\n", "")
 103         description = re.sub("IANA assigned this well-formed service .+$", "", description)
 104         description = re.sub("  +", " ", description)
 105         description = description.strip()
 106         if description == service or description == service.replace("-", " "):
 107             description = None
 108
 109         if port not in services_map:
 110             services_map[port] = collections.OrderedDict()
 111
 112         # Remove some duplicates (first entry wins)
 113         proto_exists = False
 114         for k in services_map[port].keys():
 115             if proto in services_map[port][k]:
 116                 proto_exists = True
 117                 break
 118         if proto_exists:
 119             continue
 120
 121         if service not in services_map[port]:
 122             services_map[port][service] = [description]
 123         services_map[port][service].append(proto)
 124
 125     if count < min_source_lines:
 126         exit_msg('Not enough parsed data')
 127
 128     return services_map
 129
 130 def compile_body(d):
 131     keys = list(d.keys())
 132     keys.sort()
 133     body = []
 134
 135     for port in keys:
 136         for serv in d[port].keys():
 137             line = [port, d[port][serv][1:], serv]
 138             description = d[port][serv][0]
 139             if description:
 140                 line.append(description)
 141             body.append(line)
 142
 143     return body
 144
 145 def add_entry(table, port, service_name, description):
 146     table.append([int(port), service_name, description])
 147
 148
 149  # body = [(port-range,), [proto-list], service-name, optional-description]
 150  # table = [port-number, service-name, optional-description]
 151 def compile_tables(body):
 152
 153     body.sort()
 154     tcp_udp_table = []
 155     tcp_table = []
 156     udp_table = []
 157     sctp_table = []
 158     dccp_table = []
 159
 160     for entry in body:
 161         if len(entry) == 4:
 162             port_range, proto_list, service_name, description = entry
 163         else:
 164             port_range, proto_list, service_name = entry
 165             description = None
 166
 167         for port in port_range:
 168             if 'tcp' in proto_list and 'udp' in proto_list:
 169                 add_entry(tcp_udp_table, port, service_name, description)
 170             else:
 171                 if 'tcp' in proto_list:
 172                     add_entry(tcp_table, port, service_name, description)
 173                 if 'udp' in proto_list:
 174                     add_entry(udp_table, port, service_name, description)
 175             if 'sctp' in proto_list:
 176                 add_entry(sctp_table, port, service_name, description)
 177             if 'dccp' in proto_list:
 178                 add_entry(dccp_table, port, service_name, description)
 179
 180     return tcp_udp_table, tcp_table, udp_table, sctp_table, dccp_table
 181
 182
 183 def exit_msg(msg=None, status=1):
 184     if msg is not None:
 185         sys.stderr.write(msg + '\n\n')
 186     sys.stderr.write(__doc__ + '\n')
 187     sys.exit(status)
 188
 189 def main(argv):
 190     if sys.version_info[0] < 3:
 191         print("This requires Python 3")
 192         sys.exit(2)
 193
 194     try:
 195         opts, _ = getopt.getopt(argv, "h", ["help"])
 196     except getopt.GetoptError:
 197         exit_msg()
 198     for opt, _ in opts:
 199         if opt in ("-h", "--help"):
 200             exit_msg(None, 0)
 201
 202     if (len(argv) > 0):
 203         svc_url = argv[0]
 204     else:
 205         svc_url = iana_svc_url
 206
 207     try:
 208         if not svc_url.startswith('http'):
 209             svc_fd = open(svc_url)
 210         else:
 211             req = urllib.request.urlopen(svc_url)
 212             svc_fd = codecs.getreader('utf8')(req)
 213     except Exception:
 214         exit_msg('Error opening ' + svc_url)
 215
 216     body = parse_rows(svc_fd)
 217
 218     out = open(services_file, 'w')
 219     out.write('''\
 220 /*
 221  * Wireshark - Network traffic analyzer
 222  * By Gerald Combs <gerald@wireshark.org>
 223  * Copyright 1998 Gerald Combs
 224  *
 225  * SPDX-License-Identifier: GPL-2.0-or-later
 226  *
 227  * This is a local copy of the IANA port-numbers file.
 228  *
 229  * Wireshark uses it to resolve port numbers into human readable
 230  * service names, e.g. TCP port 80 -> http.
 231  *
 232  * It is subject to copyright and being used with IANA's permission:
 233  * https://lists.wireshark.org/archives/wireshark-dev/200708/msg00160.html
 234  *
 235  * The original file can be found at:
 236  * %s
 237  *
 238  * Generated by tools/make-services.py
 239  */
 240
 241 ''' % (iana_svc_url))
 242
 243     body = compile_body(body)
 244     # body = [(port-range,), [proto-list], service-name, optional-description]
 245
 246     max_port = 0
 247
 248     tcp_udp, tcp, udp, sctp, dccp = compile_tables(body)
 249
 250     def write_entry(f, e, max_port):
 251         line = "    {{ {}, \"{}\", ".format(*e)
 252         sep_len = 32 - len(line)
 253         if sep_len <= 0:
 254             sep_len = 1
 255         line += ' ' * sep_len
 256         if len(e) == 3 and e[2]:
 257             line += "\"{}\" }},\n".format(e[2].replace('"', '\\"'))
 258         else:
 259             line += "\"\" },\n"
 260         f.write(line)
 261         if int(e[0]) > int(max_port):
 262             return e[0]
 263         return max_port
 264
 265     out.write("static const ws_services_entry_t global_tcp_udp_services_table[] = {\n")
 266     for e in tcp_udp:
 267         max_port = write_entry(out, e, max_port)
 268     out.write("};\n\n")
 269
 270     out.write("static const ws_services_entry_t global_tcp_services_table[] = {\n")
 271     for e in tcp:
 272         max_port = write_entry(out, e, max_port)
 273     out.write("};\n\n")
 274
 275     out.write("static const ws_services_entry_t global_udp_services_table[] = {\n")
 276     for e in udp:
 277         max_port = write_entry(out, e, max_port)
 278     out.write("};\n\n")
 279
 280     out.write("static const ws_services_entry_t global_sctp_services_table[] = {\n")
 281     for e in sctp:
 282         max_port = write_entry(out, e, max_port)
 283     out.write("};\n\n")
 284
 285     out.write("static const ws_services_entry_t global_dccp_services_table[] = {\n")
 286     for e in dccp:
 287         max_port = write_entry(out, e, max_port)
 288     out.write("};\n\n")
 289
 290     out.write("static const uint16_t _services_max_port = {};\n".format(max_port))
 291
 292     out.close()
 293
 294 if __name__ == "__main__":
 295     sys.exit(main(sys.argv[1:]))