compass.py

   1 #!/usr/bin/env python
   2 #
   3 # This program is free software. It comes without any warranty, to
   4 # the extent permitted by applicable law. You can redistribute it
   5 # and/or modify it under the terms of the Do What The Fuck You Want
   6 # To Public License, Version 2, as published by Sam Hocevar. See
   7 # http://sam.zoy.org/wtfpl/COPYING for more details.
   8
   9 import json
  10 import operator
  11 import sys
  12 import os
  13 from optparse import OptionParser, OptionGroup
  14 import urllib
  15 import re
  16 from abc import abstractmethod
  17
  18 class BaseFilter(object):
  19     @abstractmethod
  20     def accept(self, relay):
  21         pass
  22
  23     def load(self, relays):
  24         pass
  25
  26 class RunningFilter(BaseFilter):
  27     def accept(self, relay):
  28         return relay['running']
  29
  30 class FamilyFilter(BaseFilter):
  31     def __init__(self, family, all_relays):
  32         self._family_fingerprint = None
  33         self._family_nickname = None
  34         self._family_relays = []
  35         found_relay = None
  36         for relay in all_relays:
  37             if len(family) == 40 and relay['fingerprint'] == family:
  38                 found_relay = relay
  39                 break
  40             if len(family) < 20 and 'Named' in relay['flags'] and relay['nickname'] == family:
  41                 found_relay = relay
  42                 break
  43         if found_relay:
  44             self._family_fingerprint = '$%s' % found_relay['fingerprint']
  45             if 'Named' in found_relay['flags']:
  46                 self._family_nickname = found_relay['nickname']
  47             self._family_relays = [self._family_fingerprint] + found_relay.get('family', [])
  48
  49     def accept(self, relay):
  50         fingerprint = '$%s' % relay['fingerprint']
  51         mentions = [fingerprint] + relay.get('family', [])
  52         # Only show families as accepted by consensus (mutually listed relays)
  53         listed = fingerprint in self._family_relays
  54         listed = listed or 'Named' in relay['flags'] and relay['nickname'] in self._family_relays
  55         mentioned = self._family_fingerprint in mentions
  56         mentioned = mentioned or self._family_nickname in mentions
  57         if listed and mentioned:
  58             return True
  59         return False
  60
  61 class CountryFilter(BaseFilter):
  62     def __init__(self, countries=[]):
  63         self._countries = [x.lower() for x in countries]
  64
  65     def accept(self, relay):
  66         return relay.get('country', None) in self._countries
  67
  68 class ASFilter(BaseFilter):
  69     def __init__(self, as_sets=[]):
  70         self._as_sets = [x if not x.isdigit() else "AS" + x for x in as_sets]
  71
  72     def accept(self, relay):
  73         return relay.get('as_number', None) in self._as_sets
  74
  75 class ExitFilter(BaseFilter):
  76     def accept(self, relay):
  77         return relay.get('exit_probability', -1) > 0.0
  78
  79 class GuardFilter(BaseFilter):
  80     def accept(self, relay):
  81         return relay.get('guard_probability', -1) > 0.0
  82
  83 class FastExitFilter(BaseFilter):
  84     def __init__(self, bandwidth_rate, advertised_bandwidth, ports, inverse=False):
  85         self.bandwidth_rate = bandwidth_rate
  86         self.advertised_bandwidth = advertised_bandwidth
  87         self.ports = ports
  88         self.inverse = inverse
  89
  90     def accept(self, relay):
  91         if relay.get('bandwidth_rate', -1) < self.bandwidth_rate:
  92             return self.inverse
  93         if relay.get('advertised_bandwidth', -1) < self.advertised_bandwidth:
  94             return self.inverse
  95         relevant_ports = set(self.ports)
  96         summary = relay.get('exit_policy_summary', {})
  97         if 'accept' in summary:
  98             portlist = summary['accept']
  99         elif 'reject' in summary:
 100             portlist = summary['reject']
 101         else:
 102             return self.inverse
 103         ports = []
 104         for p in portlist:
 105             if '-' in p:
 106                 ports.extend(range(int(p.split('-')[0]),
 107                                    int(p.split('-')[1]) + 1))
 108             else:
 109                 ports.append(int(p))
 110         policy_ports = set(ports)
 111         if 'accept' in summary and not relevant_ports.issubset(policy_ports):
 112             return self.inverse
 113         if 'reject' in summary and not relevant_ports.isdisjoint(policy_ports):
 114             return self.inverse
 115         return not self.inverse
 116
 117 class RelayStats(object):
 118     def __init__(self, options):
 119         self._data = None
 120         self._filters = self._create_filters(options)
 121         self._get_group = self._get_group_function(options)
 122         self._relays = None
 123
 124     @property
 125     def data(self):
 126         if not self._data:
 127             self._data = json.load(file(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'details.json')))
 128         return self._data
 129
 130     @property
 131     def relays(self):
 132         if self._relays:
 133             return self._relays
 134
 135         self._relays = {}
 136         relays = self.data['relays']
 137         for f in self._filters:
 138             f.load(relays)
 139             relays = filter(f.accept, relays)
 140
 141         for relay in relays:
 142             self.add_relay(relay)
 143         return self._relays
 144
 145     def _create_filters(self, options):
 146         filters = []
 147         if not options.inactive:
 148             filters.append(RunningFilter())
 149         if options.family:
 150             filters.append(FamilyFilter(options.family, self.data['relays']))
 151         if options.country:
 152             filters.append(CountryFilter(options.country))
 153         if options.ases:
 154             filters.append(ASFilter(options.ases))
 155         if options.exits_only:
 156             filters.append(ExitFilter())
 157         if options.guards_only:
 158             filters.append(GuardFilter())
 159         if options.fast_exits_only:
 160             filters.append(FastExitFilter(95 * 125 * 1024, 5000 * 1024, [80, 443, 554, 1755], False))
 161         return filters
 162
 163     def _get_group_function(self, options):
 164         if options.by_country and options.by_as:
 165             return lambda relay: (relay.get('country', None), relay.get('as_number', None))
 166         elif options.by_country:
 167             return lambda relay: relay.get('country', None)
 168         elif options.by_as:
 169             return lambda relay: relay.get('as_number', None)
 170         else:
 171             return lambda relay: relay.get('fingerprint')
 172
 173     def add_relay(self, relay):
 174         key = self._get_group(relay)
 175         if key not in self._relays:
 176             self._relays[key] = []
 177         self._relays[key].append(relay)
 178
 179     def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False, links=False):
 180         formatted_groups = {}
 181         for group in grouped_relays.values():
 182             group_weights = (0, 0, 0, 0, 0)
 183             relays_in_group = 0
 184             for relay in group:
 185                 weights = (relay.get('consensus_weight_fraction', 0),
 186                            relay.get('advertised_bandwidth_fraction', 0),
 187                            relay.get('guard_probability', 0),
 188                            relay.get('middle_probability', 0),
 189                            relay.get('exit_probability', 0))
 190                 group_weights = tuple(sum(x) for x in zip(group_weights, weights))
 191                 nickname = relay['nickname']
 192                 fingerprint = relay['fingerprint'] if not links else "https://atlas.torproject.org/#details/%s" % relay['fingerprint']
 193                 exit = 'Exit' if 'Exit' in set(relay['flags']) else '-'
 194                 guard = 'Guard' if 'Guard' in set(relay['flags']) else '-'
 195                 country = relay.get('country', '')
 196                 as_number = relay.get('as_number', '')
 197                 as_name = relay.get('as_name', '')
 198                 relays_in_group += 1
 199             if by_country or by_as_number:
 200                 nickname = "(%d relays)" % relays_in_group
 201                 fingerprint = "*"
 202                 exit = "*"
 203                 guard = "*"
 204             if by_country and not by_as_number:
 205                 as_number = "*"
 206                 as_name = "*"
 207             if by_as_number and not by_country:
 208                 country = "*"
 209             if links:
 210                 format_string = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-78s %-4s %-5s %-2s %-9s %s"
 211             else:
 212                 format_string = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s"
 213             formatted_group = format_string % (
 214                               group_weights[0] * 100.0,
 215                               group_weights[1] * 100.0,
 216                               group_weights[2] * 100.0,
 217                               group_weights[3] * 100.0,
 218                               group_weights[4] * 100.0,
 219                               nickname, fingerprint,
 220                               exit, guard, country, as_number, as_name)
 221             formatted_groups[formatted_group] = group_weights
 222         sorted_groups = sorted(formatted_groups.iteritems(), key=operator.itemgetter(1))
 223         sorted_groups.reverse()
 224         return sorted_groups
 225
 226     def print_groups(self, sorted_groups, count=10, by_country=False, by_as_number=False, short=False, links=False):
 227         output_string = []
 228         if links:
 229             output_string.append("       CW    adv_bw   P_guard  P_middle    P_exit Nickname            Link                                                                           Exit Guard CC AS_num    AS_name"[:short])
 230         else:
 231             output_string.append("       CW    adv_bw   P_guard  P_middle    P_exit Nickname            Fingerprint                              Exit Guard CC AS_num    AS_name"[:short])
 232         if count < 0: count = len(sorted_groups)
 233         for formatted_group, weight in sorted_groups[:count]:
 234             output_string.append(formatted_group[:short])
 235         if len(sorted_groups) > count:
 236             if by_country and by_as_number:
 237                 type = "countries and ASes"
 238             elif by_country:
 239                 type = "countries"
 240             elif by_as_number:
 241                 type = "ASes"
 242             else:
 243                 type = "relays"
 244             other_weights = (0, 0, 0, 0, 0)
 245             for _, weights in sorted_groups[count:]:
 246                 other_weights = tuple(sum(x) for x in zip(other_weights, weights))
 247             output_string.append("%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (%d other %s)" % (
 248                   other_weights[0] * 100.0, other_weights[1] * 100.0,
 249                   other_weights[2] * 100.0, other_weights[3] * 100.0,
 250                   other_weights[4] * 100.0, len(sorted_groups) - count, type))
 251         selection_weights = (0, 0, 0, 0, 0)
 252         for _, weights in sorted_groups:
 253             selection_weights = tuple(sum(x) for x in zip(selection_weights, weights))
 254         if len(sorted_groups) > 1 and selection_weights[0] < 0.999:
 255             output_string.append("%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (total in selection)" % (
 256                   selection_weights[0] * 100.0, selection_weights[1] * 100.0,
 257                   selection_weights[2] * 100.0, selection_weights[3] * 100.0,
 258                   selection_weights[4] * 100.0))
 259         return output_string
 260
 261 def create_option_parser():
 262     parser = OptionParser()
 263     parser.add_option("-d", "--download", action="store_true",
 264                       help="download details.json from Onionoo service")
 265     group = OptionGroup(parser, "Filtering options")
 266     group.add_option("-i", "--inactive", action="store_true", default=False,
 267                      help="include relays in selection that aren't currently running")
 268     group.add_option("-a", "--as", dest="ases", action="append",
 269                      help="select only relays from autonomous system number AS",
 270                      metavar="AS")
 271     group.add_option("-c", "--country", action="append",
 272                      help="select only relays from country with code CC", metavar="CC")
 273     group.add_option("-e", "--exits-only", action="store_true",
 274                      help="select only relays suitable for exit position")
 275     group.add_option("-f", "--family", action="store", type="string", metavar="RELAY",
 276                      help="select family by fingerprint or nickname (for named relays)")
 277     group.add_option("-g", "--guards-only", action="store_true",
 278                      help="select only relays suitable for guard position")
 279     group.add_option("-x", "--fast-exits-only", action="store_true",
 280                      help="select only 100+ Mbit/s exits allowing ports 80, 443, 554, and 1755")
 281     parser.add_option_group(group)
 282     group = OptionGroup(parser, "Grouping options")
 283     group.add_option("-A", "--by-as", action="store_true", default=False,
 284                      help="group relays by AS")
 285     group.add_option("-C", "--by-country", action="store_true", default=False,
 286                      help="group relays by country")
 287     parser.add_option_group(group)
 288     group = OptionGroup(parser, "Display options")
 289     group.add_option("-l", "--links", action="store_true",
 290                      help="display links to the Atlas service instead of fingerprints")
 291     group.add_option("-t", "--top", type="int", default=10, metavar="NUM",
 292                      help="display only the top results (default: %default; -1 for all)")
 293     group.add_option("-s", "--short", action="store_true",
 294                      help="cut the length of the line output at 70 chars")
 295     parser.add_option_group(group)
 296     return parser
 297
 298 def download_details_file():
 299     url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay')
 300     details_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'details.json'), 'w')
 301     details_file.write(url.read())
 302     url.close()
 303     details_file.close()
 304
 305 if '__main__' == __name__:
 306     parser = create_option_parser()
 307     (options, args) = parser.parse_args()
 308     if len(args) > 0:
 309         parser.error("Did not understand positional argument(s), use options instead.")
 310
 311     if options.family and not re.match(r'^[A-F0-9]{40}$', options.family) and not re.match(r'^[A-Za-z0-9]{1,19}$', options.family):
 312         parser.error("Not a valid fingerprint or nickname: %s" % options.family)
 313     if options.download:
 314         download_details_file()
 315         print "Downloaded details.json.  Re-run without --download option."
 316         exit()
 317
 318     if not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'details.json')):
 319         parser.error("Did not find details.json.  Re-run with --download.")
 320
 321     stats = RelayStats(options)
 322     sorted_groups = stats.format_and_sort_groups(stats.relays,
 323                     by_country=options.by_country,
 324                     by_as_number=options.by_as,
 325                     links=options.links)
 326     output_string = stats.print_groups(sorted_groups, options.top,
 327                        by_country=options.by_country,
 328                        by_as_number=options.by_as,
 329                        short=70 if options.short else None,
 330                        links=options.links)
 331     print '\n'.join(output_string)