stem/manual.py

   1 # Copyright 2015-2020, Damian Johnson and The Tor Project
   2 # See LICENSE for licensing information
   3
   4 """
   5 Information available about Tor from `its manual
   6 <https://www.torproject.org/docs/tor-manual.html.en>`_. This provides three
   7 methods of getting this information...
   8
   9 * :func:`~stem.manual.Manual.from_cache` provides manual content bundled with
  10   Stem. This is the fastest and most reliable method but only as up-to-date as
  11   Stem's release.
  12
  13 * :func:`~stem.manual.Manual.from_man` reads Tor's local man page for
  14   information about it.
  15
  16 * :func:`~stem.manual.Manual.from_remote` fetches the latest manual information
  17   remotely. This is the slowest and least reliable method but provides the most
  18   recent information about Tor.
  19
  20 Manual information includes arguments, signals, and probably most usefully the
  21 torrc configuration options. For example, say we want a little script that told
  22 us what our torrc options do...
  23
  24 .. literalinclude:: /_static/example/manual_config_options.py
  25    :language: python
  26
  27 |
  28
  29 .. image:: /_static/manual_output.png
  30
  31 |
  32
  33 **Module Overview:**
  34
  35 ::
  36
  37   query - performs a query on our cached sqlite manual information
  38   is_important - Indicates if a configuration option is of particularly common importance.
  39   download_man_page - Downloads tor's latest man page.
  40
  41   Manual - Information about Tor available from its manual.
  42    | |- from_cache - Provides manual information cached with Stem.
  43    | |- from_man - Retrieves manual information from its man page.
  44    | +- from_remote - Retrieves manual information remotely from tor's latest manual.
  45    |
  46    +- save - writes the manual contents to a given location
  47
  48 .. versionadded:: 1.5.0
  49 """
  50
  51 import collections
  52 import functools
  53 import os
  54 import shutil
  55 import sys
  56 import tempfile
  57 import urllib.request
  58
  59 import stem
  60 import stem.util
  61 import stem.util.conf
  62 import stem.util.enum
  63 import stem.util.log
  64 import stem.util.str_tools
  65 import stem.util.system
  66
  67 from typing import Any, BinaryIO, Dict, List, Mapping, Optional, Sequence, Tuple, Union
  68
  69 Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'CIRCUIT_TIMEOUT', 'DORMANT_MODE', 'NODE_SELECTION', 'RELAY', 'STATISTIC', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN')
  70 GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/man/tor.1.txt'
  71 CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite')
  72 DATABASE = None  # cache database connections
  73 HAS_ENCODING_ARG = not stem.util.system.is_mac() and not stem.util.system.is_bsd() and not stem.util.system.is_slackware()
  74
  75 SCHEMA_VERSION = 1  # version of our scheme, bump this if you change the following
  76 SCHEMA = (
  77   'CREATE TABLE schema(version INTEGER)',
  78   'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION,
  79
  80   'CREATE TABLE metadata(name TEXT, synopsis TEXT, description TEXT, man_commit TEXT, stem_commit TEXT)',
  81   'CREATE TABLE commandline(name TEXT PRIMARY KEY, description TEXT)',
  82   'CREATE TABLE signals(name TEXT PRIMARY KEY, description TEXT)',
  83   'CREATE TABLE files(name TEXT PRIMARY KEY, description TEXT)',
  84   'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position INTEGER)',
  85 )
  86
  87 CATEGORY_SECTIONS = collections.OrderedDict((
  88   ('GENERAL OPTIONS', Category.GENERAL),
  89   ('CLIENT OPTIONS', Category.CLIENT),
  90   ('CIRCUIT TIMEOUT OPTIONS', Category.CIRCUIT_TIMEOUT),
  91   ('DORMANT MODE OPTIONS', Category.DORMANT_MODE),
  92   ('NODE SELECTION OPTIONS', Category.NODE_SELECTION),
  93   ('SERVER OPTIONS', Category.RELAY),
  94   ('STATISTICS OPTIONS', Category.STATISTIC),
  95   ('DIRECTORY SERVER OPTIONS', Category.DIRECTORY),
  96   ('DIRECTORY AUTHORITY SERVER OPTIONS', Category.AUTHORITY),
  97   ('HIDDEN SERVICE OPTIONS', Category.HIDDEN_SERVICE),
  98   ('DENIAL OF SERVICE MITIGATION OPTIONS', Category.DENIAL_OF_SERVICE),
  99   ('TESTING NETWORK OPTIONS', Category.TESTING),
 100 ))
 101
 102
 103 class SchemaMismatch(OSError):
 104   """
 105   Database schema doesn't match what Stem supports.
 106
 107   .. versionadded:: 1.6.0
 108
 109   :var int database_schema: schema of the database
 110   :var tuple supported_schemas: schemas library supports
 111   """
 112
 113   def __init__(self, message: str, database_schema: int, supported_schemas: Tuple[int]) -> None:
 114     super(SchemaMismatch, self).__init__(message)
 115     self.database_schema = database_schema
 116     self.supported_schemas = supported_schemas
 117
 118
 119 def query(query: str, *param: str) -> 'sqlite3.Cursor':  # type: ignore
 120   """
 121   Performs the given query on our sqlite manual cache. This database should
 122   be treated as being read-only. File permissions generally enforce this, and
 123   in the future will be enforced by this function as well.
 124
 125   ::
 126
 127     >>> import stem.manual
 128     >>> print(stem.manual.query('SELECT description FROM torrc WHERE key=?', 'CONTROLSOCKET').fetchone()[0])
 129     Like ControlPort, but listens on a Unix domain socket, rather than a TCP socket.  0 disables ControlSocket. (Unix and Unix-like systems only.) (Default: 0)
 130
 131   .. versionadded:: 1.6.0
 132
 133   :param query: query to run on the cache
 134   :param param: query parameters
 135
 136   :returns: :class:`sqlite3.Cursor` with the query results
 137
 138   :raises:
 139     * **ImportError** if the sqlite3 module is unavailable
 140     * **sqlite3.OperationalError** if query fails
 141   """
 142
 143   try:
 144     import sqlite3
 145   except (ImportError, ModuleNotFoundError):
 146     raise ImportError('Querying requires the sqlite3 module')
 147
 148   # The only reason to explicitly close the sqlite connection is to ensure
 149   # transactions are committed. Since we're only using read-only access this
 150   # doesn't matter, and can allow interpreter shutdown to do the needful.
 151
 152   global DATABASE
 153
 154   if DATABASE is None:
 155     DATABASE = sqlite3.connect('file:%s?mode=ro' % CACHE_PATH, uri=True)
 156
 157   return DATABASE.execute(query, param)
 158
 159
 160 class ConfigOption(object):
 161   """
 162   Tor configuration attribute found in its torrc.
 163
 164   :var str name: name of the configuration option
 165   :var stem.manual.Category category: category the config option was listed
 166     under, this is Category.UNKNOWN if we didn't recognize the category
 167   :var str usage: arguments accepted by the option
 168   :var str summary: brief description of what the option does
 169   :var str description: longer manual description with details
 170   """
 171
 172   def __init__(self, name: str, category: 'stem.manual.Category' = Category.UNKNOWN, usage: str = '', summary: str = '', description: str = '') -> None:
 173     self.name = name
 174     self.category = category
 175     self.usage = usage
 176     self.summary = summary
 177     self.description = description
 178
 179   def __hash__(self) -> int:
 180     return stem.util._hash_attr(self, 'name', 'category', 'usage', 'summary', 'description', cache = True)
 181
 182   def __eq__(self, other: Any) -> bool:
 183     return hash(self) == hash(other) if isinstance(other, ConfigOption) else False
 184
 185   def __ne__(self, other: Any) -> bool:
 186     return not self == other
 187
 188
 189 @functools.lru_cache()
 190 def _config(lowercase: bool = True) -> Dict[str, Union[List[str], str]]:
 191   """
 192   Provides a dictionary for our settings.cfg. This has a couple categories...
 193
 194     * manual.important (list) - configuration options considered to be important
 195     * manual.summary.* (str) - summary descriptions of config options
 196
 197   :param lowercase: uses lowercase keys if **True** to allow for case
 198     insensitive lookups
 199   """
 200
 201   config = stem.util.conf.Config()
 202   config_path = os.path.join(os.path.dirname(__file__), 'settings.cfg')
 203
 204   try:
 205     config.load(config_path)
 206     config_dict = dict([(key.lower() if lowercase else key, config.get_value(key)) for key in config.keys() if key.startswith('manual.summary.')])
 207     config_dict['manual.important'] = [name.lower() if lowercase else name for name in config.get_value('manual.important', [], multiple = True)]
 208     return config_dict
 209   except Exception as exc:
 210     stem.util.log.warn("BUG: stem failed to load its internal manual information from '%s': %s" % (config_path, exc))
 211     return {}
 212
 213
 214 def _manual_differences(previous_manual: 'stem.manual.Manual', new_manual: 'stem.manual.Manual') -> str:
 215   """
 216   Provides a description of how two manuals differ.
 217   """
 218
 219   lines = []
 220
 221   for attr in ('name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options'):
 222     previous_attr = getattr(previous_manual, attr)
 223     new_attr = getattr(new_manual, attr)
 224
 225     if previous_attr != new_attr:
 226       lines.append("* Manual's %s attribute changed\n" % attr)
 227
 228       if attr in ('name', 'synopsis', 'description'):
 229         lines.append('  Previously...\n\n%s\n' % previous_attr)
 230         lines.append('  Updating to...\n\n%s' % new_attr)
 231       elif attr == 'config_options':
 232         for config_name, config_attr in new_attr.items():
 233           previous = previous_attr.get(config_name)
 234
 235           if previous is None:
 236             lines.append('  adding new config option => %s' % config_name)
 237           elif config_attr != previous:
 238             for attr in ('name', 'category', 'usage', 'summary', 'description'):
 239               if getattr(config_attr, attr) != getattr(previous, attr):
 240                 lines.append('  modified %s (%s) => %s' % (config_name, attr, getattr(config_attr, attr)))
 241
 242         for config_name in set(previous_attr.keys()).difference(new_attr.keys()):
 243           lines.append('  removing config option => %s' % config_name)
 244       else:
 245         added_items = set(new_attr.items()).difference(previous_attr.items())
 246         removed_items = set(previous_attr.items()).difference(new_attr.items())
 247
 248         for added_item in added_items:
 249           lines.append('  adding %s => %s' % added_item)
 250
 251         for removed_item in removed_items:
 252           lines.append('  removing %s => %s' % removed_item)
 253
 254       lines.append('\n')
 255
 256   return '\n'.join(lines)
 257
 258
 259 def is_important(option: str) -> bool:
 260   """
 261   Indicates if a configuration option of particularly common importance or not.
 262
 263   :param option: tor configuration option to check
 264
 265   :returns: **bool** that's **True** if this is an important option and
 266     **False** otherwise
 267   """
 268
 269   return option.lower() in _config()['manual.important']
 270
 271
 272 def download_man_page(path: Optional[str] = None, file_handle: Optional[BinaryIO] = None, url: str = GITWEB_MANUAL_URL, timeout: int = 20) -> None:
 273   """
 274   Downloads tor's latest man page from `gitweb.torproject.org
 275   <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. This method is
 276   both slow and unreliable - please see the warnings on
 277   :func:`~stem.manual.Manual.from_remote`.
 278
 279   :param path: path to save tor's man page to
 280   :param file_handle: file handler to save tor's man page to
 281   :param url: url to download tor's asciidoc manual from
 282   :param timeout: seconds to wait before timing out the request
 283
 284   :raises: **OSError** if unable to retrieve the manual
 285   """
 286
 287   if not path and not file_handle:
 288     raise ValueError("Either the path or file_handle we're saving to must be provided")
 289   elif not stem.util.system.is_available('a2x'):
 290     raise OSError('We require a2x from asciidoc to provide a man page')
 291
 292   with tempfile.TemporaryDirectory() as dirpath:
 293     asciidoc_path = os.path.join(dirpath, 'tor.1.txt')
 294     manual_path = os.path.join(dirpath, 'tor.1')
 295
 296     try:
 297       with open(asciidoc_path, 'wb') as asciidoc_file:
 298         request = urllib.request.urlopen(url, timeout = timeout)
 299         shutil.copyfileobj(request, asciidoc_file)
 300     except:
 301       exc, stacktrace = sys.exc_info()[1:3]
 302       message = "Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc)
 303       raise stem.DownloadFailed(url, exc, stacktrace, message)
 304
 305     try:
 306       stem.util.system.call('a2x -f manpage %s' % asciidoc_path)
 307
 308       if not os.path.exists(manual_path):
 309         raise OSError('no man page was generated')
 310     except stem.util.system.CallError as exc:
 311       raise OSError("Unable to run '%s': %s" % (exc.command, stem.util.str_tools._to_unicode(exc.stderr)))
 312
 313     if path:
 314       try:
 315         path_dir = os.path.dirname(path)
 316
 317         if not os.path.exists(path_dir):
 318           os.makedirs(path_dir)
 319
 320         shutil.copyfile(manual_path, path)
 321       except OSError as exc:
 322         raise OSError(exc)
 323
 324     if file_handle:
 325       with open(manual_path, 'rb') as manual_file:
 326         shutil.copyfileobj(manual_file, file_handle)
 327         file_handle.flush()
 328
 329
 330 class Manual(object):
 331   """
 332   Parsed tor man page. Tor makes no guarantees about its man page format so
 333   this may not always be compatible. If not you can use the cached manual
 334   information stored with Stem.
 335
 336   This does not include every bit of information from the tor manual. For
 337   instance, I've excluded the 'THE CONFIGURATION FILE FORMAT' section. If
 338   there's a part you'd find useful then file an issue and we can add it.
 339
 340   :var str name: brief description of the tor command
 341   :var str synopsis: brief tor command usage
 342   :var str description: general description of what tor does
 343
 344   :var collections.OrderedDict commandline_options: mapping of commandline arguments to their descripton
 345   :var collections.OrderedDict signals: mapping of signals tor accepts to their description
 346   :var collections.OrderedDict files: mapping of file paths to their description
 347
 348   :var collections.OrderedDict config_options: :class:`~stem.manual.ConfigOption` tuples for tor configuration options
 349
 350   :var str man_commit: latest tor commit editing the man page when this
 351     information was cached
 352   :var str stem_commit: stem commit to cache this manual information
 353   """
 354
 355   def __init__(self, name: str, synopsis: str, description: str, commandline_options: Mapping[str, str], signals: Mapping[str, str], files: Mapping[str, str], config_options: Mapping[str, 'stem.manual.ConfigOption']) -> None:
 356     self.name = name
 357     self.synopsis = synopsis
 358     self.description = description
 359     self.commandline_options = collections.OrderedDict(commandline_options)
 360     self.signals = collections.OrderedDict(signals)
 361     self.files = collections.OrderedDict(files)
 362     self.config_options = collections.OrderedDict(config_options)
 363     self.man_commit = None
 364     self.stem_commit = None
 365     self.schema = None
 366
 367   @staticmethod
 368   def from_cache(path: Optional[str] = None) -> 'stem.manual.Manual':
 369     """
 370     Provides manual information cached with Stem. Unlike
 371     :func:`~stem.manual.Manual.from_man` and
 372     :func:`~stem.manual.Manual.from_remote` this doesn't have any system
 373     requirements, and is faster too. Only drawback is that this manual
 374     content is only as up to date as the Stem release we're using.
 375
 376     .. versionchanged:: 1.6.0
 377        Added support for sqlite cache. Support for
 378        :class:`~stem.util.conf.Config` caches will be dropped in Stem 2.x.
 379
 380     :param path: cached manual content to read, if not provided this uses
 381       the bundled manual information
 382
 383     :returns: :class:`~stem.manual.Manual` with our bundled manual information
 384
 385     :raises:
 386       * **ImportError** if cache is sqlite and the sqlite3 module is
 387         unavailable
 388       * **OSError** if a **path** was provided and we were unable to read
 389         it or the schema is out of date
 390     """
 391
 392     try:
 393       import sqlite3
 394     except (ImportError, ModuleNotFoundError):
 395       raise ImportError('Reading a sqlite cache requires the sqlite3 module')
 396
 397     if path is None:
 398       path = CACHE_PATH
 399
 400     if not os.path.exists(path):
 401       raise OSError("%s doesn't exist" % path)
 402
 403     with sqlite3.connect(path) as conn:
 404       try:
 405         schema = conn.execute('SELECT version FROM schema').fetchone()[0]
 406
 407         if schema != SCHEMA_VERSION:
 408           raise SchemaMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, (SCHEMA_VERSION,))
 409
 410         name, synopsis, description, man_commit, stem_commit = conn.execute('SELECT name, synopsis, description, man_commit, stem_commit FROM metadata').fetchone()
 411       except sqlite3.OperationalError as exc:
 412         raise OSError('Failed to read database metadata from %s: %s' % (path, exc))
 413
 414       commandline = dict(conn.execute('SELECT name, description FROM commandline').fetchall())
 415       signals = dict(conn.execute('SELECT name, description FROM signals').fetchall())
 416       files = dict(conn.execute('SELECT name, description FROM files').fetchall())
 417
 418       config_options = collections.OrderedDict()
 419
 420       for entry in conn.execute('SELECT name, category, usage, summary, description FROM torrc ORDER BY position').fetchall():
 421         option, category, usage, summary, option_description = entry
 422         config_options[option] = ConfigOption(option, category, usage, summary, option_description)
 423
 424       manual = Manual(name, synopsis, description, commandline, signals, files, config_options)
 425       manual.man_commit = man_commit
 426       manual.stem_commit = stem_commit
 427       manual.schema = schema
 428
 429       return manual
 430
 431   @staticmethod
 432   def from_man(man_path: str = 'tor') -> 'stem.manual.Manual':
 433     """
 434     Reads and parses a given man page.
 435
 436     On OSX the man command doesn't have an '--encoding' argument so its results
 437     may not quite match other platforms. For instance, it normalizes long
 438     dashes into '--'.
 439
 440     :param man_path: path argument for 'man', for example you might want
 441       '/path/to/tor/doc/tor.1' to read from tor's git repository
 442
 443     :returns: :class:`~stem.manual.Manual` for the system's man page
 444
 445     :raises: **OSError** if unable to retrieve the manual
 446     """
 447
 448     man_cmd = 'man %s -P cat %s' % ('--encoding=ascii' if HAS_ENCODING_ARG else '', man_path)
 449
 450     try:
 451       man_output = stem.util.system.call(man_cmd, env = {'MANWIDTH': '10000000'})
 452     except OSError as exc:
 453       raise OSError("Unable to run '%s': %s" % (man_cmd, exc))
 454
 455     categories = _get_categories(man_output)
 456     config_options = collections.OrderedDict()  # type: collections.OrderedDict[str, stem.manual.ConfigOption]
 457
 458     for category_header, category_enum in CATEGORY_SECTIONS.items():
 459       _add_config_options(config_options, category_enum, categories.get(category_header, []))
 460
 461     for category in categories:
 462       if category.endswith(' OPTIONS') and category not in CATEGORY_SECTIONS and category not in ('COMMAND-LINE OPTIONS', 'NON-PERSISTENT OPTIONS'):
 463         _add_config_options(config_options, Category.UNKNOWN, categories.get(category, []))
 464
 465     return Manual(
 466       _join_lines(categories.get('NAME', [])),
 467       _join_lines(categories.get('SYNOPSIS', [])),
 468       _join_lines(categories.get('DESCRIPTION', [])),
 469       _get_indented_descriptions(categories.get('COMMAND-LINE OPTIONS', [])),
 470       _get_indented_descriptions(categories.get('SIGNALS', [])),
 471       _get_indented_descriptions(categories.get('FILES', [])),
 472       config_options,
 473     )
 474
 475   @staticmethod
 476   def from_remote(timeout: int = 60) -> 'stem.manual.Manual':
 477     """
 478     Reads and parses the latest tor man page `from gitweb.torproject.org
 479     <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. Note that
 480     while convenient, this reliance on GitWeb means you should alway call with
 481     a fallback, such as...
 482
 483     ::
 484
 485       try:
 486         manual = stem.manual.from_remote()
 487       except OSError:
 488         manual = stem.manual.from_cache()
 489
 490     In addition to our GitWeb dependency this requires 'a2x' which is part of
 491     `asciidoc <http://asciidoc.org/INSTALL.html>`_ and... isn't quick.
 492     Personally this takes ~7.41s, breaking down for me as follows...
 493
 494       * 1.67s to download tor.1.txt
 495       * 5.57s to convert the asciidoc to a man page
 496       * 0.17s for stem to read and parse the manual
 497
 498     :param timeout: seconds to wait before timing out the request
 499
 500     :returns: latest :class:`~stem.manual.Manual` available for tor
 501
 502     :raises: **OSError** if unable to retrieve the manual
 503     """
 504
 505     with tempfile.NamedTemporaryFile() as tmp:
 506       download_man_page(file_handle = tmp, timeout = timeout)  # type: ignore
 507       return Manual.from_man(tmp.name)
 508
 509   def save(self, path: str) -> None:
 510     """
 511     Persists the manual content to a given location.
 512
 513     .. versionchanged:: 1.6.0
 514        Added support for sqlite cache. Support for
 515        :class:`~stem.util.conf.Config` caches will be dropped in Stem 2.x.
 516
 517     :param path: path to save our manual content to
 518
 519     :raises:
 520       * **ImportError** if saving as sqlite and the sqlite3 module is
 521         unavailable
 522       * **OSError** if unsuccessful
 523     """
 524
 525     try:
 526       import sqlite3
 527     except (ImportError, ModuleNotFoundError):
 528       raise ImportError('Saving a sqlite cache requires the sqlite3 module')
 529
 530     tmp_path = path + '.new'
 531
 532     if os.path.exists(tmp_path):
 533       os.remove(tmp_path)
 534
 535     with sqlite3.connect(tmp_path) as conn:
 536       for cmd in SCHEMA:
 537         conn.execute(cmd)
 538
 539       conn.execute('INSERT INTO metadata(name, synopsis, description, man_commit, stem_commit) VALUES (?,?,?,?,?)', (self.name, self.synopsis, self.description, self.man_commit, self.stem_commit))
 540
 541       for k, v in self.commandline_options.items():
 542         conn.execute('INSERT INTO commandline(name, description) VALUES (?,?)', (k, v))
 543
 544       for k, v in self.signals.items():
 545         conn.execute('INSERT INTO signals(name, description) VALUES (?,?)', (k, v))
 546
 547       for k, v in self.files.items():
 548         conn.execute('INSERT INTO files(name, description) VALUES (?,?)', (k, v))
 549
 550       for i, v in enumerate(self.config_options.values()):
 551         conn.execute('INSERT INTO torrc(key, name, category, usage, summary, description, position) VALUES (?,?,?,?,?,?,?)', (v.name.upper(), v.name, v.category, v.usage, v.summary, v.description, i))
 552
 553     if os.path.exists(path):
 554       os.remove(path)
 555
 556     os.rename(tmp_path, path)
 557
 558   def __hash__(self) -> int:
 559     return stem.util._hash_attr(self, 'name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options', cache = True)
 560
 561   def __eq__(self, other: Any) -> bool:
 562     return hash(self) == hash(other) if isinstance(other, Manual) else False
 563
 564   def __ne__(self, other: Any) -> bool:
 565     return not self == other
 566
 567
 568 def _get_categories(content: Sequence[str]) -> Dict[str, List[str]]:
 569   """
 570   The man page is headers followed by an indented section. First pass gets
 571   the mapping of category titles to their lines.
 572   """
 573
 574   # skip header and footer lines
 575
 576   if content and 'TOR(1)' in content[0]:
 577     content = content[1:]
 578
 579   if content and content[-1].startswith('Tor'):
 580     content = content[:-1]
 581
 582   categories = collections.OrderedDict()
 583   category = None
 584   lines = []  # type: List[str]
 585
 586   for line in content:
 587     # replace non-ascii characters
 588     #
 589     #   \u2019 - smart single quote
 590     #   \u2014 - extra long dash
 591     #   \xb7 - centered dot
 592
 593     line = line.replace(chr(0x2019), "'").replace(chr(0x2014), '-').replace(chr(0xb7), '*')
 594
 595     if line and not line.startswith(' '):
 596       if category:
 597         if lines and lines[-1] == '':
 598           lines = lines[:-1]  # sections end with an extra empty line
 599
 600         categories[category] = lines
 601
 602       category, lines = line.strip(), []
 603     else:
 604       if line.startswith('       '):
 605         line = line[7:]  # contents of a section have a seven space indentation
 606
 607       lines.append(line)
 608
 609   if category:
 610     categories[category] = lines
 611
 612   return categories
 613
 614
 615 def _get_indented_descriptions(lines: Sequence[str]) -> Dict[str, str]:
 616   """
 617   Parses the commandline argument and signal sections. These are options
 618   followed by an indented description. For example...
 619
 620   ::
 621
 622     -f FILE
 623         Specify a new configuration file to contain further Tor configuration
 624         options OR pass - to make Tor read its configuration from standard
 625         input. (Default: /usr/local/etc/tor/torrc, or $HOME/.torrc if that file
 626         is not found)
 627
 628   There can be additional paragraphs not related to any particular argument but
 629   ignoring those.
 630   """
 631
 632   options = collections.OrderedDict()  # type: collections.OrderedDict[str, List[str]]
 633   last_arg = None
 634
 635   for line in lines:
 636     if line == '    Note':
 637       last_arg = None  # manual has several indented 'Note' blocks
 638     elif line and not line.startswith(' '):
 639       options[line], last_arg = [], line
 640     elif last_arg and line.startswith('    '):
 641       options[last_arg].append(line[4:])
 642
 643   return dict([(arg, ' '.join(desc_lines)) for arg, desc_lines in options.items() if desc_lines])
 644
 645
 646 def _add_config_options(config_options: Dict[str, 'stem.manual.ConfigOption'], category: str, lines: Sequence[str]) -> None:
 647   """
 648   Parses a section of tor configuration options. These have usage information,
 649   followed by an indented description. For instance...
 650
 651   ::
 652
 653     ConnLimit NUM
 654         The minimum number of file descriptors that must be available to the
 655         Tor process before it will start. Tor will ask the OS for as many file
 656         descriptors as the OS will allow (you can find this by "ulimit -H -n").
 657         If this number is less than ConnLimit, then Tor will refuse to start.
 658
 659
 660         You probably don't need to adjust this. It has no effect on Windows
 661         since that platform lacks getrlimit(). (Default: 1000)
 662   """
 663
 664   def add_option(title: str, description: List[str]) -> None:
 665     if 'PER INSTANCE OPTIONS' in title:
 666       return  # skip, unfortunately amid the options
 667
 668     if ', ' in title:
 669       # Line actually had multiple options with the same description. For
 670       # example...
 671       #
 672       #   AlternateBridgeAuthority [nickname], AlternateDirAuthority [nickname]
 673
 674       for subtitle in title.split(', '):
 675         add_option(subtitle, description)
 676     else:
 677       name, usage = title.split(' ', 1) if ' ' in title else (title, '')
 678       summary = str(_config().get('manual.summary.%s' % name.lower(), ''))
 679       config_options[name] = ConfigOption(name, category, usage, summary, _join_lines(description).strip())
 680
 681   # Remove the section's description by finding the sentence the section
 682   # ends with.
 683
 684   end_indices = [i for (i, line) in enumerate(lines) if ('The following options' in line or 'PER SERVICE OPTIONS' in line)]
 685
 686   if end_indices:
 687     lines = lines[max(end_indices):]  # trim to the description paragrah
 688     lines = lines[lines.index(''):]  # drop the paragraph
 689
 690   last_title = None
 691   description = []  # type: List[str]
 692
 693   for line in lines:
 694     if line and not line.startswith(' '):
 695       if last_title:
 696         add_option(last_title, description)
 697
 698       last_title, description = line, []
 699     else:
 700       if line.startswith('    '):
 701         line = line[4:]
 702
 703       description.append(line)
 704
 705   if last_title:
 706     add_option(last_title, description)
 707
 708
 709 def _join_lines(lines: Sequence[str]) -> str:
 710   """
 711   Simple join, except we want empty lines to still provide a newline.
 712   """
 713
 714   result = []  # type: List[str]
 715
 716   for line in lines:
 717     if not line:
 718       if result and result[-1] != '\n':
 719         result.append('\n')
 720     else:
 721       result.append(line + '\n')
 722
 723   return ''.join(result).strip()