dev-support/git-jira-release-audit/git_jira_release_audit.py

   1 #!/usr/bin/env python3
   2 #
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Build a database from git commit histories. Can be used to audit git vs. jira. For usage,
  20 # see README.md.
  21 """An application to assist Release Managers with ensuring that histories in Git and fixVersions in
  22 JIRA are in agreement. See README.md for a detailed explanation.
  23 """
  24
  25 import argparse
  26 import csv
  27 import enum
  28 import logging
  29 import pathlib
  30 import re
  31 import sqlite3
  32 import time
  33 import os
  34
  35 import enlighten
  36 import git
  37 import jira
  38
  39
  40 LOG = logging.getLogger(os.path.basename(__file__))
  41
  42
  43 class _DB:
  44     """Manages an instance of Sqlite on behalf of the application.
  45
  46     Args:
  47         db_path (str): Path to the Sqlite database file. ':memory:' for an ephemeral database.
  48         **_kwargs: Convenience for CLI argument parsing. Ignored.
  49
  50     Attributes:
  51         conn (:obj:`sqlite3.db2api.Connection`): The underlying connection object.
  52     """
  53
  54     SQL_LOG = LOG.getChild("sql")
  55
  56     class Action(enum.Enum):
  57         """Describes an action to be taken against the database."""
  58         ADD = 'ADD'
  59         REVERT = 'REVERT'
  60         SKIP = 'SKIP'
  61
  62     def __init__(self, db_path, initialize_db, **_kwargs):
  63         self._conn = sqlite3.connect(db_path)
  64         self._conn.set_trace_callback(_DB.log_query)
  65
  66         if initialize_db:
  67             for table in 'git_commits', 'jira_versions':
  68                 self._conn.execute("DROP TABLE IF EXISTS %s" % table)
  69
  70         self._conn.execute("""
  71         CREATE TABLE IF NOT EXISTS "git_commits"(
  72           jira_id TEXT NOT NULL,
  73           branch TEXT NOT NULL,
  74           git_sha TEXT NOT NULL,
  75           git_tag TEXT,
  76           CONSTRAINT pk PRIMARY KEY (jira_id, branch, git_sha)
  77         );""")
  78         self._conn.execute("""
  79         CREATE TABLE IF NOT EXISTS "jira_versions"(
  80           jira_id TEXT NOT NULL,
  81           fix_version TEXT NOT NULL,
  82           CONSTRAINT pk PRIMARY KEY (jira_id, fix_version)
  83         );""")
  84         self._conn.commit()
  85
  86     def __enter__(self):
  87         return self
  88
  89     def __exit__(self, exc_type, exc_val, exc_tb):
  90         self._conn.close()
  91
  92     @staticmethod
  93     def log_query(query):
  94         _DB.SQL_LOG.debug(re.sub(r'\s+', ' ', query).strip())
  95
  96     @property
  97     def conn(self):
  98         """:obj:`sqlite3.db2api.Connection`: Underlying database handle."""
  99         return self._conn
 100
 101     def apply_commit(self, action, jira_id, branch, git_sha):
 102         """Apply an edit to the commits database.
 103
 104         Args:
 105             action (:obj:`_DB.Action`): The action to execute.
 106             jira_id (str): The applicable Issue ID from JIRA.
 107             branch (str): The name of the git branch from which the commit originates.
 108             git_sha (str): The commit's SHA.
 109         """
 110         if action == _DB.Action.ADD:
 111             self.conn.execute(
 112                 "INSERT INTO git_commits(jira_id, branch, git_sha) VALUES (upper(?),?,?)",
 113                 (jira_id, branch, git_sha))
 114         elif action == _DB.Action.REVERT:
 115             self.conn.execute("""
 116             DELETE FROM git_commits WHERE
 117               jira_id=upper(?)
 118               AND branch=?
 119             """, (jira_id.upper(), branch))
 120
 121     def flush_commits(self):
 122         """Commit any pending changes to the database."""
 123         self.conn.commit()
 124
 125     def apply_git_tag(self, branch, git_tag, git_shas):
 126         """Annotate a commit in the commits database as being a part of the specified release.
 127
 128         Args:
 129             branch (str): The name of the git branch from which the commit originates.
 130             git_tag (str): The first release tag following the commit.
 131             git_shas: The commits' SHAs.
 132         """
 133         self.conn.execute(
 134             (
 135                 f"UPDATE git_commits SET git_tag = ?"
 136                 f"  WHERE branch = ?"
 137                 f"    AND git_sha in ({','.join('?' for _ in git_shas)})"
 138             ),
 139             [git_tag, branch] + git_shas)
 140
 141     def apply_fix_version(self, jira_id, fix_version):
 142         """Annotate a Jira issue in the jira database as being part of the specified release
 143         version.
 144
 145         Args:
 146             jira_id (str): The applicable Issue ID from JIRA.
 147             fix_version (str): The annotated `fixVersion` as seen in JIRA.
 148         """
 149         self.conn.execute("INSERT INTO jira_versions(jira_id, fix_version) VALUES (upper(?),?)",
 150                           (jira_id, fix_version))
 151
 152     def unique_jira_ids_from_git(self):
 153         """Query the commits database for the population of Jira Issue IDs."""
 154         results = self.conn.execute("SELECT distinct jira_id FROM git_commits").fetchall()
 155         return [x[0] for x in results]
 156
 157     def backup(self, target):
 158         """Write a copy of the database to the `target` destination.
 159
 160         Args:
 161             target (str): The backup target, a filesystem path.
 162         """
 163         dst = sqlite3.connect(target)
 164         with dst:
 165             self._conn.backup(dst)
 166         dst.close()
 167
 168
 169 class _RepoReader:
 170     """This class interacts with the git repo, and encapsulates actions specific to HBase's git
 171     history.
 172
 173     Args:
 174         db (:obj:`_DB`): A handle to the database manager.
 175         fallback_actions_path (str): Path to the file containing sha-specific actions
 176             (see README.md).
 177         remote_name (str): The name of the remote to query for branches and histories
 178             (i.e., "origin").
 179         development_branch (str): The name of the branch on which active development occurs
 180             (i.e., "master").
 181         release_line_regexp (str): Filter criteria used to select "release line" branches (such
 182             as "branch-1," "branch-2," &c.).
 183         **_kwargs: Convenience for CLI argument parsing. Ignored.
 184     """
 185     _extract_release_tag_pattern = re.compile(r'^rel/(\d+\.\d+\.\d+)(\^0)?$', re.IGNORECASE)
 186     _skip_patterns = [
 187         re.compile(r'^preparing development version.+', re.IGNORECASE),
 188         re.compile(r'^preparing hbase release.+', re.IGNORECASE),
 189         re.compile(r'^\s*updated? pom.xml version (for|to) .+', re.IGNORECASE),
 190         re.compile(r'^\s*updated? chang', re.IGNORECASE),
 191         re.compile(r'^\s*updated? (book|docs|documentation)', re.IGNORECASE),
 192         re.compile(r'^\s*updating (docs|changes).+', re.IGNORECASE),
 193         re.compile(r'^\s*bump (pom )?versions?', re.IGNORECASE),
 194         re.compile(r'^\s*updated? (version|poms|changes).+', re.IGNORECASE),
 195     ]
 196     _identify_leading_jira_id_pattern = re.compile(r'^[\s\[]*(hbase-\d+)', re.IGNORECASE)
 197     _identify_backport_jira_id_patterns = [
 198         re.compile(r'^backport "(.+)".*', re.IGNORECASE),
 199         re.compile(r'^backport (.+)', re.IGNORECASE),
 200     ]
 201     _identify_revert_jira_id_pattern = re.compile(r'^revert:? "(.+)"', re.IGNORECASE)
 202     _identify_revert_revert_jira_id_pattern = re.compile(
 203         '^revert "revert "(.+)"\\.?"\\.?', re.IGNORECASE)
 204     _identify_amend_jira_id_pattern = re.compile(r'^amend (.+)', re.IGNORECASE)
 205
 206     def __init__(self, db, fallback_actions_path, remote_name, development_branch,
 207                  release_line_regexp, branch_filter_regexp, parse_release_tags, **_kwargs):
 208         self._db = db
 209         self._repo = _RepoReader._open_repo()
 210         self._fallback_actions = _RepoReader._load_fallback_actions(fallback_actions_path)
 211         self._remote_name = remote_name
 212         self._development_branch = development_branch
 213         self._release_line_regexp = release_line_regexp
 214         self._branch_filter_regexp = branch_filter_regexp
 215         self._parse_release_tags = parse_release_tags
 216
 217     @property
 218     def repo(self):
 219         """:obj:`git.repo.base.Repo`: Underlying Repo handle."""
 220         return self._repo
 221
 222     @property
 223     def remote_name(self):
 224         """str: The name of the remote used for querying branches and histories."""
 225         return self._remote_name
 226
 227     @property
 228     def development_branch_ref(self):
 229         """:obj:`git.refs.reference.Reference`: The git branch where active development occurs."""
 230         refs = self.repo.remote(self._remote_name).refs
 231         return [ref for ref in refs
 232                 if ref.name == '%s/%s' % (self._remote_name, self._development_branch)][0]
 233
 234     @property
 235     def release_line_refs(self):
 236         """:obj:`list` of :obj:`git.refs.reference.Reference`: The git branches identified as
 237         "release lines", i.e., "branch-2"."""
 238         refs = self.repo.remote(self._remote_name).refs
 239         pattern = re.compile('%s/%s' % (self._remote_name, self._release_line_regexp))
 240         return [ref for ref in refs if pattern.match(ref.name)]
 241
 242     @property
 243     def release_branch_refs(self):
 244         """:obj:`list` of :obj:`git.refs.reference.Reference`: The git branches identified as
 245         "release branches", i.e., "branch-2.2"."""
 246         refs = self.repo.remote(self._remote_name).refs
 247         release_line_refs = self.release_line_refs
 248         return [ref for ref in refs
 249                 if any([ref.name.startswith(release_line.name + '.')
 250                         for release_line in release_line_refs])]
 251
 252     @staticmethod
 253     def _open_repo():
 254         return git.Repo(pathlib.Path(__file__).parent.absolute(), search_parent_directories=True)
 255
 256     def identify_least_common_commit(self, ref_a, ref_b):
 257         """Given a pair of references, attempt to identify the commit that they have in common,
 258         i.e., the commit at which a "release branch" originates from a "release line" branch.
 259         """
 260         commits = self._repo.merge_base(ref_a, ref_b, "--all")
 261         if commits:
 262             return commits[0]
 263         raise Exception("could not identify merge base between %s, %s" % (ref_a, ref_b))
 264
 265     @staticmethod
 266     def _skip(summary):
 267         return any([p.match(summary) for p in _RepoReader._skip_patterns])
 268
 269     @staticmethod
 270     def _identify_leading_jira_id(summary):
 271         match = _RepoReader._identify_leading_jira_id_pattern.match(summary)
 272         if match:
 273             return match.groups()[0]
 274         return None
 275
 276     @staticmethod
 277     def _identify_backport_jira_id(summary):
 278         for pattern in _RepoReader._identify_backport_jira_id_patterns:
 279             match = pattern.match(summary)
 280             if match:
 281                 return _RepoReader._identify_leading_jira_id(match.groups()[0])
 282         return None
 283
 284     @staticmethod
 285     def _identify_revert_jira_id(summary):
 286         match = _RepoReader._identify_revert_jira_id_pattern.match(summary)
 287         if match:
 288             return _RepoReader._identify_leading_jira_id(match.groups()[0])
 289         return None
 290
 291     @staticmethod
 292     def _identify_revert_revert_jira_id(summary):
 293         match = _RepoReader._identify_revert_revert_jira_id_pattern.match(summary)
 294         if match:
 295             return _RepoReader._identify_leading_jira_id(match.groups()[0])
 296         return None
 297
 298     @staticmethod
 299     def _identify_amend_jira_id(summary):
 300         match = _RepoReader._identify_amend_jira_id_pattern.match(summary)
 301         if match:
 302             return _RepoReader._identify_leading_jira_id(match.groups()[0])
 303         return None
 304
 305     @staticmethod
 306     def _action_jira_id_for(summary):
 307         jira_id = _RepoReader._identify_leading_jira_id(summary)
 308         if jira_id:
 309             return _DB.Action.ADD, jira_id
 310         jira_id = _RepoReader._identify_backport_jira_id(summary)
 311         if jira_id:
 312             return _DB.Action.ADD, jira_id
 313         jira_id = _RepoReader._identify_revert_jira_id(summary)
 314         if jira_id:
 315             return _DB.Action.REVERT, jira_id
 316         jira_id = _RepoReader._identify_revert_revert_jira_id(summary)
 317         if jira_id:
 318             return _DB.Action.ADD, jira_id
 319         jira_id = _RepoReader._identify_amend_jira_id(summary)
 320         if jira_id:
 321             return _DB.Action.ADD, jira_id
 322         return None
 323
 324     def _extract_release_tag(self, commit):
 325         """works for extracting the tag, but need a way to retro-actively tag
 326         commits we've already seen."""
 327         names = self._repo.git.name_rev(commit, tags=True, refs='rel/*')
 328         for name in names.split(' '):
 329             match = _RepoReader._extract_release_tag_pattern.match(name)
 330             if match:
 331                 return match.groups()[0]
 332         return None
 333
 334     def _set_release_tag(self, branch, tag, shas):
 335         self._db.apply_git_tag(branch, tag, shas)
 336         self._db.flush_commits()
 337
 338     def _resolve_ambiguity(self, commit):
 339         if commit.hexsha not in self._fallback_actions:
 340             LOG.warning('Unable to resolve action for %s: %s', commit.hexsha, commit.summary)
 341             return _DB.Action.SKIP, None
 342         action, jira_id = self._fallback_actions[commit.hexsha]
 343         if not jira_id:
 344             jira_id = None
 345         return _DB.Action[action], jira_id
 346
 347     def _row_generator(self, branch, commit):
 348         if _RepoReader._skip(commit.summary):
 349             return None
 350         result = _RepoReader._action_jira_id_for(commit.summary)
 351         if not result:
 352             result = self._resolve_ambiguity(commit)
 353         if not result:
 354             raise Exception('Cannot resolve action for %s: %s' % (commit.hexsha, commit.summary))
 355         action, jira_id = result
 356         return action, jira_id, branch, commit.hexsha
 357
 358     def populate_db_release_branch(self, origin_commit, release_branch):
 359         """List all commits on `release_branch` since `origin_commit`, recording them as
 360         observations in the commits database.
 361
 362         Args:
 363             origin_commit (:obj:`git.objects.commit.Commit`): The sha of the first commit to
 364                 consider.
 365             release_branch (str): The name of the ref whose history is to be parsed.
 366         """
 367         global MANAGER
 368         branch_filter_pattern = re.compile('%s/%s' % (self._remote_name, self._branch_filter_regexp))
 369         if not branch_filter_pattern.match(release_branch):
 370             return
 371
 372         commits = list(self._repo.iter_commits(
 373             "%s...%s" % (origin_commit.hexsha, release_branch), reverse=True))
 374         LOG.info("%s has %d commits since its origin at %s.", release_branch, len(commits),
 375                      origin_commit)
 376         counter = MANAGER.counter(total=len(commits), desc=release_branch, unit='commit')
 377         commits_since_release = list()
 378         cnt = 0
 379         for commit in counter(commits):
 380             row = self._row_generator(release_branch, commit)
 381             if row:
 382                 self._db.apply_commit(*row)
 383             cnt += 1
 384             if cnt % 50 == 0:
 385                 self._db.flush_commits()
 386             commits_since_release.append(commit.hexsha)
 387             if self._parse_release_tags:
 388                 tag = self._extract_release_tag(commit)
 389                 if tag:
 390                     self._set_release_tag(release_branch, tag, commits_since_release)
 391                     commits_since_release = list()
 392         self._db.flush_commits()
 393
 394     @staticmethod
 395     def _load_fallback_actions(file):
 396         result = dict()
 397         if pathlib.Path(file).exists():
 398             with open(file, 'r') as handle:
 399                 reader = csv.DictReader(filter(lambda line: line[0] != '#', handle))
 400                 result = dict()
 401                 for row in reader:
 402                     result[row['hexsha']] = (row['action'], row['jira_id'])
 403         return result
 404
 405
 406 class _JiraReader:
 407     """This class interacts with the Jira instance.
 408
 409     Args:
 410         db (:obj:`_DB`): A handle to the database manager.
 411         jira_url (str): URL of the Jira instance to query.
 412         **_kwargs: Convenience for CLI argument parsing. Ignored.
 413     """
 414     def __init__(self, db, jira_url, **_kwargs):
 415         self._db = db
 416         self.client = jira.JIRA(jira_url)
 417         self.throttle_time_in_sec = 1
 418
 419     def populate_db(self):
 420         """Query Jira for issue IDs found in the commits database, writing them to the jira
 421         database."""
 422         global MANAGER
 423         jira_ids = self._db.unique_jira_ids_from_git()
 424         LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
 425         counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue')
 426         chunk_size = 50
 427         chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)]
 428
 429         cnt = 0
 430         for chunk in chunks:
 431             query = "key in (" + ",".join([("'" + jira_id + "'") for jira_id in chunk]) + ")"
 432             results = self.client.search_issues(jql_str=query, maxResults=chunk_size,
 433                                                 fields='fixVersions')
 434             for result in results:
 435                 jira_id = result.key
 436                 fix_versions = [version.name for version in result.fields.fixVersions]
 437                 for fix_version in fix_versions:
 438                     self._db.apply_fix_version(jira_id, fix_version)
 439                     cnt += 1
 440                     if cnt % 50:
 441                         self._db.flush_commits()
 442             counter.update(incr=len(chunk))
 443             time.sleep(5)
 444         self._db.flush_commits()
 445
 446     def fetch_issues(self, jira_ids):
 447         """Retrieve the specified jira Ids."""
 448         global MANAGER
 449         LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
 450         counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue')
 451         chunk_size = 50
 452         chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)]
 453         ret = list()
 454         for chunk in chunks:
 455             query = "key IN (" + ",".join([("'" + jira_id + "'") for jira_id in chunk]) + ")"\
 456                     + " ORDER BY issuetype ASC, priority DESC, key ASC"
 457             results = self.client.search_issues(
 458                 jql_str=query, maxResults=chunk_size,
 459                 fields='summary,issuetype,priority,resolution,components')
 460             for result in results:
 461                 val = dict()
 462                 val['key'] = result.key
 463                 val['summary'] = result.fields.summary.strip()
 464                 val['priority'] = result.fields.priority.name.strip()
 465                 val['issue_type'] = result.fields.issuetype.name.strip() \
 466                     if result.fields.issuetype else None
 467                 val['resolution'] = result.fields.resolution.name.strip() \
 468                     if result.fields.resolution else None
 469                 val['components'] = [x.name.strip() for x in result.fields.components if x] \
 470                     if result.fields.components else []
 471                 ret.append(val)
 472             counter.update(incr=len(chunk))
 473         return ret
 474
 475
 476 class Auditor:
 477     """This class builds databases from git and Jira, making it possible to audit the two for
 478     discrepancies. At some point, it will provide pre-canned audit queries against those databases.
 479     It is the entrypoint to this application.
 480
 481     Args:
 482         repo_reader (:obj:`_RepoReader`): An instance of the `_RepoReader`.
 483         jira_reader (:obj:`_JiraReader`): An instance of the `JiraReader`.
 484         db (:obj:`_DB`): A handle to the database manager.
 485         **_kwargs: Convenience for CLI argument parsing. Ignored.
 486     """
 487     def __init__(self, repo_reader, jira_reader, db, **_kwargs):
 488         self._repo_reader = repo_reader
 489         self._jira_reader = jira_reader
 490         self._db = db
 491         self._release_line_fix_versions = dict()
 492         for k, v in _kwargs.items():
 493             if k.endswith('_fix_version'):
 494                 release_line = k[:-len('_fix_version')]
 495                 self._release_line_fix_versions[release_line] = v
 496
 497     def populate_db_from_git(self):
 498         """Process the git repository, populating the commits database."""
 499         for release_line in self._repo_reader.release_line_refs:
 500             branch_origin = self._repo_reader.identify_least_common_commit(
 501                 self._repo_reader.development_branch_ref.name, release_line.name)
 502             self._repo_reader.populate_db_release_branch(branch_origin, release_line.name)
 503             for release_branch in self._repo_reader.release_branch_refs:
 504                 if not release_branch.name.startswith(release_line.name):
 505                     continue
 506                 self._repo_reader.populate_db_release_branch(branch_origin, release_branch.name)
 507
 508     def populate_db_from_jira(self):
 509         """Process the Jira issues identified by the commits database, populating the jira
 510         database."""
 511         self._jira_reader.populate_db()
 512
 513     @staticmethod
 514     def _write_report(filename, issues):
 515         with open(filename, 'w') as file:
 516             fieldnames = ['key', 'issue_type', 'priority', 'summary', 'resolution', 'components']
 517             writer = csv.DictWriter(file, fieldnames=fieldnames)
 518             writer.writeheader()
 519             for issue in issues:
 520                 writer.writerow(issue)
 521         LOG.info('generated report at %s', filename)
 522
 523     def report_new_for_release_line(self, release_line):
 524         """Builds a report of the Jira issues that are new on the target release line, not present
 525         on any of the associated release branches. (i.e., on branch-2 but not
 526         branch-{2.0,2.1,...})"""
 527         matches = [x for x in self._repo_reader.release_line_refs
 528                    if x.name == release_line or x.remote_head == release_line]
 529         release_line_ref = next(iter(matches), None)
 530         if not release_line_ref:
 531             LOG.error('release line %s not found. available options are %s.',
 532                           release_line, [x.name for x in self._repo_reader.release_line_refs])
 533             return
 534         cursor = self._db.conn.execute("""
 535         SELECT distinct jira_id FROM git_commits
 536         WHERE branch = ?
 537         EXCEPT SELECT distinct jira_id FROM git_commits
 538           WHERE branch LIKE ?
 539         """, (release_line_ref.name, '%s.%%' % release_line_ref.name))
 540         jira_ids = [x[0] for x in cursor.fetchall()]
 541         issues = self._jira_reader.fetch_issues(jira_ids)
 542         filename = 'new_for_%s.csv' % release_line.replace('/', '-')
 543         Auditor._write_report(filename, issues)
 544
 545     def report_new_for_release_branch(self, release_branch):
 546         """Builds a report of the Jira issues that are new on the target release branch, not present
 547         on any of the previous release branches. (i.e., on branch-2.3 but not
 548         branch-{2.0,2.1,...})"""
 549         matches = [x for x in self._repo_reader.release_branch_refs
 550                    if x.name == release_branch or x.remote_head == release_branch]
 551         release_branch_ref = next(iter(matches), None)
 552         if not release_branch_ref:
 553             LOG.error('release branch %s not found. available options are %s.',
 554                       release_branch, [x.name for x in self._repo_reader.release_branch_refs])
 555             return
 556         previous_branches = [x.name for x in self._repo_reader.release_branch_refs
 557                              if x.remote_head != release_branch_ref.remote_head]
 558         query = (
 559             "SELECT distinct jira_id FROM git_commits"
 560             " WHERE branch = ?"
 561             " EXCEPT SELECT distinct jira_id FROM git_commits"
 562             f" WHERE branch IN ({','.join('?' for _ in previous_branches)})"
 563         )
 564         cursor = self._db.conn.execute(query, tuple([release_branch_ref.name] + previous_branches))
 565         jira_ids = [x[0] for x in cursor.fetchall()]
 566         issues = self._jira_reader.fetch_issues(jira_ids)
 567         filename = 'new_for_%s.csv' % release_branch.replace('/', '-')
 568         Auditor._write_report(filename, issues)
 569
 570     @staticmethod
 571     def _str_to_bool(val):
 572         if not val:
 573             return False
 574         return val.lower() in ['true', 't', 'yes', 'y']
 575
 576     @staticmethod
 577     def _build_first_pass_parser():
 578         parser = argparse.ArgumentParser(add_help=False)
 579         building_group = parser.add_argument_group(title='Building the audit database')
 580         building_group.add_argument(
 581             '--populate-from-git',
 582             help='When true, populate the audit database from the Git repository.',
 583             type=Auditor._str_to_bool,
 584             default=True)
 585         building_group.add_argument(
 586             '--populate-from-jira',
 587             help='When true, populate the audit database from Jira.',
 588             type=Auditor._str_to_bool,
 589             default=True)
 590         building_group.add_argument(
 591             '--db-path',
 592             help='Path to the database file, or leave unspecified for a transient db.',
 593             default='audit.db')
 594         building_group.add_argument(
 595             '--initialize-db',
 596             help='When true, initialize the database tables. This is destructive to the contents'
 597             + ' of an existing database.',
 598             type=Auditor._str_to_bool,
 599             default=False)
 600         report_group = parser.add_argument_group('Generating reports')
 601         report_group.add_argument(
 602             '--report-new-for-release-line',
 603             help=Auditor.report_new_for_release_line.__doc__,
 604             type=str,
 605             default=None)
 606         report_group.add_argument(
 607             '--report-new-for-release-branch',
 608             help=Auditor.report_new_for_release_branch.__doc__,
 609             type=str,
 610             default=None)
 611         git_repo_group = parser.add_argument_group('Interactions with the Git repo')
 612         git_repo_group.add_argument(
 613             '--git-repo-path',
 614             help='Path to the git repo, or leave unspecified to infer from the current'
 615             + ' file\'s path.',
 616             default=__file__)
 617         git_repo_group.add_argument(
 618             '--remote-name',
 619             help='The name of the git remote to use when identifying branches.'
 620             + ' Default: \'origin\'',
 621             default='origin')
 622         git_repo_group.add_argument(
 623             '--development-branch',
 624             help='The name of the branch from which all release lines originate.'
 625             + ' Default: \'master\'',
 626             default='master')
 627         git_repo_group.add_argument(
 628             '--development-branch-fix-version',
 629             help='The Jira fixVersion used to indicate an issue is committed to the development'
 630             + ' branch.',
 631             default='3.0.0')
 632         git_repo_group.add_argument(
 633             '--release-line-regexp',
 634             help='A regexp used to identify release lines.',
 635             default=r'branch-\d+$')
 636         git_repo_group.add_argument(
 637             '--parse-release-tags',
 638             help='When true, look for release tags and annotate commits according to their release'
 639             + ' version. An Expensive calculation, disabled by default.',
 640             type=Auditor._str_to_bool,
 641             default=False)
 642         git_repo_group.add_argument(
 643             '--fallback-actions-path',
 644             help='Path to a file containing _DB.Actions applicable to specific git shas.',
 645             default='fallback_actions.csv')
 646         git_repo_group.add_argument(
 647             '--branch-filter-regexp',
 648             help='Limit repo parsing to branch names that match this filter expression.',
 649             default=r'.*')
 650         jira_group = parser.add_argument_group('Interactions with Jira')
 651         jira_group.add_argument(
 652             '--jira-url',
 653             help='A URL locating the target JIRA instance.',
 654             default='https://issues.apache.org/jira')
 655         return parser, git_repo_group
 656
 657     @staticmethod
 658     def _build_second_pass_parser(repo_reader, parent_parser, git_repo_group):
 659         for release_line in repo_reader.release_line_refs:
 660             name = release_line.name
 661             git_repo_group.add_argument(
 662                 '--%s-fix-version' % name[len(repo_reader.remote_name) + 1:],
 663                 help='The Jira fixVersion used to indicate an issue is committed to the specified '
 664                 + 'release line branch',
 665                 required=True)
 666         return argparse.ArgumentParser(
 667             parents=[parent_parser],
 668             formatter_class=argparse.ArgumentDefaultsHelpFormatter
 669         )
 670
 671
 672 MANAGER = None
 673
 674
 675 def main():
 676     global MANAGER
 677
 678     logging.basicConfig(level=logging.INFO)
 679     first_pass_parser, git_repo_group = Auditor._build_first_pass_parser()
 680     first_pass_args, extras = first_pass_parser.parse_known_args()
 681     first_pass_args_dict = vars(first_pass_args)
 682     with _DB(**first_pass_args_dict) as db:
 683         repo_reader = _RepoReader(db, **first_pass_args_dict)
 684         jira_reader = _JiraReader(db, **first_pass_args_dict)
 685         second_pass_parser = Auditor._build_second_pass_parser(
 686             repo_reader, first_pass_parser, git_repo_group)
 687         second_pass_args = second_pass_parser.parse_args(extras, first_pass_args)
 688         second_pass_args_dict = vars(second_pass_args)
 689         auditor = Auditor(repo_reader, jira_reader, db, **second_pass_args_dict)
 690         with enlighten.get_manager() as MANAGER:
 691             if second_pass_args.populate_from_git:
 692                 auditor.populate_db_from_git()
 693             if second_pass_args.populate_from_jira:
 694                 auditor.populate_db_from_jira()
 695             if second_pass_args.report_new_for_release_line:
 696                 release_line = second_pass_args.report_new_for_release_line
 697                 auditor.report_new_for_release_line(release_line)
 698             if second_pass_args.report_new_for_release_branch:
 699                 release_branch = second_pass_args.report_new_for_release_branch
 700                 auditor.report_new_for_release_branch(release_branch)
 701
 702
 703 if __name__ == '__main__':
 704     main()