tools/bass-o-matic

   1 #!/usr/bin/python3.9
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22 # Copyright (c) 2010, Oracle and/or it's affiliates.  All rights reserved.
  23 #
  24 #
  25 # bass-o-matic.py
  26 #  A simple program to enumerate components in the userland gate and report
  27 #  on dependency related information.
  28 #
  29
  30 import os
  31 import sys
  32 import re
  33 import glob
  34 import subprocess
  35 import argparse
  36 import logging
  37 import json
  38 import multiprocessing
  39
  40 try:
  41     from scandir import walk
  42 except ImportError:
  43     from os import walk
  44
  45 from bass.component import Component
  46
  47 logger = logging.getLogger('bass-o-matic')
  48
  49 # Locate SCM directories containing Userland components by searching from
  50 # from a supplied top of tree for .p5m files.  Once a .p5m file is located,
  51 # that directory is added to the list and no children are searched.
  52 def FindComponentPaths(path, debug=False, subdir='components',
  53                        incremental=False, begin_commit=None, end_commit=None):
  54     expression = re.compile(r'.+\.p5m$', re.IGNORECASE)
  55
  56     paths = []
  57
  58     if debug:
  59         logger.debug('searching %s for component directories', path)
  60
  61     workspace_path = os.path.join(path, subdir)
  62
  63     if incremental:
  64         cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:',
  65                '..'.join([begin_commit, end_commit])]
  66
  67         proc = subprocess.Popen(cmd,
  68                                 stdout=subprocess.PIPE,
  69                                 stderr=subprocess.PIPE,
  70                                 cwd=workspace_path,
  71                                 universal_newlines=True
  72                                 )
  73         stdout, stderr = proc.communicate()
  74         if debug:
  75             if proc.returncode != 0:
  76                 logger.debug('exit: %d, %s', proc.returncode, stderr)
  77
  78         for line in stdout.splitlines():
  79             line = line.rstrip()
  80             # git output might contain empty lines, so we skip them.
  81             if not line:
  82                 continue
  83
  84             # Every time component is added, modified or moved, Makefile has to be
  85             # present. However, this does not yet guarantee that the line is a
  86             # real component.
  87             filename = os.path.basename(line)
  88             dirname = os.path.dirname(line).rsplit(subdir + '/')[-1]
  89
  90             if filename == 'Makefile':
  91                 if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
  92                         not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
  93                     paths.append(dirname)
  94
  95         # Some components are using SCM checkout as a source code download method and
  96         # COMPONENT_REVISION is not bumped. With this, we will never rebuild them.
  97         # In order to rebuild them, we will look for such components and build them
  98         # every run. These components are located in openindiana category and we care
  99         # only about that category. One exception to this rule is meta-packages/history
 100         # component, which holds obsoleted components. We add it to paths manually for
 101         # that reason.
 102         cmd = ['git', 'grep', '-l', 'GIT_REPO *=']
 103
 104         proc = subprocess.Popen(cmd,
 105                                 stdout=subprocess.PIPE,
 106                                 stderr=subprocess.PIPE,
 107                                 cwd=workspace_path,
 108                                 universal_newlines=True
 109                                 )
 110
 111         stdout, stderr = proc.communicate()
 112         if debug:
 113             if proc.returncode != 0:
 114                 logger.debug('exit: %d, %s', proc.returncode, stderr)
 115
 116         for line in stdout.splitlines():
 117             line = line.rstrip()
 118
 119             # Only 'openindiana' category.
 120             category = line.split('/')[0]
 121             if category == 'openindiana':
 122                 continue
 123
 124             filename = os.path.basename(line)
 125             dirname = os.path.dirname(line)
 126
 127             if filename == 'Makefile':
 128                 if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
 129                         not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
 130                     paths.append(os.path.dirname(line))
 131
 132         # Add meta-packages/history only if we build the main repository, where
 133         # subdir is equal to 'components'.
 134         if subdir == 'components':
 135             paths.append('meta-packages/history')
 136         # Add encumbered/meta-packages/history only if we build the encumbered repository
 137         if subdir == 'components/encumbered':
 138             paths.append('encumbered/meta-packages/history')
 139
 140         paths = list(set(paths))
 141
 142     else:
 143         for dirpath, dirnames, filenames in walk(workspace_path):
 144             for name in filenames:
 145                 if expression.match(name):
 146                     if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')):
 147                         if debug:
 148                             logger.debug('found %s', dirpath)
 149                         paths.append(dirpath)
 150                     del dirnames[:]
 151                     break
 152
 153     return sorted(paths)
 154
 155
 156 def main():
 157     sys.stdout.flush()
 158
 159     components = {}
 160
 161     COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs']
 162     COMPONENTS_ALLOWED_FMRIS = ['fmris']
 163     COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies']
 164     COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES
 165
 166     parser = argparse.ArgumentParser()
 167     parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
 168     parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS)
 169     parser.add_argument('--make', help='Makefile target to invoke')
 170     parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata')
 171     parser.add_argument('--subdir', default='components', help='Directory holding components')
 172     parser.add_argument('-d', '--debug', action='store_true', default=False)
 173     parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1'))
 174     parser.add_argument('--end-commit', default='HEAD')
 175
 176     args = parser.parse_args()
 177
 178     workspace = args.workspace
 179     components_arg = args.components
 180     make_arg = args.make
 181     pkg5_arg = args.pkg5
 182     subdir = args.subdir
 183     begin_commit = args.begin_commit
 184     end_commit = args.end_commit
 185     debug = args.debug
 186     log_level = logging.WARNING
 187
 188     if debug:
 189         log_level = logging.DEBUG
 190
 191     logging.basicConfig(level=log_level,
 192                         format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',)
 193
 194     if make_arg:
 195         MAKE=os.getenv("MAKE","gmake")
 196
 197         # https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
 198         JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS",""))
 199         if JOBFLAGS:
 200             JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) )
 201             JOBFLAGS=[JOBFLAGS.group(1)]
 202         else:
 203             JOBFDS=()
 204             JOBFLAGS=[]
 205         proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS)
 206         rc = proc.wait()
 207         sys.exit(rc)
 208
 209     if pkg5_arg:
 210         component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
 211         # the component may not be built directly but as a dependency of another component
 212         if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')):
 213             sys.exit(0)
 214         component_pkg5 = os.path.join( os.getcwd(), 'pkg5')
 215         if os.path.isfile(component_pkg5):
 216             os.remove(component_pkg5)
 217         Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0])
 218         sys.exit(0)
 219
 220     incremental = False
 221     if os.getenv('BASS_O_MATIC_MODE') == 'incremental':
 222         incremental = True
 223
 224     if incremental:
 225         component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir,
 226                                              incremental=incremental, begin_commit=begin_commit, end_commit=end_commit)
 227     else:
 228         component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir)
 229
 230     if components_arg:
 231         if components_arg in COMPONENTS_ALLOWED_PATHS:
 232             for path in component_paths:
 233                 print('{0}'.format(path))
 234
 235         elif components_arg in COMPONENTS_ALLOWED_FMRIS:
 236             pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
 237             components = pool.map(Component, component_paths)
 238
 239             for component in components:
 240                 for fmri in component.supplied_packages:
 241                     print('{0}'.format(fmri))
 242
 243         elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES:
 244             dependencies = {}
 245
 246             pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
 247             components = pool.map(Component, component_paths)
 248
 249             with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f:
 250                      data = json.loads(f.read())
 251             component_path = {}
 252             for entry in data:
 253                 component_path[entry['fmri']] = entry['path']
 254
 255             for component in components:
 256                 selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
 257                 # Some packages from OpenSolaris only exist in binary form in the pkg repository
 258                 paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages])
 259                 # Remove self from the set of paths it depends on
 260                 paths.discard(selfpath)
 261                 dependencies[selfpath] = sorted(list(paths))
 262
 263             dependencies_file = os.path.join(workspace, subdir, 'dependencies.json')
 264             with open(dependencies_file, 'w') as f:
 265                 f.write(json.dumps(dependencies, sort_keys=True, indent=4))
 266         sys.exit(0)
 267
 268     sys.exit(1)
 269
 270
 271 if __name__ == '__main__':
 272     main()