tools/bass-o-matic

   1 #!/usr/bin/python3.9
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22 # Copyright (c) 2010, Oracle and/or it's affiliates.  All rights reserved.
  23 #
  24 #
  25 # bass-o-matic.py
  26 #  A simple program to enumerate components in the userland gate and report
  27 #  on dependency related information.
  28 #
  29
  30 import os
  31 import sys
  32 import re
  33 import glob
  34 import subprocess
  35 import argparse
  36 import logging
  37 import json
  38 import multiprocessing
  39
  40 try:
  41     from scandir import walk
  42 except ImportError:
  43     from os import walk
  44
  45 from bass.component import Component
  46
  47 logger = logging.getLogger('bass-o-matic')
  48
  49 # Locate SCM directories containing Userland components by searching from
  50 # the supplied top of tree for .p5m files.  Once a .p5m file is located,
  51 # that directory is added to the list and no children are searched.
  52 def FindComponentPaths(path, debug=False, subdir='components',
  53                        incremental=False, begin_commit=None, end_commit=None):
  54     expression = re.compile(r'.+\.p5m$', re.IGNORECASE)
  55
  56     paths = []
  57
  58     if debug:
  59         logger.debug('searching %s for component directories', path)
  60
  61     workspace_path = os.path.join(path, subdir)
  62
  63     if incremental:
  64         cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:',
  65                '..'.join([begin_commit, end_commit])]
  66
  67         proc = subprocess.Popen(cmd,
  68                                 stdout=subprocess.PIPE,
  69                                 stderr=subprocess.PIPE,
  70                                 cwd=workspace_path,
  71                                 universal_newlines=True
  72                                 )
  73         stdout, stderr = proc.communicate()
  74         if debug:
  75             if proc.returncode != 0:
  76                 logger.debug('exit: %d, %s', proc.returncode, stderr)
  77
  78         for line in stdout.splitlines():
  79             line = line.rstrip()
  80             # git output might contain empty lines, so we skip them.
  81             if not line:
  82                 continue
  83
  84             # Every time component is added, modified or moved, Makefile has to be
  85             # present. However, this does not yet guarantee that the line is a
  86             # real component.
  87             filename = os.path.basename(line)
  88             dirname = os.path.dirname(line).rsplit(subdir + '/')[-1]
  89
  90             if filename == 'Makefile':
  91                 if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
  92                         not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
  93                     paths.append(dirname)
  94
  95         # Some components are using SCM checkout as a source code download method and
  96         # COMPONENT_REVISION is not bumped. With this, we will never rebuild them.
  97         # In order to rebuild them, we will look for such components and build them
  98         # every run. These components are located in openindiana category and we care
  99         # only about that category. One exception to this rule is meta-packages/history
 100         # component, which holds obsoleted components. We add it to paths manually for
 101         # that reason.
 102         cmd = ['git', 'grep', '-l', 'GIT_REPO *=']
 103
 104         proc = subprocess.Popen(cmd,
 105                                 stdout=subprocess.PIPE,
 106                                 stderr=subprocess.PIPE,
 107                                 cwd=workspace_path,
 108                                 universal_newlines=True
 109                                 )
 110
 111         stdout, stderr = proc.communicate()
 112         if debug:
 113             if proc.returncode != 0:
 114                 logger.debug('exit: %d, %s', proc.returncode, stderr)
 115
 116         for line in stdout.splitlines():
 117             line = line.rstrip()
 118
 119             # Only 'openindiana' category.
 120             category = line.split('/')[0]
 121             if category == 'openindiana':
 122                 continue
 123
 124             filename = os.path.basename(line)
 125             dirname = os.path.dirname(line)
 126
 127             if filename == 'Makefile':
 128                 if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
 129                         not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
 130                     paths.append(os.path.dirname(line))
 131
 132         # Add meta-packages/history
 133         paths.append('meta-packages/history')
 134
 135         paths = list(set(paths))
 136
 137     else:
 138         for dirpath, dirnames, filenames in walk(workspace_path):
 139             for name in filenames:
 140                 if expression.match(name):
 141                     if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')):
 142                         if debug:
 143                             logger.debug('found %s', dirpath)
 144                         paths.append(dirpath)
 145                     del dirnames[:]
 146                     break
 147
 148     return sorted(paths)
 149
 150
 151 def main():
 152     sys.stdout.flush()
 153
 154     components = {}
 155
 156     COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs']
 157     COMPONENTS_ALLOWED_FMRIS = ['fmris']
 158     COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies']
 159     COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES
 160
 161     parser = argparse.ArgumentParser()
 162     parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
 163     parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS)
 164     parser.add_argument('--make', help='Makefile target to invoke')
 165     parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata')
 166     parser.add_argument('--subdir', default='components', help='Directory holding components')
 167     parser.add_argument('-d', '--debug', action='store_true', default=False)
 168     parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1'))
 169     parser.add_argument('--end-commit', default='HEAD')
 170
 171     args = parser.parse_args()
 172
 173     workspace = args.workspace
 174     components_arg = args.components
 175     make_arg = args.make
 176     pkg5_arg = args.pkg5
 177     subdir = args.subdir
 178     begin_commit = args.begin_commit
 179     end_commit = args.end_commit
 180     debug = args.debug
 181     log_level = logging.WARNING
 182
 183     if debug:
 184         log_level = logging.DEBUG
 185
 186     logging.basicConfig(level=log_level,
 187                         format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',)
 188
 189     if make_arg:
 190         MAKE=os.getenv("MAKE","gmake")
 191
 192         # https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
 193         JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS",""))
 194         if JOBFLAGS:
 195             JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) )
 196             JOBFLAGS=[JOBFLAGS.group(1)]
 197         else:
 198             JOBFDS=()
 199             JOBFLAGS=[]
 200         proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS)
 201         rc = proc.wait()
 202         sys.exit(rc)
 203
 204     if pkg5_arg:
 205         component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
 206         # the component may not be built directly but as a dependency of another component
 207         if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')):
 208             sys.exit(0)
 209         component_pkg5 = os.path.join( os.getcwd(), 'pkg5')
 210         if os.path.isfile(component_pkg5):
 211             os.remove(component_pkg5)
 212         Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0])
 213         sys.exit(0)
 214
 215     incremental = False
 216     if os.getenv('BASS_O_MATIC_MODE') == 'incremental':
 217         incremental = True
 218
 219     if incremental:
 220         component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir,
 221                                              incremental=incremental, begin_commit=begin_commit, end_commit=end_commit)
 222     else:
 223         component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir)
 224
 225     if components_arg:
 226         if components_arg in COMPONENTS_ALLOWED_PATHS:
 227             for path in component_paths:
 228                 print('{0}'.format(path))
 229
 230         elif components_arg in COMPONENTS_ALLOWED_FMRIS:
 231             pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
 232             components = pool.map(Component, component_paths)
 233
 234             for component in components:
 235                 for fmri in component.supplied_packages:
 236                     print('{0}'.format(fmri))
 237
 238         elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES:
 239             dependencies = {}
 240
 241             pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
 242             components = pool.map(Component, component_paths)
 243
 244             with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f:
 245                      data = json.loads(f.read())
 246             component_path = {}
 247             for entry in data:
 248                 component_path[entry['fmri']] = entry['path']
 249
 250             for component in components:
 251                 selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
 252                 # Some packages from OpenSolaris only exist in binary form in the pkg repository
 253                 paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages])
 254                 # Remove self from the set of paths it depends on
 255                 paths.discard(selfpath)
 256                 dependencies[selfpath] = sorted(list(paths))
 257
 258             dependencies_file = os.path.join(workspace, subdir, 'dependencies.json')
 259             with open(dependencies_file, 'w') as f:
 260                 f.write(json.dumps(dependencies, sort_keys=True, indent=4))
 261         sys.exit(0)
 262
 263     sys.exit(1)
 264
 265
 266 if __name__ == '__main__':
 267     main()