tools/userland-mangler

   1 #!/usr/bin/python3.9
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.illumos.org/license/CDDL.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22 # Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
  23 #
  24 #
  25 # userland-mangler - a file mangling utility
  26 #
  27 #  A simple program to mangle files to conform to Solaris WOS or Consoldation
  28 #  requirements.
  29 #
  30
  31 import os
  32 import sys
  33 import re
  34 import subprocess
  35 import shutil
  36 import stat
  37
  38
  39 import pkg.fmri
  40 import pkg.manifest
  41 import pkg.actions
  42 import pkg.elf as elf
  43
  44 attribute_oracle_table_header = """
  45 .\\\" Oracle has added the ARC stability level to this manual page"""
  46
  47 attribute_table_header = """
  48 .SH ATTRIBUTES
  49 See
  50 .BR attributes (5)
  51 for descriptions of the following attributes:
  52 .sp
  53 .TS
  54 box;
  55 cbp-1 | cbp-1
  56 l | l .
  57 ATTRIBUTE TYPE  ATTRIBUTE VALUE """
  58
  59 attribute_table_availability = """
  60 =
  61 Availability    %s"""
  62
  63 attribute_table_stability = """
  64 =
  65 Stability       %s"""
  66
  67 attribute_table_footer = """
  68 .TE
  69 .PP
  70 """
  71 def attributes_section_text(availability, stability, modified_date):
  72         result = ''
  73
  74         # is there anything to do?
  75         if availability is not None or stability is not None:
  76                 result = attribute_oracle_table_header
  77                 if modified_date is not None:
  78                         result += ("\n.\\\" on %s" % modified_date)
  79                 result += attribute_table_header
  80
  81                 if availability is not None:
  82                         result += (attribute_table_availability % availability)
  83                 if stability is not None:
  84                         result += (attribute_table_stability % stability.capitalize())
  85                 result += attribute_table_footer
  86
  87         return result
  88
  89 notes_oracle_comment = """
  90 .\\\" Oracle has added source availability information to this manual page"""
  91
  92 notes_header = """
  93 .SH NOTES
  94 """
  95
  96 notes_community = """
  97 Further information about this software can be found on the open source community website at %s.
  98 """
  99 notes_source = """
 100 This software was built from source available at https://openindiana.org/.  The original community source was downloaded from  %s
 101 """
 102
 103 def notes_section_text(header_seen, community, source, modified_date):
 104         result = ''
 105
 106         # is there anything to do?
 107         if community is not None or source is not None:
 108                 if header_seen == False:
 109                         result += notes_header
 110                 result += notes_oracle_comment
 111                 if modified_date is not None:
 112                         result += ("\n.\\\" on %s" % modified_date)
 113                 if source is not None:
 114                         result += (notes_source % source)
 115                 if community is not None:
 116                         result += (notes_community % community)
 117
 118         return result
 119
 120 so_re = re.compile('^\.so.+$', re.MULTILINE)
 121 section_re = re.compile('\.SH "?([^"]+).*$', re.IGNORECASE)
 122 TH_re = re.compile('\.TH\s+(?:"[^"]+"|\S+)\s+(\S+)', re.IGNORECASE)
 123 #
 124 # mangler.man.stability = (mangler.man.stability)
 125 # mangler.man.modified_date = (mangler.man.modified-date)
 126 # mangler.man.availability = (pkg.fmri)
 127 # mangler.man.source-url = (pkg.source-url)
 128 # mangler.man.upstream-url = (pkg.upstream-url)
 129 #
 130 def mangle_manpage(manifest, action, text):
 131         # manpages must have a taxonomy defined
 132         stability = action.attrs.pop('mangler.man.stability', None)
 133         if stability is None:
 134                 sys.stderr.write("ERROR: manpage action missing mangler.man.stability: %s" % action)
 135                 sys.exit(1)
 136
 137         # manpages may have a 'modified date'
 138         modified_date = action.attrs.pop('mangler.man.modified-date', None)
 139
 140         # Rewrite the section in the .TH line to match the section in which
 141         # we're delivering it.
 142         rewrite_sect = action.attrs.pop('mangler.man.rewrite-section', 'true')
 143
 144         attributes_written = False
 145         notes_seen = False
 146
 147         if 'pkg.fmri' in manifest.attributes:
 148                 fmri = pkg.fmri.PkgFmri(manifest.attributes['pkg.fmri'])
 149                 availability = fmri.pkg_name
 150
 151         community = None
 152         if 'info.upstream-url' in manifest.attributes:
 153                 community = manifest.attributes['info.upstream-url']
 154
 155         source = None
 156         if 'info.source-url' in manifest.attributes:
 157                 source = manifest.attributes['info.source-url']
 158         elif 'info.repository-url' in manifest.attributes:
 159                 source = manifest.attributes['info.repository-url']
 160
 161         # skip reference only pages
 162         if so_re.match(text) is not None:
 163                 return text
 164
 165         # tell man that we want tables (and eqn)
 166         result = "'\\\" te\n"
 167
 168         # write the orginal data
 169         for line in text.split('\n'):
 170                 match = section_re.match(line)
 171                 if match is not None:
 172                         section = match.group(1)
 173                         if section in ['SEE ALSO', 'NOTES']:
 174                                 if attributes_written == False:
 175                                         result += attributes_section_text(
 176                                                                  availability,
 177                                                                  stability,
 178                                                                  modified_date)
 179                                         attributes_written = True
 180                                 if section == 'NOTES':
 181                                         notes_seen = True
 182                         match = TH_re.match(line)
 183                         if match and rewrite_sect.lower() == "true":
 184                                 # Use the section defined by the filename, rather than
 185                                 # the directory in which it sits.
 186                                 sect = os.path.splitext(action.attrs["path"])[1][1:]
 187                                 line = line[:match.span(1)[0]] + sect + \
 188                                     line[match.span(1)[1]:]
 189
 190                 result += ("%s\n" % line)
 191
 192         if attributes_written == False:
 193                 result += attributes_section_text(availability, stability,
 194                     modified_date)
 195
 196         result += notes_section_text(notes_seen, community, source,
 197             modified_date)
 198
 199         return result
 200
 201 #
 202 # mangler.elf.strip_runpath = (true|false)
 203 #
 204 def mangle_elf(manifest, action, src, dest):
 205         strip_elf_runpath = action.attrs.pop('mangler.elf.strip_runpath', 'true')
 206         if strip_elf_runpath == 'false':
 207                 return
 208
 209         #
 210         # Strip any runtime linker default search path elements from the file
 211         # and replace relative paths with absolute paths
 212         #
 213         ELFEDIT = '/usr/bin/elfedit'
 214
 215         # runtime linker default search path elements + /64 link
 216         rtld_default_dirs = [ '/lib', '/usr/lib',
 217                               '/lib/64', '/usr/lib/64',
 218                               '/lib/amd64', '/usr/lib/amd64',
 219                               '/lib/sparcv9', '/usr/lib/sparcv9' ]
 220
 221         runpath_re = re.compile('.+\s(RPATH|RUNPATH)\s+\S+\s+(\S+)')
 222
 223         # Retreive the search path from the object file.  Use elfedit(1) because pkg.elf only
 224         # retrieves the RUNPATH.  Note that dyn:rpath and dyn:runpath return both values.
 225         # Both RPATH and RUNPATH are expected to be the same, but in an overabundand of caution,
 226         # process each element found separately.
 227         result = subprocess.Popen([ELFEDIT, '-re', 'dyn:runpath', src ],
 228                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 229                                   universal_newlines=True)
 230         result.wait()
 231         if result.returncode != 0:        # no RUNPATH or RPATH to potentially strip
 232                 return
 233
 234         for line in result.stdout:
 235                 result = runpath_re.match(line)
 236                 if result != None:
 237                         element = result.group(1)
 238                         original_dirs = result.group(2).split(":")
 239                         keep_dirs = []
 240                         matched_dirs = []
 241
 242                         for dir in original_dirs:
 243                                 if dir not in rtld_default_dirs:
 244                                         if dir.startswith('$ORIGIN'):
 245                                                 path = action.attrs['path']
 246                                                 dirname = os.path.dirname(path)
 247                                                 if dirname[0] != '/':
 248                                                         dirname = '/' + dirname
 249                                                 corrected_dir = dir.replace('$ORIGIN', dirname)
 250                                                 corrected_dir = os.path.realpath(corrected_dir)
 251                                                 matched_dirs.append(dir)
 252                                                 keep_dirs.append(corrected_dir)
 253                                         else:
 254                                             keep_dirs.append(dir)
 255                                 else:
 256                                         matched_dirs.append(dir)
 257
 258                         if len(matched_dirs) != 0:
 259                                 # Emit an "Error" message in case someone wants to look at the build log
 260                                 # and fix the component build so that this is a NOP.
 261                                 print("Stripping %s from %s in %s" % (":".join(matched_dirs), element, src), file=sys.stderr)
 262
 263                                 # Make sure that there is a destdir to copy the file into for mangling.
 264                                 destdir = os.path.dirname(dest)
 265                                 if not os.path.exists(destdir):
 266                                         os.makedirs(destdir)
 267                                 # Create a copy to mangle
 268                                 # Earlier the code would check that the destination file does not exist
 269                                 # yet, however internal library versioning can be different while the
 270                                 # filename remains the same.
 271                                 # When publishing from a non-clean prototype directory older libraries
 272                                 # which may be ABI incompatible would then be republished in the new
 273                                 # package instead of the new version.
 274                                 shutil.copy2(src, dest)
 275
 276                                 # Make sure we do have write permission before we try to modify the file
 277                                 os.chmod(dest, os.stat(dest).st_mode | stat.S_IWUSR)
 278
 279                                 # Mangle the copy by deleting the tag if there is nothing left to keep
 280                                 # or replacing the value if there is something left.
 281                                 elfcmd = "dyn:delete %s" % element.lower()
 282                                 if len(keep_dirs) > 0:
 283                                         elfcmd = "dyn:%s '%s'" % (element.lower(), ":".join(keep_dirs))
 284                                 subprocess.call([ELFEDIT, '-e', elfcmd, dest])
 285
 286 #
 287 # mangler.script.file-magic =
 288 #
 289 def mangle_script(manifest, action, text):
 290         return text
 291
 292 #
 293 # mangler.strip_cddl = false
 294 #
 295 def mangle_cddl(manifest, action, text):
 296         strip_cddl = action.attrs.pop('mangler.strip_cddl', 'false')
 297         if strip_cddl == 'false':
 298                 return text
 299         cddl_re = re.compile('^[^\n]*CDDL HEADER START.+CDDL HEADER END[^\n]*$',
 300                              re.MULTILINE|re.DOTALL)
 301         return cddl_re.sub('', text)
 302
 303 def do_ctfconvert(converter, file):
 304         args = [converter, '-i', '-m', '-k', file]
 305         print(*args, file=sys.stderr)
 306         subprocess.call(args)
 307
 308 def mangle_path(manifest, action, src, dest, ctfconvert):
 309         if elf.is_elf_object(src):
 310                 if ctfconvert is not None:
 311                         do_ctfconvert(ctfconvert, src)
 312                 mangle_elf(manifest, action, src, dest)
 313         else:
 314                 # a 'text' document (script, man page, config file, ...
 315                 # We treat all documents as latin-1 text to avoid
 316                 # reencoding them and loosing data
 317                 ifp = open(src, 'r', encoding='latin-1')
 318                 text = ifp.read()
 319                 ifp.close()
 320
 321                 # remove the CDDL from files
 322                 result = mangle_cddl(manifest, action, text)
 323
 324                 if 'facet.doc.man' in action.attrs:
 325                          result = mangle_manpage(manifest, action, result)
 326                 elif 'mode' in action.attrs and int(action.attrs['mode'], 8) & 0o111 != 0:
 327                         result = mangle_script(manifest, action, result)
 328
 329                 if text != result:
 330                         destdir = os.path.dirname(dest)
 331                         if not os.path.exists(destdir):
 332                                 os.makedirs(destdir)
 333                         with open(dest, 'w', encoding='latin-1') as ofp:
 334                             ofp.write(result)
 335
 336 #
 337 # mangler.bypass = (true|false)
 338 #
 339 def mangle_paths(manifest, search_paths, destination, ctfconvert):
 340         for action in manifest.gen_actions_by_type("file"):
 341                 bypass = action.attrs.pop('mangler.bypass', 'false').lower()
 342                 if bypass == 'true':
 343                         continue
 344
 345                 path = None
 346                 if 'path' in action.attrs:
 347                         path = action.attrs['path']
 348                 if action.hash and action.hash != 'NOHASH':
 349                         path = action.hash
 350                 if not path:
 351                         continue
 352
 353                 if not os.path.exists(destination):
 354                         os.makedirs(destination)
 355
 356                 dest = os.path.join(destination, path)
 357                 for directory in search_paths:
 358                         if directory != destination:
 359                                 src = os.path.join(directory, path)
 360                                 if os.path.isfile(src):
 361                                         mangle_path(manifest, action,
 362                                                     src, dest, ctfconvert)
 363                                         break
 364
 365 def load_manifest(manifest_file):
 366         manifest = pkg.manifest.Manifest()
 367         manifest.set_content(pathname=manifest_file)
 368
 369         return manifest
 370
 371 def usage():
 372         print("Usage: %s [-m|--manifest (file)] [-d|--search-directory (dir)] [-D|--destination (dir)] " % (sys.argv[0].split('/')[-1]))
 373         sys.exit(1)
 374
 375 def main():
 376         import getopt
 377
 378         sys.stdout.flush()
 379
 380         search_paths = []
 381         destination = None
 382         manifests = []
 383         ctfconvert = None
 384
 385         try:
 386                 opts, args = getopt.getopt(sys.argv[1:], "c:D:d:m:",
 387                         ["ctf=", "destination=", "search-directory=", "manifest="])
 388         except getopt.GetoptError as err:
 389                 print(str(err))
 390                 usage()
 391
 392         for opt, arg in opts:
 393                 if opt in [ "-D", "--destination" ]:
 394                         destination = arg
 395                 elif opt in [ "-d", "--search-directory" ]:
 396                         search_paths.append(arg)
 397                 elif opt in [ "-m", "--manifest" ]:
 398                         try:
 399                                 manifest = load_manifest(arg)
 400                         except IOError as err:
 401                                 print("oops, %s: %s" % (arg, str(err)))
 402                                 usage()
 403                         else:
 404                                 manifests.append(manifest)
 405                 elif opt in [ "-c", "--ctf" ]:
 406                         ctfconvert = arg
 407                 else:
 408                         usage()
 409
 410         if destination == None:
 411                 usage()
 412
 413         for manifest in manifests:
 414                 mangle_paths(manifest, search_paths, destination, ctfconvert)
 415                 print(manifest)
 416
 417         sys.exit(0)
 418
 419 if __name__ == "__main__":
 420         main()