make/scripts/version-info.py

   1 #!/usr/bin/env python
   2 #
   3 # Utility functions to access git repository info and
   4 # generate source files and binary objects using templates.
   5 #
   6 # (C) 2015, The LibrePilot Project, http://www.librepilot.org
   7 # (c) 2011, The OpenPilot Team, http://www.openpilot.org
   8 # See also: The GNU Public License (GPL) Version 3
   9 #
  10
  11 from subprocess import Popen, PIPE
  12 from re import search, MULTILINE
  13 from datetime import datetime
  14 from string import Template
  15 import optparse
  16 import hashlib
  17 import sys
  18 import os.path
  19 import json
  20
  21 class Repo:
  22     """A simple git repository HEAD commit info class
  23
  24     This simple class provides object notation to access
  25     the git repository current commit info.  If one needs
  26     better access one can try the GitPython class available
  27     here:
  28             http://packages.python.org/GitPython
  29     It is not installed by default, so we cannot rely on it.
  30
  31     Example:
  32         r = Repo('/path/to/git/repository')
  33         print "path:       ", r.path()
  34         print "origin:     ", r.origin()
  35         print "hash:       ", r.hash()
  36         print "short hash: ", r.hash(8)
  37         print "Unix time:  ", r.time()
  38         print "commit date:", r.time("%Y%m%d")
  39         print "commit tag: ", r.tag()
  40         print "branch:     ", r.branch()
  41         print "release tag:", r.reltag()
  42     """
  43
  44     def _exec(self, cmd):
  45         """Execute git using cmd as arguments"""
  46         self._git = 'git'
  47         git = Popen(self._git + " " + cmd, cwd = self._path,
  48                     shell = True, stdout = PIPE, stderr = PIPE)
  49         self._out, self._err = git.communicate()
  50         self._rc = git.poll()
  51
  52     def _get_origin(self):
  53         """Get and store the repository fetch origin path"""
  54         self._origin = None
  55         self._exec('remote -v')
  56         if self._rc == 0:
  57             m = search(r"^origin\s+(.+)\s+\(fetch\)$", self._out, MULTILINE)
  58             if m:
  59                 self._origin = m.group(1)
  60
  61     def _get_time(self):
  62         """Get and store HEAD commit timestamp in Unix format
  63
  64         We use commit timestamp rather than the build time,
  65         so it always is the same for the current commit or tag.
  66         """
  67         self._time = None
  68         self._exec('log -n1 --no-color --format=format:%ct HEAD')
  69         if self._rc == 0:
  70             self._time = self._out
  71
  72     def _get_last_tag(self):
  73         """Get and store git tag for the HEAD commit"""
  74         self._last_tag = None
  75         self._num_commits_past_tag = None
  76         self._exec('describe --tags --long')
  77         if self._rc == 0:
  78             descriptions = self._out.rsplit('-', 2)
  79             self._last_tag = descriptions[-3]
  80             self._num_commits_past_tag = descriptions[-2]
  81
  82     def _get_branch(self):
  83         """Get and store current branch containing the HEAD commit"""
  84         self._branch = None
  85         self._exec('branch --contains HEAD')
  86         if self._rc == 0:
  87             m = search(r"^\*\s+(.+)$", self._out, MULTILINE)
  88             if m:
  89                 self._branch = m.group(1)
  90
  91     def _get_dirty(self):
  92         """Check for dirty state of repository"""
  93         self._dirty = False
  94         self._exec('update-index --refresh --unmerged')
  95         self._exec('diff-index --name-only --exit-code --quiet HEAD')
  96         if self._rc:
  97             self._dirty = True
  98
  99     def _load_json(self):
 100         """Loads the repo data from version-info.json"""
 101         json_path = os.path.join(self._path, 'version-info.json')
 102         if os.path.isfile(json_path):
 103             with open(json_path) as json_file:
 104                json_data = json.load(json_file)
 105
 106             self._hash = json_data['hash']
 107             self._origin = json_data['origin']
 108             self._time = json_data['time']
 109             self._last_tag = json_data['last_tag']
 110             self._num_commits_past_tag = json_data['num_commits_past_tag']
 111             self._branch = json_data['branch']
 112             self._dirty = json_data['dirty']
 113
 114             return True
 115         return False
 116
 117     def __init__(self, path = "."):
 118         """Initialize object instance and read repo info"""
 119         self._path = path
 120         self._exec('rev-parse --verify HEAD')
 121         if self._load_json():
 122             pass
 123         elif self._rc == 0:
 124             self._hash = self._out.strip(' \t\n\r')
 125             self._get_origin()
 126             self._get_time()
 127             self._get_last_tag()
 128             self._get_branch()
 129             self._get_dirty()
 130         else:
 131             self._hash = None
 132             self._origin = None
 133             self._time = None
 134             self._last_tag = None
 135             self._num_commits_past_tag = None
 136             self._branch = None
 137             self._dirty = None
 138
 139     def path(self):
 140         """Return the repository path"""
 141         return self._path
 142
 143     def origin(self, none = None):
 144         """Return fetch origin of the repository"""
 145         if self._origin == None:
 146             return none
 147         else:
 148             return self._origin
 149
 150     def hash(self, n = 40, none = None):
 151         """Return hash of the HEAD commit"""
 152         if self._hash == None:
 153             return none
 154         else:
 155             return self._hash[:n]
 156
 157     def time(self, format = None, none = None):
 158         """Return Unix or formatted time of the HEAD commit"""
 159         if self._time == None:
 160             return none
 161         else:
 162             if format == None:
 163                 return self._time
 164             else:
 165                 return datetime.utcfromtimestamp(float(self._time)).strftime(format)
 166
 167     def tag(self, none = None):
 168         """Return git tag for the HEAD commit or given string if none"""
 169         if self._last_tag == None or self._num_commits_past_tag != "0":
 170             return none
 171         else:
 172             return self._last_tag
 173
 174     def branch(self, none = None):
 175         """Return git branch containing the HEAD or given string if none"""
 176         if self._branch == None:
 177             return none
 178         else:
 179             return self._branch
 180
 181     def dirty(self, dirty = "-dirty", clean = ""):
 182         """Return git repository dirty state or empty string"""
 183         if self._dirty:
 184             return dirty
 185         else:
 186             return clean
 187
 188     def label(self):
 189         """Return package label (similar to git describe)"""
 190         try:
 191             if self._num_commits_past_tag == "0":
 192                 return self._last_tag + self.dirty()
 193             else:
 194                 return self._last_tag + "+r" + self._num_commits_past_tag + "-g" + self.hash(7, '') + self.dirty()
 195         except:
 196             return None
 197
 198     def version_four_num(self):
 199         """Return package version in format X.X.X.X using only numbers"""
 200
 201         try:
 202             (release, junk, candidate) = self._last_tag.partition("-RC")
 203             (year, dot, month_and_patch) = release.partition(".")
 204             (month, dot, patch) = month_and_patch.partition(".")
 205
 206             if candidate == "":
 207                 candidate = "64" # Need to stay below 65536 for last part
 208
 209             if patch == "":
 210                 patch = "0"
 211
 212             return "{}.{}.{}.{}{:0>3.3}".format(year,month,patch,candidate,self._num_commits_past_tag)
 213         except:
 214             return None
 215
 216
 217     def revision(self):
 218         """Return full revison string (tag if defined, or branch:hash date time if no tag)"""
 219         try:
 220             if self._num_commits_past_tag == "0":
 221                 return self.tag('') + self.dirty()
 222             else:
 223                 return self.branch('no-branch') + ":" + self.hash(8, 'no-hash') + self.dirty() + self.time(' %Y%m%d %H:%M')
 224         except:
 225             return None
 226
 227     def info(self):
 228         """Print some repository info"""
 229         print "path:       ", self.path()
 230         print "origin:     ", self.origin()
 231         print "Unix time:  ", self.time()
 232         print "commit date:", self.time("%Y%m%d")
 233         print "hash:       ", self.hash()
 234         print "short hash: ", self.hash(8)
 235         print "branch:     ", self.branch()
 236         print "commit tag: ", self.tag('')
 237         print "dirty:      ", self.dirty('yes', 'no')
 238         print "label:      ", self.label()
 239         print "revision:   ", self.revision()
 240
 241     def save_to_json(self, path):
 242         """Saves the repo data to version-info.json"""
 243
 244         json_data = dict()
 245         json_data['hash'] = self._hash
 246         json_data['origin'] = self._origin
 247         json_data['time'] = self._time
 248         json_data['last_tag'] = self._last_tag
 249         json_data['num_commits_past_tag'] = self._num_commits_past_tag
 250         json_data['branch'] = self._branch
 251         # version-info.json is for use with git archive which doesn't take in dirty changes
 252         json_data['dirty'] = False
 253
 254         json_path = os.path.join(path, 'version-info.json')
 255
 256         write_if_different(json_path, json.dumps(json_data))
 257
 258
 259 def write_if_different(out_name, out):
 260     """Write ouput to file only if it differs from current"""
 261
 262     # Check if output file already exists
 263     try:
 264         of = open(out_name, "rb")
 265     except IOError:
 266         # No file - create new
 267         of = open(out_name, "wb")
 268         of.write(out)
 269         of.close()
 270     else:
 271         # File exists - overwite only if content is different
 272         inp = of.read()
 273         of.close()
 274         if inp != out:
 275             of = open(out_name, "wb")
 276             of.write(out)
 277             of.close()
 278
 279 def escape_dict(dictionary):
 280     """Escapes dictionary values for C"""
 281
 282     # We need to escape the strings for C
 283     for key in dictionary:
 284         # Using json.dumps and removing the surounding quotes escapes for C
 285         dictionary[key] = json.dumps(dictionary[key])[1:-1]
 286
 287
 288 def file_from_template(tpl_name, out_name, dictionary):
 289     """Create or update file from template using dictionary
 290
 291     This function reads the template, performs placeholder replacement
 292     using the dictionary and checks if output file with such content
 293     already exists.  If no such file or file data is different from
 294     expected then it will be ovewritten with new data.  Otherwise it
 295     will not be updated so make will not update dependent targets.
 296
 297     Example:
 298         # template.c:
 299         #    char source[] = "${OUTFILENAME}";
 300         #    uint32_t timestamp = ${UNIXTIME};
 301         #    uint32_t hash = 0x${HASH8};
 302
 303         r = Repo('/path/to/git/repository')
 304         tpl_name = "template.c"
 305         out_name = "output.c"
 306
 307         dictionary = dict(
 308             HASH8 = r.hash(8),
 309             UNIXTIME = r.time(),
 310             OUTFILENAME = out_name,
 311         )
 312
 313         file_from_template(tpl_name, out_name, dictionary)
 314     """
 315
 316     # Read template first
 317     tf = open(tpl_name, "rb")
 318     tpl = tf.read()
 319     tf.close()
 320
 321     # Replace placeholders using dictionary
 322     out = Template(tpl).substitute(dictionary)
 323
 324     write_if_different(out_name, out)
 325
 326
 327 def sha1(file):
 328     """Provides C source representation of sha1 sum of file"""
 329     if file == None:
 330         return ""
 331     else:
 332         sha1 = hashlib.sha1()
 333         with open(file, 'rb') as f:
 334             for chunk in iter(lambda: f.read(8192), ''):
 335                 sha1.update(chunk)
 336         hex_stream = lambda s:",".join(['0x'+hex(ord(c))[2:].zfill(2) for c in s])
 337         return hex_stream(sha1.digest())
 338
 339 def xtrim(string, suffix, length):
 340     """Return string+suffix concatenated and trimmed up to length characters
 341
 342     This function appends suffix to the end of string and returns the result
 343     up to length characters. If it does not fit then the string will be
 344     truncated and the '+' will be put between it and the suffix.
 345     """
 346     if len(string) + len(suffix) <= length:
 347         return ''.join([string, suffix])
 348     else:
 349         n = length - 1 - len(suffix)
 350         assert n > 0, "length of truncated string+suffix exceeds maximum length"
 351         return ''.join([string[:n], '+', suffix])
 352
 353 def get_hash_of_dirs(directory, verbose = 0, raw = 0, n = 40):
 354     """Return hash of XML files from UAVObject definition directory"""
 355     import hashlib, os
 356     SHAhash = hashlib.sha1()
 357
 358     if not os.path.exists(directory):
 359         return -1
 360
 361     try:
 362         for root, dirs, files in os.walk(directory):
 363             # os.walk() is unsorted. Must make sure we process files in sorted
 364             # order so that the hash is stable across invocations and across OSes.
 365             if files:
 366                 files.sort()
 367
 368             for names in files:
 369                 if names.endswith('.xml'):
 370                     if verbose == 1:
 371                         print 'Hashing', names
 372                     filepath = os.path.join(root, names)
 373                     try:
 374                         f1 = open(filepath, 'rU')
 375                     except:
 376                         # You can't open the file for some reason
 377                         continue
 378
 379                     # Compute file hash. Same as running "sha1sum <file>".
 380                     f1hash = hashlib.sha1()
 381                     while 1:
 382                         # Read file in as little chunks
 383                         buf = f1.read(4096)
 384                         if not buf:
 385                             break
 386                         f1hash.update(buf)
 387                     f1.close()
 388
 389                     if verbose == 1:
 390                         print 'Hash is', f1hash.hexdigest()
 391
 392                     # Append the hex representation of the current file's hash into the cumulative hash
 393                     SHAhash.update(f1hash.hexdigest())
 394
 395     except:
 396         import traceback
 397         # Print the stack traceback
 398         traceback.print_exc()
 399         return -2
 400
 401     if verbose == 1:
 402         print 'Final hash is', SHAhash.hexdigest()
 403
 404     if raw == 1:
 405         return SHAhash.hexdigest()[:n]
 406     else:
 407         hex_stream = lambda s:",".join(['0x'+hex(ord(c))[2:].zfill(2) for c in s])
 408         return hex_stream(SHAhash.digest())
 409
 410 def main():
 411     """This utility uses git repository in the current working directory
 412 or from the given path to extract some info about it and HEAD commit.
 413 Then some variables in the form of ${VARIABLE} could be replaced by
 414 collected data. Optional board type, board revision and sha1 sum
 415 of given image file could be applied as well or will be replaced by
 416 empty strings if not defined.
 417
 418 If --info option is given, some repository info will be printed to
 419 stdout.
 420
 421 If --format option is given then utility prints the format string
 422 after substitution to the standard output.
 423
 424 If --outfile option is given then the --template option should be
 425 defined too. In that case the utility reads a template file, performs
 426 variable substitution and writes the result into output file. Output
 427 file will be overwritten only if its content differs from expected.
 428 Otherwise it will not be touched, so make utility will not remake
 429 dependent targets.
 430
 431 Optional positional arguments may be used to add more dictionary
 432 strings for replacement. Each argument has the form:
 433     VARIABLE=replacement
 434 and each ${VARIABLE} reference will be replaced with replacement
 435 string given.
 436     """
 437
 438     # Parse command line.
 439     class RawDescriptionHelpFormatter(optparse.IndentedHelpFormatter):
 440         """optparse formatter function to pretty print raw epilog"""
 441         def format_epilog(self, epilog):
 442             if epilog:
 443                 return "\n" + epilog + "\n"
 444             else:
 445                 return ""
 446
 447     parser = optparse.OptionParser(
 448         formatter=RawDescriptionHelpFormatter(),
 449         description = "Performs variable substitution in template file or string.",
 450         epilog = main.__doc__);
 451
 452     parser.add_option('--path', default='.',
 453                         help='path to the git repository');
 454     parser.add_option('--info', action='store_true',
 455                         help='print repository info to stdout');
 456     parser.add_option('--format',
 457                         help='format string to print to stdout');
 458     parser.add_option('--template',
 459                         help='name of template file');
 460     parser.add_option('--outfile',
 461                         help='name of output file');
 462     parser.add_option('--escape', action="store_true",
 463                         help='do escape strings for C (default based on file ext)');
 464     parser.add_option('--no-escape', action="store_false", dest="escape",
 465                         help='do not escape strings for C');
 466     parser.add_option('--image',
 467                         help='name of image file for sha1 calculation');
 468     parser.add_option('--type', default="",
 469                         help='board type, for example, 0x04 for CopterControl');
 470     parser.add_option('--revision', default = "",
 471                         help='board revision, for example, 0x01');
 472     parser.add_option('--uavodir', default = "",
 473                         help='uav object definition directory');
 474     parser.add_option('--jsonpath',
 475                         help='path to save version info');
 476     (args, positional_args) = parser.parse_args()
 477
 478     # Process arguments.  No advanced error handling is here.
 479     # Any error will raise an exception and terminate process
 480     # with non-zero exit code.
 481     r = Repo(args.path)
 482
 483     dictionary = dict(
 484         TEMPLATE = args.template,
 485         OUTFILENAME = args.outfile,
 486         ORIGIN = r.origin("local repository or using build servers"),
 487         HASH = r.hash(),
 488         HASH8 = r.hash(8),
 489         TAG = r.tag(''),
 490         TAG_OR_BRANCH = r.tag(r.branch('unreleased')),
 491         TAG_OR_HASH8 = r.tag(r.hash(8, 'untagged')),
 492         LABEL = r.label(),
 493         VERSION_FOUR_NUM = r.version_four_num(),
 494         REVISION = r.revision(),
 495         DIRTY = r.dirty(),
 496         FWTAG = xtrim(r.tag(r.branch('unreleased')), r.dirty(), 25),
 497         UNIXTIME = r.time(),
 498         DATE = r.time('%Y%m%d'),
 499         DATETIME = r.time('%Y%m%d %H:%M'),
 500         DAY = r.time('%d'),
 501         MONTH = r.time('%m'),
 502         YEAR = r.time('%Y'),
 503         HOUR = r.time('%H'),
 504         MINUTE = r.time('%M'),
 505         BOARD_TYPE = args.type,
 506         BOARD_REVISION = args.revision,
 507         UAVO_HASH = get_hash_of_dirs(args.uavodir, verbose = 0, raw = 1),
 508         UAVO_HASH8 = get_hash_of_dirs(args.uavodir, verbose = 0, raw = 1, n = 8),
 509         UAVO_HASH_ARRAY = get_hash_of_dirs(args.uavodir, verbose = 0, raw = 0),
 510         IMAGE_HASH_ARRAY = sha1(args.image),
 511     )
 512
 513     # Process positional arguments in the form of:
 514     #   VAR1=str1 VAR2="string 2"
 515     for var in positional_args:
 516         (key, value) = var.split('=', 1)
 517         dictionary[key] = value
 518
 519     if args.info:
 520         r.info()
 521
 522     files_to_escape = ['.c', '.cpp']
 523
 524     if (args.escape == None and args.outfile != None and
 525             os.path.splitext(args.outfile)[1] in files_to_escape) or args.escape:
 526         escape_dict(dictionary)
 527
 528     if args.format != None:
 529         print Template(args.format).substitute(dictionary)
 530
 531     if args.outfile != None:
 532         file_from_template(args.template, args.outfile, dictionary)
 533
 534     if args.jsonpath != None:
 535         r.save_to_json(args.jsonpath)
 536
 537     return 0
 538
 539 if __name__ == "__main__":
 540     sys.exit(main())