tools/python-integrate-project

   1 #! /usr/bin/ksh
   2 #
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright 2022 Marcel Telka
  16 #
  17
  18
  19 THIS="python-integrate-project"
  20 CONF="$THIS.conf"
  21 SNIPPET="$THIS.snippet"
  22 APIURL="https://pypi.org/pypi"
  23 CURL="/usr/bin/curl -s"
  24
  25
  26 function usage
  27 {
  28         [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
  29         printf "Usage: %s [-d DIR] [-f] [-l VERSION] [-o OBSOLETE].. [-u] PROJECT\n" "$THIS" >&2
  30         [[ -n "$1" ]] && exit 1
  31         exit 0
  32 }
  33
  34
  35 OPT_VERSION=
  36 OBSOLETE=
  37 UPGRADE_ONLY=0
  38 DIRECTORY=
  39 FORCE=0
  40 while getopts ":hd:fl:o:u" OPT ; do
  41         case "$OPT" in
  42         "?"|"h")        usage ;;
  43         "d")            DIRECTORY="$OPTARG" ;;
  44         "f")            FORCE=1 ;;
  45         "l")            OPT_VERSION="$OPTARG" ;;
  46         "o")            OBSOLETE="$OBSOLETE $OPTARG" ;;
  47         "u")            UPGRADE_ONLY=1 ;;
  48         esac
  49 done
  50 shift $((OPTIND - 1))
  51
  52 (($# == 0)) && usage
  53 (($# > 1)) && usage "Too many arguments"
  54
  55 PROJECT="$1"
  56
  57
  58 # Prevent user's environment to affect the integration.
  59 # Allow one exception only: USERLAND_ARCHIVES
  60 GMAKE="env -"
  61 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
  62 GMAKE="$GMAKE gmake"
  63
  64
  65 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
  66 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
  67
  68 BASE_DIR="$WS_TOP/components"
  69 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
  70
  71
  72 # Distribution match project
  73 DISTRIBUTION="$PROJECT"
  74
  75 function get_PKGINFO_entry
  76 {
  77         typeset ENTRY="$1"
  78
  79         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/PKG-INFO" ]] || return
  80
  81         cat "$SOURCE_DIR$COMPONENT_SUBDIR/PKG-INFO" \
  82                 | sed -e '/^$/,$d' \
  83                 | awk 'END{printf("\n")}/^[^:]+: /{$0="\n"$0}1' ORS=' ' \
  84                 | grep "^$ENTRY: " \
  85                 | sed -e "s/^$ENTRY: //" -e 's/ *$//'
  86 }
  87
  88
  89 # Prepare the directory
  90 [[ -z "$DIRECTORY" ]] && DIRECTORY="python/$DISTRIBUTION"
  91 DIR="$BASE_DIR/$DIRECTORY"
  92 mkdir -p "$DIR"
  93 cd "$DIR"
  94 git restore --staged . > /dev/null 2>&1
  95 git checkout . > /dev/null 2>&1
  96
  97
  98 # Following variables could be set by the hook-begin snippet
  99 VERSION=
 100 HOMEPAGE=
 101 DOWNLOAD_URL=
 102 LICENSE_FILE=
 103 SUMMARY=
 104
 105 # Execute hook-begin snippet
 106 if [[ -f "$CONF" ]] ; then
 107         gsed -e '0,/^%hook-begin%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 108         . "./$SNIPPET"
 109         rm -f "$SNIPPET"
 110 fi
 111
 112 # Version specified as option takes precedence
 113 [[ -n "$OPT_VERSION" ]] && VERSION="$OPT_VERSION"
 114
 115
 116 # Get data from PyPI if needed
 117 if [[ -z "$VERSION" || -z "$HOMEPAGE" || -z "$SUMMARY" ]] ; then
 118         PYPI_PROJECT=$($CURL "$APIURL/$PROJECT/json")
 119         if (($? != 0)) || [[ -z "$PYPI_PROJECT" ]] ; then
 120                 printf 'WARNING: Failed to get data for project %s from PyPI\n' "$PROJECT" >&2
 121                 PYPI_PROJECT=
 122         fi
 123 fi
 124
 125
 126 # Find the latest version if not already provided
 127 if [[ -z "$VERSION" ]] ; then
 128         VERSION=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.version')
 129         if (($? != 0)) || [[ -z "$VERSION" || "$VERSION" == "null" ]] ; then
 130                 printf "FATAL: Failed to get version for project %s from pypi\n" "$PROJECT" >&2
 131                 exit 1
 132         fi
 133 fi
 134
 135
 136 # Is this new project, or just a rebuild?
 137 NEW=1
 138 PREV_VER=
 139 PREV_HVER=
 140 PREV_REV=0
 141 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
 142         NEW=0
 143         PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
 144         (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
 145         PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
 146
 147         # If we were asked to do version upgrade, but we do not have new
 148         # version, then we are done.
 149         PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
 150         ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
 151
 152         # Pre-flight environment checks
 153         if ((FORCE == 0)) ; then
 154                 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake env-check' failed!\n" >&2 && exit 1
 155                 if [[ "$($GMAKE print-value-PYTHON_TEST_BOOTSTRAP)" != "yes" ]] ; then
 156                         ! $GMAKE test-env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake test-env-check' failed!\n" >&2 && exit 1
 157                 fi
 158         fi
 159
 160         $GMAKE clobber > /dev/null 2>&1
 161 fi
 162
 163
 164 # Get project homepage if not already provided
 165 if [[ -z "$HOMEPAGE" ]] ; then
 166         HOMEPAGE=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.home_page')
 167         if (($? != 0)) || [[ -z "$HOMEPAGE" || "$HOMEPAGE" == "null" ]] ; then
 168                 HOMEPAGE=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.project_urls.Homepage')
 169                 if (($? != 0)) || [[ -z "$HOMEPAGE" || "$HOMEPAGE" == "null" ]] ; then
 170                         printf "WARNING: Failed to get homepage for project %s from pypi\n" "$PROJECT" >&2
 171                         HOMEPAGE=$(get_PKGINFO_entry "Home-page")
 172                 fi
 173         fi
 174 fi
 175
 176 # Get download url if not already provided
 177 if [[ -z "$DOWNLOAD_URL" ]] ; then
 178         # Get release data from pypi
 179         PYPI_PROJECT_RELEASE=$($CURL "$APIURL/$PROJECT/$VERSION/json")
 180         if (($? != 0)) || [[ -z "$PYPI_PROJECT_RELEASE" ]] ; then
 181                 printf "FATAL: Failed to get data for version %s from pypi\n" "$VERSION" >&2
 182                 exit 1
 183         fi
 184
 185         # Get download url
 186         DOWNLOAD_URL=$(printf "%s" "$PYPI_PROJECT_RELEASE" | /usr/bin/jq -r '.urls[]|select(.packagetype=="sdist")|.url')
 187         if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
 188                 printf "WARNING: Failed to get download url for project %s, version %s from pypi\n" "$PROJECT" "$VERSION" >&2
 189                 DOWNLOAD_URL=
 190         fi
 191 fi
 192
 193
 194 # Remove everything that is not in git
 195 rm -rf *
 196 git checkout . > /dev/null 2>&1
 197 # Remove everything from git (except known patches, files, history, and $CONF)
 198 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
 199 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do rm -f "files/$FILE" ; done
 200 rm -f history "$CONF"
 201 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
 202 rm -rf "$DIR" 2>/dev/null
 203 git checkout history > /dev/null 2>&1
 204 git checkout "$CONF" > /dev/null 2>&1
 205 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do
 206         git checkout "patches/$PATCH" > /dev/null 2>&1
 207         [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
 208 done
 209 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do
 210         git checkout "files/$FILE" > /dev/null 2>&1
 211         [[ -f "files/$FILE" ]] || printf "WARNING: File %s not found\n" "$FILE" >&2
 212 done
 213
 214
 215 # Makefile template
 216 GENERATE_CMD="\$WS_TOOLS/$THIS"
 217 [[ "$DIRECTORY" != "python/$DISTRIBUTION" ]] && GENERATE_CMD="$GENERATE_CMD -d $DIRECTORY"
 218 GENERATE_CMD="$GENERATE_CMD $PROJECT"
 219 (
 220 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
 221 cat <<EOF
 222
 223 #
 224 # This file was automatically generated using the following command:
 225 #   $GENERATE_CMD
 226 #
 227
 228 BUILD_STYLE = pyproject
 229 USE_COMMON_TEST_MASTER = no
 230 EOF
 231 [[ -f "$CONF" ]] && gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
 232 cat <<EOF
 233
 234 include ../../../make-rules/shared-macros.mk
 235
 236 COMPONENT_NAME =                $DISTRIBUTION
 237 HUMAN_VERSION =                 $VERSION
 238 COMPONENT_REVISION =            $((PREV_REV + 1))
 239 COMPONENT_SUMMARY =             TODO
 240 EOF
 241 [[ -n "$HOMEPAGE" ]] && printf "COMPONENT_PROJECT_URL =\t\t%s\n" "$HOMEPAGE"
 242 [[ -n "$DOWNLOAD_URL" ]] && printf 'DOWNLOAD_URL =\t\t\\\n\t%s\n' "$DOWNLOAD_URL"
 243 cat <<EOF
 244 COMPONENT_ARCHIVE_HASH =        \\
 245         sha256:TODO
 246 COMPONENT_LICENSE =             license:TODO
 247 COMPONENT_LICENSE_FILE =        licfile:TODO
 248
 249 _TEST_STYLE = TODO
 250 EOF
 251 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 252 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
 253 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 254 printf "\n"
 255 ) > Makefile
 256
 257 # If the automatically constructed COMPONENT_ARCHIVE_URL points to the same
 258 # location as DOWNLOAD_URL then we should use it
 259 if [[ -n "$DOWNLOAD_URL" ]] ; then
 260         COMPONENT_ARCHIVE_URL=$($GMAKE print-value-COMPONENT_ARCHIVE_URL)
 261         [[ "$COMPONENT_ARCHIVE_URL" == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 262 fi
 263 # The default COMPONENT_ARCHIVE_URL usually redirects to DOWNLOAD_URL so check
 264 # that too
 265 if [[ -n "$DOWNLOAD_URL" ]] ; then
 266         [[ $($CURL --head --write-out "%{redirect_url}\n" --output /dev/null \
 267             "$COMPONENT_ARCHIVE_URL") == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 268 fi
 269 # Use either DOWNLOAD_URL or default COMPONENT_ARCHIVE_URL
 270 if [[ -n "$DOWNLOAD_URL" ]] ; then
 271         sed -i -e $'s/^DOWNLOAD_URL =/COMPONENT_ARCHIVE_URL =/' Makefile
 272 else
 273         sed -i -e $'/^DOWNLOAD_URL =/,+1d' Makefile
 274 fi
 275
 276 # Remove COMPONENT_REVISION if not needed
 277 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
 278 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && sed -i -e '/^COMPONENT_REVISION/d' Makefile
 279 git add Makefile
 280
 281 # Calculate sham256 sum for source package
 282 $GMAKE fetch > /dev/null 2>&1
 283 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
 284 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
 285 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
 286 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
 287 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
 288 git add Makefile
 289
 290 # Unpack sources and apply patches
 291 ! $GMAKE patch > /dev/null 2>&1 && printf "FATAL: 'gmake patch' failed!\n" >&2 && exit 1
 292
 293 # Refresh patches
 294 if $GMAKE refresh-patches > /dev/null 2>&1 ; then
 295         git add patches 2>/dev/null
 296 else
 297         printf "ERROR: 'gmake refresh-patches' failed!\n" >&2
 298         git checkout patches 2>/dev/null
 299 fi
 300
 301 # Cleanup after patch refresh
 302 $GMAKE clobber > /dev/null 2>&1
 303
 304 # Prepare sources
 305 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
 306 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
 307 COMPONENT_SUBDIR=$($GMAKE print-value-COMPONENT_SUBDIR)
 308 [[ -n "$COMPONENT_SUBDIR" ]] && COMPONENT_SUBDIR="/$COMPONENT_SUBDIR"
 309
 310 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/pyproject.toml" ]] ; then
 311         [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/setup.py" ]] && printf "FATAL: Neither pyproject.toml nor setup.py found!\n" >&2 && exit 1
 312         sed -i -e 's/^\(BUILD_STYLE = \).*$/\1setup.py/' Makefile
 313 fi
 314
 315 # Get summary if not already provided
 316 if [[ -z "$SUMMARY" ]] ; then
 317         SUMMARY=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.summary')
 318         if (($? != 0)) || [[ -z "$SUMMARY" || "$SUMMARY" == "null" ]] ; then
 319                 printf "WARNING: Failed to get summary for project %s from pypi\n" "$PROJECT" >&2
 320                 SUMMARY=$(get_PKGINFO_entry "Summary")
 321                 [[ -z "$SUMMARY" ]] && SUMMARY="TODO"
 322         fi
 323 fi
 324 # Summary needs to be sanitized
 325 SUMMARY="${SUMMARY//\`/\\\\\`}"
 326 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
 327 SUMMARY="${SUMMARY//\//\/}"
 328 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
 329 SUMMARY="${SUMMARY//\&/\\&}"
 330 SUMMARY="${SUMMARY//#/\\\\#}"
 331 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
 332
 333
 334 # Try to detect license type(s)
 335 function detect_license
 336 {
 337         typeset -n L="$1"
 338         typeset F="$2"
 339         typeset D
 340
 341         D=$("$WS_TOP/tools/license-detector" "$F")
 342         [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
 343 }
 344
 345 LICENSE=
 346 LICFILE=
 347 for f in $LICENSE_FILE $(get_PKGINFO_entry "License-File") LICENSE LICENSE.rst LICENSE.txt ; do
 348         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/$f" ]] || continue
 349         LICFILE="$f"
 350
 351         detect_license LICENSE "$SOURCE_DIR$COMPONENT_SUBDIR/$LICFILE"
 352         [[ -n "$LICENSE" ]] && break
 353
 354         printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
 355 done
 356 if [[ -z "$LICFILE" ]] ; then
 357         printf "WARNING: No license file found\n" >&2
 358 else
 359         sed -i -e 's|licfile:TODO|'"$LICFILE"'|g' Makefile
 360 fi
 361
 362 if [[ -z "$LICENSE" ]] ; then
 363         # Execute hook-no-license snippet
 364         if [[ -f "$CONF" ]] ; then
 365                 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 366                 . "./$SNIPPET"
 367                 rm -f "$SNIPPET"
 368         fi
 369
 370         if [[ -f "$DISTRIBUTION.license" ]] ; then
 371                 sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
 372                 git add "$DISTRIBUTION.license"
 373                 [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
 374         fi
 375         [[ -z "$LICENSE" ]] && LICENSE="TODO"
 376 fi
 377
 378 # Store the detected license into the Makefile
 379 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
 380
 381
 382 # detect TEST_STYLE
 383 TEST_STYLE=
 384 cd "$SOURCE_DIR$COMPONENT_SUBDIR"
 385 while true ; do
 386         TOX_OUT=$(tox -l)
 387         TOX_RET=$?
 388         ((TOX_RET == 0)) && ! printf "%s" "$TOX_OUT" | grep -q 'assuming empty tox\.ini' && TEST_STYLE="tox" && break
 389
 390         # Disable some pytest plugins that almost always collects tests to run
 391         # even there are no pytest tests available otherwise.
 392         #
 393         # The system-statistics plugin is disabled because it often causes the
 394         # pytest to fail.
 395         # See also https://github.com/saltstack/pytest-system-statistics/issues/4
 396         pytest -p no:black -p no:checkdocs -p no:cov -p no:mypy -p no:relaxed -p no:system-statistics --setup-plan
 397         (($? != 5)) && TEST_STYLE="pytest" && break
 398
 399         [[ -f setup.py ]] && python setup.py test --help && TEST_STYLE="setup.py" && break
 400
 401         TEST_STYLE="none"
 402         break
 403 done > /dev/null 2>&1
 404 cd "$DIR"
 405
 406 if [[ "$TEST_STYLE" == "$($GMAKE print-value-TEST_STYLE)" ]] ; then
 407         # If the detected TEST_STYLE is same as the default value or the value
 408         # forced by the component, then we do not need to set the detected
 409         # value.
 410         sed -i -e '/^_TEST_STYLE = TODO$/,+1d' Makefile
 411 else
 412         # Set the detected TEST_STYLE value
 413         sed -i -e 's/^_\(TEST_STYLE = \)TODO$/\1'"$TEST_STYLE/" Makefile
 414
 415         # If the component forces different test style than detected, then drop
 416         # the detected value
 417         if [[ "$TEST_STYLE" != "$($GMAKE print-value-TEST_STYLE)" ]] ; then
 418                 sed -i -e '/^TEST_STYLE = '"${TEST_STYLE//./\\.}/,+1d" Makefile
 419         fi
 420 fi
 421
 422 # Warn if a testing tool is called directly by tox
 423 if [[ "$($GMAKE print-value-TEST_STYLE)" == "tox" && -f "$SOURCE_DIR$COMPONENT_SUBDIR/tox.ini" ]] ; then
 424         TOX_CALL_INDIRECTLY=$($GMAKE print-value-TOX_CALL_INDIRECTLY)
 425         for p in $TOX_CALL_INDIRECTLY ; do
 426                 sed -n -e '/^commands *=/,/^$/p' "$SOURCE_DIR$COMPONENT_SUBDIR/tox.ini" \
 427                         | tr '\t' ' ' \
 428                         | grep -q '^\(commands *=\)\{0,1\} *'"$p"'\( \{1,\}.*\)\{0,1\}$' \
 429                 && printf "WARNING: %s is called directly in tox.ini\n" "$p" >&2
 430         done
 431 fi
 432
 433
 434 # Create manifests
 435 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
 436         printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
 437 else
 438         MANIFEST="$DISTRIBUTION-PYVER.p5m"
 439         [[ "$($GMAKE print-value-SINGLE_PYTHON_VERSION)" == "yes" ]] && MANIFEST="$DISTRIBUTION.p5m"
 440         cat manifests/sample-manifest.p5m \
 441                 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
 442                 > "$MANIFEST"
 443
 444         # Execute hook-manifest snippet
 445         if [[ -f "$CONF" ]] ; then
 446                 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 447                 . "./$SNIPPET"
 448                 rm -f "$SNIPPET"
 449         fi
 450
 451         git add manifests/sample-manifest.p5m $MANIFEST
 452 fi
 453
 454
 455 # Generate REQUIRED_PACKAGES
 456 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
 457 git add Makefile
 458
 459
 460 # Check for Makefile completeness
 461 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
 462
 463
 464 # Make sure the build environment is setup properly and we do have all
 465 # requirements installed.  Otherwise we cannot continue.
 466 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
 467
 468
 469 # Handle history
 470 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
 471 PYTHON_VERSIONS_OBSOLETING=$($GMAKE print-value-PYTHON_VERSIONS_OBSOLETING)
 472 OV=
 473 OV_PLURAL=
 474 for o in $(echo $OBSOLETE $PYTHON_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
 475         PYV=${o//.}
 476         FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PYV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
 477         [[ -n "$FMRI" ]] || continue
 478         FMRI_H=${FMRI%.*}
 479         FMRI_T=${FMRI##*.}
 480         if [[ "$FMRI_H" == "$FMRI" ]] ; then
 481                 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
 482                 continue
 483         fi
 484         FMRI_T=$((FMRI_T + 1))
 485         printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
 486
 487         [[ -n "$OV" ]] && OV="${OV/ and /, } and " && OV_PLURAL="s"
 488         OV="$OV$o"
 489 done
 490 if [[ -f history ]] ; then
 491         LC_ALL=C sort -u history > history.new
 492         mv history.new history
 493         git add history
 494
 495         awk '$NF == "noincorporate" {printf("WARNING: Unincorporated package: %s\n", $1)}' < history >&2
 496 fi
 497
 498
 499 # Cleanup before we try to publish to make sure there are no leftovers from
 500 # previous steps
 501 $GMAKE clobber > /dev/null 2>&1
 502
 503 # Publish packages and create pkg5 file
 504 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
 505 git add pkg5 2>/dev/null
 506
 507
 508 PYTHON_VERSIONS=$($GMAKE print-value-PYTHON_VERSIONS)
 509 PYTHON_TEST_BOOTSTRAP=$($GMAKE print-value-PYTHON_TEST_BOOTSTRAP)
 510
 511
 512 # Run tests to make sure they pass and to create result snapshots
 513 TESTED_VERSIONS=
 514 for v in $PYTHON_VERSIONS ; do
 515         # Check the test environment
 516         if ! $GMAKE PYTHON_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
 517                 if [[ "$PYTHON_TEST_BOOTSTRAP" == "yes" ]] ; then
 518                         printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
 519                 else
 520                         printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
 521                 fi
 522                 continue
 523         fi
 524
 525         # Run the test
 526         ! $GMAKE PYTHON_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
 527
 528         # If there is no snapshot produced the component likely does not support tests
 529         COMPONENT_TEST_SNAPSHOT=$($GMAKE PYTHON_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 530         [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
 531
 532         # Empty result snapshot is suspicious
 533         [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
 534
 535         TESTED_VERSIONS="$TESTED_VERSIONS $v"
 536 done
 537
 538 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
 539 TEST_MASTERS=
 540 for common_results in yes no ; do
 541         for v in $TESTED_VERSIONS ; do
 542                 COMPONENT_TEST_SNAPSHOT=$($GMAKE PYTHON_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 543                 COMPONENT_TEST_MASTER=$($GMAKE PYTHON_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
 544
 545                 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
 546                         # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
 547                         if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
 548                                 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
 549                                 rm -f $TEST_MASTERS
 550                                 TEST_MASTERS=
 551                                 continue 2
 552                         fi
 553                 else
 554                         mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
 555                         cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
 556                         TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
 557                 fi
 558         done
 559         break
 560 done
 561 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
 562
 563 # Run tests again to confirm the results are reproducible
 564 for v in $TESTED_VERSIONS ; do
 565         $GMAKE PYTHON_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
 566 done
 567
 568 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
 569 if [[ "$common_results" == "yes" ]] ; then
 570         sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
 571         git add Makefile
 572 fi
 573
 574
 575 # Create .gitignore
 576 COMPONENT_SRC=$($GMAKE print-value-COMPONENT_SRC)
 577 printf '/%s/\n' "$COMPONENT_SRC" > .gitignore
 578 git add .gitignore
 579
 580
 581 # Construct the commit message
 582 MSG=
 583 if ((NEW)) ; then
 584         MSG="Add $PROJECT Python project"
 585 else
 586         [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && MSG="change version format"
 587         [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION"
 588
 589         REBUILDMSG=
 590
 591         if [[ "$($GMAKE print-value-SINGLE_PYTHON_VERSION)" == "no" ]] ; then
 592                 NV=
 593                 for v in $PYTHON_VERSIONS ; do
 594                         PYV=${v//.}
 595                         pkg list -avH "$COMPONENT_FMRI-$PYV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
 596                         [[ -n "$NV" ]] && NV="$NV and "
 597                         NV="$NV$v"
 598                 done
 599                 [[ -n "$NV" ]] && REBUILDMSG="rebuild for Python $NV"
 600         fi
 601
 602         if [[ -n "$OV" ]] ; then
 603                 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and "
 604                 REBUILDMSG="${REBUILDMSG}obsolete package$OV_PLURAL for Python $OV"
 605         fi
 606
 607         if [[ -n "$REBUILDMSG" ]] ; then
 608                 [[ -n "$MSG" ]] && MSG="$MSG; "
 609                 MSG="$MSG$REBUILDMSG"
 610         fi
 611         [[ -z "$MSG" ]] && MSG="rebuild"
 612
 613         MSG="$DIRECTORY: $MSG"
 614 fi
 615
 616 # Commit the results
 617 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1