tools/perl-integrate-module

   1 #! /usr/bin/ksh
   2 #
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright 2022 Marcel Telka
  16 #
  17
  18
  19 THIS="perl-integrate-module"
  20 CONF="$THIS.conf"
  21 SNIPPET="$THIS.snippet"
  22 APIURL="https://fastapi.metacpan.org/v1"
  23 CURL="/usr/bin/curl -s"
  24
  25
  26 function usage
  27 {
  28         [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
  29         printf "Usage: %s [-d DIR] [-f] [-l VERSION] [-o OBSOLETE].. [-u] MODULE\n" "$THIS" >&2
  30         [[ -n "$1" ]] && exit 1
  31         exit 0
  32 }
  33
  34
  35 OPT_VERSION=
  36 OBSOLETE=
  37 UPGRADE_ONLY=0
  38 DIRECTORY=
  39 FORCE=0
  40 while getopts ":hd:fl:o:u" OPT ; do
  41         case "$OPT" in
  42         "?"|"h")        usage ;;
  43         "d")            DIRECTORY="$OPTARG" ;;
  44         "f")            FORCE=1 ;;
  45         "l")            OPT_VERSION="$OPTARG" ;;
  46         "o")            OBSOLETE="$OBSOLETE $OPTARG" ;;
  47         "u")            UPGRADE_ONLY=1 ;;
  48         esac
  49 done
  50 shift $((OPTIND - 1))
  51
  52 (($# == 0)) && usage
  53 (($# > 1)) && usage "Too many arguments"
  54
  55 MODULE="$1"
  56
  57
  58 # Prevent user's environment to affect the integration.
  59 # Allow one exception only: USERLAND_ARCHIVES
  60 GMAKE="env -"
  61 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
  62 GMAKE="$GMAKE gmake"
  63
  64
  65 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
  66 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
  67
  68 BASE_DIR="$WS_TOP/components"
  69 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
  70
  71
  72 # Get data from metacpan
  73 METACPAN_MODULE=$($CURL "$APIURL/module/$MODULE")
  74 if (($? != 0)) || [[ -z "$METACPAN_MODULE" ]] ; then
  75         printf "FATAL: Failed to get data from metacpan\n" >&2
  76         exit 1
  77 fi
  78
  79 # Detect distribution for module
  80 DISTRIBUTION=$(printf "%s" "$METACPAN_MODULE" | /usr/bin/jq -r '.distribution')
  81 if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
  82         printf "FATAL: Failed to get distribution for module %s from metacpan\n" "$MODULE" >&2
  83         exit 1
  84 fi
  85 if [[ "$DISTRIBUTION" != "${MODULE//::/-}" ]] then
  86         NEW_MODULE="${DISTRIBUTION//-/::}"
  87         NEW_METACPAN_MODULE=$($CURL "$APIURL/module/$NEW_MODULE")
  88         NEW_DISTRIBUTION=$(printf "%s" "$NEW_METACPAN_MODULE" | /usr/bin/jq -r '.distribution')
  89         if [[ "$NEW_DISTRIBUTION" == "$DISTRIBUTION" ]] ; then
  90                 printf "WARNING: Module %s does not match distribution %s\n" "$MODULE" "$DISTRIBUTION" >&2
  91                 printf "WARNING: Continue with module %s instead of %s\n" "$NEW_MODULE" "$MODULE" >&2
  92                 MODULE="$NEW_MODULE"
  93                 METACPAN_MODULE="$NEW_METACPAN_MODULE"
  94         fi
  95 fi
  96
  97
  98 # Prepare the directory
  99 [[ -z "$DIRECTORY" ]] && DIRECTORY="perl/$DISTRIBUTION"
 100 DIR="$BASE_DIR/$DIRECTORY"
 101 mkdir -p "$DIR"
 102 cd "$DIR"
 103 git restore --staged . > /dev/null 2>&1
 104 git checkout . > /dev/null 2>&1
 105
 106
 107 # Following variables could be set by the hook-begin snippet
 108 VERSION=
 109 DOWNLOAD_URL=
 110 LICENSE_FILE=
 111 SUMMARY=
 112
 113 # Execute hook-begin snippet
 114 if [[ -f "$CONF" ]] ; then
 115         gsed -e '0,/^%hook-begin%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 116         . "./$SNIPPET"
 117         rm -f "$SNIPPET"
 118 fi
 119
 120 # Version specified as option takes precedence
 121 [[ -n "$OPT_VERSION" ]] && VERSION="$OPT_VERSION"
 122
 123 # Find the latest version if not already provided
 124 if [[ -z "$VERSION" ]] ; then
 125         VERSION=$(printf "%s" "$METACPAN_MODULE" | /usr/bin/jq -r '.version')
 126         if (($? != 0)) || [[ -z "$VERSION" || "$VERSION" == "null" ]] ; then
 127                 printf "FATAL: Failed to get version for module %s from metacpan\n" "$MODULE" >&2
 128                 exit 1
 129         fi
 130         VERSION=${VERSION#v}
 131 fi
 132
 133
 134 # Is this new module, or just a rebuild?
 135 NEW=1
 136 PREV_VER=
 137 PREV_HVER=
 138 PREV_REV=0
 139 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
 140         NEW=0
 141         PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
 142         (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
 143         PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
 144
 145         # If we were asked to do version upgrade, but we do not have new
 146         # version, then we are done.
 147         PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
 148         ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
 149
 150         # Pre-flight environment checks
 151         if ((FORCE == 0)) ; then
 152                 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake env-check' failed!\n" >&2 && exit 1
 153                 if [[ "$($GMAKE print-value-PERL_TEST_BOOTSTRAP)" != "yes" ]] ; then
 154                         ! $GMAKE test-env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake test-env-check' failed!\n" >&2 && exit 1
 155                 fi
 156         fi
 157
 158         $GMAKE clobber > /dev/null 2>&1
 159 fi
 160
 161
 162 # Get download_url if not already provided
 163 if [[ -z "$DOWNLOAD_URL" ]] ; then
 164         # Get module author
 165         # TODO: here we get author of the latest version, not the specified version
 166         AUTHOR=$(printf "%s" "$METACPAN_MODULE" | /usr/bin/jq -r '.author')
 167         if (($? != 0)) || [[ -z "$AUTHOR" || "$AUTHOR" == "null" ]] ; then
 168                 printf "FATAL: Failed to get author for module %s from metacpan\n" "$MODULE" >&2
 169                 exit 1
 170         fi
 171
 172         # Get download_url
 173         DOWNLOAD_URL=$($CURL "$APIURL/download_url/$MODULE?version===$VERSION" | /usr/bin/jq -r '.download_url')
 174         if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
 175                 printf "WARNING: Failed to get download_url for module %s, version %s from metacpan\n" "$MODULE" "$VERSION" >&2
 176                 DOWNLOAD_URL=
 177         fi
 178 fi
 179
 180
 181 # Remove everything that is not in git
 182 rm -rf *
 183 git checkout . > /dev/null 2>&1
 184 # Remove everything from git (except known patches, files, history, and $CONF)
 185 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
 186 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do rm -f "files/$FILE" ; done
 187 rm -f history "$CONF"
 188 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
 189 rm -rf "$DIR" 2>/dev/null
 190 git checkout history > /dev/null 2>&1
 191 git checkout "$CONF" > /dev/null 2>&1
 192 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do
 193         git checkout "patches/$PATCH" > /dev/null 2>&1
 194         [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
 195 done
 196 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do
 197         git checkout "files/$FILE" > /dev/null 2>&1
 198         [[ -f "files/$FILE" ]] || printf "WARNING: File %s not found\n" "$FILE" >&2
 199 done
 200
 201
 202 # Makefile template
 203 GENERATE_CMD="\$WS_TOOLS/$THIS"
 204 [[ "$DIRECTORY" != "perl/$DISTRIBUTION" ]] && GENERATE_CMD="$GENERATE_CMD -d $DIRECTORY"
 205 GENERATE_CMD="$GENERATE_CMD $MODULE"
 206 (
 207 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
 208 cat <<EOF
 209
 210 #
 211 # This file was automatically generated using the following command:
 212 #   $GENERATE_CMD
 213 #
 214
 215 BUILD_STYLE = makemaker
 216 USE_COMMON_TEST_MASTER = no
 217 EOF
 218 [[ -f "$CONF" ]] && gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
 219 cat <<EOF
 220
 221 include ../../../make-rules/shared-macros.mk
 222
 223 COMPONENT_PERL_MODULE =         $MODULE
 224 EOF
 225 [[ "$DISTRIBUTION" != "${MODULE//::/-}" ]] && printf "COMPONENT_PERL_DISTRIBUTION =\t%s\n" "$DISTRIBUTION"
 226 cat <<EOF
 227 HUMAN_VERSION =                 $VERSION
 228 COMPONENT_REVISION =            $((PREV_REV + 1))
 229 COMPONENT_SUMMARY =             $MODULE - TODO
 230 COMPONENT_CPAN_AUTHOR =         $AUTHOR
 231 COMPONENT_ARCHIVE_HASH =        \\
 232         sha256:TODO
 233 COMPONENT_LICENSE =             license:TODO
 234 COMPONENT_LICENSE_FILE =        licfile:TODO
 235 EOF
 236 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 237 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
 238 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 239 printf "\n"
 240 ) > Makefile
 241
 242 # If the automatically constructed COMPONENT_ARCHIVE_URL is not correct then we
 243 # do not need COMPONENT_CPAN_AUTHOR.  We need COMPONENT_ARCHIVE_URL instead.
 244 if [[ -n "$DOWNLOAD_URL" ]] ; then
 245         COMPONENT_ARCHIVE_URL=$($GMAKE print-value-COMPONENT_ARCHIVE_URL)
 246         [[ "$COMPONENT_ARCHIVE_URL" == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 247 fi
 248 [[ -n "$DOWNLOAD_URL" ]] && sed -i -e $'s|^COMPONENT_CPAN_AUTHOR.*|COMPONENT_ARCHIVE_URL =\t\t\\\\\\\n\t'"$DOWNLOAD_URL"'|' Makefile
 249
 250 # Remove COMPONENT_REVISION if not needed
 251 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
 252 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && sed -i -e '/^COMPONENT_REVISION/d' Makefile
 253 git add Makefile
 254
 255 # Calculate sham256 sum for source package
 256 $GMAKE fetch > /dev/null 2>&1
 257 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
 258 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
 259 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
 260 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
 261 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
 262 git add Makefile
 263
 264 # Unpack sources and apply patches
 265 ! $GMAKE patch > /dev/null 2>&1 && printf "FATAL: 'gmake patch' failed!\n" >&2 && exit 1
 266
 267 # Refresh patches
 268 if $GMAKE refresh-patches > /dev/null 2>&1 ; then
 269         git add patches 2>/dev/null
 270 else
 271         printf "ERROR: 'gmake refresh-patches' failed!\n" >&2
 272         git checkout patches 2>/dev/null
 273 fi
 274
 275 # Cleanup after patch refresh
 276 $GMAKE clobber > /dev/null 2>&1
 277
 278 # Prepare sources
 279 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
 280 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
 281 COMPONENT_SUBDIR=$($GMAKE print-value-COMPONENT_SUBDIR)
 282 [[ -n "$COMPONENT_SUBDIR" ]] && COMPONENT_SUBDIR="/$COMPONENT_SUBDIR"
 283
 284 # Switch to modulebuild if possible
 285 [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/Build.PL" ]] && sed -i -e 's/makemaker/modulebuild/g' Makefile
 286
 287 # Get summary if not already provided
 288 if [[ -z "$SUMMARY" ]] ; then
 289         # Get abstract and use it as summary.  Either from metacpan, or directly from sources.
 290         ABSTRACT=$(printf "%s" "$METACPAN_MODULE" | /usr/bin/jq -r '.abstract')
 291         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 292                 printf "WARNING: Failed to get abstract for module %s from metacpan\n" "$MODULE" >&2
 293                 ABSTRACT="TODO"
 294         fi
 295         if [[ "$ABSTRACT" == "TODO" ]] ; then
 296                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" ]] ; then
 297                         printf "WARNING: META.json missing\n" >&2
 298                 else
 299                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" | /usr/bin/jq -r '.abstract')
 300                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 301                                 printf "WARNING: Failed to get abstract from META.json\n" >&2
 302                                 ABSTRACT="TODO"
 303                         fi
 304                 fi
 305         fi
 306         if [[ "$ABSTRACT" == "TODO" ]] ; then
 307                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" ]] ; then
 308                         printf "WARNING: META.yml missing\n" >&2
 309                 else
 310                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' | /usr/bin/jq -r '.abstract')
 311                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 312                                 printf "WARNING: Failed to get abstract from META.yml\n" >&2
 313                                 ABSTRACT="TODO"
 314                         fi
 315                 fi
 316         fi
 317         SUMMARY="$ABSTRACT"
 318 fi
 319 # Summary needs to be sanitized
 320 SUMMARY="${SUMMARY//\`/\\\\\`}"
 321 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
 322 SUMMARY="${SUMMARY//\//\/}"
 323 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
 324 SUMMARY="${SUMMARY//\&/\\&}"
 325 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
 326
 327
 328 # Try to detect license type(s)
 329 function detect_license
 330 {
 331         typeset -n L="$1"
 332         typeset F="$2"
 333         typeset D
 334
 335         D=$("$WS_TOP/tools/license-detector" "$F")
 336         [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
 337 }
 338
 339 LICENSE=
 340 LICFILE=
 341 COPYRIGHT=
 342 for f in $LICENSE_FILE LICENSE LICENCE COPYING COPYRIGHT ; do
 343         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/$f" ]] || continue
 344         LICFILE="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 345
 346         detect_license LICENSE "$LICFILE"
 347
 348         if [[ -n "$LICENSE" ]] ; then
 349                 sed -i -e 's|licfile:TODO|'"$f"'|g' Makefile
 350                 break
 351         fi
 352
 353         printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
 354
 355         # Since the license file does not contain any known license we will use
 356         # its content as Copyright notice only
 357         COPYRIGHT=$(<"$LICFILE")
 358 done
 359 if [[ -z "$LICFILE" ]] ; then
 360         printf "WARNING: No license file found\n" >&2
 361 fi
 362
 363 if [[ -z "$LICENSE" ]] ; then
 364         # Since the distibution does not provide own license file (or we failed
 365         # to find it) we will use default Perl license with added Copyright
 366         # notice from this distribution
 367
 368         sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
 369
 370         # Try to find Copyright notice if we do not have one yet
 371         [[ -z "$COPYRIGHT" ]] && for f in README README.md ; do
 372                 f="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 373                 [[ -f "$f" ]] || continue
 374
 375                 COPYRIGHT=$(gsed -e '0,/^# LICENSE/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 376                 [[ -n "$COPYRIGHT" ]] && break
 377                 COPYRIGHT=$(gsed -e '0,/^# COPYRIGHT/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 378                 [[ -n "$COPYRIGHT" ]] && break
 379                 COPYRIGHT=$(gsed -e '0,/LICENSE/d' -e '/^REPOSITORY/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 380                 [[ -n "$COPYRIGHT" ]] && break
 381                 COPYRIGHT=$(gsed -e '0,/COPYING/d' -e '/^BUGS/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 382                 [[ -n "$COPYRIGHT" ]] && break
 383                 COPYRIGHT=$(gsed -e '0,/COPYRIGHT/d' -e '/^AUTHOR/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 384                 [[ -n "$COPYRIGHT" ]] && break
 385                 COPYRIGHT=$(gsed -e '0,/^## Copyright/d' -e '/./,$!d' "$f" 2>/dev/null)
 386                 [[ -n "$COPYRIGHT" ]] && break
 387         done
 388         if [[ -z "$COPYRIGHT" ]] ; then
 389                 printf "WARNING: No copyright notice found at standard locations\n" >&2
 390                 for f in $(find "$SOURCE_DIR$COMPONENT_SUBDIR" -type f -name "*.pm" | LC_ALL=C sort | while read f ; do egrep -q "^=head1 (LICENSE|LICENCE|COPYRIGHT)" "$f" && echo "$f" ; done) ; do
 391                         COPYRIGHT=$(sed -e '1,/^=head1 LICENSE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 392                         if [[ -n "$COPYRIGHT" ]] ; then
 393                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 394                                 break
 395                         fi
 396                         COPYRIGHT=$(sed -e '1,/^=head1 LICENCE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 397                         if [[ -n "$COPYRIGHT" ]] ; then
 398                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 399                                 break
 400                         fi
 401                         COPYRIGHT=$(sed -e '1,/^=head1 COPYRIGHT/d' -e '/^=/,$d' "$f" 2>/dev/null)
 402                         if [[ -n "$COPYRIGHT" ]] ; then
 403                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 404                                 break
 405                         fi
 406                 done
 407         fi
 408         if [[ -z "$COPYRIGHT" ]] ; then
 409                 printf "WARNING: No copyright notice found\n" >&2
 410                 > "$DISTRIBUTION.license"
 411         else
 412                 (printf "%s\n\n" "$COPYRIGHT" | dos2unix -ascii
 413                 i=75 ; while ((i)) ; do printf "=" ; i=$((i-1)) ; done
 414                 printf "\n\n") > "$DISTRIBUTION.license"
 415         fi
 416
 417         USE_DEFAULT_PERL_LICENSE=1
 418
 419         # Execute hook-no-license snippet
 420         if [[ -f "$CONF" ]] ; then
 421                 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 422                 . "./$SNIPPET"
 423                 rm -f "$SNIPPET"
 424         fi
 425
 426
 427         if ((USE_DEFAULT_PERL_LICENSE)) ; then
 428                 # Confirm the package is distributed under the same terms as Perl itself
 429                 D=1
 430                 ((D)) && (printf "%s\n" "$COPYRIGHT" | grep -q -i "under the same terms as Perl itself") && D=0
 431                 ((D)) && grep -q "license *=> *'http://dev\.perl\.org/licenses/'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 432                 ((D)) && grep -q "LICENSE *=> *'perl'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 433                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" && "$(/usr/bin/jq -r '.license[]' < "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" 2>/dev/null)" == "perl_5" ]] && D=0
 434                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" && "$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" \
 435                         | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' \
 436                         | /usr/bin/jq -r '.license' 2>/dev/null)" == "perl" ]] && D=0
 437
 438                 ((D)) && printf "ERROR: Heuristics failed to detect license type, using default Perl license\n" >&2
 439
 440                 # Make a copy of license so we can use it during publish
 441                 cat "$WS_TOP/tools/perl-license" | grep -v "^#" >> "$DISTRIBUTION.license"
 442         fi
 443         git add "$DISTRIBUTION.license"
 444
 445         [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
 446         [[ -z "$LICENSE" ]] && LICENSE="TODO"
 447 fi
 448
 449 # Store the detected license into the Makefile
 450 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
 451
 452
 453 # Create manifests
 454 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
 455         printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
 456 else
 457         MANIFEST="$DISTRIBUTION-PERLVER.p5m"
 458         [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "yes" ]] && MANIFEST="$DISTRIBUTION.p5m"
 459         cat manifests/sample-manifest.p5m \
 460                 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
 461                 > "$MANIFEST"
 462
 463         # Execute hook-manifest snippet
 464         if [[ -f "$CONF" ]] ; then
 465                 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 466                 . "./$SNIPPET"
 467                 rm -f "$SNIPPET"
 468         fi
 469
 470         git add manifests/sample-manifest.p5m $MANIFEST
 471 fi
 472
 473
 474 # Generate REQUIRED_PACKAGES
 475 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
 476 git add Makefile
 477
 478
 479 # Check for Makefile completeness
 480 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
 481
 482
 483 # Make sure the build environment is setup properly and we do have all
 484 # requirements installed.  Otherwise we cannot continue.
 485 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
 486
 487
 488 # Handle history
 489 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
 490 PERL_VERSIONS_OBSOLETING=$($GMAKE print-value-PERL_VERSIONS_OBSOLETING)
 491 OV=
 492 OV_PLURAL=
 493 for o in $(echo $OBSOLETE $PERL_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
 494         PLV=${o//.}
 495         FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
 496         [[ -n "$FMRI" ]] || continue
 497         FMRI_H=${FMRI%.*}
 498         FMRI_T=${FMRI##*.}
 499         if [[ "$FMRI_H" == "$FMRI" ]] ; then
 500                 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
 501                 continue
 502         fi
 503         FMRI_T=$((FMRI_T + 1))
 504         printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
 505
 506         [[ -n "$OV" ]] && OV="${OV/ and /, } and " && OV_PLURAL="s"
 507         OV="$OV$o"
 508 done
 509 if [[ -f history ]] ; then
 510         LC_ALL=C sort -u history > history.new
 511         mv history.new history
 512         git add history
 513
 514         awk '$NF == "noincorporate" {printf("WARNING: Unincorporated package: %s\n", $1)}' < history >&2
 515 fi
 516
 517
 518 # Cleanup before we try to publish to make sure there are no leftovers from
 519 # previous steps
 520 $GMAKE clobber > /dev/null 2>&1
 521
 522 # Publish packages and create pkg5 file
 523 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
 524 git add pkg5 2>/dev/null
 525
 526
 527 PERL_VERSIONS=$($GMAKE print-value-PERL_VERSIONS)
 528 PERL_TEST_BOOTSTRAP=$($GMAKE print-value-PERL_TEST_BOOTSTRAP)
 529
 530
 531 # Run tests to make sure they pass and to create result snapshots
 532 TESTED_VERSIONS=
 533 for v in $PERL_VERSIONS ; do
 534         # Check the test environment
 535         if ! $GMAKE PERL_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
 536                 if [[ "$PERL_TEST_BOOTSTRAP" == "yes" ]] ; then
 537                         printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
 538                 else
 539                         printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
 540                 fi
 541                 continue
 542         fi
 543
 544         # Run the test
 545         ! $GMAKE PERL_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
 546
 547         # If there is no snapshot produced the component likely does not support tests
 548         COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 549         [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
 550
 551         # Empty result snapshot is suspicious
 552         [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
 553
 554         TESTED_VERSIONS="$TESTED_VERSIONS $v"
 555 done
 556
 557 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
 558 TEST_MASTERS=
 559 for common_results in yes no ; do
 560         for v in $TESTED_VERSIONS ; do
 561                 COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 562                 COMPONENT_TEST_MASTER=$($GMAKE PERL_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
 563
 564                 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
 565                         # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
 566                         if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
 567                                 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
 568                                 rm -f $TEST_MASTERS
 569                                 TEST_MASTERS=
 570                                 continue 2
 571                         fi
 572                 else
 573                         mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
 574                         cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
 575                         TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
 576                 fi
 577         done
 578         break
 579 done
 580 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
 581
 582 # Run tests again to confirm the results are reproducible
 583 for v in $TESTED_VERSIONS ; do
 584         $GMAKE PERL_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
 585 done
 586
 587 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
 588 if [[ "$common_results" == "yes" ]] ; then
 589         sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
 590         git add Makefile
 591 fi
 592
 593
 594 # Create .gitignore
 595 COMPONENT_SRC=$($GMAKE print-value-COMPONENT_SRC)
 596 printf '/%s/\n' "$COMPONENT_SRC" > .gitignore
 597 git add .gitignore
 598
 599
 600 # Construct the commit message
 601 MSG=
 602 if ((NEW)) ; then
 603         MSG="Add $MODULE Perl module"
 604 else
 605         [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && MSG="change version format"
 606         [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION"
 607
 608         REBUILDMSG=
 609
 610         if [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "no" ]] ; then
 611                 NV=
 612                 for v in $PERL_VERSIONS ; do
 613                         PLV=${v//.}
 614                         pkg list -avH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
 615                         [[ -n "$NV" ]] && NV="$NV and "
 616                         NV="$NV$v"
 617                 done
 618                 [[ -n "$NV" ]] && REBUILDMSG="rebuild for Perl $NV"
 619         fi
 620
 621         if [[ -n "$OV" ]] ; then
 622                 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and "
 623                 REBUILDMSG="${REBUILDMSG}obsolete package$OV_PLURAL for Perl $OV"
 624         fi
 625
 626         if [[ -n "$REBUILDMSG" ]] ; then
 627                 [[ -n "$MSG" ]] && MSG="$MSG; "
 628                 MSG="$MSG$REBUILDMSG"
 629         fi
 630         [[ -z "$MSG" ]] && MSG="rebuild"
 631
 632         MSG="$DIRECTORY: $MSG"
 633 fi
 634
 635 # Commit the results
 636 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1