tools/perl-integrate-module

   1 #! /usr/bin/ksh
   2 #
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright 2022 Marcel Telka
  16 #
  17
  18
  19 THIS="perl-integrate-module"
  20 CONF="$THIS.conf"
  21 SNIPPET="$THIS.snippet"
  22 APIURL="https://fastapi.metacpan.org/v1"
  23 CURL="/usr/bin/curl -s"
  24
  25
  26 function usage
  27 {
  28         [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
  29         printf "Usage: %s [-d DIR] [-f] [-l VERSION] [-o OBSOLETE].. [-u] DISTRIBUTION|MODULE\n" "$THIS" >&2
  30         [[ -n "$1" ]] && exit 1
  31         exit 0
  32 }
  33
  34
  35 OPT_VERSION=
  36 OBSOLETE=
  37 UPGRADE_ONLY=0
  38 DIRECTORY=
  39 FORCE=0
  40 while getopts ":hd:fl:o:u" OPT ; do
  41         case "$OPT" in
  42         "?"|"h")        usage ;;
  43         "d")            DIRECTORY="$OPTARG" ;;
  44         "f")            FORCE=1 ;;
  45         "l")            OPT_VERSION="$OPTARG" ;;
  46         "o")            OBSOLETE="$OBSOLETE $OPTARG" ;;
  47         "u")            UPGRADE_ONLY=1 ;;
  48         esac
  49 done
  50 shift $((OPTIND - 1))
  51
  52 (($# == 0)) && usage
  53 (($# > 1)) && usage "Too many arguments"
  54
  55 MODULE="$1"
  56
  57
  58 # Prevent user's environment to affect the integration.
  59 # Allow one exception only: USERLAND_ARCHIVES
  60 GMAKE="env -"
  61 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
  62 GMAKE="$GMAKE gmake"
  63
  64
  65 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
  66 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
  67
  68 BASE_DIR="$WS_TOP/components"
  69 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
  70
  71
  72 # Query MetaCPAN
  73 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  74         printf '{"term":{"distribution":"%s"}}],' "$MODULE" ; \
  75         printf '"must_not":{"term":{"status":"backpan"}}}},' ; \
  76         printf '"size":1,"sort":[{"date":"desc"}],' ; \
  77         printf '"fields":["distribution","version","author","download_url","abstract"]}') \
  78                 | $CURL "$APIURL/release/_search" -d @- \
  79                 | /usr/bin/jq -r '.hits.hits[].fields')
  80 if (($? != 0)) ; then
  81         printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
  82         exit 1
  83 fi
  84
  85 # If we didn't found the distribution directly try to find the module
  86 if [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
  87         DISTRIBUTION=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  88                 printf '{"term":{"module.name":"%s"}}],' "$MODULE" ; \
  89                 printf '"must_not":{"term":{"status":"backpan"}}}},' ; \
  90                 printf '"size":1,"fields":["distribution"]}') \
  91                         | $CURL "$APIURL/module/_search" -d @- \
  92                         | /usr/bin/jq -r '.hits.hits[].fields.distribution')
  93         if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
  94                 printf 'FATAL: Failed to find distribution for %s at MetaCPAN\n' "$MODULE" >&2
  95                 exit 1
  96         fi
  97         METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  98                 printf '{"term":{"distribution":"%s"}}],' "$DISTRIBUTION" ; \
  99                 printf '"must_not":{"term":{"status":"backpan"}}}},' ; \
 100                 printf '"size":1,"sort":[{"date":"desc"}],' ; \
 101                 printf '"fields":["distribution","version","author","download_url","abstract"]}') \
 102                         | $CURL "$APIURL/release/_search" -d @- \
 103                         | /usr/bin/jq -r '.hits.hits[].fields')
 104         if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 105                 printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
 106                 exit 1
 107         fi
 108 fi
 109
 110 # Get distribution
 111 DISTRIBUTION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.distribution')
 112 if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
 113         printf 'FATAL: Failed to confirm or detect distribution\n' >&2
 114         exit 1
 115 fi
 116 if [[ "$DISTRIBUTION" != "$MODULE" ]] then
 117         printf 'WARNING: Using distribution %s for module %s\n' "$DISTRIBUTION" "$MODULE" >&2
 118 fi
 119
 120
 121 # Prepare the directory
 122 [[ -z "$DIRECTORY" ]] && DIRECTORY="perl/$DISTRIBUTION"
 123 DIR="$BASE_DIR/$DIRECTORY"
 124 mkdir -p "$DIR"
 125 cd "$DIR"
 126 git restore --staged . > /dev/null 2>&1
 127 git checkout . > /dev/null 2>&1
 128
 129
 130 # Following variables could be set by the hook-begin snippet
 131 VERSION=
 132 DOWNLOAD_URL=
 133 LICENSE_FILE=
 134 SUMMARY=
 135
 136 # Execute hook-begin snippet
 137 if [[ -f "$CONF" ]] ; then
 138         gsed -e '0,/^%hook-begin%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 139         . "./$SNIPPET"
 140         rm -f "$SNIPPET"
 141 fi
 142
 143 # Version specified as option takes precedence
 144 [[ -n "$OPT_VERSION" ]] && VERSION="$OPT_VERSION"
 145
 146 # Find the latest version
 147 LATEST_VERSION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.version')
 148 if (($? != 0)) || [[ -z "$LATEST_VERSION" || "$LATEST_VERSION" == "null" ]] ; then
 149         printf 'FATAL: Failed to detect the latest version\n' >&2
 150         exit 1
 151 fi
 152
 153 # Use the latest version if version was not provided
 154 if [[ -z "$VERSION" ]] ; then
 155         VERSION=$LATEST_VERSION
 156 elif [[ "$VERSION" != "$LATEST_VERSION" ]] ; then
 157         # MetaCPAN data are no longer valid because we need different version
 158         METACPAN_DATA=
 159 fi
 160
 161
 162 # Is this new module, or just a rebuild?
 163 NEW=1
 164 PREV_VER=
 165 PREV_HVER=
 166 PREV_REV=0
 167 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
 168         NEW=0
 169         PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
 170         (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
 171         PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
 172
 173         # If we were asked to do version upgrade, but we do not have new
 174         # version, then we are done.
 175         PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
 176         ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
 177
 178         # Pre-flight environment checks
 179         if ((FORCE == 0)) ; then
 180                 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake env-check' failed!\n" >&2 && exit 1
 181                 if [[ "$($GMAKE print-value-PERL_TEST_BOOTSTRAP)" != "yes" ]] ; then
 182                         ! $GMAKE test-env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake test-env-check' failed!\n" >&2 && exit 1
 183                 fi
 184         fi
 185
 186         $GMAKE clobber > /dev/null 2>&1
 187 fi
 188
 189
 190 # Get download_url if not already provided
 191 if [[ -z "$DOWNLOAD_URL" ]] ; then
 192         # Get new MetaCPAN data if needed
 193         if [[ -z "$METACPAN_DATA" ]] ; then
 194                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 195                         printf '{"term":{"maturity":"released"}},' ; \
 196                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 197                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 198                         printf '"size":1,' ; \
 199                         printf '"fields":["author","download_url","abstract"]}') \
 200                                 | $CURL "$APIURL/release/_search" -d @- \
 201                                 | /usr/bin/jq -r '.hits.hits[].fields')
 202                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 203                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 204                         exit 1
 205                 fi
 206         fi
 207
 208         # Get author
 209         AUTHOR=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.author')
 210         if (($? != 0)) || [[ -z "$AUTHOR" || "$AUTHOR" == "null" ]] ; then
 211                 printf 'FATAL: Failed to get author from MetaCPAN\n' >&2
 212                 exit 1
 213         fi
 214
 215         # Get download_url
 216         DOWNLOAD_URL=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.download_url')
 217         if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
 218                 printf 'WARNING: Failed to get download_url for version %s from MetaCPAN\n' "$VERSION" >&2
 219                 DOWNLOAD_URL=
 220         fi
 221 fi
 222
 223
 224 # Remove everything that is not in git
 225 rm -rf *
 226 git checkout . > /dev/null 2>&1
 227 # Remove everything from git (except known patches, files, history, and $CONF)
 228 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
 229 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do rm -f "files/$FILE" ; done
 230 rm -f history "$CONF"
 231 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
 232 rm -rf "$DIR" 2>/dev/null
 233 git checkout history > /dev/null 2>&1
 234 git checkout "$CONF" > /dev/null 2>&1
 235 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do
 236         git checkout "patches/$PATCH" > /dev/null 2>&1
 237         [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
 238 done
 239 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do
 240         git checkout "files/$FILE" > /dev/null 2>&1
 241         [[ -f "files/$FILE" ]] || printf "WARNING: File %s not found\n" "$FILE" >&2
 242 done
 243
 244
 245 # Makefile template
 246 GENERATE_CMD="\$WS_TOOLS/$THIS"
 247 [[ "$DIRECTORY" != "perl/$DISTRIBUTION" ]] && GENERATE_CMD="$GENERATE_CMD -d $DIRECTORY"
 248 GENERATE_CMD="$GENERATE_CMD $DISTRIBUTION"
 249 (
 250 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
 251 cat <<EOF
 252
 253 #
 254 # This file was automatically generated using the following command:
 255 #   $GENERATE_CMD
 256 #
 257
 258 BUILD_STYLE = makemaker
 259 USE_COMMON_TEST_MASTER = no
 260 EOF
 261 [[ -f "$CONF" ]] && gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
 262 cat <<EOF
 263
 264 include ../../../make-rules/shared-macros.mk
 265
 266 COMPONENT_NAME =                $DISTRIBUTION
 267 HUMAN_VERSION =                 $VERSION
 268 COMPONENT_REVISION =            $((PREV_REV + 1))
 269 COMPONENT_SUMMARY =             TODO
 270 COMPONENT_CPAN_AUTHOR =         $AUTHOR
 271 COMPONENT_ARCHIVE_HASH =        \\
 272         sha256:TODO
 273 COMPONENT_LICENSE =             license:TODO
 274 COMPONENT_LICENSE_FILE =        licfile:TODO
 275 EOF
 276 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 277 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
 278 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 279 printf "\n"
 280 ) > Makefile
 281
 282 # If the automatically constructed COMPONENT_ARCHIVE_URL is not correct then we
 283 # do not need COMPONENT_CPAN_AUTHOR.  We need COMPONENT_ARCHIVE_URL instead.
 284 if [[ -n "$DOWNLOAD_URL" ]] ; then
 285         COMPONENT_ARCHIVE_URL=$($GMAKE print-value-COMPONENT_ARCHIVE_URL)
 286         [[ "$COMPONENT_ARCHIVE_URL" == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 287 fi
 288 [[ -n "$DOWNLOAD_URL" ]] && sed -i -e $'s|^COMPONENT_CPAN_AUTHOR.*|COMPONENT_ARCHIVE_URL =\t\t\\\\\\\n\t'"$DOWNLOAD_URL"'|' Makefile
 289
 290 # Remove COMPONENT_REVISION if not needed
 291 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
 292 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && sed -i -e '/^COMPONENT_REVISION/d' Makefile
 293 git add Makefile
 294
 295 # Calculate sham256 sum for source package
 296 $GMAKE fetch > /dev/null 2>&1
 297 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
 298 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
 299 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
 300 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
 301 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
 302 git add Makefile
 303
 304 # Unpack sources and apply patches
 305 ! $GMAKE patch > /dev/null 2>&1 && printf "FATAL: 'gmake patch' failed!\n" >&2 && exit 1
 306
 307 # Refresh patches
 308 if $GMAKE refresh-patches > /dev/null 2>&1 ; then
 309         git add patches 2>/dev/null
 310 else
 311         printf "ERROR: 'gmake refresh-patches' failed!\n" >&2
 312         git checkout patches 2>/dev/null
 313 fi
 314
 315 # Cleanup after patch refresh
 316 $GMAKE clobber > /dev/null 2>&1
 317
 318 # Prepare sources
 319 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
 320 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
 321 COMPONENT_SUBDIR=$($GMAKE print-value-COMPONENT_SUBDIR)
 322 [[ -n "$COMPONENT_SUBDIR" ]] && COMPONENT_SUBDIR="/$COMPONENT_SUBDIR"
 323
 324 # Switch to modulebuild if possible
 325 [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/Build.PL" ]] && sed -i -e 's/makemaker/modulebuild/g' Makefile
 326
 327 # Get summary if not already provided
 328 if [[ -z "$SUMMARY" ]] ; then
 329         # Get new MetaCPAN data if needed
 330         if [[ -z "$METACPAN_DATA" ]] ; then
 331                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 332                         printf '{"term":{"maturity":"released"}},' ; \
 333                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 334                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 335                         printf '"size":1,"fields":["abstract"]}') \
 336                                 | $CURL "$APIURL/release/_search" -d @- \
 337                                 | /usr/bin/jq -r '.hits.hits[].fields')
 338                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 339                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 340                         exit 1
 341                 fi
 342         fi
 343
 344         # Get abstract and use it as summary.  Either from MetaCPAN, or directly from sources.
 345         ABSTRACT=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.abstract')
 346         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 347                 printf 'WARNING: Failed to get abstract from MetaCPAN\n' >&2
 348                 ABSTRACT="TODO"
 349         fi
 350         if [[ "$ABSTRACT" == "TODO" ]] ; then
 351                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" ]] ; then
 352                         printf "WARNING: META.json missing\n" >&2
 353                 else
 354                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" | /usr/bin/jq -r '.abstract')
 355                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 356                                 printf "WARNING: Failed to get abstract from META.json\n" >&2
 357                                 ABSTRACT="TODO"
 358                         fi
 359                 fi
 360         fi
 361         if [[ "$ABSTRACT" == "TODO" ]] ; then
 362                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" ]] ; then
 363                         printf "WARNING: META.yml missing\n" >&2
 364                 else
 365                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' | /usr/bin/jq -r '.abstract')
 366                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 367                                 printf "WARNING: Failed to get abstract from META.yml\n" >&2
 368                                 ABSTRACT="TODO"
 369                         fi
 370                 fi
 371         fi
 372         SUMMARY="$ABSTRACT"
 373 fi
 374 # Summary needs to be sanitized
 375 SUMMARY="${SUMMARY//\`/\\\\\`}"
 376 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
 377 SUMMARY="${SUMMARY//\//\/}"
 378 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
 379 SUMMARY="${SUMMARY//\&/\\&}"
 380 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
 381
 382
 383 # Try to detect license type(s)
 384 function detect_license
 385 {
 386         typeset -n L="$1"
 387         typeset F="$2"
 388         typeset D
 389
 390         D=$("$WS_TOP/tools/license-detector" "$F")
 391         [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
 392 }
 393
 394 LICENSE=
 395 LICFILE=
 396 COPYRIGHT=
 397 for f in $LICENSE_FILE LICENSE LICENCE COPYING COPYRIGHT ; do
 398         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/$f" ]] || continue
 399         LICFILE="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 400
 401         detect_license LICENSE "$LICFILE"
 402
 403         if [[ -n "$LICENSE" ]] ; then
 404                 sed -i -e 's|licfile:TODO|'"$f"'|g' Makefile
 405                 break
 406         fi
 407
 408         printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
 409
 410         # Since the license file does not contain any known license we will use
 411         # its content as Copyright notice only
 412         COPYRIGHT=$(<"$LICFILE")
 413 done
 414 if [[ -z "$LICFILE" ]] ; then
 415         printf "WARNING: No license file found\n" >&2
 416 fi
 417
 418 if [[ -z "$LICENSE" ]] ; then
 419         # Since the distibution does not provide own license file (or we failed
 420         # to find it) we will use default Perl license with added Copyright
 421         # notice from this distribution
 422
 423         sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
 424
 425         # Try to find Copyright notice if we do not have one yet
 426         [[ -z "$COPYRIGHT" ]] && for f in README README.md ; do
 427                 f="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 428                 [[ -f "$f" ]] || continue
 429
 430                 COPYRIGHT=$(gsed -e '0,/^# LICENSE/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 431                 [[ -n "$COPYRIGHT" ]] && break
 432                 COPYRIGHT=$(gsed -e '0,/^# COPYRIGHT/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 433                 [[ -n "$COPYRIGHT" ]] && break
 434                 COPYRIGHT=$(gsed -e '0,/LICENSE/d' -e '/^REPOSITORY/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 435                 [[ -n "$COPYRIGHT" ]] && break
 436                 COPYRIGHT=$(gsed -e '0,/COPYING/d' -e '/^BUGS/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 437                 [[ -n "$COPYRIGHT" ]] && break
 438                 COPYRIGHT=$(gsed -e '0,/COPYRIGHT/d' -e '/^AUTHOR/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 439                 [[ -n "$COPYRIGHT" ]] && break
 440                 COPYRIGHT=$(gsed -e '0,/^## Copyright/d' -e '/./,$!d' "$f" 2>/dev/null)
 441                 [[ -n "$COPYRIGHT" ]] && break
 442         done
 443         if [[ -z "$COPYRIGHT" ]] ; then
 444                 printf "WARNING: No copyright notice found at standard locations\n" >&2
 445                 for f in $(find "$SOURCE_DIR$COMPONENT_SUBDIR" -type f -name "*.pm" | LC_ALL=C sort | while read f ; do egrep -q "^=head1 (LICENSE|LICENCE|COPYRIGHT)" "$f" && echo "$f" ; done) ; do
 446                         COPYRIGHT=$(sed -e '1,/^=head1 LICENSE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 447                         if [[ -n "$COPYRIGHT" ]] ; then
 448                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 449                                 break
 450                         fi
 451                         COPYRIGHT=$(sed -e '1,/^=head1 LICENCE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 452                         if [[ -n "$COPYRIGHT" ]] ; then
 453                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 454                                 break
 455                         fi
 456                         COPYRIGHT=$(sed -e '1,/^=head1 COPYRIGHT/d' -e '/^=/,$d' "$f" 2>/dev/null)
 457                         if [[ -n "$COPYRIGHT" ]] ; then
 458                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 459                                 break
 460                         fi
 461                 done
 462         fi
 463         if [[ -z "$COPYRIGHT" ]] ; then
 464                 printf "WARNING: No copyright notice found\n" >&2
 465                 > "$DISTRIBUTION.license"
 466         else
 467                 (printf "%s\n\n" "$COPYRIGHT" | dos2unix -ascii
 468                 i=75 ; while ((i)) ; do printf "=" ; i=$((i-1)) ; done
 469                 printf "\n\n") > "$DISTRIBUTION.license"
 470         fi
 471
 472         USE_DEFAULT_PERL_LICENSE=1
 473
 474         # Execute hook-no-license snippet
 475         if [[ -f "$CONF" ]] ; then
 476                 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 477                 . "./$SNIPPET"
 478                 rm -f "$SNIPPET"
 479         fi
 480
 481
 482         if ((USE_DEFAULT_PERL_LICENSE)) ; then
 483                 # Confirm the package is distributed under the same terms as Perl itself
 484                 D=1
 485                 ((D)) && (printf "%s\n" "$COPYRIGHT" | grep -q -i "under the same terms as Perl itself") && D=0
 486                 ((D)) && grep -q "license *=> *'http://dev\.perl\.org/licenses/'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 487                 ((D)) && grep -q "LICENSE *=> *'perl'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 488                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" && "$(/usr/bin/jq -r '.license[]' < "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" 2>/dev/null)" == "perl_5" ]] && D=0
 489                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" && "$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" \
 490                         | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' \
 491                         | /usr/bin/jq -r '.license' 2>/dev/null)" == "perl" ]] && D=0
 492
 493                 ((D)) && printf "ERROR: Heuristics failed to detect license type, using default Perl license\n" >&2
 494
 495                 # Make a copy of license so we can use it during publish
 496                 cat "$WS_TOP/tools/perl-license" | grep -v "^#" >> "$DISTRIBUTION.license"
 497         fi
 498         git add "$DISTRIBUTION.license"
 499
 500         [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
 501         [[ -z "$LICENSE" ]] && LICENSE="TODO"
 502 fi
 503
 504 # Store the detected license into the Makefile
 505 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
 506
 507
 508 # Create manifests
 509 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
 510         printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
 511 else
 512         MANIFEST="$DISTRIBUTION-PERLVER.p5m"
 513         [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "yes" ]] && MANIFEST="$DISTRIBUTION.p5m"
 514         cat manifests/sample-manifest.p5m \
 515                 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
 516                 > "$MANIFEST"
 517
 518         # Execute hook-manifest snippet
 519         if [[ -f "$CONF" ]] ; then
 520                 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 521                 . "./$SNIPPET"
 522                 rm -f "$SNIPPET"
 523         fi
 524
 525         git add manifests/sample-manifest.p5m $MANIFEST
 526 fi
 527
 528
 529 # Generate REQUIRED_PACKAGES
 530 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
 531 git add Makefile
 532
 533
 534 # Check for Makefile completeness
 535 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
 536
 537
 538 # Make sure the build environment is setup properly and we do have all
 539 # requirements installed.  Otherwise we cannot continue.
 540 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
 541
 542
 543 # Handle history
 544 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
 545 PERL_VERSIONS_OBSOLETING=$($GMAKE print-value-PERL_VERSIONS_OBSOLETING)
 546 OV=
 547 OV_PLURAL=
 548 for o in $(echo $OBSOLETE $PERL_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
 549         PLV=${o//.}
 550         FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
 551         [[ -n "$FMRI" ]] || continue
 552         FMRI_H=${FMRI%.*}
 553         FMRI_T=${FMRI##*.}
 554         if [[ "$FMRI_H" == "$FMRI" ]] ; then
 555                 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
 556                 continue
 557         fi
 558         FMRI_T=$((FMRI_T + 1))
 559         printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
 560
 561         [[ -n "$OV" ]] && OV="${OV/ and /, } and " && OV_PLURAL="s"
 562         OV="$OV$o"
 563 done
 564 if [[ -f history ]] ; then
 565         LC_ALL=C sort -u history > history.new
 566         mv history.new history
 567         git add history
 568
 569         awk '$NF == "noincorporate" {printf("WARNING: Unincorporated package: %s\n", $1)}' < history >&2
 570 fi
 571
 572
 573 # Cleanup before we try to publish to make sure there are no leftovers from
 574 # previous steps
 575 $GMAKE clobber > /dev/null 2>&1
 576
 577 # Publish packages and create pkg5 file
 578 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
 579 git add pkg5 2>/dev/null
 580
 581
 582 PERL_VERSIONS=$($GMAKE print-value-PERL_VERSIONS)
 583 PERL_TEST_BOOTSTRAP=$($GMAKE print-value-PERL_TEST_BOOTSTRAP)
 584
 585
 586 # Run tests to make sure they pass and to create result snapshots
 587 TESTED_VERSIONS=
 588 for v in $PERL_VERSIONS ; do
 589         # Check the test environment
 590         if ! $GMAKE PERL_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
 591                 if [[ "$PERL_TEST_BOOTSTRAP" == "yes" ]] ; then
 592                         printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
 593                 else
 594                         printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
 595                 fi
 596                 continue
 597         fi
 598
 599         # Run the test
 600         ! $GMAKE PERL_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
 601
 602         # If there is no snapshot produced the component likely does not support tests
 603         COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 604         [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
 605
 606         # Empty result snapshot is suspicious
 607         [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
 608
 609         TESTED_VERSIONS="$TESTED_VERSIONS $v"
 610 done
 611
 612 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
 613 TEST_MASTERS=
 614 for common_results in yes no ; do
 615         for v in $TESTED_VERSIONS ; do
 616                 COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 617                 COMPONENT_TEST_MASTER=$($GMAKE PERL_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
 618
 619                 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
 620                         # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
 621                         if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
 622                                 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
 623                                 rm -f $TEST_MASTERS
 624                                 TEST_MASTERS=
 625                                 continue 2
 626                         fi
 627                 else
 628                         mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
 629                         cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
 630                         TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
 631                 fi
 632         done
 633         break
 634 done
 635 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
 636
 637 # Run tests again to confirm the results are reproducible
 638 for v in $TESTED_VERSIONS ; do
 639         $GMAKE PERL_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
 640 done
 641
 642 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
 643 if [[ "$common_results" == "yes" ]] ; then
 644         sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
 645         git add Makefile
 646 fi
 647
 648
 649 # Create .gitignore
 650 COMPONENT_SRC=$($GMAKE print-value-COMPONENT_SRC)
 651 printf '/%s/\n' "$COMPONENT_SRC" > .gitignore
 652 git add .gitignore
 653
 654
 655 # Construct the commit message
 656 MSG=
 657 if ((NEW)) ; then
 658         MSG="Add $DISTRIBUTION Perl distribution"
 659 else
 660         [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && MSG="change version format"
 661         [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION"
 662
 663         REBUILDMSG=
 664
 665         if [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "no" ]] ; then
 666                 NV=
 667                 for v in $PERL_VERSIONS ; do
 668                         PLV=${v//.}
 669                         pkg list -avH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
 670                         [[ -n "$NV" ]] && NV="$NV and "
 671                         NV="$NV$v"
 672                 done
 673                 [[ -n "$NV" ]] && REBUILDMSG="rebuild for Perl $NV"
 674         fi
 675
 676         if [[ -n "$OV" ]] ; then
 677                 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and "
 678                 REBUILDMSG="${REBUILDMSG}obsolete package$OV_PLURAL for Perl $OV"
 679         fi
 680
 681         if [[ -n "$REBUILDMSG" ]] ; then
 682                 [[ -n "$MSG" ]] && MSG="$MSG; "
 683                 MSG="$MSG$REBUILDMSG"
 684         fi
 685         [[ -z "$MSG" ]] && MSG="rebuild"
 686
 687         MSG="$DIRECTORY: $MSG"
 688 fi
 689
 690 # Commit the results
 691 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1