tools/perl-integrate-module

   1 #! /usr/bin/ksh
   2 #
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright 2022 Marcel Telka
  16 #
  17
  18
  19 THIS="perl-integrate-module"
  20 CONF="$THIS.conf"
  21 SNIPPET="$THIS.snippet"
  22 APIURL="https://fastapi.metacpan.org/v1"
  23 CURL="/usr/bin/curl -s"
  24
  25
  26 function usage
  27 {
  28         [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
  29         printf "Usage: %s [-d DIR] [-f] [-l VERSION] [-o OBSOLETE].. [-u] DISTRIBUTION|MODULE\n" "$THIS" >&2
  30         [[ -n "$1" ]] && exit 1
  31         exit 0
  32 }
  33
  34
  35 OPT_VERSION=
  36 OBSOLETE=
  37 UPGRADE_ONLY=0
  38 DIRECTORY=
  39 FORCE=0
  40 while getopts ":hd:fl:o:u" OPT ; do
  41         case "$OPT" in
  42         "?"|"h")        usage ;;
  43         "d")            DIRECTORY="$OPTARG" ;;
  44         "f")            FORCE=1 ;;
  45         "l")            OPT_VERSION="$OPTARG" ;;
  46         "o")            OBSOLETE="$OBSOLETE $OPTARG" ;;
  47         "u")            UPGRADE_ONLY=1 ;;
  48         esac
  49 done
  50 shift $((OPTIND - 1))
  51
  52 (($# == 0)) && usage
  53 (($# > 1)) && usage "Too many arguments"
  54
  55 MODULE="$1"
  56
  57
  58 # Prevent user's environment to affect the integration.
  59 # Allow one exception only: USERLAND_ARCHIVES
  60 GMAKE="env -"
  61 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
  62 GMAKE="$GMAKE gmake"
  63
  64
  65 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
  66 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
  67
  68 BASE_DIR="$WS_TOP/components"
  69 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
  70
  71
  72 # Query MetaCPAN
  73 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  74         printf '{"bool":{"should":[{"term":{"distribution":"%s"}},' "$MODULE" ; \
  75         printf '{"term":{"main_module":"%s"}}]}}]}},' "$MODULE" ; \
  76         printf '"size":1,"sort":[{"date":"desc"}],' ; \
  77         printf '"fields":["distribution","version","author","download_url","abstract"]}') \
  78                 | $CURL "$APIURL/release/_search" -d @- \
  79                 | /usr/bin/jq -r '.hits.hits[].fields')
  80 if (($? != 0)) ; then
  81         printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
  82         exit 1
  83 fi
  84
  85 # If we didn't found the distribution directly try to find the module
  86 if [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
  87         DISTRIBUTION=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  88                 printf '{"term":{"module.name":"%s"}}]}},' "$MODULE" ; \
  89                 printf '"size":1,"fields":["distribution"]}') \
  90                         | $CURL "$APIURL/module/_search" -d @- \
  91                         | /usr/bin/jq -r '.hits.hits[].fields.distribution')
  92         if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
  93                 printf 'FATAL: Failed to find distribution for %s at MetaCPAN\n' "$MODULE" >&2
  94                 exit 1
  95         fi
  96         METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  97                 printf '{"term":{"distribution":"%s"}}]}},' "$DISTRIBUTION" ; \
  98                 printf '"size":1,"sort":[{"date":"desc"}],' ; \
  99                 printf '"fields":["distribution","version","author","download_url","abstract"]}') \
 100                         | $CURL "$APIURL/release/_search" -d @- \
 101                         | /usr/bin/jq -r '.hits.hits[].fields')
 102         if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 103                 printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
 104                 exit 1
 105         fi
 106 fi
 107
 108 # Get distribution
 109 DISTRIBUTION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.distribution')
 110 if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
 111         printf 'FATAL: Failed to confirm or detect distribution\n' >&2
 112         exit 1
 113 fi
 114 if [[ "$DISTRIBUTION" != "$MODULE" ]] then
 115         printf 'WARNING: Using distribution %s for module %s\n' "$DISTRIBUTION" "$MODULE" >&2
 116 fi
 117
 118
 119 # Prepare the directory
 120 [[ -z "$DIRECTORY" ]] && DIRECTORY="perl/$DISTRIBUTION"
 121 DIR="$BASE_DIR/$DIRECTORY"
 122 mkdir -p "$DIR"
 123 cd "$DIR"
 124 git restore --staged . > /dev/null 2>&1
 125 git checkout . > /dev/null 2>&1
 126
 127
 128 # Following variables could be set by the hook-begin snippet
 129 VERSION=
 130 DOWNLOAD_URL=
 131 LICENSE_FILE=
 132 SUMMARY=
 133
 134 # Execute hook-begin snippet
 135 if [[ -f "$CONF" ]] ; then
 136         gsed -e '0,/^%hook-begin%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 137         . "./$SNIPPET"
 138         rm -f "$SNIPPET"
 139 fi
 140
 141 # Version specified as option takes precedence
 142 [[ -n "$OPT_VERSION" ]] && VERSION="$OPT_VERSION"
 143
 144 # Find the latest version
 145 LATEST_VERSION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.version')
 146 if (($? != 0)) || [[ -z "$LATEST_VERSION" || "$LATEST_VERSION" == "null" ]] ; then
 147         printf 'FATAL: Failed to detect the latest version\n' >&2
 148         exit 1
 149 fi
 150
 151 # Use the latest version if version was not provided
 152 if [[ -z "$VERSION" ]] ; then
 153         VERSION=$LATEST_VERSION
 154 elif [[ "$VERSION" != "$LATEST_VERSION" ]] ; then
 155         # MetaCPAN data are no longer valid because we need different version
 156         METACPAN_DATA=
 157 fi
 158
 159
 160 # Is this new module, or just a rebuild?
 161 NEW=1
 162 PREV_VER=
 163 PREV_HVER=
 164 PREV_REV=0
 165 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
 166         NEW=0
 167         PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
 168         (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
 169         PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
 170
 171         # If we were asked to do version upgrade, but we do not have new
 172         # version, then we are done.
 173         PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
 174         ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
 175
 176         # Pre-flight environment checks
 177         if ((FORCE == 0)) ; then
 178                 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake env-check' failed!\n" >&2 && exit 1
 179                 if [[ "$($GMAKE print-value-PERL_TEST_BOOTSTRAP)" != "yes" ]] ; then
 180                         ! $GMAKE test-env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake test-env-check' failed!\n" >&2 && exit 1
 181                 fi
 182         fi
 183
 184         $GMAKE clobber > /dev/null 2>&1
 185 fi
 186
 187
 188 # Get download_url if not already provided
 189 if [[ -z "$DOWNLOAD_URL" ]] ; then
 190         # Get new MetaCPAN data if needed
 191         if [[ -z "$METACPAN_DATA" ]] ; then
 192                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 193                         printf '{"term":{"maturity":"released"}},' ; \
 194                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 195                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 196                         printf '"size":1,' ; \
 197                         printf '"fields":["author","download_url","abstract"]}') \
 198                                 | $CURL "$APIURL/release/_search" -d @- \
 199                                 | /usr/bin/jq -r '.hits.hits[].fields')
 200                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 201                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 202                         exit 1
 203                 fi
 204         fi
 205
 206         # Get author
 207         AUTHOR=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.author')
 208         if (($? != 0)) || [[ -z "$AUTHOR" || "$AUTHOR" == "null" ]] ; then
 209                 printf 'FATAL: Failed to get author from MetaCPAN\n' >&2
 210                 exit 1
 211         fi
 212
 213         # Get download_url
 214         DOWNLOAD_URL=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.download_url')
 215         if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
 216                 printf 'WARNING: Failed to get download_url for version %s from MetaCPAN\n' "$VERSION" >&2
 217                 DOWNLOAD_URL=
 218         fi
 219 fi
 220
 221
 222 # Remove everything that is not in git
 223 rm -rf *
 224 git checkout . > /dev/null 2>&1
 225 # Remove everything from git (except known patches, files, history, and $CONF)
 226 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
 227 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do rm -f "files/$FILE" ; done
 228 rm -f history "$CONF"
 229 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
 230 rm -rf "$DIR" 2>/dev/null
 231 git checkout history > /dev/null 2>&1
 232 git checkout "$CONF" > /dev/null 2>&1
 233 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do
 234         git checkout "patches/$PATCH" > /dev/null 2>&1
 235         [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
 236 done
 237 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do
 238         git checkout "files/$FILE" > /dev/null 2>&1
 239         [[ -f "files/$FILE" ]] || printf "WARNING: File %s not found\n" "$FILE" >&2
 240 done
 241
 242
 243 # Makefile template
 244 GENERATE_CMD="\$WS_TOOLS/$THIS"
 245 [[ "$DIRECTORY" != "perl/$DISTRIBUTION" ]] && GENERATE_CMD="$GENERATE_CMD -d $DIRECTORY"
 246 GENERATE_CMD="$GENERATE_CMD $DISTRIBUTION"
 247 (
 248 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
 249 cat <<EOF
 250
 251 #
 252 # This file was automatically generated using the following command:
 253 #   $GENERATE_CMD
 254 #
 255
 256 BUILD_STYLE = makemaker
 257 USE_COMMON_TEST_MASTER = no
 258 EOF
 259 [[ -f "$CONF" ]] && gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
 260 cat <<EOF
 261
 262 include ../../../make-rules/shared-macros.mk
 263
 264 COMPONENT_NAME =                $DISTRIBUTION
 265 HUMAN_VERSION =                 $VERSION
 266 COMPONENT_REVISION =            $((PREV_REV + 1))
 267 COMPONENT_SUMMARY =             TODO
 268 COMPONENT_CPAN_AUTHOR =         $AUTHOR
 269 COMPONENT_ARCHIVE_HASH =        \\
 270         sha256:TODO
 271 COMPONENT_LICENSE =             license:TODO
 272 COMPONENT_LICENSE_FILE =        licfile:TODO
 273 EOF
 274 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 275 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
 276 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 277 printf "\n"
 278 ) > Makefile
 279
 280 # If the automatically constructed COMPONENT_ARCHIVE_URL is not correct then we
 281 # do not need COMPONENT_CPAN_AUTHOR.  We need COMPONENT_ARCHIVE_URL instead.
 282 if [[ -n "$DOWNLOAD_URL" ]] ; then
 283         COMPONENT_ARCHIVE_URL=$($GMAKE print-value-COMPONENT_ARCHIVE_URL)
 284         [[ "$COMPONENT_ARCHIVE_URL" == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 285 fi
 286 [[ -n "$DOWNLOAD_URL" ]] && sed -i -e $'s|^COMPONENT_CPAN_AUTHOR.*|COMPONENT_ARCHIVE_URL =\t\t\\\\\\\n\t'"$DOWNLOAD_URL"'|' Makefile
 287
 288 # Remove COMPONENT_REVISION if not needed
 289 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
 290 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && sed -i -e '/^COMPONENT_REVISION/d' Makefile
 291 git add Makefile
 292
 293 # Calculate sham256 sum for source package
 294 $GMAKE fetch > /dev/null 2>&1
 295 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
 296 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
 297 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
 298 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
 299 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
 300 git add Makefile
 301
 302 # Unpack sources and apply patches
 303 ! $GMAKE patch > /dev/null 2>&1 && printf "FATAL: 'gmake patch' failed!\n" >&2 && exit 1
 304
 305 # Refresh patches
 306 if $GMAKE refresh-patches > /dev/null 2>&1 ; then
 307         git add patches 2>/dev/null
 308 else
 309         printf "ERROR: 'gmake refresh-patches' failed!\n" >&2
 310         git checkout patches 2>/dev/null
 311 fi
 312
 313 # Cleanup after patch refresh
 314 $GMAKE clobber > /dev/null 2>&1
 315
 316 # Prepare sources
 317 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
 318 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
 319 COMPONENT_SUBDIR=$($GMAKE print-value-COMPONENT_SUBDIR)
 320 [[ -n "$COMPONENT_SUBDIR" ]] && COMPONENT_SUBDIR="/$COMPONENT_SUBDIR"
 321
 322 # Switch to modulebuild if possible
 323 [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/Build.PL" ]] && sed -i -e 's/makemaker/modulebuild/g' Makefile
 324
 325 # Get summary if not already provided
 326 if [[ -z "$SUMMARY" ]] ; then
 327         # Get new MetaCPAN data if needed
 328         if [[ -z "$METACPAN_DATA" ]] ; then
 329                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 330                         printf '{"term":{"maturity":"released"}},' ; \
 331                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 332                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 333                         printf '"size":1,"fields":["abstract"]}') \
 334                                 | $CURL "$APIURL/release/_search" -d @- \
 335                                 | /usr/bin/jq -r '.hits.hits[].fields')
 336                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 337                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 338                         exit 1
 339                 fi
 340         fi
 341
 342         # Get abstract and use it as summary.  Either from MetaCPAN, or directly from sources.
 343         ABSTRACT=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.abstract')
 344         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 345                 printf 'WARNING: Failed to get abstract from MetaCPAN\n' >&2
 346                 ABSTRACT="TODO"
 347         fi
 348         if [[ "$ABSTRACT" == "TODO" ]] ; then
 349                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" ]] ; then
 350                         printf "WARNING: META.json missing\n" >&2
 351                 else
 352                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" | /usr/bin/jq -r '.abstract')
 353                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 354                                 printf "WARNING: Failed to get abstract from META.json\n" >&2
 355                                 ABSTRACT="TODO"
 356                         fi
 357                 fi
 358         fi
 359         if [[ "$ABSTRACT" == "TODO" ]] ; then
 360                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" ]] ; then
 361                         printf "WARNING: META.yml missing\n" >&2
 362                 else
 363                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' | /usr/bin/jq -r '.abstract')
 364                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 365                                 printf "WARNING: Failed to get abstract from META.yml\n" >&2
 366                                 ABSTRACT="TODO"
 367                         fi
 368                 fi
 369         fi
 370         SUMMARY="$ABSTRACT"
 371 fi
 372 # Summary needs to be sanitized
 373 SUMMARY="${SUMMARY//\`/\\\\\`}"
 374 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
 375 SUMMARY="${SUMMARY//\//\/}"
 376 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
 377 SUMMARY="${SUMMARY//\&/\\&}"
 378 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
 379
 380
 381 # Try to detect license type(s)
 382 function detect_license
 383 {
 384         typeset -n L="$1"
 385         typeset F="$2"
 386         typeset D
 387
 388         D=$("$WS_TOP/tools/license-detector" "$F")
 389         [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
 390 }
 391
 392 LICENSE=
 393 LICFILE=
 394 COPYRIGHT=
 395 for f in $LICENSE_FILE LICENSE LICENCE COPYING COPYRIGHT ; do
 396         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/$f" ]] || continue
 397         LICFILE="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 398
 399         detect_license LICENSE "$LICFILE"
 400
 401         if [[ -n "$LICENSE" ]] ; then
 402                 sed -i -e 's|licfile:TODO|'"$f"'|g' Makefile
 403                 break
 404         fi
 405
 406         printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
 407
 408         # Since the license file does not contain any known license we will use
 409         # its content as Copyright notice only
 410         COPYRIGHT=$(<"$LICFILE")
 411 done
 412 if [[ -z "$LICFILE" ]] ; then
 413         printf "WARNING: No license file found\n" >&2
 414 fi
 415
 416 if [[ -z "$LICENSE" ]] ; then
 417         # Since the distibution does not provide own license file (or we failed
 418         # to find it) we will use default Perl license with added Copyright
 419         # notice from this distribution
 420
 421         sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
 422
 423         # Try to find Copyright notice if we do not have one yet
 424         [[ -z "$COPYRIGHT" ]] && for f in README README.md ; do
 425                 f="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 426                 [[ -f "$f" ]] || continue
 427
 428                 COPYRIGHT=$(gsed -e '0,/^# LICENSE/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 429                 [[ -n "$COPYRIGHT" ]] && break
 430                 COPYRIGHT=$(gsed -e '0,/^# COPYRIGHT/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 431                 [[ -n "$COPYRIGHT" ]] && break
 432                 COPYRIGHT=$(gsed -e '0,/LICENSE/d' -e '/^REPOSITORY/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 433                 [[ -n "$COPYRIGHT" ]] && break
 434                 COPYRIGHT=$(gsed -e '0,/COPYING/d' -e '/^BUGS/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 435                 [[ -n "$COPYRIGHT" ]] && break
 436                 COPYRIGHT=$(gsed -e '0,/COPYRIGHT/d' -e '/^AUTHOR/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 437                 [[ -n "$COPYRIGHT" ]] && break
 438                 COPYRIGHT=$(gsed -e '0,/^## Copyright/d' -e '/./,$!d' "$f" 2>/dev/null)
 439                 [[ -n "$COPYRIGHT" ]] && break
 440         done
 441         if [[ -z "$COPYRIGHT" ]] ; then
 442                 printf "WARNING: No copyright notice found at standard locations\n" >&2
 443                 for f in $(find "$SOURCE_DIR$COMPONENT_SUBDIR" -type f -name "*.pm" | LC_ALL=C sort | while read f ; do egrep -q "^=head1 (LICENSE|LICENCE|COPYRIGHT)" "$f" && echo "$f" ; done) ; do
 444                         COPYRIGHT=$(sed -e '1,/^=head1 LICENSE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 445                         if [[ -n "$COPYRIGHT" ]] ; then
 446                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 447                                 break
 448                         fi
 449                         COPYRIGHT=$(sed -e '1,/^=head1 LICENCE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 450                         if [[ -n "$COPYRIGHT" ]] ; then
 451                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 452                                 break
 453                         fi
 454                         COPYRIGHT=$(sed -e '1,/^=head1 COPYRIGHT/d' -e '/^=/,$d' "$f" 2>/dev/null)
 455                         if [[ -n "$COPYRIGHT" ]] ; then
 456                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 457                                 break
 458                         fi
 459                 done
 460         fi
 461         if [[ -z "$COPYRIGHT" ]] ; then
 462                 printf "WARNING: No copyright notice found\n" >&2
 463                 > "$DISTRIBUTION.license"
 464         else
 465                 (printf "%s\n\n" "$COPYRIGHT" | dos2unix -ascii
 466                 i=75 ; while ((i)) ; do printf "=" ; i=$((i-1)) ; done
 467                 printf "\n\n") > "$DISTRIBUTION.license"
 468         fi
 469
 470         USE_DEFAULT_PERL_LICENSE=1
 471
 472         # Execute hook-no-license snippet
 473         if [[ -f "$CONF" ]] ; then
 474                 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 475                 . "./$SNIPPET"
 476                 rm -f "$SNIPPET"
 477         fi
 478
 479
 480         if ((USE_DEFAULT_PERL_LICENSE)) ; then
 481                 # Confirm the package is distributed under the same terms as Perl itself
 482                 D=1
 483                 ((D)) && (printf "%s\n" "$COPYRIGHT" | grep -q -i "under the same terms as Perl itself") && D=0
 484                 ((D)) && grep -q "license *=> *'http://dev\.perl\.org/licenses/'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 485                 ((D)) && grep -q "LICENSE *=> *'perl'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 486                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" && "$(/usr/bin/jq -r '.license[]' < "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" 2>/dev/null)" == "perl_5" ]] && D=0
 487                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" && "$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" \
 488                         | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' \
 489                         | /usr/bin/jq -r '.license' 2>/dev/null)" == "perl" ]] && D=0
 490
 491                 ((D)) && printf "ERROR: Heuristics failed to detect license type, using default Perl license\n" >&2
 492
 493                 # Make a copy of license so we can use it during publish
 494                 cat "$WS_TOP/tools/perl-license" | grep -v "^#" >> "$DISTRIBUTION.license"
 495         fi
 496         git add "$DISTRIBUTION.license"
 497
 498         [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
 499         [[ -z "$LICENSE" ]] && LICENSE="TODO"
 500 fi
 501
 502 # Store the detected license into the Makefile
 503 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
 504
 505
 506 # Create manifests
 507 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
 508         printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
 509 else
 510         MANIFEST="$DISTRIBUTION-PERLVER.p5m"
 511         [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "yes" ]] && MANIFEST="$DISTRIBUTION.p5m"
 512         cat manifests/sample-manifest.p5m \
 513                 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
 514                 > "$MANIFEST"
 515
 516         # Execute hook-manifest snippet
 517         if [[ -f "$CONF" ]] ; then
 518                 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 519                 . "./$SNIPPET"
 520                 rm -f "$SNIPPET"
 521         fi
 522
 523         git add manifests/sample-manifest.p5m $MANIFEST
 524 fi
 525
 526
 527 # Generate REQUIRED_PACKAGES
 528 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
 529 git add Makefile
 530
 531
 532 # Check for Makefile completeness
 533 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
 534
 535
 536 # Make sure the build environment is setup properly and we do have all
 537 # requirements installed.  Otherwise we cannot continue.
 538 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
 539
 540
 541 # Handle history
 542 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
 543 PERL_VERSIONS_OBSOLETING=$($GMAKE print-value-PERL_VERSIONS_OBSOLETING)
 544 OV=
 545 OV_PLURAL=
 546 for o in $(echo $OBSOLETE $PERL_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
 547         PLV=${o//.}
 548         FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
 549         [[ -n "$FMRI" ]] || continue
 550         FMRI_H=${FMRI%.*}
 551         FMRI_T=${FMRI##*.}
 552         if [[ "$FMRI_H" == "$FMRI" ]] ; then
 553                 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
 554                 continue
 555         fi
 556         FMRI_T=$((FMRI_T + 1))
 557         printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
 558
 559         [[ -n "$OV" ]] && OV="${OV/ and /, } and " && OV_PLURAL="s"
 560         OV="$OV$o"
 561 done
 562 if [[ -f history ]] ; then
 563         LC_ALL=C sort -u history > history.new
 564         mv history.new history
 565         git add history
 566
 567         awk '$NF == "noincorporate" {printf("WARNING: Unincorporated package: %s\n", $1)}' < history >&2
 568 fi
 569
 570
 571 # Cleanup before we try to publish to make sure there are no leftovers from
 572 # previous steps
 573 $GMAKE clobber > /dev/null 2>&1
 574
 575 # Publish packages and create pkg5 file
 576 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
 577 git add pkg5 2>/dev/null
 578
 579
 580 PERL_VERSIONS=$($GMAKE print-value-PERL_VERSIONS)
 581 PERL_TEST_BOOTSTRAP=$($GMAKE print-value-PERL_TEST_BOOTSTRAP)
 582
 583
 584 # Run tests to make sure they pass and to create result snapshots
 585 TESTED_VERSIONS=
 586 for v in $PERL_VERSIONS ; do
 587         # Check the test environment
 588         if ! $GMAKE PERL_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
 589                 if [[ "$PERL_TEST_BOOTSTRAP" == "yes" ]] ; then
 590                         printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
 591                 else
 592                         printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
 593                 fi
 594                 continue
 595         fi
 596
 597         # Run the test
 598         ! $GMAKE PERL_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
 599
 600         # If there is no snapshot produced the component likely does not support tests
 601         COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 602         [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
 603
 604         # Empty result snapshot is suspicious
 605         [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
 606
 607         TESTED_VERSIONS="$TESTED_VERSIONS $v"
 608 done
 609
 610 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
 611 TEST_MASTERS=
 612 for common_results in yes no ; do
 613         for v in $TESTED_VERSIONS ; do
 614                 COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 615                 COMPONENT_TEST_MASTER=$($GMAKE PERL_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
 616
 617                 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
 618                         # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
 619                         if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
 620                                 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
 621                                 rm -f $TEST_MASTERS
 622                                 TEST_MASTERS=
 623                                 continue 2
 624                         fi
 625                 else
 626                         mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
 627                         cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
 628                         TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
 629                 fi
 630         done
 631         break
 632 done
 633 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
 634
 635 # Run tests again to confirm the results are reproducible
 636 for v in $TESTED_VERSIONS ; do
 637         $GMAKE PERL_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
 638 done
 639
 640 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
 641 if [[ "$common_results" == "yes" ]] ; then
 642         sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
 643         git add Makefile
 644 fi
 645
 646
 647 # Create .gitignore
 648 COMPONENT_SRC=$($GMAKE print-value-COMPONENT_SRC)
 649 printf '/%s/\n' "$COMPONENT_SRC" > .gitignore
 650 git add .gitignore
 651
 652
 653 # Construct the commit message
 654 MSG=
 655 if ((NEW)) ; then
 656         MSG="Add $DISTRIBUTION Perl distribution"
 657 else
 658         [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && MSG="change version format"
 659         [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION"
 660
 661         REBUILDMSG=
 662
 663         if [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "no" ]] ; then
 664                 NV=
 665                 for v in $PERL_VERSIONS ; do
 666                         PLV=${v//.}
 667                         pkg list -avH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
 668                         [[ -n "$NV" ]] && NV="$NV and "
 669                         NV="$NV$v"
 670                 done
 671                 [[ -n "$NV" ]] && REBUILDMSG="rebuild for Perl $NV"
 672         fi
 673
 674         if [[ -n "$OV" ]] ; then
 675                 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and "
 676                 REBUILDMSG="${REBUILDMSG}obsolete package$OV_PLURAL for Perl $OV"
 677         fi
 678
 679         if [[ -n "$REBUILDMSG" ]] ; then
 680                 [[ -n "$MSG" ]] && MSG="$MSG; "
 681                 MSG="$MSG$REBUILDMSG"
 682         fi
 683         [[ -z "$MSG" ]] && MSG="rebuild"
 684
 685         MSG="$DIRECTORY: $MSG"
 686 fi
 687
 688 # Commit the results
 689 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1