tools/perl-integrate-module

   1 #! /usr/bin/ksh
   2 #
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright 2022 Marcel Telka
  16 #
  17
  18
  19 THIS="perl-integrate-module"
  20 CONF="$THIS.conf"
  21 SNIPPET="$THIS.snippet"
  22 APIURL="https://fastapi.metacpan.org/v1"
  23 CURL="/usr/bin/curl -s"
  24
  25
  26 function usage
  27 {
  28         [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
  29         printf "Usage: %s [-d DIR] [-f] [-l VERSION] [-o OBSOLETE].. [-u] DISTRIBUTION|MODULE\n" "$THIS" >&2
  30         [[ -n "$1" ]] && exit 1
  31         exit 0
  32 }
  33
  34
  35 OPT_VERSION=
  36 OBSOLETE=
  37 UPGRADE_ONLY=0
  38 DIRECTORY=
  39 FORCE=0
  40 while getopts ":hd:fl:o:u" OPT ; do
  41         case "$OPT" in
  42         "?"|"h")        usage ;;
  43         "d")            DIRECTORY="$OPTARG" ;;
  44         "f")            FORCE=1 ;;
  45         "l")            OPT_VERSION="$OPTARG" ;;
  46         "o")            OBSOLETE="$OBSOLETE $OPTARG" ;;
  47         "u")            UPGRADE_ONLY=1 ;;
  48         esac
  49 done
  50 shift $((OPTIND - 1))
  51
  52 (($# == 0)) && usage
  53 (($# > 1)) && usage "Too many arguments"
  54
  55 MODULE="$1"
  56
  57
  58 # Prevent user's environment to affect the integration.
  59 # Allow one exception only: USERLAND_ARCHIVES
  60 GMAKE="env -"
  61 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
  62 GMAKE="$GMAKE gmake"
  63
  64
  65 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
  66 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
  67
  68 BASE_DIR="$WS_TOP/components"
  69 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
  70
  71
  72 # Query MetaCPAN
  73 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  74         printf '{"bool":{"should":[{"term":{"distribution":"%s"}},' "$MODULE" ; \
  75         printf '{"term":{"main_module":"%s"}}]}}]}},' "$MODULE" ; \
  76         printf '"size":1,"sort":[{"date":"desc"}],' ; \
  77         printf '"fields":["distribution","version","author","download_url","abstract"]}') \
  78                 | $CURL "$APIURL/release/_search" -d @- \
  79                 | /usr/bin/jq -r '.hits.hits[].fields')
  80 if (($? != 0)) ; then
  81         printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
  82         exit 1
  83 fi
  84
  85 # If we didn't found the distribution directly try to find the module
  86 if [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
  87         DISTRIBUTION=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  88                 printf '{"term":{"module.name":"%s"}}]}},' "$MODULE" ; \
  89                 printf '"size":1,"fields":["distribution"]}') \
  90                         | $CURL "$APIURL/module/_search" -d @- \
  91                         | /usr/bin/jq -r '.hits.hits[].fields.distribution')
  92         if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
  93                 printf 'FATAL: Failed to find distribution for %s at MetaCPAN\n' "$MODULE" >&2
  94                 exit 1
  95         fi
  96         METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[{"term":{"maturity":"released"}},' ; \
  97                 printf '{"term":{"distribution":"%s"}}]}},' "$DISTRIBUTION" ; \
  98                 printf '"size":1,"sort":[{"date":"desc"}],' ; \
  99                 printf '"fields":["distribution","version","author","download_url","abstract"]}') \
 100                         | $CURL "$APIURL/release/_search" -d @- \
 101                         | /usr/bin/jq -r '.hits.hits[].fields')
 102         if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 103                 printf 'FATAL: Failed to get distribution info from MetaCPAN\n' >&2
 104                 exit 1
 105         fi
 106 fi
 107
 108 # Get distribution
 109 DISTRIBUTION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.distribution')
 110 if (($? != 0)) || [[ -z "$DISTRIBUTION" || "$DISTRIBUTION" == "null" ]] ; then
 111         printf 'FATAL: Failed to confirm or detect distribution\n' >&2
 112         exit 1
 113 fi
 114 if [[ "$DISTRIBUTION" != "$MODULE" ]] then
 115         printf 'WARNING: Using distribution %s for module %s\n' "$DISTRIBUTION" "$MODULE" >&2
 116 fi
 117
 118
 119 # Prepare the directory
 120 [[ -z "$DIRECTORY" ]] && DIRECTORY="perl/$DISTRIBUTION"
 121 DIR="$BASE_DIR/$DIRECTORY"
 122 mkdir -p "$DIR"
 123 cd "$DIR"
 124 git restore --staged . > /dev/null 2>&1
 125 git checkout . > /dev/null 2>&1
 126
 127
 128 # Following variables could be set by the hook-begin snippet
 129 VERSION=
 130 DOWNLOAD_URL=
 131 LICENSE_FILE=
 132 SUMMARY=
 133
 134 # Execute hook-begin snippet
 135 if [[ -f "$CONF" ]] ; then
 136         gsed -e '0,/^%hook-begin%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 137         . "./$SNIPPET"
 138         rm -f "$SNIPPET"
 139 fi
 140
 141 # Version specified as option takes precedence
 142 [[ -n "$OPT_VERSION" ]] && VERSION="$OPT_VERSION"
 143
 144 # Find the latest version
 145 LATEST_VERSION=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.version')
 146 if (($? != 0)) || [[ -z "$LATEST_VERSION" || "$LATEST_VERSION" == "null" ]] ; then
 147         printf 'FATAL: Failed to detect the latest version\n' >&2
 148         exit 1
 149 fi
 150
 151 # Use the latest version if version was not provided
 152 if [[ -z "$VERSION" ]] ; then
 153         VERSION=$LATEST_VERSION
 154         VERSION=${VERSION#v}
 155 elif [[ "$VERSION" != "$LATEST_VERSION" ]] ; then
 156         # MetaCPAN data are no longer valid because we need different version
 157         METACPAN_DATA=
 158 fi
 159
 160
 161 # Is this new module, or just a rebuild?
 162 NEW=1
 163 PREV_VER=
 164 PREV_HVER=
 165 PREV_REV=0
 166 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
 167         NEW=0
 168         PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
 169         (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
 170         PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
 171
 172         # If we were asked to do version upgrade, but we do not have new
 173         # version, then we are done.
 174         PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
 175         ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
 176
 177         # Pre-flight environment checks
 178         if ((FORCE == 0)) ; then
 179                 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake env-check' failed!\n" >&2 && exit 1
 180                 if [[ "$($GMAKE print-value-PERL_TEST_BOOTSTRAP)" != "yes" ]] ; then
 181                         ! $GMAKE test-env-check > /dev/null 2>&1 && printf "FATAL: Pre-flight 'gmake test-env-check' failed!\n" >&2 && exit 1
 182                 fi
 183         fi
 184
 185         $GMAKE clobber > /dev/null 2>&1
 186 fi
 187
 188
 189 # Get download_url if not already provided
 190 if [[ -z "$DOWNLOAD_URL" ]] ; then
 191         # Get new MetaCPAN data if needed
 192         if [[ -z "$METACPAN_DATA" ]] ; then
 193                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 194                         printf '{"term":{"maturity":"released"}},' ; \
 195                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 196                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 197                         printf '"size":1,' ; \
 198                         printf '"fields":["author","download_url","abstract"]}') \
 199                                 | $CURL "$APIURL/release/_search" -d @- \
 200                                 | /usr/bin/jq -r '.hits.hits[].fields')
 201                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 202                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 203                         exit 1
 204                 fi
 205         fi
 206
 207         # Get author
 208         AUTHOR=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.author')
 209         if (($? != 0)) || [[ -z "$AUTHOR" || "$AUTHOR" == "null" ]] ; then
 210                 printf 'FATAL: Failed to get author from MetaCPAN\n' >&2
 211                 exit 1
 212         fi
 213
 214         # Get download_url
 215         DOWNLOAD_URL=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.download_url')
 216         if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
 217                 printf 'WARNING: Failed to get download_url for version %s from MetaCPAN\n' "$VERSION" >&2
 218                 DOWNLOAD_URL=
 219         fi
 220 fi
 221
 222
 223 # Remove everything that is not in git
 224 rm -rf *
 225 git checkout . > /dev/null 2>&1
 226 # Remove everything from git (except known patches, files, history, and $CONF)
 227 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
 228 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do rm -f "files/$FILE" ; done
 229 rm -f history "$CONF"
 230 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
 231 rm -rf "$DIR" 2>/dev/null
 232 git checkout history > /dev/null 2>&1
 233 git checkout "$CONF" > /dev/null 2>&1
 234 [[ -f "$CONF" ]] && grep "^%patch%" "$CONF" | while read TAG PATCH ; do
 235         git checkout "patches/$PATCH" > /dev/null 2>&1
 236         [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
 237 done
 238 [[ -f "$CONF" ]] && grep "^%file%" "$CONF" | while read TAG FILE ; do
 239         git checkout "files/$FILE" > /dev/null 2>&1
 240         [[ -f "files/$FILE" ]] || printf "WARNING: File %s not found\n" "$FILE" >&2
 241 done
 242
 243
 244 # Makefile template
 245 GENERATE_CMD="\$WS_TOOLS/$THIS"
 246 [[ "$DIRECTORY" != "perl/$DISTRIBUTION" ]] && GENERATE_CMD="$GENERATE_CMD -d $DIRECTORY"
 247 GENERATE_CMD="$GENERATE_CMD $DISTRIBUTION"
 248 (
 249 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
 250 cat <<EOF
 251
 252 #
 253 # This file was automatically generated using the following command:
 254 #   $GENERATE_CMD
 255 #
 256
 257 BUILD_STYLE = makemaker
 258 USE_COMMON_TEST_MASTER = no
 259 EOF
 260 [[ -f "$CONF" ]] && gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
 261 cat <<EOF
 262
 263 include ../../../make-rules/shared-macros.mk
 264
 265 COMPONENT_NAME =                $DISTRIBUTION
 266 HUMAN_VERSION =                 $VERSION
 267 COMPONENT_REVISION =            $((PREV_REV + 1))
 268 COMPONENT_SUMMARY =             TODO
 269 COMPONENT_CPAN_AUTHOR =         $AUTHOR
 270 COMPONENT_ARCHIVE_HASH =        \\
 271         sha256:TODO
 272 COMPONENT_LICENSE =             license:TODO
 273 COMPONENT_LICENSE_FILE =        licfile:TODO
 274 EOF
 275 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 276 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
 277 [[ -f "$CONF" ]] && cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
 278 printf "\n"
 279 ) > Makefile
 280
 281 # If the automatically constructed COMPONENT_ARCHIVE_URL is not correct then we
 282 # do not need COMPONENT_CPAN_AUTHOR.  We need COMPONENT_ARCHIVE_URL instead.
 283 if [[ -n "$DOWNLOAD_URL" ]] ; then
 284         COMPONENT_ARCHIVE_URL=$($GMAKE print-value-COMPONENT_ARCHIVE_URL)
 285         [[ "$COMPONENT_ARCHIVE_URL" == "$DOWNLOAD_URL" ]] && DOWNLOAD_URL=
 286 fi
 287 [[ -n "$DOWNLOAD_URL" ]] && sed -i -e $'s|^COMPONENT_CPAN_AUTHOR.*|COMPONENT_ARCHIVE_URL =\t\t\\\\\\\n\t'"$DOWNLOAD_URL"'|' Makefile
 288
 289 # Remove COMPONENT_REVISION if not needed
 290 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
 291 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && sed -i -e '/^COMPONENT_REVISION/d' Makefile
 292 git add Makefile
 293
 294 # Calculate sham256 sum for source package
 295 $GMAKE fetch > /dev/null 2>&1
 296 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
 297 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
 298 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
 299 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
 300 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
 301 git add Makefile
 302
 303 # Unpack sources and apply patches
 304 ! $GMAKE patch > /dev/null 2>&1 && printf "FATAL: 'gmake patch' failed!\n" >&2 && exit 1
 305
 306 # Refresh patches
 307 if $GMAKE refresh-patches > /dev/null 2>&1 ; then
 308         git add patches 2>/dev/null
 309 else
 310         printf "ERROR: 'gmake refresh-patches' failed!\n" >&2
 311         git checkout patches 2>/dev/null
 312 fi
 313
 314 # Cleanup after patch refresh
 315 $GMAKE clobber > /dev/null 2>&1
 316
 317 # Prepare sources
 318 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
 319 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
 320 COMPONENT_SUBDIR=$($GMAKE print-value-COMPONENT_SUBDIR)
 321 [[ -n "$COMPONENT_SUBDIR" ]] && COMPONENT_SUBDIR="/$COMPONENT_SUBDIR"
 322
 323 # Switch to modulebuild if possible
 324 [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/Build.PL" ]] && sed -i -e 's/makemaker/modulebuild/g' Makefile
 325
 326 # Get summary if not already provided
 327 if [[ -z "$SUMMARY" ]] ; then
 328         # Get new MetaCPAN data if needed
 329         if [[ -z "$METACPAN_DATA" ]] ; then
 330                 METACPAN_DATA=$( (printf '{"query":{"bool":{"must":[' ; \
 331                         printf '{"term":{"maturity":"released"}},' ; \
 332                         printf '{"term":{"distribution":"%s"}},' "$DISTRIBUTION" ; \
 333                         printf '{"term":{"version":"%s"}}]}},' "$VERSION" ; \
 334                         printf '"size":1,"fields":["abstract"]}') \
 335                                 | $CURL "$APIURL/release/_search" -d @- \
 336                                 | /usr/bin/jq -r '.hits.hits[].fields')
 337                 if (($? != 0)) || [[ -z "$METACPAN_DATA" || "$METACPAN_DATA" == "null" ]] ; then
 338                         printf 'FATAL: Failed to get data from MetaCPAN\n' >&2
 339                         exit 1
 340                 fi
 341         fi
 342
 343         # Get abstract and use it as summary.  Either from MetaCPAN, or directly from sources.
 344         ABSTRACT=$(printf '%s' "$METACPAN_DATA" | /usr/bin/jq -r '.abstract')
 345         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 346                 printf 'WARNING: Failed to get abstract from MetaCPAN\n' >&2
 347                 ABSTRACT="TODO"
 348         fi
 349         if [[ "$ABSTRACT" == "TODO" ]] ; then
 350                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" ]] ; then
 351                         printf "WARNING: META.json missing\n" >&2
 352                 else
 353                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" | /usr/bin/jq -r '.abstract')
 354                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 355                                 printf "WARNING: Failed to get abstract from META.json\n" >&2
 356                                 ABSTRACT="TODO"
 357                         fi
 358                 fi
 359         fi
 360         if [[ "$ABSTRACT" == "TODO" ]] ; then
 361                 if [[ ! -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" ]] ; then
 362                         printf "WARNING: META.yml missing\n" >&2
 363                 else
 364                         ABSTRACT=$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' | /usr/bin/jq -r '.abstract')
 365                         if (($? != 0)) || [[ -z "$ABSTRACT" || "$ABSTRACT" == "null" ]] ; then
 366                                 printf "WARNING: Failed to get abstract from META.yml\n" >&2
 367                                 ABSTRACT="TODO"
 368                         fi
 369                 fi
 370         fi
 371         SUMMARY="$ABSTRACT"
 372 fi
 373 # Summary needs to be sanitized
 374 SUMMARY="${SUMMARY//\`/\\\\\`}"
 375 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
 376 SUMMARY="${SUMMARY//\//\/}"
 377 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
 378 SUMMARY="${SUMMARY//\&/\\&}"
 379 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
 380
 381
 382 # Try to detect license type(s)
 383 function detect_license
 384 {
 385         typeset -n L="$1"
 386         typeset F="$2"
 387         typeset D
 388
 389         D=$("$WS_TOP/tools/license-detector" "$F")
 390         [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
 391 }
 392
 393 LICENSE=
 394 LICFILE=
 395 COPYRIGHT=
 396 for f in $LICENSE_FILE LICENSE LICENCE COPYING COPYRIGHT ; do
 397         [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/$f" ]] || continue
 398         LICFILE="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 399
 400         detect_license LICENSE "$LICFILE"
 401
 402         if [[ -n "$LICENSE" ]] ; then
 403                 sed -i -e 's|licfile:TODO|'"$f"'|g' Makefile
 404                 break
 405         fi
 406
 407         printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
 408
 409         # Since the license file does not contain any known license we will use
 410         # its content as Copyright notice only
 411         COPYRIGHT=$(<"$LICFILE")
 412 done
 413 if [[ -z "$LICFILE" ]] ; then
 414         printf "WARNING: No license file found\n" >&2
 415 fi
 416
 417 if [[ -z "$LICENSE" ]] ; then
 418         # Since the distibution does not provide own license file (or we failed
 419         # to find it) we will use default Perl license with added Copyright
 420         # notice from this distribution
 421
 422         sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
 423
 424         # Try to find Copyright notice if we do not have one yet
 425         [[ -z "$COPYRIGHT" ]] && for f in README README.md ; do
 426                 f="$SOURCE_DIR$COMPONENT_SUBDIR/$f"
 427                 [[ -f "$f" ]] || continue
 428
 429                 COPYRIGHT=$(gsed -e '0,/^# LICENSE/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 430                 [[ -n "$COPYRIGHT" ]] && break
 431                 COPYRIGHT=$(gsed -e '0,/^# COPYRIGHT/d' -e '/^#/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 432                 [[ -n "$COPYRIGHT" ]] && break
 433                 COPYRIGHT=$(gsed -e '0,/LICENSE/d' -e '/^REPOSITORY/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 434                 [[ -n "$COPYRIGHT" ]] && break
 435                 COPYRIGHT=$(gsed -e '0,/COPYING/d' -e '/^BUGS/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 436                 [[ -n "$COPYRIGHT" ]] && break
 437                 COPYRIGHT=$(gsed -e '0,/COPYRIGHT/d' -e '/^AUTHOR/,$d' -e '/^SEE/,$d' -e '/./,$!d' "$f" 2>/dev/null)
 438                 [[ -n "$COPYRIGHT" ]] && break
 439                 COPYRIGHT=$(gsed -e '0,/^## Copyright/d' -e '/./,$!d' "$f" 2>/dev/null)
 440                 [[ -n "$COPYRIGHT" ]] && break
 441         done
 442         if [[ -z "$COPYRIGHT" ]] ; then
 443                 printf "WARNING: No copyright notice found at standard locations\n" >&2
 444                 for f in $(find "$SOURCE_DIR$COMPONENT_SUBDIR" -type f -name "*.pm" | LC_ALL=C sort | while read f ; do egrep -q "^=head1 (LICENSE|LICENCE|COPYRIGHT)" "$f" && echo "$f" ; done) ; do
 445                         COPYRIGHT=$(sed -e '1,/^=head1 LICENSE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 446                         if [[ -n "$COPYRIGHT" ]] ; then
 447                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 448                                 break
 449                         fi
 450                         COPYRIGHT=$(sed -e '1,/^=head1 LICENCE/d' -e '/^=/,$d' "$f" 2>/dev/null)
 451                         if [[ -n "$COPYRIGHT" ]] ; then
 452                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 453                                 break
 454                         fi
 455                         COPYRIGHT=$(sed -e '1,/^=head1 COPYRIGHT/d' -e '/^=/,$d' "$f" 2>/dev/null)
 456                         if [[ -n "$COPYRIGHT" ]] ; then
 457                                 printf "WARNING: Using copyright notice from %s\n" "$f" >&2
 458                                 break
 459                         fi
 460                 done
 461         fi
 462         if [[ -z "$COPYRIGHT" ]] ; then
 463                 printf "WARNING: No copyright notice found\n" >&2
 464                 > "$DISTRIBUTION.license"
 465         else
 466                 (printf "%s\n\n" "$COPYRIGHT" | dos2unix -ascii
 467                 i=75 ; while ((i)) ; do printf "=" ; i=$((i-1)) ; done
 468                 printf "\n\n") > "$DISTRIBUTION.license"
 469         fi
 470
 471         USE_DEFAULT_PERL_LICENSE=1
 472
 473         # Execute hook-no-license snippet
 474         if [[ -f "$CONF" ]] ; then
 475                 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 476                 . "./$SNIPPET"
 477                 rm -f "$SNIPPET"
 478         fi
 479
 480
 481         if ((USE_DEFAULT_PERL_LICENSE)) ; then
 482                 # Confirm the package is distributed under the same terms as Perl itself
 483                 D=1
 484                 ((D)) && (printf "%s\n" "$COPYRIGHT" | grep -q -i "under the same terms as Perl itself") && D=0
 485                 ((D)) && grep -q "license *=> *'http://dev\.perl\.org/licenses/'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 486                 ((D)) && grep -q "LICENSE *=> *'perl'" "$SOURCE_DIR$COMPONENT_SUBDIR/Makefile.PL" 2>/dev/null && D=0
 487                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" && "$(/usr/bin/jq -r '.license[]' < "$SOURCE_DIR$COMPONENT_SUBDIR/META.json" 2>/dev/null)" == "perl_5" ]] && D=0
 488                 ((D)) && [[ -f "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" && "$(cat "$SOURCE_DIR$COMPONENT_SUBDIR/META.yml" \
 489                         | python -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' \
 490                         | /usr/bin/jq -r '.license' 2>/dev/null)" == "perl" ]] && D=0
 491
 492                 ((D)) && printf "ERROR: Heuristics failed to detect license type, using default Perl license\n" >&2
 493
 494                 # Make a copy of license so we can use it during publish
 495                 cat "$WS_TOP/tools/perl-license" | grep -v "^#" >> "$DISTRIBUTION.license"
 496         fi
 497         git add "$DISTRIBUTION.license"
 498
 499         [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
 500         [[ -z "$LICENSE" ]] && LICENSE="TODO"
 501 fi
 502
 503 # Store the detected license into the Makefile
 504 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
 505
 506
 507 # Create manifests
 508 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
 509         printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
 510 else
 511         MANIFEST="$DISTRIBUTION-PERLVER.p5m"
 512         [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "yes" ]] && MANIFEST="$DISTRIBUTION.p5m"
 513         cat manifests/sample-manifest.p5m \
 514                 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
 515                 > "$MANIFEST"
 516
 517         # Execute hook-manifest snippet
 518         if [[ -f "$CONF" ]] ; then
 519                 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
 520                 . "./$SNIPPET"
 521                 rm -f "$SNIPPET"
 522         fi
 523
 524         git add manifests/sample-manifest.p5m $MANIFEST
 525 fi
 526
 527
 528 # Generate REQUIRED_PACKAGES
 529 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
 530 git add Makefile
 531
 532
 533 # Check for Makefile completeness
 534 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
 535
 536
 537 # Make sure the build environment is setup properly and we do have all
 538 # requirements installed.  Otherwise we cannot continue.
 539 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
 540
 541
 542 # Handle history
 543 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
 544 PERL_VERSIONS_OBSOLETING=$($GMAKE print-value-PERL_VERSIONS_OBSOLETING)
 545 OV=
 546 OV_PLURAL=
 547 for o in $(echo $OBSOLETE $PERL_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
 548         PLV=${o//.}
 549         FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
 550         [[ -n "$FMRI" ]] || continue
 551         FMRI_H=${FMRI%.*}
 552         FMRI_T=${FMRI##*.}
 553         if [[ "$FMRI_H" == "$FMRI" ]] ; then
 554                 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
 555                 continue
 556         fi
 557         FMRI_T=$((FMRI_T + 1))
 558         printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
 559
 560         [[ -n "$OV" ]] && OV="${OV/ and /, } and " && OV_PLURAL="s"
 561         OV="$OV$o"
 562 done
 563 if [[ -f history ]] ; then
 564         LC_ALL=C sort -u history > history.new
 565         mv history.new history
 566         git add history
 567
 568         awk '$NF == "noincorporate" {printf("WARNING: Unincorporated package: %s\n", $1)}' < history >&2
 569 fi
 570
 571
 572 # Cleanup before we try to publish to make sure there are no leftovers from
 573 # previous steps
 574 $GMAKE clobber > /dev/null 2>&1
 575
 576 # Publish packages and create pkg5 file
 577 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
 578 git add pkg5 2>/dev/null
 579
 580
 581 PERL_VERSIONS=$($GMAKE print-value-PERL_VERSIONS)
 582 PERL_TEST_BOOTSTRAP=$($GMAKE print-value-PERL_TEST_BOOTSTRAP)
 583
 584
 585 # Run tests to make sure they pass and to create result snapshots
 586 TESTED_VERSIONS=
 587 for v in $PERL_VERSIONS ; do
 588         # Check the test environment
 589         if ! $GMAKE PERL_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
 590                 if [[ "$PERL_TEST_BOOTSTRAP" == "yes" ]] ; then
 591                         printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
 592                 else
 593                         printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
 594                 fi
 595                 continue
 596         fi
 597
 598         # Run the test
 599         ! $GMAKE PERL_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
 600
 601         # If there is no snapshot produced the component likely does not support tests
 602         COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 603         [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
 604
 605         # Empty result snapshot is suspicious
 606         [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
 607
 608         TESTED_VERSIONS="$TESTED_VERSIONS $v"
 609 done
 610
 611 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
 612 TEST_MASTERS=
 613 for common_results in yes no ; do
 614         for v in $TESTED_VERSIONS ; do
 615                 COMPONENT_TEST_SNAPSHOT=$($GMAKE PERL_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
 616                 COMPONENT_TEST_MASTER=$($GMAKE PERL_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
 617
 618                 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
 619                         # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
 620                         if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
 621                                 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
 622                                 rm -f $TEST_MASTERS
 623                                 TEST_MASTERS=
 624                                 continue 2
 625                         fi
 626                 else
 627                         mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
 628                         cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
 629                         TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
 630                 fi
 631         done
 632         break
 633 done
 634 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
 635
 636 # Run tests again to confirm the results are reproducible
 637 for v in $TESTED_VERSIONS ; do
 638         $GMAKE PERL_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
 639 done
 640
 641 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
 642 if [[ "$common_results" == "yes" ]] ; then
 643         sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
 644         git add Makefile
 645 fi
 646
 647
 648 # Create .gitignore
 649 COMPONENT_SRC=$($GMAKE print-value-COMPONENT_SRC)
 650 printf '/%s/\n' "$COMPONENT_SRC" > .gitignore
 651 git add .gitignore
 652
 653
 654 # Construct the commit message
 655 MSG=
 656 if ((NEW)) ; then
 657         MSG="Add $DISTRIBUTION Perl distribution"
 658 else
 659         [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && MSG="change version format"
 660         [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION"
 661
 662         REBUILDMSG=
 663
 664         if [[ "$($GMAKE print-value-SINGLE_PERL_VERSION)" == "no" ]] ; then
 665                 NV=
 666                 for v in $PERL_VERSIONS ; do
 667                         PLV=${v//.}
 668                         pkg list -avH "$COMPONENT_FMRI-$PLV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
 669                         [[ -n "$NV" ]] && NV="$NV and "
 670                         NV="$NV$v"
 671                 done
 672                 [[ -n "$NV" ]] && REBUILDMSG="rebuild for Perl $NV"
 673         fi
 674
 675         if [[ -n "$OV" ]] ; then
 676                 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and "
 677                 REBUILDMSG="${REBUILDMSG}obsolete package$OV_PLURAL for Perl $OV"
 678         fi
 679
 680         if [[ -n "$REBUILDMSG" ]] ; then
 681                 [[ -n "$MSG" ]] && MSG="$MSG; "
 682                 MSG="$MSG$REBUILDMSG"
 683         fi
 684         [[ -z "$MSG" ]] && MSG="rebuild"
 685
 686         MSG="$DIRECTORY: $MSG"
 687 fi
 688
 689 # Commit the results
 690 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1