openexr: make architecture independent
[oi-userland.git] / tools / python-integrate-project
blob3da518f9ed9eef0b19b4daae87143b72cf6f0832
1 #! /usr/bin/ksh
4 # This file and its contents are supplied under the terms of the
5 # Common Development and Distribution License ("CDDL"), version 1.0.
6 # You may only use this file in accordance with the terms of version
7 # 1.0 of the CDDL.
9 # A full copy of the text of the CDDL should have accompanied this
10 # source. A copy of the CDDL is also available via the Internet at
11 # http://www.illumos.org/license/CDDL.
15 # Copyright 2022 Marcel Telka
19 THIS="python-integrate-project"
20 CONF="$THIS.conf"
21 SNIPPET="$THIS.snippet"
22 APIURL="https://pypi.org/pypi"
23 CURL="/usr/bin/curl -s"
26 function usage
28 [[ -n "$1" ]] && printf "ERROR: %s\n\n" "$1" >&2
29 printf "Usage: %s [-d DIR] [-l VERSION] [-o OBSOLETE].. [-u] PROJECT\n" "$THIS" >&2
30 [[ -n "$1" ]] && exit 1
31 exit 0
35 VERSION=
36 OBSOLETE=
37 UPGRADE_ONLY=0
38 DIRECTORY=
39 while getopts ":hd:l:o:u" OPT ; do
40 case "$OPT" in
41 "?"|"h") usage ;;
42 "d") DIRECTORY="$OPTARG" ;;
43 "l") VERSION="$OPTARG" ;;
44 "o") OBSOLETE="$OBSOLETE $OPTARG" ;;
45 "u") UPGRADE_ONLY=1 ;;
46 esac
47 done
48 shift $((OPTIND - 1))
50 (($# == 0)) && usage
51 (($# > 1)) && usage "Too many arguments"
53 PROJECT="$1"
56 # Prevent user's environment to affect the integration.
57 # Allow one exception only: USERLAND_ARCHIVES
58 GMAKE="env -"
59 [[ -n "$USERLAND_ARCHIVES" ]] && GMAKE="$GMAKE USERLAND_ARCHIVES=$USERLAND_ARCHIVES"
60 GMAKE="$GMAKE gmake"
63 WS_TOP=$(git rev-parse --show-toplevel 2>/dev/null)
64 [[ -z "$WS_TOP" ]] && usage "The script must be run in git repo"
66 BASE_DIR="$WS_TOP/components"
67 [[ -d "$BASE_DIR" ]] || usage "Directory $BASE_DIR not found"
70 # Get data from pypi
71 PYPI_PROJECT=$($CURL "$APIURL/$PROJECT/json")
72 if (($? != 0)) || [[ -z "$PYPI_PROJECT" ]] ; then
73 printf "FATAL: Failed to get data from pypi\n" >&2
74 exit 1
77 # Distribution match project
78 DISTRIBUTION="$PROJECT"
80 function get_PKGINFO_entry
82 typeset ENTRY="$1"
84 [[ -f "$SOURCE_DIR/PKG-INFO" ]] || return
86 cat "$SOURCE_DIR/PKG-INFO" \
87 | sed -e '/^$/,$d' \
88 | awk 'END{printf("\n")}/^[^:]+: /{$0="\n"$0}1' ORS=' ' \
89 | grep "^$ENTRY: " \
90 | sed -e "s/^$ENTRY: //" -e 's/ *$//'
93 # Get project homepage
94 HOMEPAGE=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.home_page')
95 if (($? != 0)) || [[ -z "$HOMEPAGE" || "$HOMEPAGE" == "null" ]] ; then
96 HOMEPAGE=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.project_urls.Homepage')
97 if (($? != 0)) || [[ -z "$HOMEPAGE" || "$HOMEPAGE" == "null" ]] ; then
98 printf "WARNING: Failed to get homepage for project %s from pypi\n" "$PROJECT" >&2
99 HOMEPAGE=$(get_PKGINFO_entry "Home-page")
103 # Find the latest version if not provided by user
104 if [[ -z "$VERSION" ]] ; then
105 VERSION=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.version')
106 if (($? != 0)) || [[ -z "$VERSION" || "$VERSION" == "null" ]] ; then
107 printf "FATAL: Failed to get version for project %s from pypi\n" "$PROJECT" >&2
108 exit 1
112 # Get release data from pypi
113 PYPI_PROJECT_RELEASE=$($CURL "$APIURL/$PROJECT/$VERSION/json")
114 if (($? != 0)) || [[ -z "$PYPI_PROJECT_RELEASE" ]] ; then
115 printf "FATAL: Failed to get data for version %s from pypi\n" "$VERSION" >&2
116 exit 1
119 # Get download url
120 DOWNLOAD_URL=$(printf "%s" "$PYPI_PROJECT_RELEASE" | /usr/bin/jq -r '.urls[]|select(.packagetype=="sdist")|.url')
121 if (($? != 0)) || [[ -z "$DOWNLOAD_URL" || "$DOWNLOAD_URL" == "null" ]] ; then
122 printf "WARNING: Failed to get download url for project %s, version %s from pypi\n" "$PROJECT" "$VERSION" >&2
123 DOWNLOAD_URL=
127 # Prepare the directory
128 [[ -z "$DIRECTORY" ]] && DIRECTORY="python/$DISTRIBUTION"
129 DIR="$BASE_DIR/$DIRECTORY"
130 mkdir -p "$DIR"
131 cd "$DIR"
132 git restore --staged . > /dev/null 2>&1
133 git checkout . > /dev/null 2>&1
135 # Is this new project, or just a rebuild?
136 NEW=1
137 REBUILD=0
138 PREV_VER=
139 PREV_HVER=
140 PREV_REV=0
141 if git ls-files --error-unmatch Makefile > /dev/null 2>&1 ; then
142 NEW=0
143 REBUILD=1
144 PREV_VER=$($GMAKE print-value-COMPONENT_VERSION 2>/dev/null)
145 (($? != 0)) && printf "FATAL: 'gmake print-value-COMPONENT_VERSION' failed!\n" >&2 && exit 1
146 PREV_REV=$($GMAKE print-value-COMPONENT_REVISION 2>/dev/null)
148 # If we were asked to do version upgrade, but we do not have new
149 # version, then we are done.
150 PREV_HVER=$($GMAKE print-value-HUMAN_VERSION 2>/dev/null)
151 ((UPGRADE_ONLY)) && [[ "$PREV_HVER" == "$VERSION" ]] && exit 0
153 $GMAKE clobber > /dev/null 2>&1
156 # Remove everything from git (except known patches, history, and $CONF)
157 touch "$CONF"
158 grep "^%patch%" "$CONF" | while read TAG PATCH ; do rm -f "patches/$PATCH" ; done
159 rm -f history "$CONF"
160 find . -type f | while read f ; do git rm "$f" > /dev/null 2>&1 ; done
161 rm -rf "$DIR" 2>/dev/null
162 git checkout history > /dev/null 2>&1
163 git checkout "$CONF" > /dev/null 2>&1
164 touch "$CONF"
165 grep "^%patch%" "$CONF" | while read TAG PATCH ; do
166 git checkout "patches/$PATCH" > /dev/null 2>&1
167 [[ -f "patches/$PATCH" ]] || printf "WARNING: Patch %s not found\n" "$PATCH" >&2
168 done
171 # Makefile template
173 cat $WS_TOP/transforms/copyright-template | sed -e '/^$/,$d'
174 cat <<EOF
177 # This file was automatically generated using the following command:
178 # \$WS_TOOLS/$THIS $PROJECT
181 BUILD_STYLE = pyproject
182 USE_COMMON_TEST_MASTER = no
184 gsed -e '0,/^%include-1%/d' -e '/^%/,$d' < "$CONF"
185 cat <<EOF
187 include ../../../make-rules/shared-macros.mk
189 COMPONENT_NAME = $DISTRIBUTION
190 HUMAN_VERSION = $VERSION
191 COMPONENT_REVISION = $((PREV_REV + 1))
192 COMPONENT_SUMMARY = $PROJECT - TODO
194 [[ -n "$HOMEPAGE" ]] && printf "COMPONENT_PROJECT_URL =\t\t%s\n" "$HOMEPAGE"
195 [[ -n "$DOWNLOAD_URL" ]] && printf 'COMPONENT_ARCHIVE_URL =\t\t\\\n\t%s\n' "$DOWNLOAD_URL"
196 cat <<EOF
197 COMPONENT_ARCHIVE_HASH = \\
198 sha256:TODO
199 COMPONENT_LICENSE = license:TODO
200 COMPONENT_LICENSE_FILE = licfile:TODO
202 _TEST_STYLE = TODO
204 cat "$CONF" | gsed -e '0,/^%include-2%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
205 printf "\ninclude \$(WS_MAKE_RULES)/common.mk\n"
206 cat "$CONF" | gsed -e '0,/^%include-3%/d' -e '/^%/,$d' | gsed -e '1s/^./\n&/'
207 printf "\n"
208 ) > Makefile
210 # Remove COMPONENT_REVISION if not needed
211 COMPONENT_VERSION=$($GMAKE print-value-COMPONENT_VERSION)
212 [[ "$PREV_VER" != "$COMPONENT_VERSION" ]] && REBUILD=0 && sed -i -e '/^COMPONENT_REVISION/d' Makefile
213 git add Makefile
215 # Calculate sham256 sum for source package
216 $GMAKE fetch > /dev/null 2>&1
217 USERLAND_ARCHIVES=$($GMAKE print-value-USERLAND_ARCHIVES)
218 COMPONENT_ARCHIVE=$($GMAKE print-value-COMPONENT_ARCHIVE)
219 [[ ! -f "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE" ]] && printf "FATAL: 'gmake fetch' failed!\n" >&2 && exit 1
220 SHA256=$(digest -a sha256 "$USERLAND_ARCHIVES$COMPONENT_ARCHIVE")
221 sed -i -e 's/sha256:TODO/sha256:'"$SHA256"'/g' Makefile
222 git add Makefile
224 # Unpack sources
225 ! $GMAKE prep > /dev/null 2>&1 && printf "FATAL: 'gmake prep' failed!\n" >&2 && exit 1
226 SOURCE_DIR=$($GMAKE print-value-SOURCE_DIR)
228 if [[ ! -f "$SOURCE_DIR/pyproject.toml" ]] ; then
229 [[ ! -f "$SOURCE_DIR/setup.py" ]] && printf "FATAL: Neither pyproject.toml nor setup.py found!\n" >&2 && exit 1
230 sed -i -e 's/^\(BUILD_STYLE = \).*$/\1setup.py/' Makefile
233 # Get summary
234 SUMMARY=$(printf "%s" "$PYPI_PROJECT" | /usr/bin/jq -r '.info.summary')
235 if (($? != 0)) || [[ -z "$SUMMARY" || "$SUMMARY" == "null" ]] ; then
236 printf "WARNING: Failed to get summary for project %s from pypi\n" "$PROJECT" >&2
237 SUMMARY=$(get_PKGINFO_entry "Summary")
238 [[ -z "$SUMMARY" ]] && SUMMARY="TODO"
240 # Summary needs to be sanitized
241 SUMMARY="${SUMMARY//\`/\\\\\`}"
242 SUMMARY="${SUMMARY//\"/\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"}"
243 SUMMARY="${SUMMARY//\//\/}"
244 SUMMARY="${SUMMARY//\$/\\\\\$\$}"
245 SUMMARY="${SUMMARY//\&/\\&}"
246 sed -i -e 's/\(COMPONENT_SUMMARY.*\)TODO$/\1'"$SUMMARY"'/g' Makefile
249 # Try to detect license type(s)
250 function detect_license
252 typeset -n L="$1"
253 typeset F="$2"
254 typeset D
256 D=$("$WS_TOP/tools/license-detector" "$F")
257 [[ -n "$L" ]] && L="$L OR " ; L="$L$D"
260 LICENSE=
261 LICFILE=
262 for f in $(get_PKGINFO_entry "License-File") LICENSE LICENSE.rst LICENSE.txt ; do
263 [[ -f "$SOURCE_DIR/$f" ]] || continue
264 LICFILE="$f"
266 detect_license LICENSE "$SOURCE_DIR/$LICFILE"
267 [[ -n "$LICENSE" ]] && break
269 printf "WARNING: Failed to detect license type in %s file\n" "$f" >&2
270 done
271 if [[ -z "$LICFILE" ]] ; then
272 printf "WARNING: No license file found\n" >&2
273 else
274 sed -i -e 's|licfile:TODO|'"$LICFILE"'|g' Makefile
277 if [[ -z "$LICENSE" ]] ; then
278 # Execute hook-no-license snippet
279 gsed -e '0,/^%hook-no-license%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
280 . "./$SNIPPET"
281 rm -f "$SNIPPET"
283 if [[ -f "$DISTRIBUTION.license" ]] ; then
284 sed -i -e '/^COMPONENT_LICENSE_FILE/d' Makefile
285 git add "$DISTRIBUTION.license"
286 [[ -z "$LICENSE" ]] && detect_license LICENSE "$DISTRIBUTION.license"
288 [[ -z "$LICENSE" ]] && LICENSE="TODO"
291 # Store the detected license into the Makefile
292 sed -i -e 's/license:TODO/'"$LICENSE"'/g' Makefile
295 # detect TEST_STYLE
296 TEST_STYLE=
297 cd "$SOURCE_DIR"
298 while true ; do
299 TOX_OUT=$(tox -l)
300 TOX_RET=$?
301 ((TOX_RET == 0)) && ! printf "%s" "$TOX_OUT" | grep -q 'assuming empty tox\.ini' && TEST_STYLE="tox" && break
303 pytest -p no:checkdocs --setup-plan
304 (($? != 5)) && TEST_STYLE="pytest" && break
306 [[ -f setup.py ]] && python setup.py test --help && TEST_STYLE="setup.py" && break
308 TEST_STYLE="none"
309 break
310 done > /dev/null 2>&1
311 cd "$DIR"
313 if [[ "$TEST_STYLE" == "$($GMAKE print-value-TEST_STYLE)" ]] ; then
314 # If the detected TEST_STYLE is same as the default value or the value
315 # forced by the component, then we do not need to set the detected
316 # value.
317 sed -i -e '/^_TEST_STYLE = TODO$/,+1d' Makefile
318 else
319 # Set the detected TEST_STYLE value
320 sed -i -e 's/^_\(TEST_STYLE = \)TODO$/\1'"$TEST_STYLE/" Makefile
322 # If the component forces different test style than detected, then drop
323 # the detected value
324 if [[ "$TEST_STYLE" != "$($GMAKE print-value-TEST_STYLE)" ]] ; then
325 sed -i -e '/^TEST_STYLE = '"${TEST_STYLE//./\\.}/,+1d" Makefile
329 # Warn if a testing tool is called directly by tox
330 if [[ "$($GMAKE print-value-TEST_STYLE)" == "tox" && -f "$SOURCE_DIR/tox.ini" ]] ; then
331 TOX_CALL_INDIRECTLY=$($GMAKE print-value-TOX_CALL_INDIRECTLY)
332 for p in $TOX_CALL_INDIRECTLY ; do
333 sed -n -e '/^commands *=/,/^$/p' "$SOURCE_DIR/tox.ini" \
334 | tr '\t' ' ' \
335 | grep -q '^\(commands *=\)\{0,1\} *'"$p"'\( \{1,\}.*\)\{0,1\}$' \
336 && printf "WARNING: %s is called directly in tox.ini\n" "$p" >&2
337 done
341 # Create manifests
342 if ! $GMAKE sample-manifest > /dev/null 2>&1 ; then
343 printf "ERROR: 'gmake sample-manifest' failed!\n" >&2
344 else
345 cat manifests/sample-manifest.p5m \
346 | sed -e 's/^#.*Copyright.*<contributor>.*$/# This file was automatically generated using '"$THIS"'/g' \
347 > "$DISTRIBUTION-PYVER.p5m"
349 # Execute hook-manifest snippet
350 gsed -e '0,/^%hook-manifest%/d' -e '/^%/,$d' < "$CONF" > "$SNIPPET"
351 . "./$SNIPPET"
352 rm -f "$SNIPPET"
354 git add manifests/sample-manifest.p5m "$DISTRIBUTION-PYVER.p5m"
358 # $CONF is no longer needed
359 rm -f "$CONF"
360 git checkout "$CONF" > /dev/null 2>&1
363 # Generate REQUIRED_PACKAGES
364 $GMAKE REQUIRED_PACKAGES > /dev/null 2>&1 || printf "ERROR: 'gmake REQUIRED_PACKAGES' failed!\n" >&2
365 git add Makefile
368 # Check for Makefile completeness
369 grep -q "TODO" Makefile && printf "ERROR: Makefile is not complete (TODO found)\n" >&2
372 # Make sure the build environment is setup properly and we do have all
373 # requirements installed. Otherwise we cannot continue.
374 ! $GMAKE env-check > /dev/null 2>&1 && printf "FATAL: 'gmake env-check' failed!\n" >&2 && exit 1
377 # Publish packages and create pkg5 file
378 $GMAKE publish > /dev/null 2>&1 || printf "ERROR: 'gmake publish' failed!\n" >&2
379 git add pkg5 2>/dev/null
382 PYTHON_VERSIONS=$($GMAKE print-value-PYTHON_VERSIONS)
383 PYTHON_TEST_BOOTSTRAP=$($GMAKE print-value-PYTHON_TEST_BOOTSTRAP)
386 # Run tests to make sure they pass and to create result snapshots
387 TESTED_VERSIONS=
388 for v in $PYTHON_VERSIONS ; do
389 # Check the test environment
390 if ! $GMAKE PYTHON_VERSIONS=$v test-env-check > /dev/null 2>&1 ; then
391 if [[ "$PYTHON_TEST_BOOTSTRAP" == "yes" ]] ; then
392 printf "WARNING: Test environment for %s is not ready yet (bootstrap)\n" "$v" >&2
393 else
394 printf "ERROR: 'gmake test-env-check' failed for %s!\n" "$v" >&2
396 continue
399 # Run the test
400 ! $GMAKE PYTHON_VERSIONS=$v test > /dev/null 2>&1 && printf "ERROR: Testing failed for %s!\n" "$v" >&2 && continue
402 # If there is no snapshot produced the component likely does not support tests
403 COMPONENT_TEST_SNAPSHOT=$($GMAKE PYTHON_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
404 [[ ! -f "$COMPONENT_TEST_SNAPSHOT" ]] && printf "WARNING: Testing unsupported for %s\n" "$v" >&2 && continue
406 # Empty result snapshot is suspicious
407 [[ -s "$COMPONENT_TEST_SNAPSHOT" ]] || printf "WARNING: Empty test results for %s\n" "$v" >&2
409 TESTED_VERSIONS="$TESTED_VERSIONS $v"
410 done
412 # Save result snapshots and detect USE_COMMON_TEST_MASTER value
413 TEST_MASTERS=
414 for common_results in yes no ; do
415 for v in $TESTED_VERSIONS ; do
416 COMPONENT_TEST_SNAPSHOT=$($GMAKE PYTHON_VERSION=$v print-value-COMPONENT_TEST_SNAPSHOT)
417 COMPONENT_TEST_MASTER=$($GMAKE PYTHON_VERSION=$v USE_COMMON_TEST_MASTER=$common_results print-value-COMPONENT_TEST_MASTER)
419 if [[ -f "$COMPONENT_TEST_MASTER" ]] ; then
420 # Switch to 'USE_COMMON_TEST_MASTER = no' if test results differ
421 if ! diff "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER" > /dev/null ; then
422 printf "WARNING: Test results differ so switch to 'USE_COMMON_TEST_MASTER = no'\n" >&2
423 rm -f $TEST_MASTERS
424 TEST_MASTERS=
425 continue 2
427 else
428 mkdir -p $(dirname "$COMPONENT_TEST_MASTER")
429 cp -p "$COMPONENT_TEST_SNAPSHOT" "$COMPONENT_TEST_MASTER"
430 TEST_MASTERS="$TEST_MASTERS $COMPONENT_TEST_MASTER"
432 done
433 break
434 done
435 [[ -n "$TEST_MASTERS" ]] && git add $TEST_MASTERS
437 # Run tests again to confirm the results are reproducible
438 for v in $TESTED_VERSIONS ; do
439 $GMAKE PYTHON_VERSIONS=$v USE_COMMON_TEST_MASTER=$common_results test > /dev/null 2>&1 || printf "ERROR: Testing for %s is not reproducible!\n" "$v" >&2
440 done
442 # Remove USE_COMMON_TEST_MASTER from Makefile if it should be set to (default) 'yes'
443 if [[ "$common_results" == "yes" ]] ; then
444 sed -i -e '/^USE_COMMON_TEST_MASTER/d' Makefile
445 git add Makefile
449 # Handle history
450 COMPONENT_FMRI=$($GMAKE print-value-COMPONENT_FMRI)
451 PYTHON_VERSIONS_OBSOLETING=$($GMAKE print-value-PYTHON_VERSIONS_OBSOLETING)
453 OV_PLURAL=
454 for o in $(echo $OBSOLETE $PYTHON_VERSIONS_OBSOLETING | LC_ALL=C sort -u) ; do
455 PYV=${o//.}
456 FMRI=$(pkg list -nvH "$COMPONENT_FMRI-$PYV" 2>/dev/null | egrep -v '(o|r)$' | sed -e 's|^.*\('"$COMPONENT_FMRI"'\)|\1|g' -e 's/:[^:]*$//g' -e 's/\(-[^-]*\)$/,5.11\1/g')
457 [[ -n "$FMRI" ]] || continue
458 FMRI_H=${FMRI%.*}
459 FMRI_T=${FMRI##*.}
460 if [[ "$FMRI_H" == "$FMRI" ]] ; then
461 printf "WARNING: Wrong fmri format: %s\n" "$FMRI" >&2
462 continue
464 FMRI_T=$((FMRI_T + 1))
465 printf "%s.%s noincorporate\n" "$FMRI_H" "$FMRI_T" >> history
467 [[ -n "$OV" ]] && OV="$OV and " && OV_PLURAL="s"
468 OV="$OV$o"
469 done
470 if [[ -f history ]] ; then
471 LC_ALL=C sort -u history > history.new
472 mv history.new history
473 git add history
477 # Construct the commit message
478 MSG=
479 if ((NEW)) ; then
480 MSG="Add $PROJECT python project"
481 else
482 if ((REBUILD == 0)) ; then
483 [[ "$PREV_HVER" != "$VERSION" ]] && MSG="update to $VERSION" || MSG="change version format"
487 for v in $PYTHON_VERSIONS ; do
488 PYV=${v//.}
489 pkg list -avH "$COMPONENT_FMRI-$PYV" 2>/dev/null | egrep -q -v '(o|r)$' && continue
490 [[ -n "$NV" ]] && NV="$NV and "
491 NV="$NV$v"
492 done
494 REBUILDMSG=
495 [[ -n "$NV" ]] && REBUILDMSG="rebuild for python $NV"
496 if [[ -n "$OV" ]] ; then
497 [[ -n "$REBUILDMSG" ]] && REBUILDMSG="$REBUILDMSG and" || REBUILDMSG="rebuild"
498 REBUILDMSG="$REBUILDMSG to get package$OV_PLURAL for python $OV obsoleted"
501 if [[ -n "$REBUILDMSG" ]] ; then
502 [[ -n "$MSG" ]] && MSG="$MSG; "
503 MSG="$MSG$REBUILDMSG"
505 [[ -z "$MSG" ]] && MSG="rebuild"
507 MSG="$DIRECTORY: $MSG"
510 # Commit the results
511 ! git commit -m "$MSG" > /dev/null 2>&1 && printf "FATAL: 'git commit' failed!\n" >&2 && exit 1