scripts/Build-Target

   1 #!/usr/bin/env bash
   2 #
   3 # --- T2-COPYRIGHT-NOTE-BEGIN ---
   4 # T2 SDE: scripts/Build-Target
   5 # Copyright (C) 2004 - 2024 The T2 SDE Project
   6 # Copyright (C) 1998 - 2003 ROCK Linux Project
   7 #
   8 # This Copyright note is generated by scripts/Create-CopyPatch,
   9 # more information can be found in the files COPYING and README.
  10 #
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License version 2.
  13 # --- T2-COPYRIGHT-NOTE-END ---
  14 #
  15 #   Run this command from the T2 SDE base directory as scripts/Build-Target
  16 #   after running the scripts/Config and the now optional
  17 #   scripts/Download commands.
  18 #
  19 #   It compiles/builds all the packages and stores their binary package
  20 #   ssuitable for distribution.
  21 #
  22 #   This script (and Build-Pkg) is T2 SDE's work-horse. It builds in a
  23 #   chroot environment (stage 0..2) and goes through a number of build
  24 #   stages (stage 3..9).
  25 #
  26
  27 config=default
  28 build_only_this_job=
  29 daemon_mode=0
  30 autodownload=1
  31 options="$*"
  32 downloadopt="-q"
  33 verbose=
  34 xtrace=
  35 noclearsrc=
  36
  37 while [ $# -gt 0 ]; do
  38         case "$1" in
  39                 -cfg)           config=$2; shift ;;
  40                 -job)           build_only_this_job=$2; shift ;;
  41                 -daemon)        daemon_mode=1 ;;
  42                 -nodaemon)      daemon_mode=0 ;;
  43                 -nodownload)    autodownload=0 ;;
  44                 -noclearsrc)    noclearsrc="-noclearsrc" ;;
  45                 -xtrace)        xtrace="-xtrace" ;;
  46                 -v)             verbose="-v" ;;
  47                 [0-9]-*)        build_only_this_job=$1 ;;
  48                 *)              echo "Usage: $0 [ -daemon ] [ -nodownload ] [ -noclearsrc ] [ -cfg config ]" \
  49                                      "[ -job <stage>-<package> ]"; exit 1 ;;
  50         esac
  51         shift
  52 done
  53
  54 if [ "$daemon_mode" = 1 ]; then
  55         . config/$config/config
  56         echo "Running $0 in the background (messages go to logfile only)"
  57         echo "Logfile: build/$SDECFG_ID/TOOLCHAIN/logs/build_target.log"
  58         nohup $0 $options -nodaemon > /dev/null 2> /dev/null < /dev/null &
  59         exit 0
  60 fi
  61
  62 . scripts/parse-config
  63 . scripts/functions.in
  64
  65 build_root="$base/build/$SDECFG_ID"
  66 build_toolchain="$base/build/$SDECFG_ID/TOOLCHAIN"
  67 build_logs="$build_toolchain/logs"; mkdir -p "${build_logs}"
  68 build_pkgs="$build_toolchain/pkgs"; mkdir -p "${build_root}"
  69
  70 if [ "$SDECFG_PARANOIA_CHECK" = 1 ]; then
  71         scripts/Check-System || exit 1
  72 fi
  73
  74 # Package Build loop - executed by build-target
  75 #
  76 pkgloop() {
  77         local x= y=
  78
  79         if [ "$SDECFG_NOBROKENDEPS" = 1 ]; then
  80                 nobrokendeps="-nobrokendeps"
  81         else
  82                 nobrokendeps=""
  83         fi
  84         if [ -z "$build_only_this_job" ]; then
  85
  86           if [ "`ls ${build_root}/var/adm/logs/*.err 2> /dev/null`" ]; then
  87                 echo_header "Removing old error logs ..."
  88                 for y in 0 1 2 3 4 5 6 7 8 9; do
  89                         if [ "$SDECFG_RETRY_BROKEN" -eq 1 -o $y -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
  90                                 for x in ${build_root}/var/adm/logs/$y-*.err; do
  91                                         if [ -f $x ]; then
  92                                                 echo_status "Removing ${x#$build_root/} ..."
  93                                                 rm -f $x
  94                                         fi
  95                                 done
  96                         fi
  97                 done
  98           fi
  99           if [ "`ls ${build_root}/var/adm/logs/*.out 2> /dev/null`" ]; then
 100                 echo_header "Removing old output logs ..."
 101                 for x in ${build_root}/var/adm/logs/*.out; do
 102                         echo_status "Removing ${x#$build_root/} ..."
 103                         rm -f $x
 104                 done
 105           fi
 106
 107           while
 108                 next="`scripts/Create-PkgQueue \
 109                         -cfg "$config" -single $nobrokendeps`"
 110                 [ "$next" ]
 111           do
 112                 pkgloop_package $next
 113           done
 114
 115         else
 116                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.log"
 117                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.err"
 118                 next="$(awk 'BEGIN { FS=" "; }
 119                               $5 == "'${build_only_this_job#*-}'" &&
 120                               $2 ~ /'${build_only_this_job%%-*}'/ \
 121                               { $1="'${build_only_this_job%%-*}' 0";
 122                               print; exit; }' < config/$config/packages)"
 123                 if [ ! "$next" ]; then
 124                         echo_error "No job matching job spec: $build_only_this_job"
 125                         exit 1
 126                 fi
 127
 128                 pkgloop_package $next
 129                 exit 0
 130         fi
 131
 132         # wait for background compression jobs to finish
 133         for j in $(jobs -p); do wait $j; done
 134
 135         local pkglst=`mktemp` errors=0; rm -f src/invalid-files.lst
 136         echo_header "Searching for old lingering files ..."
 137         sed '/^[^X]/d; s,.*=,,' config/$config/packages | cut -d' ' -f5 |
 138         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 139                 while read p; do
 140                         v=$(grep '^Package Name and Version:' \
 141                                 build/$SDECFG_ID/var/adm/packages/$p \
 142                                 2>/dev/null | cut -f6 -d' ')
 143                         echo "$p-$v"
 144                 done
 145         else
 146                 cat
 147         fi > $pkglst
 148         for file in $(ls build/$SDECFG_ID/TOOLCHAIN/pkgs/ 2> /dev/null); do
 149                 x="$file"
 150                 case $SDECFG_PKGFILE_TYPE in
 151                   tar.*)        x=${x%.$SDECFG_PKGFILE_TYPE} ;;
 152                   none)         : ;;
 153                 esac
 154                 if ! grep -qx "$x" $pkglst && ! test "$x" = packages.db; then
 155                         file="build/$SDECFG_ID/TOOLCHAIN/pkgs/$file"
 156                         echo_error "$file should not be present" \
 157                                    "(now in src/invalid-files.lst)!"
 158                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 159                         errors=1
 160                 fi
 161         done
 162         for dir in build/$SDECFG_ID/var/adm/{cache,dependencies,descs,flists,md5sums,packages}; do
 163                 for file in $(ls $dir 2> /dev/null); do
 164                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 165                                 x="$file-"
 166                         else
 167                                 x="$file"
 168                         fi
 169                         if ! grep -q "$x" $pkglst; then
 170                                 echo_error "$dir/$file should not be present (now in src/invalid-files.lst)!"
 171                                 mkdir -p src; echo "$dir/$file" >> src/invalid-files.lst
 172                                 errors=1
 173                         fi
 174                 done
 175         done
 176         for file in $(ls build/$SDECFG_ID/var/adm/logs/); do
 177                         x="`echo $file | sed -e 's/^.-//' -e 's/\.log//' -e 's/\.err//' -e s'/\.out//'`"
 178                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 179                                 x=$x-
 180                         else
 181                                 x=$x
 182                         fi
 183
 184                 if ! grep -q "$x" $pkglst; then
 185                         file="build/$SDECFG_ID/var/adm/logs/$file"
 186                         echo_error "$file should not be present (now in src/invalid-files.lst)!"
 187                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 188                         errors=1
 189                 fi
 190         done
 191         [ $errors = 0 ] && echo_status "None found."
 192         rm $pkglst
 193 }
 194
 195 # Process one line of output generated by Create-PkgQueue
 196 #
 197 pkgloop_package() {
 198         stagelevel="$1" pkg_depnr="$2" pkg_stages="$3" pkg_pri="$4" pkg_tree="$5"
 199         pkg_name="$6" pkg_ver="$7" pkg_prefix="$8" pkg_extra="$9"
 200
 201         [ "$build_only_this_job" -a \
 202           "$stagelevel-$pkg_name" != "$build_only_this_job" ] && return
 203
 204         [ $(expr "$pkg_stages" : ".*$stagelevel.*") -eq 0 ] && return
 205         pkg_laststage=$(echo "$pkg_stages" | sed "s,-,,g; s,.*\(.\),\1,")
 206
 207         cmd_root="-root auto"
 208         [ $stagelevel -gt 2 ] && cmd_root="$cmd_root -chroot"
 209
 210         if [ "$pkg_prefix" != "/" ]; then
 211                 cmd_prefix="-prefix $pkg_prefix"
 212         else cmd_prefix=""; fi
 213
 214         if [ "$autodownload" == 1 ]; then
 215                 scripts/Download -cfg $config $downloadopt $pkg_name
 216         fi
 217
 218         cmd_buildpkg="scripts/Build-Pkg -$stagelevel -cfg $config $verbose"
 219         cmd_buildpkg="$cmd_buildpkg $noclearsrc $xtrace $cmd_root $cmd_prefix $pkg_name"
 220
 221         # Execute action handler
 222         if ! pkgloop_action &&
 223            [ $stagelevel -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
 224                 exit 1
 225         fi
 226
 227         if [ ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.log -a \
 228              ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 229         then
 230                 echo_header "Package build ended abnormally!"
 231                 echo_error "Usually a package build creates either a *.log"
 232                 echo_error "or a *.err file. As neither is there, creating a"
 233                 echo_error "*.err file and aborting the build process."
 234                 touch ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err
 235                 exit 1
 236         fi
 237
 238         if [ $stagelevel -gt 0 -a $pkg_laststage -eq $stagelevel -a \
 239              "$SDECFG_PKGFILE_TYPE" != none ]
 240         then
 241                 if [ -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 242                 then echo_error "Creation of binary package isn't possible, \
 243 because the package was not"
 244                      echo_error "built successfully in (at least) the current \
 245 stage."
 246                 else
 247                         mkdir -p "${build_pkgs}"
 248
 249                         local file=${pkg_name}
 250                         if [ "$SDECFG_PKGFILE_VER" = 1 ]; then
 251                                 v="$(grep '^Package Name and Version:' \
 252                                       ${build_root}/var/adm/packages/$pkg_name | cut -f6 -d' ')"
 253                                 file="$file-$v"
 254                         fi
 255
 256                         [[ $SDECFG_PKGFILE_TYPE = tar.* ]] &&
 257                                 file=$file.$SDECFG_PKGFILE_TYPE
 258                         local compressor=${SDECFG_PKGFILE_TYPE#tar.}
 259                         case $compressor in
 260                           br)   compressor=brotli ;;
 261                           gz)   compressor=gzip ;;
 262                           bz2)  compressor=bzip2 ;;
 263                           lzo)  compressor=lzop ;;
 264                           zst)  [ "$arch_sizeof_char_p" = 8 ] &&
 265                                 compressor="zstd -T0 --ultra -20" || compressor="zstd -T0 -19"
 266                                 ;;
 267                           # lzma, xz et al. are named after the extension
 268                           gem)  compressor=bzip2 file=$file.tar.bz2 ;;
 269                         esac
 270
 271                         echo_status "Creating ${build_pkgs#$base/}/$file"
 272                         (
 273                         # sort var/adm before the rest - so they can be extracted, quickly
 274                         (cd "$build_root/"
 275                           (grep    ' var/adm' var/adm/flists/$pkg_name
 276                            grep -v ' var/adm' var/adm/flists/$pkg_name) | cut -f2- -d' ' |
 277                           tar -cf- --no-recursion --files-from=- | $compressor
 278                         ) > ${build_pkgs}/$file.tmp
 279                         mv ${build_pkgs}/$file{.tmp,}
 280
 281                         if [ "$SDECFG_PKGFILE_TYPE" = gem ]; then
 282                                 mine -C "$build_root/var/adm" $build_pkgs/$file $pkg_name \
 283                                      $build_pkgs/${file%.tar.bz2}.gem.tmp
 284                                 mv $build_pkgs/${file%.tar.bz2}.gem{.tmp,}
 285
 286                                 echo_status "Removing temporary tar.bz2."
 287                                 rm -f $build_pkgs/$file
 288                         fi
 289                         ) &
 290                 fi
 291         fi
 292 }
 293
 294 # Action executed by pkgloop(). This function may be redefined
 295 # before calling pkgloop().
 296 #
 297 pkgloop_action() {
 298         $cmd_buildpkg
 299 }
 300
 301 # Try to umount any directories mounted by Build-Pkg -chroot
 302 # if we are the last process using them.
 303 #
 304 umount_chroot() {
 305         local ret=$?
 306         exec 201> /dev/null
 307         if ! (cd ${build_logs}; fuser *.log > /dev/null 2>&1); then
 308                 echo_status "Unmounting loop mounts ..."
 309                 umount -d -f    $build_toolchain/{loop,config,download} 2> /dev/null
 310                 umount -d -f -l $build_toolchain/{loop,config,download} 2> /dev/null
 311                 umount -d -f    $build_root/proc 2> /dev/null
 312                 umount -d -f -l $build_root/proc 2> /dev/null
 313         fi
 314         return $ret
 315 }
 316
 317 # must trap outside the group command
 318 trap 'umount_chroot' EXIT
 319
 320 {
 321         ln -sf build_target_$$.log ${build_logs}/build_target.log
 322         scripts/Build-Tools -1 -cfg $config
 323         _built=0
 324         for x in $(get_expanded target/%/build.sh $targetchain); do
 325                 if [ -f $x ]; then
 326                         . $x
 327                         _built=1
 328                         break
 329                 fi
 330         done
 331         [ $_built = 1 ] || echo_warning "No target/*/build.sh controlling the build!"
 332 } 2>&1 201>> "${build_logs}/build_target_$$.log" #| tee -a "${build_logs}/build_target_$$.log"