scripts/Build-Target

   1 #!/usr/bin/env bash
   2 #
   3 # --- T2-COPYRIGHT-NOTE-BEGIN ---
   4 # T2 SDE: scripts/Build-Target
   5 # Copyright (C) 2004 - 2024 The T2 SDE Project
   6 # Copyright (C) 1998 - 2003 ROCK Linux Project
   7 #
   8 # This Copyright note is generated by scripts/Create-CopyPatch,
   9 # more information can be found in the files COPYING and README.
  10 #
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License version 2.
  13 # --- T2-COPYRIGHT-NOTE-END ---
  14 #
  15 #   Run this command from the T2 SDE base directory as scripts/Build-Target
  16 #   after running the scripts/Config and the now optional
  17 #   scripts/Download commands.
  18 #
  19 #   It compiles/builds all the packages and stores their binary package
  20 #   ssuitable for distribution.
  21 #
  22 #   This script (and Build-Pkg) is T2 SDE's work-horse. It builds in a
  23 #   chroot environment (stage 0..2) and goes through a number of build
  24 #   stages (stage 3..9).
  25 #
  26
  27 config=default
  28 build_only_this_job=
  29 daemon_mode=0
  30 autodownload=1
  31 options="$*"
  32 downloadopt="-q"
  33 verbose=
  34 noclearsrc=
  35
  36 while [ $# -gt 0 ]; do
  37         case "$1" in
  38                 -cfg)           config=$2; shift ;;
  39                 -job)           build_only_this_job=$2; shift ;;
  40                 -daemon)        daemon_mode=1 ;;
  41                 -nodaemon)      daemon_mode=0 ;;
  42                 -nodownload)    autodownload=0 ;;
  43                 -noclearsrc)    noclearsrc="-noclearsrc" ;;
  44                 -v)             verbose="-v" ;;
  45                 [0-9]-*)        build_only_this_job=$1 ;;
  46                 *)              echo "Usage: $0 [ -daemon ] [ -nodownload ] [ -noclearsrc ] [ -cfg config ]" \
  47                                      "[ -job <stage>-<package> ]"; exit 1 ;;
  48         esac
  49         shift
  50 done
  51
  52 if [ "$daemon_mode" = 1 ]; then
  53         . config/$config/config
  54         echo "Running $0 in the background (messages go to logfile only)"
  55         echo "Logfile: build/$SDECFG_ID/TOOLCHAIN/logs/build_target.log"
  56         nohup $0 $options -nodaemon > /dev/null 2> /dev/null < /dev/null &
  57         exit 0
  58 fi
  59
  60 . scripts/parse-config
  61 . scripts/functions.in
  62
  63 build_root="$base/build/$SDECFG_ID"
  64 build_toolchain="$base/build/$SDECFG_ID/TOOLCHAIN"
  65 build_logs="$build_toolchain/logs"; mkdir -p "${build_logs}"
  66 build_pkgs="$build_toolchain/pkgs"; mkdir -p "${build_root}"
  67
  68 if [ "$SDECFG_PARANOIA_CHECK" = 1 ]; then
  69         scripts/Check-System || exit 1
  70 fi
  71
  72 # Package Build loop - executed by build-target
  73 #
  74 pkgloop() {
  75         local x= y=
  76
  77         if [ "$SDECFG_NOBROKENDEPS" = 1 ]; then
  78                 nobrokendeps="-nobrokendeps"
  79         else
  80                 nobrokendeps=""
  81         fi
  82         if [ -z "$build_only_this_job" ]; then
  83
  84           if [ "`ls ${build_root}/var/adm/logs/*.err 2> /dev/null`" ]; then
  85                 echo_header "Removing old error logs ..."
  86                 for y in 0 1 2 3 4 5 6 7 8 9; do
  87                         if [ "$SDECFG_RETRY_BROKEN" -eq 1 -o $y -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
  88                                 for x in ${build_root}/var/adm/logs/$y-*.err; do
  89                                         if [ -f $x ]; then
  90                                                 echo_status "Removing ${x#$build_root/} ..."
  91                                                 rm -f $x
  92                                         fi
  93                                 done
  94                         fi
  95                 done
  96           fi
  97           if [ "`ls ${build_root}/var/adm/logs/*.out 2> /dev/null`" ]; then
  98                 echo_header "Removing old output logs ..."
  99                 for x in ${build_root}/var/adm/logs/*.out; do
 100                         echo_status "Removing ${x#$build_root/} ..."
 101                         rm -f $x
 102                 done
 103           fi
 104
 105           while
 106                 next="`scripts/Create-PkgQueue \
 107                         -cfg "$config" -single $nobrokendeps`"
 108                 [ "$next" ]
 109           do
 110                 pkgloop_package $next
 111           done
 112
 113         else
 114                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.log"
 115                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.err"
 116                 next="$(awk 'BEGIN { FS=" "; }
 117                               $5 == "'${build_only_this_job#*-}'" &&
 118                               $2 ~ /'${build_only_this_job%%-*}'/ \
 119                               { $1="'${build_only_this_job%%-*}' 0";
 120                               print; exit; }' < config/$config/packages)"
 121                 if [ ! "$next" ]; then
 122                         echo_error "No job matching job spec: $build_only_this_job"
 123                         exit 1
 124                 fi
 125
 126                 pkgloop_package $next
 127                 exit 0
 128         fi
 129
 130         # wait for background compression jobs to finish
 131         for j in $(jobs -p); do wait $j; done
 132
 133         local pkglst=`mktemp` errors=0; rm -f src/invalid-files.lst
 134         echo_header "Searching for old lingering files ..."
 135         sed '/^[^X]/d; s,.*=,,' config/$config/packages | cut -d' ' -f5 |
 136         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 137                 while read p; do
 138                         v=$(grep '^Package Name and Version:' \
 139                                 build/$SDECFG_ID/var/adm/packages/$p \
 140                                 2>/dev/null | cut -f6 -d' ')
 141                         echo "$p-$v"
 142                 done
 143         else
 144                 cat
 145         fi > $pkglst
 146         for file in $(ls build/$SDECFG_ID/TOOLCHAIN/pkgs/ 2> /dev/null); do
 147                 x="$file"
 148                 case $SDECFG_PKGFILE_TYPE in
 149                   tar.*)        x=${x%.$SDECFG_PKGFILE_TYPE} ;;
 150                   none)         : ;;
 151                 esac
 152                 if ! grep -qx "$x" $pkglst && ! test "$x" = packages.db; then
 153                         file="build/$SDECFG_ID/TOOLCHAIN/pkgs/$file"
 154                         echo_error "$file should not be present" \
 155                                    "(now in src/invalid-files.lst)!"
 156                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 157                         errors=1
 158                 fi
 159         done
 160         for dir in build/$SDECFG_ID/var/adm/{cache,dependencies,descs,flists,md5sums,packages}; do
 161                 for file in $(ls $dir 2> /dev/null); do
 162                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 163                                 x="$file-"
 164                         else
 165                                 x="$file"
 166                         fi
 167                         if ! grep -q "$x" $pkglst; then
 168                                 echo_error "$dir/$file should not be present (now in src/invalid-files.lst)!"
 169                                 mkdir -p src; echo "$dir/$file" >> src/invalid-files.lst
 170                                 errors=1
 171                         fi
 172                 done
 173         done
 174         for file in $(ls build/$SDECFG_ID/var/adm/logs/); do
 175                         x="`echo $file | sed -e 's/^.-//' -e 's/\.log//' -e 's/\.err//' -e s'/\.out//'`"
 176                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 177                                 x=$x-
 178                         else
 179                                 x=$x
 180                         fi
 181
 182                 if ! grep -q "$x" $pkglst; then
 183                         file="build/$SDECFG_ID/var/adm/logs/$file"
 184                         echo_error "$file should not be present (now in src/invalid-files.lst)!"
 185                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 186                         errors=1
 187                 fi
 188         done
 189         [ $errors = 0 ] && echo_status "None found."
 190         rm $pkglst
 191 }
 192
 193 # Process one line of output generated by Create-PkgQueue
 194 #
 195 pkgloop_package() {
 196         stagelevel="$1" pkg_depnr="$2" pkg_stages="$3" pkg_pri="$4" pkg_tree="$5"
 197         pkg_name="$6" pkg_ver="$7" pkg_prefix="$8" pkg_extra="$9"
 198
 199         [ "$build_only_this_job" -a \
 200           "$stagelevel-$pkg_name" != "$build_only_this_job" ] && return
 201
 202         [ $(expr "$pkg_stages" : ".*$stagelevel.*") -eq 0 ] && return
 203         pkg_laststage=$(echo "$pkg_stages" | sed "s,-,,g; s,.*\(.\),\1,")
 204
 205         cmd_root="-root auto"
 206         [ $stagelevel -gt 2 ] && cmd_root="$cmd_root -chroot"
 207
 208         if [ "$pkg_prefix" != "/" ]; then
 209                 cmd_prefix="-prefix $pkg_prefix"
 210         else cmd_prefix=""; fi
 211
 212         if [ "$autodownload" == 1 ]; then
 213                 scripts/Download -cfg $config $downloadopt $pkg_name
 214         fi
 215
 216         cmd_buildpkg="scripts/Build-Pkg -$stagelevel -cfg $config $verbose"
 217         cmd_buildpkg="$cmd_buildpkg $noclearsrc $cmd_root $cmd_prefix $pkg_name"
 218
 219         # Execute action handler
 220         if ! pkgloop_action &&
 221            [ $stagelevel -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
 222                 exit 1
 223         fi
 224
 225         if [ ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.log -a \
 226              ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 227         then
 228                 echo_header "Package build ended abnormally!"
 229                 echo_error "Usually a package build creates either a *.log"
 230                 echo_error "or a *.err file. As neither is there, creating a"
 231                 echo_error "*.err file and aborting the build process."
 232                 touch ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err
 233                 exit 1
 234         fi
 235
 236         if [ $stagelevel -gt 0 -a $pkg_laststage -eq $stagelevel -a \
 237              "$SDECFG_PKGFILE_TYPE" != none ]
 238         then
 239                 if [ -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 240                 then echo_error "Creation of binary package isn't possible, \
 241 because the package was not"
 242                      echo_error "built successfully in (at least) the current \
 243 stage."
 244                 else
 245                         mkdir -p "${build_pkgs}"
 246
 247                         local file=${pkg_name}
 248                         if [ "$SDECFG_PKGFILE_VER" = 1 ]; then
 249                                 v="$(grep '^Package Name and Version:' \
 250                                       ${build_root}/var/adm/packages/$pkg_name | cut -f6 -d' ')"
 251                                 file="$file-$v"
 252                         fi
 253
 254                         [[ $SDECFG_PKGFILE_TYPE = tar.* ]] &&
 255                                 file=$file.$SDECFG_PKGFILE_TYPE
 256                         local compressor=${SDECFG_PKGFILE_TYPE#tar.}
 257                         case $compressor in
 258                           br)   compressor=brotli ;;
 259                           gz)   compressor=gzip ;;
 260                           bz2)  compressor=bzip2 ;;
 261                           lzo)  compressor=lzop ;;
 262                           zst)  [ "$arch_sizeof_char_p" = 8 ] &&
 263                                 compressor="zstd -T0 --ultra -20" || compressor="zstd -T0 -19"
 264                                 ;;
 265                           # lzma, xz et al. are named after the extension
 266                           gem)  compressor=bzip2 file=$file.tar.bz2 ;;
 267                         esac
 268
 269                         echo_status "Creating ${build_pkgs#$base/}/$file"
 270                         (
 271                         # sort var/adm before the rest - so they can be extracted, quickly
 272                         (cd "$build_root/"
 273                           (grep    ' var/adm' var/adm/flists/$pkg_name
 274                            grep -v ' var/adm' var/adm/flists/$pkg_name) | cut -f2- -d' ' |
 275                           tar -cf- --no-recursion --files-from=- | $compressor
 276                         ) > ${build_pkgs}/$file.tmp
 277                         mv ${build_pkgs}/$file{.tmp,}
 278
 279                         if [ "$SDECFG_PKGFILE_TYPE" = gem ]; then
 280                                 mine -C "$build_root/var/adm" $build_pkgs/$file $pkg_name \
 281                                      $build_pkgs/${file%.tar.bz2}.gem.tmp
 282                                 mv $build_pkgs/${file%.tar.bz2}.gem{.tmp,}
 283
 284                                 echo_status "Removing temporary tar.bz2."
 285                                 rm -f $build_pkgs/$file
 286                         fi
 287                         ) &
 288                 fi
 289         fi
 290 }
 291
 292 # Action executed by pkgloop(). This function may be redefined
 293 # before calling pkgloop().
 294 #
 295 pkgloop_action() {
 296         $cmd_buildpkg
 297 }
 298
 299 # Try to umount any directories mounted by Build-Pkg -chroot
 300 # if we are the last process using them.
 301 #
 302 umount_chroot() {
 303         local ret=$?
 304         exec 201> /dev/null
 305         if ! (cd ${build_logs}; fuser *.log > /dev/null 2>&1); then
 306                 echo_status "Unmounting loop mounts ..."
 307                 umount -d -f    $build_toolchain/{loop,config,download} 2> /dev/null
 308                 umount -d -f -l $build_toolchain/{loop,config,download} 2> /dev/null
 309                 umount -d -f    $build_root/proc 2> /dev/null
 310                 umount -d -f -l $build_root/proc 2> /dev/null
 311         fi
 312         return $ret
 313 }
 314
 315 # must trap outside the group command
 316 trap 'umount_chroot' EXIT
 317
 318 {
 319         ln -sf build_target_$$.log ${build_logs}/build_target.log
 320         scripts/Build-Tools -1 -cfg $config
 321         _built=0
 322         for x in $(get_expanded target/%/build.sh $targetchain); do
 323                 if [ -f $x ]; then
 324                         . $x
 325                         _built=1
 326                         break
 327                 fi
 328         done
 329         [ $_built = 1 ] || echo_warning "No target/*/build.sh controlling the build!"
 330 } 2>&1 201>> "${build_logs}/build_target_$$.log" #| tee -a "${build_logs}/build_target_$$.log"