scripts/Build-Target

   1 #!/bin/bash
   2 #
   3 # --- T2-COPYRIGHT-NOTE-BEGIN ---
   4 # This copyright note is auto-generated by scripts/Create-CopyPatch.
   5 #
   6 # T2 SDE: scripts/Build-Target
   7 # Copyright (C) 2004 - 2021 The T2 SDE Project
   8 # Copyright (C) 1998 - 2003 ROCK Linux Project
   9 #
  10 # More information can be found in the files COPYING and README.
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; version 2 of the License. A copy of the
  15 # GNU General Public License can be found in the file COPYING.
  16 # --- T2-COPYRIGHT-NOTE-END ---
  17 #
  18 #   Run this command from the T2 SDE base directory as scripts/Build-Target
  19 #   after running the scripts/Config and the now optional
  20 #   scripts/Download commands.
  21 #
  22 #   It compiles/builds all the packages and stores their binary package
  23 #   ssuitable for distribution.
  24 #
  25 #   This script (and Build-Pkg) is T2 SDE's work-horse. It builds in a
  26 #   chroot environment (stage 0..1) and goes through a number of build
  27 #   stages (stage 2..9).
  28 #
  29
  30 config=default
  31 build_only_this_job=
  32 daemon_mode=0
  33 autodownload=1
  34 options="$*"
  35 downloadopt="-q"
  36 verbose=
  37
  38 while [ $# -gt 0 ]; do
  39         case "$1" in
  40                 -cfg)           config=$2; shift ;;
  41                 -job)           build_only_this_job=$2; shift ;;
  42                 -daemon)        daemon_mode=1 ;;
  43                 -nodaemon)      daemon_mode=0 ;;
  44                 -nodownload)    autodownload=0 ;;
  45                 -v)             verbose="-v" ;;
  46                 *)              echo "Usage: $0 [ -daemon ] [ -nodownload ] [ -cfg config ]" \
  47                                      "[ -job <stage>-<package> ]"; exit 1 ;;
  48         esac
  49         shift
  50 done
  51
  52 if [ "$daemon_mode" = 1 ]; then
  53         . config/$config/config
  54         echo "Running $0 in the background (messages go to logfile only)"
  55         echo "Logfile: build/$SDECFG_ID/TOOLCHAIN/logs/build_target.log"
  56         nohup $0 $options -nodaemon > /dev/null 2> /dev/null < /dev/null &
  57         exit 0
  58 fi
  59
  60 . scripts/parse-config
  61 . scripts/functions.in
  62
  63 build_root="$base/build/$SDECFG_ID"
  64 build_toolchain="$base/build/$SDECFG_ID/TOOLCHAIN"
  65 build_logs="$build_toolchain/logs"; mkdir -p "${build_logs}"
  66 build_pkgs="$build_toolchain/pkgs"; mkdir -p "${build_root}"
  67
  68 if [ "$SDECFG_PARANOIA_CHECK" = 1 ]; then
  69         scripts/Check-System || exit 1
  70 fi
  71
  72 # Package Build loop - executed by build-target
  73 #
  74 pkgloop() {
  75         local x= y=
  76
  77         if [ "$SDECFG_NOBROKENDEPS" = 1 ]; then
  78                 nobrokendeps="-nobrokendeps"
  79         else
  80                 nobrokendeps=""
  81         fi
  82         if [ -z "$build_only_this_job" ]; then
  83
  84           if [ "`ls ${build_root}/var/adm/logs/*.err 2> /dev/null`" ]; then
  85                 echo_header "Removing old error logs ..."
  86                 for y in 0 1 2 3 4 5 6 7 8 9; do
  87                         if [ "$SDECFG_RETRY_BROKEN" -eq 1 -o $y -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
  88                                 for x in ${build_root}/var/adm/logs/$y-*.err; do
  89                                         if [ -f $x ]; then
  90                                                 echo_status "Removing ${x#$build_root/} ..."
  91                                                 rm -f $x
  92                                         fi
  93                                 done
  94                         fi
  95                 done
  96           fi
  97           if [ "`ls ${build_root}/var/adm/logs/*.out 2> /dev/null`" ]; then
  98                 echo_header "Removing old output logs ..."
  99                 for x in ${build_root}/var/adm/logs/*.out; do
 100                         echo_status "Removing ${x#$build_root/} ..."
 101                         rm -f $x
 102                 done
 103           fi
 104
 105           while
 106                 next="`scripts/Create-PkgQueue \
 107                         -cfg "$config" -single $nobrokendeps`"
 108                 [ "$next" ]
 109           do
 110                 pkgloop_package $next
 111           done
 112
 113         else
 114                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.log"
 115                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.err"
 116                 next="$( awk 'BEGIN { FS=" "; }
 117                               $5 == "'${build_only_this_job#*-}'" &&
 118                               $2 ~ /'${build_only_this_job%%-*}'/ \
 119                               { $1="'${build_only_this_job%%-*}' 0";
 120                               print; exit; }' < config/$config/packages )"
 121                 if [ ! "$next" ]; then
 122                         echo_error "No job matching job spec: $build_only_this_job"
 123                         exit 1
 124                 fi
 125
 126                 pkgloop_package $next
 127                 exit 0
 128         fi
 129
 130         # wait for background compression jobs to finish
 131         for j in $(jobs -p); do wait $j; done
 132
 133         local pkglst=`mktemp` errors=0; rm -f src/invalid-files.lst
 134         echo_header "Searching for old lingering files ..."
 135         sed '/^[^X]/d; s,.*=,,' config/$config/packages | cut -d' ' -f5 |
 136         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 137                 while read p; do
 138                         v=$( grep '^Package Name and Version:' \
 139                                 build/$SDECFG_ID/var/adm/packages/$p \
 140                                 2>/dev/null | cut -f6 -d' ' )
 141                         echo "$p-$v"
 142                 done
 143         else
 144                 cat
 145         fi > $pkglst
 146         for file in $( ls build/$SDECFG_ID/TOOLCHAIN/pkgs/ 2> /dev/null ); do
 147                 x="$file"
 148                 case $SDECFG_PKGFILE_TYPE in
 149                   tar.*)        x=${x%.$SDECFG_PKGFILE_TYPE} ;;
 150                   none)         : ;;
 151                 esac
 152                 if ! grep -qx "$x" $pkglst && ! test "$x" = packages.db; then
 153                         file="build/$SDECFG_ID/TOOLCHAIN/pkgs/$file"
 154                         echo_error "$file should not be present" \
 155                                    "(now in src/invalid-files.lst)!"
 156                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 157                         errors=1
 158                 fi
 159         done
 160         for dir in build/$SDECFG_ID/var/adm/{cache,dependencies,descs,flists,md5sums,packages}; do
 161                 for file in $( ls $dir 2> /dev/null ); do
 162                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 163                                 x="$file-"
 164                         else
 165                                 x="$file"
 166                         fi
 167                         if ! grep -q "$x" $pkglst; then
 168                                 echo_error "$dir/$file should not be present (now in src/invalid-files.lst)!"
 169                                 mkdir -p src; echo "$dir/$file" >> src/invalid-files.lst
 170                                 errors=1
 171                         fi
 172                 done
 173         done
 174         for file in $( ls build/$SDECFG_ID/var/adm/logs/ ); do
 175                         x="`echo $file | sed -e 's/^.-//' -e 's/\.log//' -e 's/\.err//' -e s'/\.out//'`"
 176                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 177                                 x=$x-
 178                         else
 179                                 x=$x
 180                         fi
 181
 182                 if ! grep -q "$x" $pkglst; then
 183                         file="build/$SDECFG_ID/var/adm/logs/$file"
 184                         echo_error "$file should not be present (now in src/invalid-files.lst)!"
 185                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 186                         errors=1
 187                 fi
 188         done
 189         [ $errors = 0 ] && echo_status "None found."
 190         rm $pkglst
 191 }
 192
 193 # Process one line of output generated by Create-PkgQueue
 194 #
 195 pkgloop_package() {
 196         for x in stagelevel pkg_depnr pkg_stages pkg_pri pkg_tree \
 197                  pkg_name pkg_ver pkg_prefix pkg_extra
 198         do eval "$x=\$1"; shift; done
 199
 200         [ "$build_only_this_job" -a \
 201           "$stagelevel-$pkg_name" != "$build_only_this_job" ] && return
 202
 203         [ $(expr "$pkg_stages" : ".*$stagelevel.*") -eq 0 ] && return
 204         pkg_laststage=$(echo "$pkg_stages" | sed "s,-,,g; s,.*\(.\),\1,")
 205
 206         cmd_root="-root auto"
 207         [ $stagelevel -gt 2 ] && cmd_root="$cmd_root -chroot"
 208
 209         if [ "$pkg_prefix" != "/" ]; then
 210                 cmd_prefix="-prefix $pkg_prefix"
 211         else cmd_prefix=""; fi
 212
 213         if [ "$autodownload" == 1 ]; then
 214                 scripts/Download -cfg $config $downloadopt $pkg_name
 215         fi
 216
 217         cmd_buildpkg="scripts/Build-Pkg -$stagelevel -cfg $config $verbose"
 218         cmd_buildpkg="$cmd_buildpkg $cmd_root $cmd_prefix $pkg_name"
 219
 220         # Execute action handler
 221         if ! pkgloop_action &&
 222            [ $stagelevel -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
 223                 exit 1
 224         fi
 225
 226         if [ ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.log -a \
 227              ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 228         then
 229                 echo_header "Package build ended abnormally!"
 230                 echo_error "Usually a package build creates either a *.log"
 231                 echo_error "or a *.err file. Neither the 1st nor the 2nd is"
 232                 echo_error "there. So I'm going to create a *.err file now"
 233                 echo_error "and abort the build process."
 234                 touch ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err
 235                 exit 1
 236         fi
 237
 238         if [ $stagelevel -gt 0 -a $pkg_laststage -eq $stagelevel -a \
 239              "$SDECFG_PKGFILE_TYPE" != none ]
 240         then
 241                 if [ -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 242                 then echo_error "Creation of binary package isn't possible, \
 243 because the package was not"
 244                      echo_error "built successfully in (at least) the current \
 245 stage."
 246                 else
 247                         mkdir -p "${build_pkgs}"
 248
 249                         file=${pkg_name}
 250                         if [ "$SDECFG_PKGFILE_VER" = 1 ]; then
 251                                 v="$( grep '^Package Name and Version:' \
 252                                       ${build_root}/var/adm/packages/$pkg_name | cut -f6 -d' ' )"
 253                                 file="$file-$v"
 254                         fi
 255
 256                         [[ $SDECFG_PKGFILE_TYPE = tar.* ]] &&
 257                                 file=$file.$SDECFG_PKGFILE_TYPE
 258                         compressor=${SDECFG_PKGFILE_TYPE#tar.}
 259                         case $compressor in
 260                           gz)   compressor=gzip ;;
 261                           bz2)  compressor=bzip2 ;;
 262                           lzo)  comrpessor=lzop ;;
 263                           zst)  compressor="zstd --ultra -20 -T0" ;;
 264                           # lzma, xz et al. are named after the extension
 265                           gem)  compressor=bzip2; file=$file.tar.bz2 ;;
 266                         esac
 267
 268                         echo_status "Creating binary package build/.../pkgs/$file"
 269                         (
 270                         # sort var/adm before the rest - so they can be extracted, quickly
 271                         ( cd "$build_root/"
 272                           (grep    ' var/adm' var/adm/flists/$pkg_name
 273                            grep -v ' var/adm' var/adm/flists/$pkg_name) | cut -f2- -d' ' |
 274                           tar -cf- --no-recursion --files-from=- | $compressor
 275                         ) > ${build_pkgs}/$file.tmp
 276                         mv ${build_pkgs}/$file{.tmp,}
 277
 278                         if [ "$SDECFG_PKGFILE_TYPE" = gem ]; then
 279                                 mine -C "$build_root/var/adm" $build_pkgs/$file $pkg_name \
 280                                      $build_pkgs/${file%.tar.bz2}.gem.tmp
 281                                 mv $build_pkgs/${file%.tar.bz2}.gem{.tmp,}
 282
 283                                 echo_status "Removing temporary tar.bz2."
 284                                 rm -f $build_pkgs/$file
 285                         fi
 286                         ) &
 287                 fi
 288         fi
 289 }
 290
 291 # Action executed by pkgloop(). This function may be redefined
 292 # before calling pkgloop().
 293 #
 294 pkgloop_action() {
 295         $cmd_buildpkg
 296 }
 297
 298 # Try to umount any directories mounted by Build-Pkg -chroot
 299 # if we are the last process using them.
 300 #
 301 umount_chroot() {
 302         exec 201> /dev/null
 303         if ! ( cd ${build_logs}; fuser *.log > /dev/null 2>&1 ); then
 304                 echo_status "Unmounting loop mounts ..."
 305                 umount -d -f    $build_toolchain/{loop,config,download} 2> /dev/null
 306                 umount -d -f -l $build_toolchain/{loop,config,download} 2> /dev/null
 307                 umount -d -f    $build_root/proc 2> /dev/null
 308                 umount -d -f -l $build_root/proc 2> /dev/null
 309         fi
 310 }
 311
 312 # must trap outside the group command
 313 trap 'umount_chroot' EXIT
 314
 315 {
 316         ln -sf build_target_$$.log ${build_logs}/build_target.log
 317         scripts/Build-Tools -1 -cfg $config
 318         _built=0
 319         for x in $( get_expanded ./target/%/build.sh $targetchain ); do
 320                 if [ -f $x ]; then
 321                         . $x
 322                         _built=1
 323                         break
 324                 fi
 325         done
 326         [ $_built = 1 ] || echo_warning "No target/*/build.sh controlling the build!"
 327 } 2>&1 201>> "${build_logs}/build_target_$$.log" | \
 328         tee -a "${build_logs}/build_target_$$.log"