scripts/Build-Target

   1 #!/bin/bash
   2 #
   3 # --- T2-COPYRIGHT-NOTE-BEGIN ---
   4 # T2 SDE: scripts/Build-Target
   5 # Copyright (C) 2004 - 2021 The T2 SDE Project
   6 # Copyright (C) 1998 - 2003 ROCK Linux Project
   7 #
   8 # This Copyright note is generated by scripts/Create-CopyPatch,
   9 # more information can be found in the files COPYING and README.
  10 #
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License version 2.
  13 # --- T2-COPYRIGHT-NOTE-END ---
  14 #
  15 #   Run this command from the T2 SDE base directory as scripts/Build-Target
  16 #   after running the scripts/Config and the now optional
  17 #   scripts/Download commands.
  18 #
  19 #   It compiles/builds all the packages and stores their binary package
  20 #   ssuitable for distribution.
  21 #
  22 #   This script (and Build-Pkg) is T2 SDE's work-horse. It builds in a
  23 #   chroot environment (stage 0..1) and goes through a number of build
  24 #   stages (stage 2..9).
  25 #
  26
  27 config=default
  28 build_only_this_job=
  29 daemon_mode=0
  30 autodownload=1
  31 options="$*"
  32 downloadopt="-q"
  33 verbose=
  34
  35 while [ $# -gt 0 ]; do
  36         case "$1" in
  37                 -cfg)           config=$2; shift ;;
  38                 -job)           build_only_this_job=$2; shift ;;
  39                 -daemon)        daemon_mode=1 ;;
  40                 -nodaemon)      daemon_mode=0 ;;
  41                 -nodownload)    autodownload=0 ;;
  42                 -v)             verbose="-v" ;;
  43                 *)              echo "Usage: $0 [ -daemon ] [ -nodownload ] [ -cfg config ]" \
  44                                      "[ -job <stage>-<package> ]"; exit 1 ;;
  45         esac
  46         shift
  47 done
  48
  49 if [ "$daemon_mode" = 1 ]; then
  50         . config/$config/config
  51         echo "Running $0 in the background (messages go to logfile only)"
  52         echo "Logfile: build/$SDECFG_ID/TOOLCHAIN/logs/build_target.log"
  53         nohup $0 $options -nodaemon > /dev/null 2> /dev/null < /dev/null &
  54         exit 0
  55 fi
  56
  57 . scripts/parse-config
  58 . scripts/functions.in
  59
  60 build_root="$base/build/$SDECFG_ID"
  61 build_toolchain="$base/build/$SDECFG_ID/TOOLCHAIN"
  62 build_logs="$build_toolchain/logs"; mkdir -p "${build_logs}"
  63 build_pkgs="$build_toolchain/pkgs"; mkdir -p "${build_root}"
  64
  65 if [ "$SDECFG_PARANOIA_CHECK" = 1 ]; then
  66         scripts/Check-System || exit 1
  67 fi
  68
  69 # Package Build loop - executed by build-target
  70 #
  71 pkgloop() {
  72         local x= y=
  73
  74         if [ "$SDECFG_NOBROKENDEPS" = 1 ]; then
  75                 nobrokendeps="-nobrokendeps"
  76         else
  77                 nobrokendeps=""
  78         fi
  79         if [ -z "$build_only_this_job" ]; then
  80
  81           if [ "`ls ${build_root}/var/adm/logs/*.err 2> /dev/null`" ]; then
  82                 echo_header "Removing old error logs ..."
  83                 for y in 0 1 2 3 4 5 6 7 8 9; do
  84                         if [ "$SDECFG_RETRY_BROKEN" -eq 1 -o $y -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
  85                                 for x in ${build_root}/var/adm/logs/$y-*.err; do
  86                                         if [ -f $x ]; then
  87                                                 echo_status "Removing ${x#$build_root/} ..."
  88                                                 rm -f $x
  89                                         fi
  90                                 done
  91                         fi
  92                 done
  93           fi
  94           if [ "`ls ${build_root}/var/adm/logs/*.out 2> /dev/null`" ]; then
  95                 echo_header "Removing old output logs ..."
  96                 for x in ${build_root}/var/adm/logs/*.out; do
  97                         echo_status "Removing ${x#$build_root/} ..."
  98                         rm -f $x
  99                 done
 100           fi
 101
 102           while
 103                 next="`scripts/Create-PkgQueue \
 104                         -cfg "$config" -single $nobrokendeps`"
 105                 [ "$next" ]
 106           do
 107                 pkgloop_package $next
 108           done
 109
 110         else
 111                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.log"
 112                 rm -f "${build_root}"/var/adm/logs/${build_only_this_job}.err"
 113                 next="$( awk 'BEGIN { FS=" "; }
 114                               $5 == "'${build_only_this_job#*-}'" &&
 115                               $2 ~ /'${build_only_this_job%%-*}'/ \
 116                               { $1="'${build_only_this_job%%-*}' 0";
 117                               print; exit; }' < config/$config/packages )"
 118                 if [ ! "$next" ]; then
 119                         echo_error "No job matching job spec: $build_only_this_job"
 120                         exit 1
 121                 fi
 122
 123                 pkgloop_package $next
 124                 exit 0
 125         fi
 126
 127         # wait for background compression jobs to finish
 128         for j in $(jobs -p); do wait $j; done
 129
 130         local pkglst=`mktemp` errors=0; rm -f src/invalid-files.lst
 131         echo_header "Searching for old lingering files ..."
 132         sed '/^[^X]/d; s,.*=,,' config/$config/packages | cut -d' ' -f5 |
 133         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 134                 while read p; do
 135                         v=$( grep '^Package Name and Version:' \
 136                                 build/$SDECFG_ID/var/adm/packages/$p \
 137                                 2>/dev/null | cut -f6 -d' ' )
 138                         echo "$p-$v"
 139                 done
 140         else
 141                 cat
 142         fi > $pkglst
 143         for file in $( ls build/$SDECFG_ID/TOOLCHAIN/pkgs/ 2> /dev/null ); do
 144                 x="$file"
 145                 case $SDECFG_PKGFILE_TYPE in
 146                   tar.*)        x=${x%.$SDECFG_PKGFILE_TYPE} ;;
 147                   none)         : ;;
 148                 esac
 149                 if ! grep -qx "$x" $pkglst && ! test "$x" = packages.db; then
 150                         file="build/$SDECFG_ID/TOOLCHAIN/pkgs/$file"
 151                         echo_error "$file should not be present" \
 152                                    "(now in src/invalid-files.lst)!"
 153                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 154                         errors=1
 155                 fi
 156         done
 157         for dir in build/$SDECFG_ID/var/adm/{cache,dependencies,descs,flists,md5sums,packages}; do
 158                 for file in $( ls $dir 2> /dev/null ); do
 159                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 160                                 x="$file-"
 161                         else
 162                                 x="$file"
 163                         fi
 164                         if ! grep -q "$x" $pkglst; then
 165                                 echo_error "$dir/$file should not be present (now in src/invalid-files.lst)!"
 166                                 mkdir -p src; echo "$dir/$file" >> src/invalid-files.lst
 167                                 errors=1
 168                         fi
 169                 done
 170         done
 171         for file in $( ls build/$SDECFG_ID/var/adm/logs/ ); do
 172                         x="`echo $file | sed -e 's/^.-//' -e 's/\.log//' -e 's/\.err//' -e s'/\.out//'`"
 173                         if [ $SDECFG_PKGFILE_VER = 1 ]; then
 174                                 x=$x-
 175                         else
 176                                 x=$x
 177                         fi
 178
 179                 if ! grep -q "$x" $pkglst; then
 180                         file="build/$SDECFG_ID/var/adm/logs/$file"
 181                         echo_error "$file should not be present (now in src/invalid-files.lst)!"
 182                         mkdir -p src; echo "$file" >> src/invalid-files.lst
 183                         errors=1
 184                 fi
 185         done
 186         [ $errors = 0 ] && echo_status "None found."
 187         rm $pkglst
 188 }
 189
 190 # Process one line of output generated by Create-PkgQueue
 191 #
 192 pkgloop_package() {
 193         for x in stagelevel pkg_depnr pkg_stages pkg_pri pkg_tree \
 194                  pkg_name pkg_ver pkg_prefix pkg_extra
 195         do eval "$x=\$1"; shift; done
 196
 197         [ "$build_only_this_job" -a \
 198           "$stagelevel-$pkg_name" != "$build_only_this_job" ] && return
 199
 200         [ $(expr "$pkg_stages" : ".*$stagelevel.*") -eq 0 ] && return
 201         pkg_laststage=$(echo "$pkg_stages" | sed "s,-,,g; s,.*\(.\),\1,")
 202
 203         cmd_root="-root auto"
 204         [ $stagelevel -gt 2 ] && cmd_root="$cmd_root -chroot"
 205
 206         if [ "$pkg_prefix" != "/" ]; then
 207                 cmd_prefix="-prefix $pkg_prefix"
 208         else cmd_prefix=""; fi
 209
 210         if [ "$autodownload" == 1 ]; then
 211                 scripts/Download -cfg $config $downloadopt $pkg_name
 212         fi
 213
 214         cmd_buildpkg="scripts/Build-Pkg -$stagelevel -cfg $config $verbose"
 215         cmd_buildpkg="$cmd_buildpkg $cmd_root $cmd_prefix $pkg_name"
 216
 217         # Execute action handler
 218         if ! pkgloop_action &&
 219            [ $stagelevel -le $SDECFG_CONTINUE_ON_ERROR_AFTER ]; then
 220                 exit 1
 221         fi
 222
 223         if [ ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.log -a \
 224              ! -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 225         then
 226                 echo_header "Package build ended abnormally!"
 227                 echo_error "Usually a package build creates either a *.log"
 228                 echo_error "or a *.err file. Neither the 1st nor the 2nd is"
 229                 echo_error "there. So I'm going to create a *.err file now"
 230                 echo_error "and abort the build process."
 231                 touch ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err
 232                 exit 1
 233         fi
 234
 235         if [ $stagelevel -gt 0 -a $pkg_laststage -eq $stagelevel -a \
 236              "$SDECFG_PKGFILE_TYPE" != none ]
 237         then
 238                 if [ -f ${build_root}/var/adm/logs/$stagelevel-$pkg_name.err ]
 239                 then echo_error "Creation of binary package isn't possible, \
 240 because the package was not"
 241                      echo_error "built successfully in (at least) the current \
 242 stage."
 243                 else
 244                         mkdir -p "${build_pkgs}"
 245
 246                         file=${pkg_name}
 247                         if [ "$SDECFG_PKGFILE_VER" = 1 ]; then
 248                                 v="$( grep '^Package Name and Version:' \
 249                                       ${build_root}/var/adm/packages/$pkg_name | cut -f6 -d' ' )"
 250                                 file="$file-$v"
 251                         fi
 252
 253                         [[ $SDECFG_PKGFILE_TYPE = tar.* ]] &&
 254                                 file=$file.$SDECFG_PKGFILE_TYPE
 255                         compressor=${SDECFG_PKGFILE_TYPE#tar.}
 256                         case $compressor in
 257                           gz)   compressor=gzip ;;
 258                           bz2)  compressor=bzip2 ;;
 259                           lzo)  comrpessor=lzop ;;
 260                           zst)  [ "$arch_sizeof_char_p" = 8 ] &&
 261                                 compressor="zstd -T0 --ultra -20" || compressor="zstd -T0 -16"
 262                                 ;;
 263                           # lzma, xz et al. are named after the extension
 264                           gem)  compressor=bzip2; file=$file.tar.bz2 ;;
 265                         esac
 266
 267                         echo_status "Creating binary package build/.../pkgs/$file"
 268                         (
 269                         # sort var/adm before the rest - so they can be extracted, quickly
 270                         ( cd "$build_root/"
 271                           (grep    ' var/adm' var/adm/flists/$pkg_name
 272                            grep -v ' var/adm' var/adm/flists/$pkg_name) | cut -f2- -d' ' |
 273                           tar -cf- --no-recursion --files-from=- | $compressor
 274                         ) > ${build_pkgs}/$file.tmp
 275                         mv ${build_pkgs}/$file{.tmp,}
 276
 277                         if [ "$SDECFG_PKGFILE_TYPE" = gem ]; then
 278                                 mine -C "$build_root/var/adm" $build_pkgs/$file $pkg_name \
 279                                      $build_pkgs/${file%.tar.bz2}.gem.tmp
 280                                 mv $build_pkgs/${file%.tar.bz2}.gem{.tmp,}
 281
 282                                 echo_status "Removing temporary tar.bz2."
 283                                 rm -f $build_pkgs/$file
 284                         fi
 285                         ) &
 286                 fi
 287         fi
 288 }
 289
 290 # Action executed by pkgloop(). This function may be redefined
 291 # before calling pkgloop().
 292 #
 293 pkgloop_action() {
 294         $cmd_buildpkg
 295 }
 296
 297 # Try to umount any directories mounted by Build-Pkg -chroot
 298 # if we are the last process using them.
 299 #
 300 umount_chroot() {
 301         exec 201> /dev/null
 302         if ! ( cd ${build_logs}; fuser *.log > /dev/null 2>&1 ); then
 303                 echo_status "Unmounting loop mounts ..."
 304                 umount -d -f    $build_toolchain/{loop,config,download} 2> /dev/null
 305                 umount -d -f -l $build_toolchain/{loop,config,download} 2> /dev/null
 306                 umount -d -f    $build_root/proc 2> /dev/null
 307                 umount -d -f -l $build_root/proc 2> /dev/null
 308         fi
 309 }
 310
 311 # must trap outside the group command
 312 trap 'umount_chroot' EXIT
 313
 314 {
 315         ln -sf build_target_$$.log ${build_logs}/build_target.log
 316         scripts/Build-Tools -1 -cfg $config
 317         _built=0
 318         for x in $( get_expanded ./target/%/build.sh $targetchain ); do
 319                 if [ -f $x ]; then
 320                         . $x
 321                         _built=1
 322                         break
 323                 fi
 324         done
 325         [ $_built = 1 ] || echo_warning "No target/*/build.sh controlling the build!"
 326 } 2>&1 201>> "${build_logs}/build_target_$$.log" | \
 327         tee -a "${build_logs}/build_target_$$.log"