fetchDependencies

   1 #!/bin/bash
   2 #
   3 # Copyright (c) 2016-2024 The Brenwill Workshop Ltd.
   4 #
   5 # fetchDependencies - Retrieves the correct versions of all dependencies
   6 #
   7 # macOS usage: ./fetchDependencies [--macos] [--ios] [--iossim] [--maccat] [--tvos] [--tvossim]
   8 #                                  [--visionos] [--visionossim] [--all] [--none]
   9 #                                  [-v] [--debug] [--build-spirv-tools]
  10 #                                  [--v-headers-root path] [--spirv-cross-root path] [--spirv-tools-root path]
  11 #
  12 #      --macos
  13 #              Build the external libraries for the macOS platform.
  14 #
  15 #      --ios
  16 #              Build the external libraries for the iOS platform.
  17 #
  18 #      --iossim
  19 #              Build the external libraries for the iOS Simulator platform.
  20 #
  21 #      --maccat
  22 #              Build the external libraries for the Mac Catalyst platform.
  23 #
  24 #      --tvos
  25 #              Build the external libraries for the tvOS platform.
  26 #
  27 #      --tvossim
  28 #              Build the external libraries for the tvOS Simulator platform.
  29 #
  30 #      --visionos
  31 #              Build the external libraries for the visionOS platform.
  32 #
  33 #      --visionossim
  34 #              Build the external libraries for the visionOS Simulator platform.
  35 #
  36 #      --all
  37 #              Equivalent to specifying [--macos --ios --iossim --maccat --tvos --tvossim].
  38 #              Results in one XCFramework for each external library, each containing
  39 #              binaries for all supported platforms.
  40 #              Currently excludes --visionos and --visionossim targets because those
  41 #              require Xcode 15+ and will abort a multi-platform build.
  42 #
  43 #      --none
  44 #              Don't build the external libraries for any platform (this is the default).
  45 #
  46 #      Multiple platform options may be specified. At least one platform option must be specified.
  47 #
  48 #      --asan
  49 #              Build the external libraries against Address Sanitizer, which may be useful when debugging
  50 #              and tracing calls into those libraries.
  51 #
  52 #      --tsan
  53 #              Build the external libraries against Thread Sanitizer, which may be useful when debugging
  54 #              and tracing calls into those libraries.
  55 #
  56 #      --ubsan
  57 #              Build the external libraries against Undefined Behavior Sanitizer, which may be useful when debugging
  58 #              and tracing calls into those libraries.
  59 #
  60 #      --debug
  61 #              Build the external libraries in Debug mode, which may be useful when debugging
  62 #              and tracing calls into those libraries.
  63 #
  64 #      --parallel-build
  65 #              Build the external libraries in parallel using background processes.
  66 #
  67 #      --no-parallel-build
  68 #              Build the external libraries serially instead of in parallel. This is the default.
  69 #
  70 #      --keep-cache
  71 #              Do not remove the External/build/Intermediates cache directory after building.
  72 #              Removing the Intermediates directory returns significant disk space after the
  73 #              build, and is the default behaviour. Use this option if you intend to run this
  74 #              script repeatedly to incrementally build one platform at a time.
  75 #
  76 #      --spirv-tools-root path
  77 #              "path" specifies a directory path to a KhronosGroup/SPIRV-tools repository.
  78 #              This repository does need to be built and the build directory must be in the
  79 #              specified directory. It should be built the same way this script builds it.
  80 #
  81 #      --build-spirv-tools
  82 #              Build the full spirv-tools distribution. Normally this is not needed, because
  83 #              MoltenVK includes a template of pre-generated SPIRV-Tools header files, which
  84 #              is all that is needed. Avoiding the spirv-tools build saves significant time
  85 #              during the running of this script, and is necessary during CI because Travis CI
  86 #              cannot support the required use of Python3 by the spirv-tools build. This flag
  87 #              is used by the packagePregenSpirvToolsHeaders script which regenerates the
  88 #              spirv-tools header files and repackages the Templates/spirv-tools/build.zip
  89 #              file when the spirv-tools library version is upgraded.
  90 #
  91 #      --spirv-cross-root path
  92 #              "path" specifies a directory path to a KhronosGroup/SPIRV-Cross repository.
  93 #              This repository does not have to be built.
  94 #
  95 #      -v      verbose output
  96 #
  97 #      --v-headers-root path
  98 #              "path" specifies a directory path to a KhronosGroup/Vulkan-Headers repository.
  99 #              This repository does not have to be built.
 100 #
 101
 102 set -e
 103
 104 # ----------------- Functions -------------------
 105
 106 BLD_NONE=""
 107 BLD_IOS=""
 108 BLD_IOS_SIM=""
 109 BLD_MAC_CAT=""
 110 BLD_TVOS=""
 111 BLD_TVOS_SIM=""
 112 BLD_VISIONOS=""
 113 BLD_VISIONOS_SIM=""
 114 BLD_MACOS=""
 115 BLD_SPECIFIED=""
 116 XC_CONFIG="Release"
 117 XC_BUILD_VERBOSITY="-quiet"
 118 XC_USE_BCKGND=""
 119 XC_USE_ASAN="NO"
 120 XC_USE_TSAN="NO"
 121 XC_USE_UBSAN="NO"
 122 V_HEADERS_ROOT=""
 123 SPIRV_CROSS_ROOT=""
 124 SPIRV_TOOLS_ROOT=""
 125 BLD_SPV_TLS=""
 126 export KEEP_CACHE=""
 127
 128 while (( "$#" )); do
 129   case "$1" in
 130        --ios)
 131          BLD_IOS="Y"
 132          shift 1
 133          ;;
 134        --iossim)
 135          BLD_IOS_SIM="Y"
 136          shift 1
 137          ;;
 138        --maccat)
 139          BLD_MAC_CAT="Y"
 140          shift 1
 141          ;;
 142        --tvos)
 143          BLD_TVOS="Y"
 144          shift 1
 145          ;;
 146        --tvossim)
 147          BLD_TVOS_SIM="Y"
 148          shift 1
 149          ;;
 150        --visionos)
 151          BLD_VISIONOS="Y"
 152          shift 1
 153          ;;
 154        --visionossim)
 155          BLD_VISIONOS_SIM="Y"
 156          shift 1
 157          ;;
 158        --macos)
 159          BLD_MACOS="Y"
 160          shift 1
 161          ;;
 162        --all)
 163          BLD_MACOS="Y"
 164          BLD_IOS="Y"
 165          BLD_IOS_SIM="Y"
 166          BLD_MAC_CAT="Y"
 167          BLD_TVOS="Y"
 168          BLD_TVOS_SIM="Y"
 169 #         BLD_VISIONOS="Y"              # Requires Xcode 15+
 170 #         BLD_VISIONOS_SIM="Y"          # Requires Xcode 15+
 171          shift 1
 172          ;;
 173        --none)
 174          BLD_NONE="Y"
 175          shift 1
 176          ;;
 177        --debug)
 178          XC_CONFIG="Debug"
 179          shift 1
 180          ;;
 181        --asan)
 182          XC_USE_ASAN="YES"
 183          shift 1
 184          ;;
 185        --tsan)
 186          XC_USE_TSAN="YES"
 187          shift 1
 188          ;;
 189        --ubsan)
 190          XC_USE_UBSAN="YES"
 191          shift 1
 192          ;;
 193        --parallel-build)
 194          XC_USE_BCKGND="Y"
 195          shift 1
 196          ;;
 197        --no-parallel-build)
 198          XC_USE_BCKGND=""
 199          shift 1
 200          ;;
 201        --keep-cache)
 202          KEEP_CACHE="Y"
 203          shift 1
 204          ;;
 205        -v)
 206          XC_BUILD_VERBOSITY=""
 207          shift 1
 208          ;;
 209        --build-spirv-tools)
 210          BLD_SPV_TLS="Y"
 211          shift 1
 212          ;;
 213        --skip-spirv-tools-build)        #deprecated
 214          BLD_SPV_TLS=""
 215          shift 1
 216          ;;
 217        --v-headers-root)
 218          V_HEADERS_ROOT=$2
 219          shift 2
 220          ;;
 221        --spirv-cross-root)
 222          SPIRV_CROSS_ROOT=$2
 223          shift 2
 224          ;;
 225        --spirv-tools-root)
 226          SPIRV_TOOLS_ROOT=$2
 227          shift 2
 228          ;;
 229        *)
 230          echo "Error: Unsupported option $1" >&2
 231          exit 1
 232          ;;
 233   esac
 234 done
 235
 236 # Update a repository. If it exists, fetch it; if not, clone it.
 237 # $1 repo name
 238 # $2 repo url
 239 # $3 repo revision (commit SHA)
 240 function update_repo() {
 241         echo "$1 repo: $2"
 242         echo "$1 revision: $3"
 243
 244         if [ -d $1 -a -d $1/.git ]; then
 245                 cd $1
 246                 git cat-file -e $3 || git fetch --all
 247                 git checkout --force $3
 248                 cd -  > /dev/null
 249         else
 250                 rm -rf $1
 251                 git clone $2 $1
 252                 cd $1
 253                 git checkout $3
 254                 cd -  > /dev/null
 255         fi
 256 }
 257
 258 # Build a repository
 259 # $1 repo name
 260 function build_repo() {
 261         echo "Building $1"
 262
 263         mkdir -p $1/build
 264         cd $1/build
 265         if type ninja >/dev/null 2>&1 ; then
 266                 cmake -G Ninja -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=install ..
 267                 ninja
 268         else
 269                 cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=install ..
 270                 make -j $(sysctl -n hw.activecpu)
 271         fi
 272
 273         cd -  > /dev/null
 274 }
 275
 276
 277 # ----------------- Main -------------------
 278
 279 EXT_DIR=External
 280 EXT_REV_DIR=../ExternalRevisions
 281
 282 echo
 283 echo ========== Retrieving MoltenVK dependencies into ${EXT_DIR} ==========
 284
 285 mkdir -p ${EXT_DIR}
 286 cd ${EXT_DIR}
 287
 288
 289 # ----------------- Cereal -------------------
 290
 291 echo
 292 echo ========== Cereal ==========
 293 echo
 294
 295 REPO_NAME=cereal
 296 REPO_URL="https://github.com/USCiLab/${REPO_NAME}.git"
 297 REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 298
 299 update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 300
 301
 302 # ----------------- Vulkan-Headers -------------------
 303
 304 echo
 305 echo ========== Vulkan-Headers ==========
 306 echo
 307
 308 # When MoltenVK is built by something that already has
 309 # a copy of this repo, use it by creating a symlink.
 310
 311 REPO_NAME=Vulkan-Headers
 312
 313 if [ ! "$V_HEADERS_ROOT" = "" ]; then
 314         rm -rf ${REPO_NAME}
 315         ln -sfn ${V_HEADERS_ROOT} ${REPO_NAME}
 316 else
 317         REPO_URL="https://github.com/KhronosGroup/${REPO_NAME}.git"
 318         REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 319
 320         update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 321 fi
 322
 323
 324 # ----------------- SPIRV-Cross -------------------
 325
 326 echo
 327 echo ========== SPIRV-Cross ==========
 328 echo
 329
 330 # When MoltenVK is built by something that already has
 331 # a copy of this repo, use it by creating a symlink.
 332
 333 REPO_NAME=SPIRV-Cross
 334
 335 if [ ! "$SPIRV_CROSS_ROOT" = "" ]; then
 336         rm -rf ${REPO_NAME}
 337         ln -sfn ${SPIRV_CROSS_ROOT} ${REPO_NAME}
 338 else
 339         REPO_URL="https://github.com/KhronosGroup/${REPO_NAME}.git"
 340         REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 341
 342         update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 343 fi
 344
 345
 346 # ----------------- SPIRV-Tools -------------------
 347
 348 echo
 349 echo ========== SPIRV-Tools and SPIRV-Headers ==========
 350 echo
 351
 352 # When MoltenVK is built by something that already has
 353 # a copy of this repo, use it by creating a symlink.
 354
 355 REPO_NAME=SPIRV-Tools
 356
 357 if [ ! "$SPIRV_TOOLS_ROOT" = "" ]; then
 358         rm -rf ${REPO_NAME}
 359         ln -sfn ${SPIRV_TOOLS_ROOT} ${REPO_NAME}
 360 else
 361         REPO_URL="https://github.com/KhronosGroup/${REPO_NAME}.git"
 362         REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 363
 364         update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 365
 366         SPIRV_HEADERS_REV=$(cat "${EXT_REV_DIR}/SPIRV-Headers_repo_revision")
 367         update_repo SPIRV-Tools/external/spirv-headers "https://github.com/KhronosGroup/SPIRV-Headers.git" ${SPIRV_HEADERS_REV}
 368
 369         # Build spirv-tools, or use option of pre-generated headers
 370         if [ "$BLD_SPV_TLS" = "Y" ]; then
 371                 build_repo "${REPO_NAME}"
 372         else
 373                 unzip -o -q -d "${REPO_NAME}" ../Templates/spirv-tools/build.zip
 374                 rm -rf "${REPO_NAME}/__MACOSX"
 375         fi
 376 fi
 377
 378
 379 # ----------------- Vulkan-Tools -------------------
 380
 381 echo
 382 echo ========== Vulkan-Tools ==========
 383 echo
 384
 385 REPO_NAME=Vulkan-Tools
 386 REPO_URL="https://github.com/KhronosGroup/${REPO_NAME}.git"
 387 REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 388
 389 update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 390
 391
 392 # ----------------- Volk -------------------
 393
 394 echo
 395 echo ========== Volk ==========
 396 echo
 397
 398 REPO_NAME=Volk
 399 REPO_URL="https://github.com/zeux/${REPO_NAME}.git"
 400 REPO_REV=$(cat "${EXT_REV_DIR}/${REPO_NAME}_repo_revision")
 401
 402 update_repo ${REPO_NAME} ${REPO_URL} ${REPO_REV}
 403
 404
 405 # ----------------- Cleanup -------------------
 406
 407 cd ..
 408
 409
 410 # -------------- Build MoltenVK external library dependencies -----------------
 411
 412 echo
 413 echo ========== Started building dependency libraries at `date +"%r"` ==========
 414 echo Please be patient on first build
 415
 416 function execute_xcodebuild_command () {
 417         if [ -n "${XCPRETTY}" ]; then
 418                 set -o pipefail && xcodebuild "$@" | tee -a "dependenciesbuild.log" | ${XCPRETTY}
 419         else
 420                 xcodebuild "$@"
 421         fi
 422 }
 423
 424 # Build an Xcode scheme for an OS and platform
 425 # 1 - OS
 426 # 2 - Platform
 427 # 3 - Destination (Optional. Defaults to same as platform)
 428 function build_impl() {
 429         XC_OS=${1}
 430         XC_PLTFM=${2}
 431         if [ "${3}" != "" ]; then
 432                 XC_DEST=${3};
 433         else
 434                 XC_DEST=${XC_PLTFM};
 435         fi
 436
 437         XC_SCHEME="${EXT_DEPS}-${XC_OS}"
 438         XC_LCL_DD_PATH="${XC_DD_PATH}/Intermediates/${XC_PLTFM}"
 439
 440         echo Building external libraries for platform ${XC_PLTFM} and destination ${XC_DEST}
 441
 442         execute_xcodebuild_command                                                                      \
 443                 -project "${XC_PROJ}"                                                                   \
 444                 -scheme "${XC_SCHEME}"                                                                  \
 445                 -destination "generic/platform=${XC_DEST}"                              \
 446                 -configuration "${XC_CONFIG}"                                                   \
 447                 -enableAddressSanitizer "${XC_USE_ASAN}"                                \
 448                 -enableThreadSanitizer "${XC_USE_TSAN}"                                 \
 449                 -enableUndefinedBehaviorSanitizer "${XC_USE_UBSAN}"             \
 450                 -derivedDataPath "${XC_LCL_DD_PATH}"                                    \
 451                 ${XC_BUILD_VERBOSITY}                                                                   \
 452                 build
 453
 454         echo Completed building external libraries for ${XC_PLTFM}
 455 }
 456
 457 # Select whether or not to run the build in parallel.
 458 # 1 - OS
 459 # 2 - platform
 460 # 3 - Destination (Optional. Defaults to same as platform)
 461 function build() {
 462     BLD_SPECIFIED="Y"
 463         if [ "$XC_USE_BCKGND" != "" ]; then
 464                 build_impl "${1}" "${2}" "${3}" &
 465         else
 466                 build_impl "${1}" "${2}" "${3}"
 467         fi
 468 }
 469
 470 EXT_DEPS=ExternalDependencies
 471 XC_PROJ="${EXT_DEPS}.xcodeproj"
 472 XC_DD_PATH="${EXT_DIR}/build"
 473 export SKIP_PACKAGING="Y"
 474
 475 # Determine if xcpretty is present
 476 XCPRETTY_PATH=$(command -v xcpretty 2> /dev/null || true) # ignore failures
 477
 478 # If xcpretty is present, use it to format xcodebuild output
 479 XCPRETTY=""
 480 if [ -n "$XCPRETTY_PATH" ]; then
 481         XCPRETTY="xcpretty -c"
 482 fi
 483 if [ "$XC_USE_BCKGND" != "" ]; then
 484         # For now, avoid using xcpretty if parallel background tasks are being used
 485         XCPRETTY=""
 486 fi
 487
 488 # Structure build tasks by platform so they can be built in parallel per platform.
 489 # Content for each platform must be built in series to avoid
 490 if [ "$XC_USE_BCKGND" != "" ]; then
 491         trap "exit" INT TERM ERR
 492         trap "kill 0" EXIT
 493 fi
 494
 495 if [ "$BLD_MACOS" != "" ]; then
 496         build "macOS" "macOS"
 497 fi
 498
 499 if [ "$BLD_IOS" != "" ]; then
 500         build "iOS" "iOS"
 501 fi
 502
 503 if [ "$BLD_IOS_SIM" != "" ]; then
 504         build "iOS" "iOS Simulator"
 505 fi
 506
 507 if [ "$BLD_MAC_CAT" != "" ]; then
 508         build "iOS" "Mac Catalyst" "macOS,variant=Mac Catalyst"
 509 fi
 510
 511 if [ "$BLD_TVOS" != "" ]; then
 512         build "tvOS" "tvOS"
 513 fi
 514
 515 if [ "$BLD_TVOS_SIM" != "" ]; then
 516         build "tvOS" "tvOS Simulator"
 517 fi
 518
 519 if [ "$BLD_VISIONOS" != "" ]; then
 520         build "xrOS" "xrOS"
 521 fi
 522
 523 if [ "$BLD_VISIONOS_SIM" != "" ]; then
 524         build "xrOS" "xrOS Simulator"
 525 fi
 526
 527 # Wait for any background process (if selected) to finish
 528 if [ "$XC_USE_BCKGND" != "" ]; then
 529         wait
 530 fi
 531
 532 if [ "$BLD_SPECIFIED" != "" ]; then
 533         # Build XCFrameworks, update latest symlink, remove intermediates, and clean MoltenVK for rebuild
 534         PROJECT_DIR="."
 535         CONFIGURATION=${XC_CONFIG}
 536         SKIP_PACKAGING=""
 537         . "./Scripts/create_ext_lib_xcframeworks.sh"
 538         . "./Scripts/package_ext_libs_finish.sh"
 539 else
 540         if [ "$BLD_NONE" != "" ]; then
 541                 echo Not building any platforms
 542         else
 543                 echo "WARNING: You did not specify a platform to build."
 544                 echo "To build the external libraries, include one or"
 545                 echo "more of the following platform options:"
 546                 echo "    --macos --ios --iossim --maccat --tvos --tvossim --visionos --visionossim --all"
 547                 echo "See the instructions in the fetchDependencies script for more info."
 548         fi
 549 fi
 550
 551 echo ========== Finished at `date +"%r"` ==========
 552 exit 0