.github/workflows/release-binaries.yml

   1 name: Release Binaries
   2
   3 on:
   4   workflow_dispatch:
   5     inputs:
   6       release-version:
   7         description: 'Release Version'
   8         required: false
   9         type: string
  10       upload:
  11         description: 'Upload binaries to the release page'
  12         required: true
  13         default: false
  14         type: boolean
  15       runs-on:
  16         description: "Runner to use for the build"
  17         required: true
  18         type: choice
  19         options:
  20           - ubuntu-22.04
  21           - windows-2022
  22           - macos-13
  23           - macos-14
  24
  25   workflow_call:
  26     inputs:
  27       release-version:
  28         description: 'Release Version'
  29         required: false
  30         type: string
  31       upload:
  32         description: 'Upload binaries to the release page'
  33         required: true
  34         default: false
  35         type: boolean
  36       runs-on:
  37         description: "Runner to use for the build"
  38         required: true
  39         type: string
  40     secrets:
  41       RELEASE_TASKS_USER_TOKEN:
  42         description: "Secret used to check user permissions."
  43         required: false
  44
  45
  46 permissions:
  47   contents: read # Default everything to read-only
  48
  49 jobs:
  50   prepare:
  51     name: Prepare to build binaries
  52     runs-on: ${{ inputs.runs-on }}
  53     if: github.repository == 'llvm/llvm-project'
  54     outputs:
  55       release-version: ${{ steps.vars.outputs.release-version }}
  56       ref: ${{ steps.vars.outputs.ref }}
  57       upload: ${{ steps.vars.outputs.upload }}
  58       target-cmake-flags: ${{ steps.vars.outputs.target-cmake-flags }}
  59       build-flang: ${{ steps.vars.outputs.build-flang }}
  60       enable-pgo: ${{ steps.vars.outputs.enable-pgo }}
  61       release-binary-basename: ${{ steps.vars.outputs.release-binary-basename }}
  62       release-binary-filename: ${{ steps.vars.outputs.release-binary-filename }}
  63
  64     steps:
  65     # It's good practice to use setup-python, but this is also required on macos-14
  66     # due to https://github.com/actions/runner-images/issues/10385
  67     - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
  68       with:
  69         python-version: '3.12'
  70
  71     - name: Checkout LLVM
  72       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  73
  74     - name: Install Dependencies
  75       shell: bash
  76       run: |
  77         pip install --require-hashes -r ./llvm/utils/git/requirements.txt
  78
  79     - name: Check Permissions
  80       if: github.event_name != 'pull_request'
  81       env:
  82         GITHUB_TOKEN: ${{ github.token }}
  83         USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
  84       shell: bash
  85       run: |
  86         ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions
  87
  88     - name: Collect Variables
  89       id: vars
  90       shell: bash
  91       # In order for the test-release.sh script to run correctly, the LLVM
  92       # source needs to be at the following location relative to the build dir:
  93       # | X.Y.Z-rcN | ./rcN/llvm-project
  94       # | X.Y.Z     | ./final/llvm-project
  95       #
  96       # We also need to set divergent flags based on the release version:
  97       # | X.Y.Z-rcN | -rc N -test-asserts
  98       # | X.Y.Z     | -final
  99       run: |
 100         trimmed=$(echo ${{ inputs.release-version }} | xargs)
 101         if [ -n "$trimmed" ]; then
 102           release_version="$trimmed"
 103           ref="llvmorg-$release_version"
 104         else
 105           release_version="${{ (github.event_name == 'pull_request' && format('PR{0}', github.event.pull_request.number)) || 'CI'}}-${{ github.sha }}"
 106           ref=${{ github.sha }}
 107         fi
 108         if [ -n "${{ inputs.upload }}" ]; then
 109           upload="${{ inputs.upload }}"
 110         else
 111           upload="false"
 112         fi
 113         echo "release-version=$release_version">> $GITHUB_OUTPUT
 114         echo "ref=$ref" >> $GITHUB_OUTPUT
 115         echo "upload=$upload" >> $GITHUB_OUTPUT
 116
 117         release_binary_basename="LLVM-$release_version-${{ runner.os }}-${{ runner.arch }}"
 118         echo "release-binary-basename=$release_binary_basename" >> $GITHUB_OUTPUT
 119         echo "release-binary-filename=$release_binary_basename.tar.xz" >> $GITHUB_OUTPUT
 120
 121         # Detect necessary CMake flags
 122         target="${{ runner.os }}-${{ runner.arch }}"
 123         echo "enable-pgo=false" >> $GITHUB_OUTPUT
 124         target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF"
 125         # The macOS builds try to cross compile some libraries so we need to
 126         # add extra CMake args to disable them.
 127         # See https://github.com/llvm/llvm-project/issues/99767
 128         if [ "${{ runner.os }}" = "macOS" ]; then
 129           target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_COMPILER_RT_ENABLE_IOS=OFF"
 130           if [ "${{ runner.arch }}" = "ARM64" ]; then
 131             arches=arm64
 132           else
 133             arches=x86_64
 134           fi
 135           target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_DARWIN_osx_ARCHS=$arches -DBOOTSTRAP_DARWIN_osx_BUILTIN_ARCHS=$arches"
 136         fi
 137
 138         build_flang="true"
 139
 140         if [ "${{ runner.os }}" = "Windows" ]; then
 141           # The build times out on Windows, so we need to disable LTO.
 142           target_cmake_flags="$target_cmake_flags -DLLVM_RELEASE_ENABLE_LTO=OFF"
 143         fi
 144
 145         echo "target-cmake-flags=$target_cmake_flags" >> $GITHUB_OUTPUT
 146         echo "build-flang=$build_flang" >> $GITHUB_OUTPUT
 147
 148   build-stage1:
 149     name: "Build Stage 1"
 150     needs: prepare
 151     if: github.repository == 'llvm/llvm-project'
 152     runs-on: ${{ inputs.runs-on }}
 153     steps:
 154
 155     - name: Checkout Actions
 156       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 157       with:
 158         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 159         sparse-checkout: |
 160           .github/workflows/
 161         sparse-checkout-cone-mode: false
 162         # Check out outside of working directory so the source checkout doesn't
 163         # remove it.
 164         path: workflows
 165
 166     # actions/checkout does not support paths outside of the GITHUB_WORKSPACE.
 167     # Also, anything that we put inside of GITHUB_WORKSPACE will be overwritten
 168     # by future actions/checkout steps.  Therefore, in order to checkout the
 169     # latest actions from main, we need to first checkout out the actions inside of
 170     # GITHUB_WORKSPACE (see previous step), then use actions/checkout to checkout
 171     # the code being built and the move the actions from main back into GITHUB_WORKSPACE,
 172     # becasue the uses on composite actions only reads workflows from inside GITHUB_WORKSPACE.
 173     - shell: bash
 174       run: mv workflows  ../workflows-main
 175
 176     - name: Checkout LLVM
 177       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 178       with:
 179         ref: ${{ needs.prepare.outputs.ref }}
 180
 181     - name: Copy main workflows
 182       shell: bash
 183       run: |
 184         mv ../workflows-main .
 185
 186     - name: Setup Stage
 187       id: setup-stage
 188       uses: ./workflows-main/.github/workflows/release-binaries-setup-stage
 189
 190     - name: Setup sccache
 191       uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
 192       with:
 193         # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
 194         max-size: 2G
 195         key: sccache-${{ runner.os }}-${{ runner.arch }}-release
 196         variant: sccache
 197
 198     - name: Build Stage 1 Clang
 199       id: build
 200       shell: bash
 201       run: |
 202         # There were some issues on the ARM64 MacOS runners with trying to build x86 object,
 203         # so we need to set some extra cmake flags to disable this.
 204         cmake -G Ninja -S llvm -B ${{ steps.setup-stage.outputs.build-prefix }}/build \
 205             ${{ needs.prepare.outputs.target-cmake-flags }} \
 206             -C clang/cmake/caches/Release.cmake \
 207             -DBOOTSTRAP_LLVM_PARALLEL_LINK_JOBS=1 \
 208             -DBOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \
 209             -DCMAKE_C_COMPILER_LAUNCHER=sccache \
 210             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 211         ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build
 212         # There is a race condition on the MacOS builders and this command is here
 213         # to help debug that when it happens.
 214         ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
 215
 216     - name: Save Stage
 217       uses: ./workflows-main/.github/workflows/release-binaries-save-stage
 218       with:
 219         build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
 220
 221   build-stage2:
 222     name: "Build Stage 2"
 223     needs:
 224       - prepare
 225       - build-stage1
 226     if: github.repository == 'llvm/llvm-project'
 227     runs-on: ${{ inputs.runs-on }}
 228     steps:
 229     - name: Checkout Actions
 230       uses: actions/checkout@v4
 231       with:
 232         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 233         sparse-checkout: |
 234           .github/workflows/
 235         sparse-checkout-cone-mode: false
 236         path: workflows
 237     - name: Setup Stage
 238       id: setup-stage
 239       uses: ./workflows/.github/workflows/release-binaries-setup-stage
 240       with:
 241         previous-artifact: build-stage1
 242
 243     - name: Build Stage 2
 244       # Re-enable once PGO builds are supported.
 245       if: needs.prepare.outputs.enable-pgo == 'true'
 246       shell: bash
 247       run: |
 248         ninja -C ${{ steps.setup-stage.outputs.build-prefix}}/build stage2-instrumented
 249
 250     - name: Save Stage
 251       uses: ./workflows/.github/workflows/release-binaries-save-stage
 252       with:
 253         build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
 254
 255   build-stage3-clang:
 256     name: "Build Stage 3 LLVM/Clang"
 257     needs:
 258       - prepare
 259       - build-stage2
 260     if: github.repository == 'llvm/llvm-project'
 261     runs-on: ${{ inputs.runs-on }}
 262     steps:
 263     - name: Checkout Actions
 264       uses: actions/checkout@v4
 265       with:
 266         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 267         sparse-checkout: |
 268           .github/workflows/
 269         sparse-checkout-cone-mode: false
 270         path: workflows
 271     - name: Setup Stage
 272       id: setup-stage
 273       uses: ./workflows/.github/workflows/release-binaries-setup-stage
 274       with:
 275         previous-artifact: build-stage2
 276
 277     - name: Build LLVM/Clang
 278       shell: bash
 279       run: |
 280         # There is a race condition on the MacOS builders and this command is here
 281         # to help debug that when it happens.
 282         ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build
 283         ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-clang
 284         # Build some of the larger binaries here too.
 285         ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ \
 286             clang-scan-deps \
 287             modularize clangd \
 288             clangd-indexer \
 289             clang-check \
 290             ${{ (runner.os == 'Linux' && 'clangd-fuzzer') || '' }} \
 291             clang-tidy \
 292             llc \
 293             lli \
 294             llvm-exegesis \
 295             llvm-opt-fuzzer \
 296             llvm-reduce \
 297             llvm-lto \
 298             dsymutil
 299
 300     - name: Save Stage
 301       uses: ./workflows/.github/workflows/release-binaries-save-stage
 302       with:
 303         build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
 304
 305   build-stage3-flang:
 306     name: "Build Stage 3 Flang/MLIR/Bolt"
 307     needs:
 308       - prepare
 309       - build-stage3-clang
 310     runs-on: ${{ inputs.runs-on }}
 311     steps:
 312     - name: Checkout Actions
 313       uses: actions/checkout@v4
 314       with:
 315         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 316         sparse-checkout: |
 317           .github/workflows/
 318         sparse-checkout-cone-mode: false
 319         path: workflows
 320     - name: Setup Stage
 321       id: setup-stage
 322       uses: ./workflows/.github/workflows/release-binaries-setup-stage
 323       with:
 324         previous-artifact: build-stage3-clang
 325
 326     - name: Build Flang / MLIR / Bolt
 327       shell: bash
 328       run: |
 329         # Build some of the mlir tools that take a long time to link
 330         if [ "${{ needs.prepare.outputs.build-flang }}" = "true" ]; then
 331           ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ -j2 flang bbc
 332         fi
 333         ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/ \
 334             mlir-bytecode-parser-fuzzer \
 335             mlir-cpu-runner \
 336             mlir-lsp-server \
 337             mlir-opt \
 338             mlir-query \
 339             mlir-reduce \
 340             mlir-text-parser-fuzzer \
 341             mlir-translate \
 342             mlir-transform-opt \
 343             mlir-cat \
 344             mlir-minimal-opt \
 345             mlir-minimal-opt-canonicalize \
 346             mlir-pdll-lsp-server \
 347             llvm-bolt \
 348             llvm-bolt-heatmap
 349
 350     - name: Save Stage
 351       uses: ./workflows/.github/workflows/release-binaries-save-stage
 352       with:
 353         build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
 354
 355   build-stage3-all:
 356     name: "Build Stage 3"
 357     needs:
 358       - prepare
 359       - build-stage3-flang
 360     runs-on: ${{ inputs.runs-on }}
 361     steps:
 362     - name: Checkout Actions
 363       uses: actions/checkout@v4
 364       with:
 365         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 366         sparse-checkout: |
 367           .github/workflows/
 368         sparse-checkout-cone-mode: false
 369         path: workflows
 370     - name: Setup Stage
 371       id: setup-stage
 372       uses: ./workflows/.github/workflows/release-binaries-setup-stage
 373       with:
 374         previous-artifact: build-stage3-flang
 375
 376     - name: Build Release Package
 377       shell: bash
 378       run: |
 379         which cmake
 380         ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package
 381         # Copy Release artifact to the workspace so it is easier to upload.
 382         # This is necessary, because on Windows, the build-prefix path can
 383         # only be used on bash steps, because it uses the form of /d/files/
 384         # and other steps expect D:\files.
 385         mv ${{ steps.setup-stage.outputs.build-prefix  }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} .
 386
 387     - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
 388       with:
 389         name: ${{ runner.os }}-${{ runner.arch }}-release-binary
 390         # Due to path differences on Windows when running in bash vs running on node,
 391         # we need to search for files in the current workspace.
 392         path: |
 393           ${{ needs.prepare.outputs.release-binary-filename }}
 394
 395     # Clean up some build files to reduce size of artifact.
 396     - name: Clean Up Build Directory
 397       shell: bash
 398       run: |
 399         find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete
 400         rm -Rf ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/_CPack_Packages
 401
 402     - name: Save Stage
 403       uses: ./workflows/.github/workflows/release-binaries-save-stage
 404       with:
 405         build-prefix: ${{ steps.setup-stage.outputs.build-prefix }}
 406
 407   upload-release-binaries:
 408     name: "Upload Release Binaries"
 409     needs:
 410       - prepare
 411       - build-stage3-all
 412     if: >-
 413       always() &&
 414       github.event_name != 'pull_request' &&
 415       needs.prepare.outputs.upload == 'true'
 416     runs-on: ubuntu-22.04
 417     permissions:
 418       contents: write # For release uploads
 419       id-token: write     # For artifact attestations
 420       attestations: write # For artifact attestations
 421
 422     steps:
 423     - name: Checkout Release Scripts
 424       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 425       with:
 426         sparse-checkout: |
 427           llvm/utils/release/github-upload-release.py
 428           llvm/utils/git/requirements.txt
 429         sparse-checkout-cone-mode: false
 430
 431     - name: 'Download artifact'
 432       uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
 433       with:
 434         pattern: '*-release-binary'
 435         merge-multiple: true
 436
 437     - name: Attest Build Provenance
 438       id: provenance
 439       uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
 440       with:
 441         subject-path: ${{ needs.prepare.outputs.release-binary-filename }}
 442
 443     - name: Rename attestation file
 444       run:
 445         mv ${{ steps.provenance.outputs.bundle-path }} ${{ needs.prepare.outputs.release-binary-filename }}.jsonl
 446
 447     - name: Upload Build Provenance
 448       uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3
 449       with:
 450         name: ${{ needs.prepare.outputs.release-binary-filename }}-attestation
 451         path: ${{ needs.prepare.outputs.release-binary-filename }}.jsonl
 452
 453     - name: Install Python Requirements
 454       run: |
 455         pip install --require-hashes -r ./llvm/utils/git/requirements.txt
 456
 457     - name: Upload Release
 458       shell: bash
 459       run: |
 460         ./llvm/utils/release/github-upload-release.py \
 461         --token ${{ github.token }} \
 462         --release ${{ needs.prepare.outputs.release-version }} \
 463         upload \
 464         --files ${{ needs.prepare.outputs.release-binary-filename }}*
 465
 466   test-stage3:
 467     name: "Test Stage 3"
 468     needs:
 469       - prepare
 470       - build-stage3-all
 471     if: >-
 472       github.repository == 'llvm/llvm-project'
 473     runs-on: ${{ inputs.runs-on }}
 474     steps:
 475     - name: Checkout Actions
 476       uses: actions/checkout@v4
 477       with:
 478         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
 479         sparse-checkout: |
 480           .github/workflows/
 481         sparse-checkout-cone-mode: false
 482         path: workflows
 483     - name: Setup Stage
 484       id: setup-stage
 485       uses: ./workflows/.github/workflows/release-binaries-setup-stage
 486       with:
 487         previous-artifact: build-stage3-all
 488
 489     - name: Run Tests
 490       shell: bash
 491       run: |
 492         ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-check-all