.github/workflows/build.yml

   1 name: Build Artifacts
   2 on:
   3   workflow_call:
   4     inputs:
   5       version:
   6         required: true
   7         type: string
   8       channel:
   9         required: false
  10         default: stable
  11         type: string
  12       unix:
  13         default: true
  14         type: boolean
  15       linux_static:
  16         default: true
  17         type: boolean
  18       linux_arm:
  19         default: true
  20         type: boolean
  21       macos:
  22         default: true
  23         type: boolean
  24       macos_legacy:
  25         default: true
  26         type: boolean
  27       windows:
  28         default: true
  29         type: boolean
  30       windows32:
  31         default: true
  32         type: boolean
  33       origin:
  34         required: false
  35         default: ''
  36         type: string
  37     secrets:
  38       GPG_SIGNING_KEY:
  39         required: false
  40
  41   workflow_dispatch:
  42     inputs:
  43       version:
  44         description: |
  45           VERSION: yyyy.mm.dd[.rev] or rev
  46         required: true
  47         type: string
  48       channel:
  49         description: |
  50           SOURCE of this build's updates: stable/nightly/master/<repo>
  51         required: true
  52         default: stable
  53         type: string
  54       unix:
  55         description: yt-dlp, yt-dlp.tar.gz
  56         default: true
  57         type: boolean
  58       linux_static:
  59         description: yt-dlp_linux
  60         default: true
  61         type: boolean
  62       linux_arm:
  63         description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
  64         default: true
  65         type: boolean
  66       macos:
  67         description: yt-dlp_macos, yt-dlp_macos.zip
  68         default: true
  69         type: boolean
  70       macos_legacy:
  71         description: yt-dlp_macos_legacy
  72         default: true
  73         type: boolean
  74       windows:
  75         description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
  76         default: true
  77         type: boolean
  78       windows32:
  79         description: yt-dlp_x86.exe
  80         default: true
  81         type: boolean
  82       origin:
  83         description: Origin
  84         required: false
  85         default: 'current repo'
  86         type: choice
  87         options:
  88         - 'current repo'
  89
  90 permissions:
  91   contents: read
  92
  93 jobs:
  94   process:
  95     runs-on: ubuntu-latest
  96     outputs:
  97       origin: ${{ steps.process_origin.outputs.origin }}
  98     steps:
  99       - name: Process origin
 100         id: process_origin
 101         run: |
 102           echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
 103
 104   unix:
 105     needs: process
 106     if: inputs.unix
 107     runs-on: ubuntu-latest
 108     steps:
 109       - uses: actions/checkout@v4
 110         with:
 111           fetch-depth: 0  # Needed for changelog
 112       - uses: actions/setup-python@v5
 113         with:
 114           python-version: "3.10"
 115       - name: Install Requirements
 116         run: |
 117           sudo apt -y install zip pandoc man sed
 118       - name: Prepare
 119         run: |
 120           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 121           python devscripts/update_changelog.py -vv
 122           python devscripts/make_lazy_extractors.py
 123       - name: Build Unix platform-independent binary
 124         run: |
 125           make all tar
 126       - name: Verify --update-to
 127         if: vars.UPDATE_TO_VERIFICATION
 128         run: |
 129           chmod +x ./yt-dlp
 130           cp ./yt-dlp ./yt-dlp_downgraded
 131           version="$(./yt-dlp --version)"
 132           ./yt-dlp_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 133           downgraded_version="$(./yt-dlp_downgraded --version)"
 134           [[ "$version" != "$downgraded_version" ]]
 135       - name: Upload artifacts
 136         uses: actions/upload-artifact@v4
 137         with:
 138           name: build-bin-${{ github.job }}
 139           path: |
 140             yt-dlp
 141             yt-dlp.tar.gz
 142           compression-level: 0
 143
 144   linux_static:
 145     needs: process
 146     if: inputs.linux_static
 147     runs-on: ubuntu-latest
 148     steps:
 149       - uses: actions/checkout@v4
 150       - name: Build static executable
 151         env:
 152           channel: ${{ inputs.channel }}
 153           origin: ${{ needs.process.outputs.origin }}
 154           version: ${{ inputs.version }}
 155         run: |
 156           mkdir ~/build
 157           cd bundle/docker
 158           docker compose up --build static
 159           sudo chown "${USER}:docker" ~/build/yt-dlp_linux
 160       - name: Verify --update-to
 161         if: vars.UPDATE_TO_VERIFICATION
 162         run: |
 163           chmod +x ~/build/yt-dlp_linux
 164           cp ~/build/yt-dlp_linux ~/build/yt-dlp_linux_downgraded
 165           version="$(~/build/yt-dlp_linux --version)"
 166           ~/build/yt-dlp_linux_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 167           downgraded_version="$(~/build/yt-dlp_linux_downgraded --version)"
 168           [[ "$version" != "$downgraded_version" ]]
 169       - name: Upload artifacts
 170         uses: actions/upload-artifact@v4
 171         with:
 172           name: build-bin-${{ github.job }}
 173           path: |
 174             ~/build/yt-dlp_linux
 175           compression-level: 0
 176
 177   linux_arm:
 178     needs: process
 179     if: inputs.linux_arm
 180     permissions:
 181       contents: read
 182       packages: write # for creating cache
 183     runs-on: ubuntu-latest
 184     strategy:
 185       matrix:
 186         architecture:
 187           - armv7
 188           - aarch64
 189
 190     steps:
 191       - uses: actions/checkout@v4
 192         with:
 193           path: ./repo
 194       - name: Virtualized Install, Prepare & Build
 195         uses: yt-dlp/run-on-arch-action@v2
 196         with:
 197           # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
 198           env: |
 199             GITHUB_WORKFLOW: build
 200           githubToken: ${{ github.token }} # To cache image
 201           arch: ${{ matrix.architecture }}
 202           distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
 203           dockerRunArgs: --volume "${PWD}/repo:/repo"
 204           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
 205             apt update
 206             apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
 207             python3.8 -m pip install -U pip setuptools wheel
 208             # Cannot access any files from the repo directory at this stage
 209             python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
 210
 211           run: |
 212             cd repo
 213             python3.8 devscripts/install_deps.py -o --include build
 214             python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage  # Cached version may be out of date
 215             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 216             python3.8 devscripts/make_lazy_extractors.py
 217             python3.8 -m bundle.pyinstaller
 218
 219             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
 220               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
 221               chmod +x ./dist/yt-dlp_linux_${arch}
 222               cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded
 223               version="$(./dist/yt-dlp_linux_${arch} --version)"
 224               ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 225               downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)"
 226               [[ "$version" != "$downgraded_version" ]]
 227             fi
 228
 229       - name: Upload artifacts
 230         uses: actions/upload-artifact@v4
 231         with:
 232           name: build-bin-linux_${{ matrix.architecture }}
 233           path: | # run-on-arch-action designates armv7l as armv7
 234             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 235           compression-level: 0
 236
 237   macos:
 238     needs: process
 239     if: inputs.macos
 240     permissions:
 241       contents: read
 242       actions: write  # For cleaning up cache
 243     runs-on: macos-13
 244
 245     steps:
 246       - uses: actions/checkout@v4
 247       # NB: Building universal2 does not work with python from actions/setup-python
 248
 249       - name: Restore cached requirements
 250         id: restore-cache
 251         uses: actions/cache/restore@v4
 252         env:
 253           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1
 254         with:
 255           path: |
 256             ~/yt-dlp-build-venv
 257           key: cache-reqs-${{ github.job }}
 258
 259       - name: Install Requirements
 260         run: |
 261           brew install coreutils
 262           python3 -m venv ~/yt-dlp-build-venv
 263           source ~/yt-dlp-build-venv/bin/activate
 264           python3 devscripts/install_deps.py -o --include build
 265           python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
 266           # We need to ignore wheels otherwise we break universal2 builds
 267           python3 -m pip install -U --no-binary :all: -r requirements.txt
 268           # We need to fuse our own universal2 wheels for curl_cffi
 269           python3 -m pip install -U 'delocate==0.11.0'
 270           mkdir curl_cffi_whls curl_cffi_universal2
 271           python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt
 272           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
 273             python3 -m pip download \
 274               --only-binary=:all: \
 275               --platform "${platform}" \
 276               -d curl_cffi_whls \
 277               -r requirements.txt
 278           done
 279           ( # Overwrite x86_64-only libs with fat/universal2 libs or else Pyinstaller will do the opposite
 280             # See https://github.com/yt-dlp/yt-dlp/pull/10069
 281             cd curl_cffi_whls
 282             mkdir -p curl_cffi/.dylibs
 283             python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
 284             for dylib in lib{ssl,crypto}.3.dylib; do
 285               cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
 286               for wheel in curl_cffi*macos*x86_64.whl; do
 287                 zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
 288               done
 289             done
 290           )
 291           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
 292           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
 293           for wheel in curl_cffi_universal2/*cffi*.whl; do
 294             mv -n -- "${wheel}" "${wheel/x86_64/universal2}"
 295           done
 296           python3 -m pip install --force-reinstall -U curl_cffi_universal2/*cffi*.whl
 297
 298       - name: Prepare
 299         run: |
 300           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 301           python3 devscripts/make_lazy_extractors.py
 302       - name: Build
 303         run: |
 304           source ~/yt-dlp-build-venv/bin/activate
 305           python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
 306           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
 307           python3 -m bundle.pyinstaller --target-architecture universal2
 308
 309       - name: Verify --update-to
 310         if: vars.UPDATE_TO_VERIFICATION
 311         run: |
 312           chmod +x ./dist/yt-dlp_macos
 313           cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded
 314           version="$(./dist/yt-dlp_macos --version)"
 315           ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 316           downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)"
 317           [[ "$version" != "$downgraded_version" ]]
 318
 319       - name: Upload artifacts
 320         uses: actions/upload-artifact@v4
 321         with:
 322           name: build-bin-${{ github.job }}
 323           path: |
 324             dist/yt-dlp_macos
 325             dist/yt-dlp_macos.zip
 326           compression-level: 0
 327
 328       - name: Cleanup cache
 329         if: steps.restore-cache.outputs.cache-hit == 'true'
 330         env:
 331           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 332           cache_key: cache-reqs-${{ github.job }}
 333           repository: ${{ github.repository }}
 334           branch: ${{ github.ref }}
 335         run: |
 336           gh extension install actions/gh-actions-cache
 337           gh actions-cache delete "${cache_key}" -R "${repository}" -B "${branch}" --confirm
 338
 339       - name: Cache requirements
 340         uses: actions/cache/save@v4
 341         with:
 342           path: |
 343             ~/yt-dlp-build-venv
 344           key: cache-reqs-${{ github.job }}
 345
 346   macos_legacy:
 347     needs: process
 348     if: inputs.macos_legacy
 349     runs-on: macos-13
 350
 351     steps:
 352       - uses: actions/checkout@v4
 353       - name: Install Python
 354         # We need the official Python, because the GA ones only support newer macOS versions
 355         env:
 356           PYTHON_VERSION: 3.10.5
 357           MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
 358         run: |
 359           # Hack to get the latest patch version. Uncomment if needed
 360           #brew install python@3.10
 361           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
 362           curl "https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg" -o "python.pkg"
 363           sudo installer -pkg python.pkg -target /
 364           python3 --version
 365       - name: Install Requirements
 366         run: |
 367           brew install coreutils
 368           python3 devscripts/install_deps.py --user -o --include build
 369           python3 devscripts/install_deps.py --user --include pyinstaller
 370
 371       - name: Prepare
 372         run: |
 373           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 374           python3 devscripts/make_lazy_extractors.py
 375       - name: Build
 376         run: |
 377           python3 -m bundle.pyinstaller
 378           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 379
 380       - name: Verify --update-to
 381         if: vars.UPDATE_TO_VERIFICATION
 382         run: |
 383           chmod +x ./dist/yt-dlp_macos_legacy
 384           cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded
 385           version="$(./dist/yt-dlp_macos_legacy --version)"
 386           ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 387           downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)"
 388           [[ "$version" != "$downgraded_version" ]]
 389
 390       - name: Upload artifacts
 391         uses: actions/upload-artifact@v4
 392         with:
 393           name: build-bin-${{ github.job }}
 394           path: |
 395             dist/yt-dlp_macos_legacy
 396           compression-level: 0
 397
 398   windows:
 399     needs: process
 400     if: inputs.windows
 401     runs-on: windows-latest
 402
 403     steps:
 404       - uses: actions/checkout@v4
 405       - uses: actions/setup-python@v5
 406         with: # 3.8 is used for Win7 support
 407           python-version: "3.8"
 408       - name: Install Requirements
 409         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
 410           python devscripts/install_deps.py -o --include build
 411           python devscripts/install_deps.py --include curl-cffi
 412           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.10.0-py3-none-any.whl"
 413
 414       - name: Prepare
 415         run: |
 416           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 417           python devscripts/make_lazy_extractors.py
 418       - name: Build
 419         run: |
 420           python -m bundle.pyinstaller
 421           python -m bundle.pyinstaller --onedir
 422           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 423
 424       - name: Add migration executable for py2exe
 425         run: |
 426           Copy-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
 427
 428       - name: Verify --update-to
 429         if: vars.UPDATE_TO_VERIFICATION
 430         run: |
 431           foreach ($name in @("yt-dlp")) {
 432             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 433             $version = & "./dist/${name}.exe" --version
 434             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 435             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 436             if ($version -eq $downgraded_version) {
 437               exit 1
 438             }
 439           }
 440
 441       - name: Upload artifacts
 442         uses: actions/upload-artifact@v4
 443         with:
 444           name: build-bin-${{ github.job }}
 445           path: |
 446             dist/yt-dlp.exe
 447             dist/yt-dlp_min.exe
 448             dist/yt-dlp_win.zip
 449           compression-level: 0
 450
 451   windows32:
 452     needs: process
 453     if: inputs.windows32
 454     runs-on: windows-latest
 455
 456     steps:
 457       - uses: actions/checkout@v4
 458       - uses: actions/setup-python@v5
 459         with:
 460           python-version: "3.8"
 461           architecture: "x86"
 462       - name: Install Requirements
 463         run: |
 464           python devscripts/install_deps.py -o --include build
 465           python devscripts/install_deps.py
 466           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.10.0-py3-none-any.whl"
 467
 468       - name: Prepare
 469         run: |
 470           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 471           python devscripts/make_lazy_extractors.py
 472       - name: Build
 473         run: |
 474           python -m bundle.pyinstaller
 475
 476       - name: Verify --update-to
 477         if: vars.UPDATE_TO_VERIFICATION
 478         run: |
 479           foreach ($name in @("yt-dlp_x86")) {
 480             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 481             $version = & "./dist/${name}.exe" --version
 482             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 483             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 484             if ($version -eq $downgraded_version) {
 485               exit 1
 486             }
 487           }
 488
 489       - name: Upload artifacts
 490         uses: actions/upload-artifact@v4
 491         with:
 492           name: build-bin-${{ github.job }}
 493           path: |
 494             dist/yt-dlp_x86.exe
 495           compression-level: 0
 496
 497   meta_files:
 498     if: always() && !cancelled()
 499     needs:
 500       - process
 501       - unix
 502       - linux_static
 503       - linux_arm
 504       - macos
 505       - macos_legacy
 506       - windows
 507       - windows32
 508     runs-on: ubuntu-latest
 509     steps:
 510       - uses: actions/download-artifact@v4
 511         with:
 512           path: artifact
 513           pattern: build-bin-*
 514           merge-multiple: true
 515
 516       - name: Make SHA2-SUMS files
 517         run: |
 518           cd ./artifact/
 519           # make sure SHA sums are also printed to stdout
 520           sha256sum -- * | tee ../SHA2-256SUMS
 521           sha512sum -- * | tee ../SHA2-512SUMS
 522           # also print as permanent annotations to the summary page
 523           while read -r shasum; do
 524             echo "::notice title=${shasum##* }::sha256: ${shasum% *}"
 525           done < ../SHA2-256SUMS
 526
 527       - name: Make Update spec
 528         run: |
 529           cat >> _update_spec << EOF
 530           # This file is used for regulating self-update
 531           lock 2022.08.18.36 .+ Python 3\.6
 532           lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 533           lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 534           lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
 535           lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 536           lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 537           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
 538           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
 539           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
 540           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
 541           EOF
 542
 543       - name: Sign checksum files
 544         env:
 545           GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 546         if: env.GPG_SIGNING_KEY != ''
 547         run: |
 548           gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}"
 549           for signfile in ./SHA*SUMS; do
 550             gpg --batch --detach-sign "$signfile"
 551           done
 552
 553       - name: Upload artifacts
 554         uses: actions/upload-artifact@v4
 555         with:
 556           name: build-${{ github.job }}
 557           path: |
 558             _update_spec
 559             SHA*SUMS*
 560           compression-level: 0
 561           overwrite: true