Release 2024.11.18
[yt-dlp3.git] / .github / workflows / build.yml
bloba211ae1652c3e345106e810b3bd09da5cdbd17bb
1 name: Build Artifacts
2 on:
3   workflow_call:
4     inputs:
5       version:
6         required: true
7         type: string
8       channel:
9         required: false
10         default: stable
11         type: string
12       unix:
13         default: true
14         type: boolean
15       linux_static:
16         default: true
17         type: boolean
18       linux_arm:
19         default: true
20         type: boolean
21       macos:
22         default: true
23         type: boolean
24       macos_legacy:
25         default: true
26         type: boolean
27       windows:
28         default: true
29         type: boolean
30       windows32:
31         default: true
32         type: boolean
33       origin:
34         required: false
35         default: ''
36         type: string
37     secrets:
38       GPG_SIGNING_KEY:
39         required: false
41   workflow_dispatch:
42     inputs:
43       version:
44         description: |
45           VERSION: yyyy.mm.dd[.rev] or rev
46         required: true
47         type: string
48       channel:
49         description: |
50           SOURCE of this build's updates: stable/nightly/master/<repo>
51         required: true
52         default: stable
53         type: string
54       unix:
55         description: yt-dlp, yt-dlp.tar.gz
56         default: true
57         type: boolean
58       linux_static:
59         description: yt-dlp_linux
60         default: true
61         type: boolean
62       linux_arm:
63         description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
64         default: true
65         type: boolean
66       macos:
67         description: yt-dlp_macos, yt-dlp_macos.zip
68         default: true
69         type: boolean
70       macos_legacy:
71         description: yt-dlp_macos_legacy
72         default: true
73         type: boolean
74       windows:
75         description: yt-dlp.exe, yt-dlp_win.zip
76         default: true
77         type: boolean
78       windows32:
79         description: yt-dlp_x86.exe
80         default: true
81         type: boolean
82       origin:
83         description: Origin
84         required: false
85         default: 'current repo'
86         type: choice
87         options:
88         - 'current repo'
90 permissions:
91   contents: read
93 jobs:
94   process:
95     runs-on: ubuntu-latest
96     outputs:
97       origin: ${{ steps.process_origin.outputs.origin }}
98     steps:
99       - name: Process origin
100         id: process_origin
101         run: |
102           echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
104   unix:
105     needs: process
106     if: inputs.unix
107     runs-on: ubuntu-latest
108     steps:
109       - uses: actions/checkout@v4
110         with:
111           fetch-depth: 0  # Needed for changelog
112       - uses: actions/setup-python@v5
113         with:
114           python-version: "3.10"
115       - name: Install Requirements
116         run: |
117           sudo apt -y install zip pandoc man sed
118       - name: Prepare
119         run: |
120           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
121           python devscripts/update_changelog.py -vv
122           python devscripts/make_lazy_extractors.py
123       - name: Build Unix platform-independent binary
124         run: |
125           make all tar
126       - name: Verify --update-to
127         if: vars.UPDATE_TO_VERIFICATION
128         run: |
129           chmod +x ./yt-dlp
130           cp ./yt-dlp ./yt-dlp_downgraded
131           version="$(./yt-dlp --version)"
132           ./yt-dlp_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
133           downgraded_version="$(./yt-dlp_downgraded --version)"
134           [[ "$version" != "$downgraded_version" ]]
135       - name: Upload artifacts
136         uses: actions/upload-artifact@v4
137         with:
138           name: build-bin-${{ github.job }}
139           path: |
140             yt-dlp
141             yt-dlp.tar.gz
142           compression-level: 0
144   linux_static:
145     needs: process
146     if: inputs.linux_static
147     runs-on: ubuntu-latest
148     steps:
149       - uses: actions/checkout@v4
150       - name: Build static executable
151         env:
152           channel: ${{ inputs.channel }}
153           origin: ${{ needs.process.outputs.origin }}
154           version: ${{ inputs.version }}
155         run: |
156           mkdir ~/build
157           cd bundle/docker
158           docker compose up --build static
159           sudo chown "${USER}:docker" ~/build/yt-dlp_linux
160       - name: Verify --update-to
161         if: vars.UPDATE_TO_VERIFICATION
162         run: |
163           chmod +x ~/build/yt-dlp_linux
164           cp ~/build/yt-dlp_linux ~/build/yt-dlp_linux_downgraded
165           version="$(~/build/yt-dlp_linux --version)"
166           ~/build/yt-dlp_linux_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
167           downgraded_version="$(~/build/yt-dlp_linux_downgraded --version)"
168           [[ "$version" != "$downgraded_version" ]]
169       - name: Upload artifacts
170         uses: actions/upload-artifact@v4
171         with:
172           name: build-bin-${{ github.job }}
173           path: |
174             ~/build/yt-dlp_linux
175           compression-level: 0
177   linux_arm:
178     needs: process
179     if: inputs.linux_arm
180     permissions:
181       contents: read
182       packages: write # for creating cache
183     runs-on: ubuntu-latest
184     strategy:
185       matrix:
186         architecture:
187           - armv7
188           - aarch64
190     steps:
191       - uses: actions/checkout@v4
192         with:
193           path: ./repo
194       - name: Virtualized Install, Prepare & Build
195         uses: yt-dlp/run-on-arch-action@v2
196         with:
197           # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
198           env: |
199             GITHUB_WORKFLOW: build
200           githubToken: ${{ github.token }} # To cache image
201           arch: ${{ matrix.architecture }}
202           distro: ubuntu20.04 # Standalone executable should be built on minimum supported OS
203           dockerRunArgs: --volume "${PWD}/repo:/repo"
204           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
205             apt update
206             apt -y install zlib1g-dev libffi-dev python3.9 python3.9-dev python3.9-distutils python3-pip \
207               python3-secretstorage  # Cannot build cryptography wheel in virtual armv7 environment
208             python3.9 -m pip install -U pip wheel 'setuptools>=71.0.2'
209             # XXX: Keep this in sync with pyproject.toml (it can't be accessed at this stage) and exclude secretstorage
210             python3.9 -m pip install -U Pyinstaller mutagen pycryptodomex brotli certifi cffi \
211               'requests>=2.32.2,<3' 'urllib3>=1.26.17,<3' 'websockets>=13.0'
213           run: |
214             cd repo
215             python3.9 devscripts/install_deps.py -o --include build
216             python3.9 devscripts/install_deps.py --include pyinstaller  # Cached versions may be out of date
217             python3.9 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
218             python3.9 devscripts/make_lazy_extractors.py
219             python3.9 -m bundle.pyinstaller
221             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
222               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
223               chmod +x ./dist/yt-dlp_linux_${arch}
224               cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded
225               version="$(./dist/yt-dlp_linux_${arch} --version)"
226               ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
227               downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)"
228               [[ "$version" != "$downgraded_version" ]]
229             fi
231       - name: Upload artifacts
232         uses: actions/upload-artifact@v4
233         with:
234           name: build-bin-linux_${{ matrix.architecture }}
235           path: | # run-on-arch-action designates armv7l as armv7
236             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
237           compression-level: 0
239   macos:
240     needs: process
241     if: inputs.macos
242     permissions:
243       contents: read
244       actions: write  # For cleaning up cache
245     runs-on: macos-13
247     steps:
248       - uses: actions/checkout@v4
249       # NB: Building universal2 does not work with python from actions/setup-python
251       - name: Restore cached requirements
252         id: restore-cache
253         uses: actions/cache/restore@v4
254         env:
255           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1
256         with:
257           path: |
258             ~/yt-dlp-build-venv
259           key: cache-reqs-${{ github.job }}
261       - name: Install Requirements
262         run: |
263           brew install coreutils
264           python3 -m venv ~/yt-dlp-build-venv
265           source ~/yt-dlp-build-venv/bin/activate
266           python3 devscripts/install_deps.py -o --include build
267           python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
268           # We need to ignore wheels otherwise we break universal2 builds
269           python3 -m pip install -U --no-binary :all: -r requirements.txt
270           # We need to fuse our own universal2 wheels for curl_cffi
271           python3 -m pip install -U 'delocate==0.11.0'
272           mkdir curl_cffi_whls curl_cffi_universal2
273           python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt
274           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
275             python3 -m pip download \
276               --only-binary=:all: \
277               --platform "${platform}" \
278               -d curl_cffi_whls \
279               -r requirements.txt
280           done
281           ( # Overwrite x86_64-only libs with fat/universal2 libs or else Pyinstaller will do the opposite
282             # See https://github.com/yt-dlp/yt-dlp/pull/10069
283             cd curl_cffi_whls
284             mkdir -p curl_cffi/.dylibs
285             python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
286             for dylib in lib{ssl,crypto}.3.dylib; do
287               cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
288               for wheel in curl_cffi*macos*x86_64.whl; do
289                 zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
290               done
291             done
292           )
293           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
294           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
295           for wheel in curl_cffi_universal2/*cffi*.whl; do
296             mv -n -- "${wheel}" "${wheel/x86_64/universal2}"
297           done
298           python3 -m pip install --force-reinstall -U curl_cffi_universal2/*cffi*.whl
300       - name: Prepare
301         run: |
302           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
303           python3 devscripts/make_lazy_extractors.py
304       - name: Build
305         run: |
306           source ~/yt-dlp-build-venv/bin/activate
307           python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
308           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
309           python3 -m bundle.pyinstaller --target-architecture universal2
311       - name: Verify --update-to
312         if: vars.UPDATE_TO_VERIFICATION
313         run: |
314           chmod +x ./dist/yt-dlp_macos
315           cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded
316           version="$(./dist/yt-dlp_macos --version)"
317           ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
318           downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)"
319           [[ "$version" != "$downgraded_version" ]]
321       - name: Upload artifacts
322         uses: actions/upload-artifact@v4
323         with:
324           name: build-bin-${{ github.job }}
325           path: |
326             dist/yt-dlp_macos
327             dist/yt-dlp_macos.zip
328           compression-level: 0
330       - name: Cleanup cache
331         if: steps.restore-cache.outputs.cache-hit == 'true'
332         env:
333           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
334           cache_key: cache-reqs-${{ github.job }}
335           repository: ${{ github.repository }}
336           branch: ${{ github.ref }}
337         run: |
338           gh extension install actions/gh-actions-cache
339           gh actions-cache delete "${cache_key}" -R "${repository}" -B "${branch}" --confirm
341       - name: Cache requirements
342         uses: actions/cache/save@v4
343         with:
344           path: |
345             ~/yt-dlp-build-venv
346           key: cache-reqs-${{ github.job }}
348   macos_legacy:
349     needs: process
350     if: inputs.macos_legacy
351     runs-on: macos-13
353     steps:
354       - uses: actions/checkout@v4
355       - name: Install Python
356         # We need the official Python, because the GA ones only support newer macOS versions
357         env:
358           PYTHON_VERSION: 3.10.5
359           MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
360         run: |
361           # Hack to get the latest patch version. Uncomment if needed
362           #brew install python@3.10
363           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
364           curl "https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg" -o "python.pkg"
365           sudo installer -pkg python.pkg -target /
366           python3 --version
367       - name: Install Requirements
368         run: |
369           brew install coreutils
370           python3 devscripts/install_deps.py --user -o --include build
371           python3 devscripts/install_deps.py --user --include pyinstaller
373       - name: Prepare
374         run: |
375           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
376           python3 devscripts/make_lazy_extractors.py
377       - name: Build
378         run: |
379           python3 -m bundle.pyinstaller
380           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
382       - name: Verify --update-to
383         if: vars.UPDATE_TO_VERIFICATION
384         run: |
385           chmod +x ./dist/yt-dlp_macos_legacy
386           cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded
387           version="$(./dist/yt-dlp_macos_legacy --version)"
388           ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
389           downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)"
390           [[ "$version" != "$downgraded_version" ]]
392       - name: Upload artifacts
393         uses: actions/upload-artifact@v4
394         with:
395           name: build-bin-${{ github.job }}
396           path: |
397             dist/yt-dlp_macos_legacy
398           compression-level: 0
400   windows:
401     needs: process
402     if: inputs.windows
403     runs-on: windows-latest
405     steps:
406       - uses: actions/checkout@v4
407       - uses: actions/setup-python@v5
408         with:
409           python-version: "3.10"
410       - name: Install Requirements
411         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
412           python devscripts/install_deps.py -o --include build
413           python devscripts/install_deps.py --include curl-cffi
414           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.11.1-py3-none-any.whl"
416       - name: Prepare
417         run: |
418           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
419           python devscripts/make_lazy_extractors.py
420       - name: Build
421         run: |
422           python -m bundle.pyinstaller
423           python -m bundle.pyinstaller --onedir
424           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
426       - name: Verify --update-to
427         if: vars.UPDATE_TO_VERIFICATION
428         run: |
429           foreach ($name in @("yt-dlp")) {
430             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
431             $version = & "./dist/${name}.exe" --version
432             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
433             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
434             if ($version -eq $downgraded_version) {
435               exit 1
436             }
437           }
439       - name: Upload artifacts
440         uses: actions/upload-artifact@v4
441         with:
442           name: build-bin-${{ github.job }}
443           path: |
444             dist/yt-dlp.exe
445             dist/yt-dlp_win.zip
446           compression-level: 0
448   windows32:
449     needs: process
450     if: inputs.windows32
451     runs-on: windows-latest
453     steps:
454       - uses: actions/checkout@v4
455       - uses: actions/setup-python@v5
456         with:
457           python-version: "3.10"
458           architecture: "x86"
459       - name: Install Requirements
460         run: |
461           python devscripts/install_deps.py -o --include build
462           python devscripts/install_deps.py
463           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.11.1-py3-none-any.whl"
465       - name: Prepare
466         run: |
467           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
468           python devscripts/make_lazy_extractors.py
469       - name: Build
470         run: |
471           python -m bundle.pyinstaller
473       - name: Verify --update-to
474         if: vars.UPDATE_TO_VERIFICATION
475         run: |
476           foreach ($name in @("yt-dlp_x86")) {
477             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
478             $version = & "./dist/${name}.exe" --version
479             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
480             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
481             if ($version -eq $downgraded_version) {
482               exit 1
483             }
484           }
486       - name: Upload artifacts
487         uses: actions/upload-artifact@v4
488         with:
489           name: build-bin-${{ github.job }}
490           path: |
491             dist/yt-dlp_x86.exe
492           compression-level: 0
494   meta_files:
495     if: always() && !cancelled()
496     needs:
497       - process
498       - unix
499       - linux_static
500       - linux_arm
501       - macos
502       - macos_legacy
503       - windows
504       - windows32
505     runs-on: ubuntu-latest
506     steps:
507       - name: Download artifacts
508         uses: actions/download-artifact@v4
509         with:
510           path: artifact
511           pattern: build-bin-*
512           merge-multiple: true
514       - name: Make SHA2-SUMS files
515         run: |
516           cd ./artifact/
517           # make sure SHA sums are also printed to stdout
518           sha256sum -- * | tee ../SHA2-256SUMS
519           sha512sum -- * | tee ../SHA2-512SUMS
520           # also print as permanent annotations to the summary page
521           while read -r shasum; do
522             echo "::notice title=${shasum##* }::sha256: ${shasum% *}"
523           done < ../SHA2-256SUMS
525       - name: Make Update spec
526         run: |
527           cat >> _update_spec << EOF
528           # This file is used for regulating self-update
529           lock 2022.08.18.36 .+ Python 3\.6
530           lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
531           lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
532           lock 2024.10.22 py2exe .+
533           lock 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
534           lock 2024.10.22 (?!\w+_exe).+ Python 3\.8
535           lock 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
536           lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
537           lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
538           lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
539           lockV2 yt-dlp/yt-dlp 2024.10.22 py2exe .+
540           lockV2 yt-dlp/yt-dlp 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
541           lockV2 yt-dlp/yt-dlp 2024.10.22 (?!\w+_exe).+ Python 3\.8
542           lockV2 yt-dlp/yt-dlp 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
543           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
544           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
545           lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 py2exe .+
546           lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
547           lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 (?!\w+_exe).+ Python 3\.8
548           lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
549           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
550           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
551           lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.045052 py2exe .+
552           lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b
553           lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 (?!\w+_exe).+ Python 3\.8
554           lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2)
555           EOF
557       - name: Sign checksum files
558         env:
559           GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
560         if: env.GPG_SIGNING_KEY != ''
561         run: |
562           gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}"
563           for signfile in ./SHA*SUMS; do
564             gpg --batch --detach-sign "$signfile"
565           done
567       - name: Upload artifacts
568         uses: actions/upload-artifact@v4
569         with:
570           name: build-${{ github.job }}
571           path: |
572             _update_spec
573             SHA*SUMS*
574           compression-level: 0
575           overwrite: true