1 # Maintainer: Sven-Hendrik Haase <svenstaro@archlinux.org>
2 # Maintainer: Torsten Keßler <tpkessler@archlinux.org>
3 # Contributor: Stephen Zhang <zsrkmyn at gmail dot com>
6 pkgbase="python-${_pkgname}"
7 pkgname=("${pkgbase}" "${pkgbase}-opt" "${pkgbase}-cuda" "${pkgbase}-opt-cuda" "${pkgbase}-rocm" "${pkgbase}-opt-rocm")
11 _pkgdesc='Tensors and Dynamic neural networks in Python with strong GPU acceleration'
14 url="https://pytorch.org"
16 depends=('google-glog' 'gflags' 'opencv' 'openmp' 'nccl' 'pybind11' 'python' 'python-yaml' 'libuv'
17 'python-numpy' 'python-sympy' 'protobuf' 'ffmpeg4.4' 'python-future' 'qt6-base'
18 'intel-oneapi-mkl' 'python-typing_extensions' 'numactl')
19 # Exclude the magma package here and add the corresponding {cuda, rocm/hip} version
20 # to makedepends of the split packages.
21 # The magma package does not allow to build the cuda and rocm/hip code at the same time,
22 # so we need to work with the split packages magma-{cuda,hip}.
23 makedepends=('python' 'python-setuptools' 'python-yaml' 'python-numpy' 'cmake' 'cuda'
24 'cudnn' 'git' 'rocm-hip-sdk' 'roctracer' 'miopen'
25 'ninja' 'pkgconfig' 'doxygen' 'vulkan-headers' 'shaderc')
26 source=("${_pkgname}-${pkgver}::git+https://github.com/pytorch/pytorch.git#tag=v$_pkgver"
27 # generated using parse-submodules
28 "${pkgname}-ARM_NEON_2_x86_SSE::git+https://github.com/intel/ARM_NEON_2_x86_SSE.git"
29 "${pkgname}-FP16::git+https://github.com/Maratyszcza/FP16.git"
30 "${pkgname}-FXdiv::git+https://github.com/Maratyszcza/FXdiv.git"
31 "${pkgname}-NNPACK::git+https://github.com/Maratyszcza/NNPACK.git"
32 "${pkgname}-PeachPy::git+https://github.com/malfet/PeachPy.git"
33 "${pkgname}-QNNPACK::git+https://github.com/pytorch/QNNPACK"
34 "${pkgname}-VulkanMemoryAllocator::git+https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git"
35 "${pkgname}-XNNPACK::git+https://github.com/google/XNNPACK.git"
36 "${pkgname}-benchmark::git+https://github.com/google/benchmark.git"
37 "${pkgname}-cpuinfo::git+https://github.com/pytorch/cpuinfo.git"
38 "${pkgname}-cub::git+https://github.com/NVlabs/cub.git"
39 "${pkgname}-cudnn-frontend::git+https://github.com/NVIDIA/cudnn-frontend.git"
40 "${pkgname}-cutlass::git+https://github.com/NVIDIA/cutlass.git"
41 "${pkgname}-eigen::git+https://gitlab.com/libeigen/eigen.git"
42 "${pkgname}-enum34::git+https://github.com/PeachPy/enum34.git"
43 "${pkgname}-fbgemm::git+https://github.com/pytorch/fbgemm"
44 "${pkgname}-fbjni::git+https://github.com/facebookincubator/fbjni.git"
45 "${pkgname}-flatbuffers::git+https://github.com/google/flatbuffers.git"
46 "${pkgname}-fmt::git+https://github.com/fmtlib/fmt.git"
47 "${pkgname}-foxi::git+https://github.com/houseroad/foxi.git"
48 "${pkgname}-gemmlowp::git+https://github.com/google/gemmlowp.git"
49 "${pkgname}-gloo::git+https://github.com/facebookincubator/gloo"
50 "${pkgname}-googletest::git+https://github.com/google/googletest.git"
51 "${pkgname}-ideep::git+https://github.com/intel/ideep"
52 "${pkgname}-ios-cmake::git+https://github.com/Yangqing/ios-cmake.git"
53 "${pkgname}-ittapi::git+https://github.com/intel/ittapi.git"
54 "${pkgname}-json::git+https://github.com/nlohmann/json.git"
55 "${pkgname}-kineto::git+https://github.com/pytorch/kineto"
56 "${pkgname}-nccl::git+https://github.com/NVIDIA/nccl"
57 "${pkgname}-onnx-tensorrt::git+https://github.com/onnx/onnx-tensorrt"
58 "${pkgname}-onnx::git+https://github.com/onnx/onnx.git"
59 "${pkgname}-pocketfft::git+https://github.com/mreineck/pocketfft"
60 "${pkgname}-protobuf::git+https://github.com/protocolbuffers/protobuf.git"
61 "${pkgname}-psimd::git+https://github.com/Maratyszcza/psimd.git"
62 "${pkgname}-pthreadpool::git+https://github.com/Maratyszcza/pthreadpool.git"
63 "${pkgname}-pybind11::git+https://github.com/pybind/pybind11.git"
64 "${pkgname}-six::git+https://github.com/benjaminp/six.git"
65 "${pkgname}-sleef::git+https://github.com/shibatch/sleef"
66 "${pkgname}-tbb::git+https://github.com/01org/tbb"
67 "${pkgname}-tensorpipe::git+https://github.com/pytorch/tensorpipe.git"
68 "${pkgname}-zstd::git+https://github.com/facebook/zstd.git"
69 fix_include_system.patch
70 use-system-libuv.patch
71 fix-building-for-torchvision.patch
78 rocblas-batched.patch)
121 '77f85808e480bd37dfb5f072d565466ae30a8f827f49ef97591fc2fc03bea54944eb1adeaa4a1e3466518a5640f575eda88d15b4c4d549a6f41f0bf4f2cfb086'
122 '1f7ce593fa9fc62535ca1c3d85c996a73006cc614c7b7258160c3fc53cd52a1cfddcb18baf897f2e1223ecdfee52ca1471b91c9f845368ed6ac51b66f6e0e676'
123 'fdea0b815d7750a4233c1d4668593020da017aea43cf4cb63b4c00d0852c7d34f0333e618fcf98b8df2185313a2089b8c2e9fe8ec3cfb0bf693598f9c61461a8'
124 '0a8fc110a306e81beeb9ddfb3a1ddfd26aeda5e3f7adfb0f7c9bc3fd999c2dde62e0b407d3eca573097a53fd97329214e30e8767fb38d770197c7ec2b53daf18'
125 '844d0b7b39777492a6d456fa845d5399f673b4bb37b62473393449c9ad0c29dca3c33276dc3980f2e766680100335c0acfb69d51781b79575f4da112d9c4018c'
126 '985e331b2025e1ca5a4fba5188af0900f1f38bd0fd32c9173deb8bed7358af01e387d4654c7e0389e5f98b6f7cbed053226934d180b8b3b1270bdbbb36fc89b2'
127 '96de2729b29c7ce3e4fdd8008f575d24c2c3ef9f85d6217e607902d7b870ac71b9290fde71e87a68d75bb75ef28eacbf5ce04e071146809ccf1e76a03f97b479'
128 'eea86bbed0a37e1661035913536456f90e0cd1e687c7e4103011f0688bc8347b6fc2ff82019909c41e7c89ddbc3b80dde641e88abf406f4faebc71b0bb693d25'
129 '6286b05d5b5143f117363e3ce3c7d693910f53845aeb6f501b3eea64aa71778cb2d7dcd4ac945d5321ef23b4da02446e86dedc6a9b6a998df4a7f3b1ce50550a'
130 '232d2aca7cae8da511d1451890f8696d47da72276929ac5731a1a1a481d2a515fa7288bf33730d8ea2c892616551a74ca2439b53de6b1dfee156c30919120741')
131 options=('!lto' '!debug')
134 python -c 'import sys; print(str(sys.version_info[0]) + "." + str(sys.version_info[1]))'
138 cd "${srcdir}/${_pkgname}-${pkgver}"
140 # generated using parse-submodules
143 git config submodule."android/libs/fbjni".url "${srcdir}/${pkgname}"-fbjni
144 git config submodule."third_party/NNPACK".url "${srcdir}/${pkgname}"-NNPACK
145 git config submodule."third_party/NNPACK_deps/FP16".url "${srcdir}/${pkgname}"-FP16
146 git config submodule."third_party/NNPACK_deps/FXdiv".url "${srcdir}/${pkgname}"-FXdiv
147 git config submodule."third_party/NNPACK_deps/psimd".url "${srcdir}/${pkgname}"-psimd
148 git config submodule."third_party/NNPACK_deps/pthreadpool".url "${srcdir}/${pkgname}"-pthreadpool
149 git config submodule."third_party/QNNPACK".url "${srcdir}/${pkgname}"-QNNPACK
150 git config submodule."third_party/VulkanMemoryAllocator".url "${srcdir}/${pkgname}"-VulkanMemoryAllocator
151 git config submodule."third_party/XNNPACK".url "${srcdir}/${pkgname}"-XNNPACK
152 git config submodule."third_party/benchmark".url "${srcdir}/${pkgname}"-benchmark
153 git config submodule."third_party/cpuinfo".url "${srcdir}/${pkgname}"-cpuinfo
154 git config submodule."third_party/cub".url "${srcdir}/${pkgname}"-cub
155 git config submodule."third_party/cudnn_frontend".url "${srcdir}/${pkgname}"-cudnn-frontend
156 git config submodule."third_party/cutlass".url "${srcdir}/${pkgname}"-cutlass
157 git config submodule."third_party/eigen".url "${srcdir}/${pkgname}"-eigen
158 git config submodule."third_party/fbgemm".url "${srcdir}/${pkgname}"-fbgemm
159 git config submodule."third_party/flatbuffers".url "${srcdir}/${pkgname}"-flatbuffers
160 git config submodule."third_party/fmt".url "${srcdir}/${pkgname}"-fmt
161 git config submodule."third_party/foxi".url "${srcdir}/${pkgname}"-foxi
162 git config submodule."third_party/gemmlowp/gemmlowp".url "${srcdir}/${pkgname}"-gemmlowp
163 git config submodule."third_party/gloo".url "${srcdir}/${pkgname}"-gloo
164 git config submodule."third_party/googletest".url "${srcdir}/${pkgname}"-googletest
165 git config submodule."third_party/ideep".url "${srcdir}/${pkgname}"-ideep
166 git config submodule."third_party/ios-cmake".url "${srcdir}/${pkgname}"-ios-cmake
167 git config submodule."third_party/ittapi".url "${srcdir}/${pkgname}"-ittapi
168 git config submodule."third_party/kineto".url "${srcdir}/${pkgname}"-kineto
169 git config submodule."third_party/nccl/nccl".url "${srcdir}/${pkgname}"-nccl
170 git config submodule."third_party/neon2sse".url "${srcdir}/${pkgname}"-ARM_NEON_2_x86_SSE
171 git config submodule."third_party/nlohmann".url "${srcdir}/${pkgname}"-json
172 git config submodule."third_party/onnx".url "${srcdir}/${pkgname}"-onnx
173 git config submodule."third_party/onnx-tensorrt".url "${srcdir}/${pkgname}"-onnx-tensorrt
174 git config submodule."third_party/pocketfft".url "${srcdir}/${pkgname}"-pocketfft
175 git config submodule."third_party/protobuf".url "${srcdir}/${pkgname}"-protobuf
176 git config submodule."third_party/pybind11".url "${srcdir}/${pkgname}"-pybind11
177 git config submodule."third_party/python-enum".url "${srcdir}/${pkgname}"-enum34
178 git config submodule."third_party/python-peachpy".url "${srcdir}/${pkgname}"-PeachPy
179 git config submodule."third_party/python-six".url "${srcdir}/${pkgname}"-six
180 git config submodule."third_party/sleef".url "${srcdir}/${pkgname}"-sleef
181 git config submodule."third_party/tbb".url "${srcdir}/${pkgname}"-tbb
182 git config submodule."third_party/tensorpipe".url "${srcdir}/${pkgname}"-tensorpipe
183 git config submodule."third_party/zstd".url "${srcdir}/${pkgname}"-zstd
185 git -c protocol.file.allow=always submodule update --init --recursive
187 # Fix include with GCC 12
188 sed "1i#include <mutex>" -i third_party/kineto/libkineto/src/RoctracerActivityApi.h
190 # https://bugs.archlinux.org/task/64981
191 patch -N torch/utils/cpp_extension.py "${srcdir}"/fix_include_system.patch
194 patch -Np1 -i "${srcdir}"/use-system-libuv.patch
196 # fix https://github.com/pytorch/vision/issues/3695
197 patch -Np1 -i "${srcdir}/fix-building-for-torchvision.patch"
199 # Fix building against glog 0.6
200 patch -Np1 -i "${srcdir}/87773.patch"
203 patch -Np1 -d third_party/fbgemm -i "${srcdir}/disable-werror1.patch"
204 patch -Np1 -d third_party/benchmark -i "${srcdir}/disable-werror2.patch"
205 patch -Np1 -d third_party/ideep/mkl-dnn -i "${srcdir}/disable-werror3.patch"
206 patch -Np1 -i "${srcdir}/disable-werror4.patch"
208 # build against ffmpeg4.4
209 patch -Np1 -i "${srcdir}/ffmpeg4.4.patch"
211 # fix https://github.com/pytorch/pytorch/issues/97640
212 patch -Np1 -i "${srcdir}/rocblas-batched.patch"
216 cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-opt"
217 cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-cuda"
218 cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-opt-cuda"
219 cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-rocm"
220 cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-opt-rocm"
223 # Common build configuration, called in all package() functions.
226 export PYTORCH_BUILD_VERSION="${pkgver}"
227 export PYTORCH_BUILD_NUMBER=1
229 # Check tools/setup_helpers/cmake.py, setup.py and CMakeLists.txt for a list of flags that can be set via env vars.
230 export ATEN_NO_TEST=ON # do not build ATen tests
232 export BUILD_CUSTOM_PROTOBUF=OFF
233 export BUILD_CAFFE2=ON
234 export BUILD_CAFFE2_OPS=ON
235 # export BUILD_SHARED_LIBS=OFF
240 export BUILD_BINARY=ON
241 export USE_OBSERVERS=ON
243 # export USE_SYSTEM_LIBS=ON # experimental, not all libs present in repos
244 export USE_SYSTEM_NCCL=ON
245 export NCCL_VERSION=$(pkg-config nccl --modversion)
246 export NCCL_VER_CODE=$(sed -n 's/^#define NCCL_VERSION_CODE\s*\(.*\).*/\1/p' /usr/include/nccl.h)
247 # export BUILD_SPLIT_CUDA=ON # modern preferred build, but splits libs and symbols, ABI break
248 # export USE_FAST_NVCC=ON # parallel build with nvcc, spawns too many processes
249 export USE_CUPTI_SO=ON # make sure cupti.so is used as shared lib
250 export CC=/usr/bin/gcc-12
251 export CXX=/usr/bin/g++-12
252 export CUDAHOSTCXX=/opt/cuda/bin/g++
253 export CUDA_HOST_COMPILER="${CUDAHOSTCXX}"
254 export CUDA_HOME=/opt/cuda
255 # hide build-time CUDA devices
256 export CUDA_VISIBLE_DEVICES=""
257 export CUDNN_LIB_DIR=/usr/lib
258 export CUDNN_INCLUDE_DIR=/usr/include
259 export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
260 # CUDA arch 8.7 is not supported (needed by Jetson boards, etc.)
261 export TORCH_CUDA_ARCH_LIST="5.2;5.3;6.0;6.1;6.2;7.0;7.2;7.5;8.0;8.6;8.9;9.0;9.0+PTX" #include latest PTX for future compat
262 export OVERRIDE_TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}"
263 export PYTORCH_ROCM_ARCH="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1030"
265 # Hack to make sure that the generated dnnl_config.h from onednn can be inlcuded.
266 export CXXFLAGS="${CXXFLAGS} -I third_party/ideep/mkl-dnn/third_party/oneDNN/include/"
270 # Important note on the missing build() function
272 # We build the pytorch packages for the different backends directly in
273 # corresponding package() functions. This change became necessary when
274 # merging the two different GPU backends (CUDA and ROCm) into one package.
275 # Both share a dependency on the magma package but compiled against
276 # different GPU backends. This leads to two incompatible magma-{cuda,hip}
277 # packages that cannot be installed side-by-side.
278 # Therefore, we need to separately add magma-{cuda,hip} as (make-) dependencies
279 # of pytorch-{cuda,rocm}.
282 # Prevent setup.py from re-running CMake and rebuilding
283 sed -e 's/RUN_BUILD_DEPS = True/RUN_BUILD_DEPS = False/g' -i setup.py
285 python setup.py install --root="${pkgdir}"/ --optimize=1 --skip-build
287 install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
289 pytorchpath="usr/lib/python$(get_pyver)/site-packages/torch"
290 install -d "${pkgdir}/usr/lib"
292 # put CMake files in correct place
293 mv "${pkgdir}/${pytorchpath}/share/cmake" "${pkgdir}/usr/lib/cmake"
295 # put C++ API in correct place
296 mv "${pkgdir}/${pytorchpath}/include" "${pkgdir}/usr/include"
297 find "${pkgdir}/${pytorchpath}"/lib/ -type f,l \( -iname '*.so' -or -iname '*.so*' \) -print0 | while read -rd $'\0' _lib; do
298 mv "${_lib}" "${pkgdir}"/usr/lib/
301 # clean up duplicates
302 rm -r "${pkgdir}/usr/include/pybind11"
303 rm "${pkgdir}"/usr/include/*.h
305 # Python module is hardcoded so look there at runtime
306 ln -s /usr/include "${pkgdir}/${pytorchpath}/include"
307 find "${pkgdir}"/usr/lib -maxdepth 1 -type f,l \( -iname '*.so' -or -iname '*.so*' \) -print0 | while read -rd $'\0' _lib; do
308 ln -s ${_lib#"$pkgdir"} "${pkgdir}/${pytorchpath}/lib/"
312 package_python-pytorch() {
313 pkgdesc="${_pkgdesc}"
315 cd "${srcdir}/${_pkgname}-${pkgver}"
316 echo "Building without cuda or rocm and without non-x86-64 optimizations"
321 echo "add_definitions(-march=x86-64)" >> cmake/MiscCheck.cmake
322 # this horrible hack is necessary because the current release
323 # ships inconsistent CMake which tries to build objects before
324 # thier dependencies, build twice when dependencies are available
325 python setup.py build || python setup.py build
330 package_python-pytorch-opt() {
331 pkgdesc="${_pkgdesc} (with AVX2 CPU optimizations)"
332 conflicts=(python-pytorch)
333 provides=(python-pytorch)
335 cd "${srcdir}/${_pkgname}-${pkgver}-opt"
336 echo "Building without cuda or rocm and with non-x86-64 optimizations"
341 echo "add_definitions(-march=haswell)" >> cmake/MiscCheck.cmake
342 # same horrible hack as above
343 python setup.py build || python setup.py build
348 package_python-pytorch-cuda() {
349 pkgdesc="${_pkgdesc} (with CUDA)"
350 depends+=(cuda cudnn magma-cuda)
351 conflicts=(python-pytorch)
352 provides=(python-pytorch)
354 cd "${srcdir}/${_pkgname}-${pkgver}-cuda"
355 echo "Building with cuda and without non-x86-64 optimizations"
360 cd "${srcdir}/${_pkgname}-${pkgver}-cuda"
361 echo "add_definitions(-march=x86-64)" >> cmake/MiscCheck.cmake
362 # same horrible hack as above
363 python setup.py build || python setup.py build
366 # oneDNN from the repos conflicts with the version in the ideep submodule,
367 # so we have to add the dependency after building the package.
371 package_python-pytorch-opt-cuda() {
372 pkgdesc="${_pkgdesc} (with CUDA and AVX2 CPU optimizations)"
373 depends+=(cuda cudnn magma-cuda)
374 conflicts=(python-pytorch)
375 provides=(python-pytorch python-pytorch-cuda)
377 cd "${srcdir}/${_pkgname}-${pkgver}-opt-cuda"
378 echo "Building with cuda and with non-x86-64 optimizations"
383 echo "add_definitions(-march=haswell)" >> cmake/MiscCheck.cmake
384 # same horrible hack as above
385 python setup.py build || python setup.py build
392 package_python-pytorch-rocm() {
393 pkgdesc="${_pkgdesc} (with ROCm)"
394 depends+=(rocm-hip-sdk roctracer miopen magma-hip)
395 conflicts=(python-pytorch)
396 provides=(python-pytorch)
398 cd "${srcdir}/${_pkgname}-${pkgver}-rocm"
399 echo "Building with rocm and without non-x86-64 optimizations"
404 echo "add_definitions(-march=x86-64)" >> cmake/MiscCheck.cmake
405 # Conversion of CUDA to ROCm source files
406 python tools/amd_build/build_amd.py
407 # same horrible hack as above
408 python setup.py build || python setup.py build
415 package_python-pytorch-opt-rocm() {
416 pkgdesc="${_pkgdesc} (with ROCm and AVX2 CPU optimizations)"
417 depends+=(rocm-hip-sdk roctracer miopen magma-hip)
418 conflicts=(python-pytorch)
419 provides=(python-pytorch python-pytorch-rocm)
421 cd "${srcdir}/${_pkgname}-${pkgver}-opt-rocm"
422 echo "Building with rocm and with non-x86-64 optimizations"
427 echo "add_definitions(-march=haswell)" >> cmake/MiscCheck.cmake
428 # Conversion of CUDA to ROCm source files
429 python tools/amd_build/build_amd.py
430 # same horrible hack as above
431 python setup.py build || python setup.py build
438 # vim:set ts=2 sw=2 et: