python-pytorch-rocm/trunk/PKGBUILD

   1 # Maintainer: Torsten Keßler <tpkessler at archlinux dot org>
   2 # Contributor: acxz <akashpatel2008 at yahoo dot com>
   3 # Contributor: Sven-Hendrik Haase <svenstaro@gmail.com>
   4 # Contributor: Stephen Zhang <zsrkmyn at gmail dot com>
   5 # Contributor: Federico Zappone <federico dot zappone at unimol dot it>
   6
   7 _pkgname="pytorch"
   8 pkgbase="python-${_pkgname}-rocm"
   9 pkgname=("${pkgbase}" "python-pytorch-opt-rocm")
  10 pkgver=1.13.1
  11 _pkgver=1.13.1
  12 pkgrel=2
  13 _pkgdesc="Tensors and Dynamic neural networks in Python with strong GPU acceleration"
  14 pkgdesc="${_pkgdesc}"
  15 arch=('x86_64')
  16 url="https://pytorch.org"
  17 license=('BSD')
  18 depends=('google-glog' 'gflags' 'opencv' 'openmp' 'rccl' 'pybind11' 'python' 'python-yaml' 'libuv'
  19          'python-numpy' 'protobuf' 'ffmpeg4.4' 'python-future' 'qt5-base' 'intel-oneapi-mkl'
  20          'python-typing_extensions')
  21 makedepends=('python' 'python-setuptools' 'python-yaml' 'python-numpy' 'cmake' 'rocm-hip-sdk' 'roctracer'
  22              'miopen' 'git' 'hipmagma' 'ninja' 'pkgconfig' 'doxygen' 'gcc11' 'onednn'
  23              'vulkan-headers' 'shaderc')
  24 source=("${_pkgname}-${pkgver}::git+https://github.com/pytorch/pytorch.git#tag=v$_pkgver"
  25         # generated using parse-submodules
  26         "${pkgname}-cub::git+https://github.com/NVlabs/cub.git"
  27         "${pkgname}-NNPACK::git+https://github.com/Maratyszcza/NNPACK.git"
  28         "${pkgname}-ios-cmake::git+https://github.com/Yangqing/ios-cmake.git"
  29         "${pkgname}-pthreadpool::git+https://github.com/Maratyszcza/pthreadpool.git"
  30         "${pkgname}-FXdiv::git+https://github.com/Maratyszcza/FXdiv.git"
  31         "${pkgname}-gloo::git+https://github.com/facebookincubator/gloo"
  32         "${pkgname}-benchmark::git+https://github.com/google/benchmark.git"
  33         "${pkgname}-pybind11::git+https://github.com/pybind/pybind11.git"
  34         "${pkgname}-googletest::git+https://github.com/google/googletest.git"
  35         "${pkgname}-enum34::git+https://github.com/PeachPy/enum34.git"
  36         "${pkgname}-six::git+https://github.com/benjaminp/six.git"
  37         "${pkgname}-cpuinfo::git+https://github.com/pytorch/cpuinfo.git"
  38         "${pkgname}-protobuf::git+https://github.com/protocolbuffers/protobuf.git"
  39         "${pkgname}-foxi::git+https://github.com/houseroad/foxi.git"
  40         "${pkgname}-sleef::git+https://github.com/shibatch/sleef"
  41         "${pkgname}-ideep::git+https://github.com/intel/ideep"
  42         "${pkgname}-nccl::git+https://github.com/NVIDIA/nccl"
  43         "${pkgname}-zstd::git+https://github.com/facebook/zstd.git"
  44         "${pkgname}-gemmlowp::git+https://github.com/google/gemmlowp.git"
  45         "${pkgname}-QNNPACK::git+https://github.com/pytorch/QNNPACK"
  46         "${pkgname}-onnx-tensorrt::git+https://github.com/onnx/onnx-tensorrt"
  47         "${pkgname}-onnx::git+https://github.com/onnx/onnx.git"
  48         "${pkgname}-cudnn-frontend::git+https://github.com/NVIDIA/cudnn-frontend.git"
  49         "${pkgname}-ARM_NEON_2_x86_SSE::git+https://github.com/intel/ARM_NEON_2_x86_SSE.git"
  50         "${pkgname}-ittapi::git+https://github.com/intel/ittapi.git"
  51         "${pkgname}-fbjni::git+https://github.com/facebookincubator/fbjni.git"
  52         "${pkgname}-json::git+https://github.com/nlohmann/json.git"
  53         "${pkgname}-fbgemm::git+https://github.com/pytorch/fbgemm"
  54         "${pkgname}-fmt::git+https://github.com/fmtlib/fmt.git"
  55         "${pkgname}-kineto::git+https://github.com/pytorch/kineto"
  56         "${pkgname}-tbb::git+https://github.com/01org/tbb"
  57         "${pkgname}-tensorpipe::git+https://github.com/pytorch/tensorpipe.git"
  58         "${pkgname}-psimd::git+https://github.com/Maratyszcza/psimd.git"
  59         "${pkgname}-XNNPACK::git+https://github.com/google/XNNPACK.git"
  60         "${pkgname}-FP16::git+https://github.com/Maratyszcza/FP16.git"
  61         "${pkgname}-flatbuffers::git+https://github.com/google/flatbuffers.git"
  62         "${pkgname}-eigen::git+https://gitlab.com/libeigen/eigen.git"
  63         "${pkgname}-PeachPy::git+https://github.com/malfet/PeachPy.git"
  64         "${pkgname}-pocketfft::git+https://github.com/mreineck/pocketfft"
  65         fix_include_system.patch
  66         use-system-libuv.patch
  67         fix-building-for-torchvision.patch
  68         87773.patch
  69         ffmpeg4.4.patch
  70         gloo-linux-6.patch)
  71 sha256sums=('SKIP'
  72             'SKIP'
  73             'SKIP'
  74             'SKIP'
  75             'SKIP'
  76             'SKIP'
  77             'SKIP'
  78             'SKIP'
  79             'SKIP'
  80             'SKIP'
  81             'SKIP'
  82             'SKIP'
  83             'SKIP'
  84             'SKIP'
  85             'SKIP'
  86             'SKIP'
  87             'SKIP'
  88             'SKIP'
  89             'SKIP'
  90             'SKIP'
  91             'SKIP'
  92             'SKIP'
  93             'SKIP'
  94             'SKIP'
  95             'SKIP'
  96             'SKIP'
  97             'SKIP'
  98             'SKIP'
  99             'SKIP'
 100             'SKIP'
 101             'SKIP'
 102             'SKIP'
 103             'SKIP'
 104             'SKIP'
 105             'SKIP'
 106             'SKIP'
 107             'SKIP'
 108             'SKIP'
 109             'SKIP'
 110             'SKIP'
 111             '557761502bbd994d9795bef46779e4b8c60ba0b45e7d60841f477d3b7f28a00a'
 112             'cd9ac4aaa9f946ac5eafc57cf66c5c16b3ea7ac8af32c2558fad0705411bb669'
 113             '600bd6a4bbcec9f99ab815d82cee1c2875530b2b75f4010da5ba72ce9bf31aff'
 114             'bbc61ba2392bd4c6766cca820d4c5ca51ef9abd188e53884b95abcbe4d89a20e'
 115             '75001b59e76831b0c93a547f851cb980e00b0d8cc7b66fb507eaeac217dc6ff9'
 116             '91061f2ec36b5aa2002053654e1517d04bc2defa1ce3985ac554944a6497ce52')
 117 options=('!lto')
 118
 119 get_pyver () {
 120   python -c 'import sys; print(str(sys.version_info[0]) + "." + str(sys.version_info[1]))'
 121 }
 122
 123 prepare() {
 124   cd "${srcdir}/${_pkgname}-${pkgver}"
 125
 126   # generated using parse-submodules
 127   git submodule init
 128
 129   git config submodule."third_party/pybind11".url "${srcdir}/${pkgname}"-pybind11
 130   git config submodule."third_party/cub".url "${srcdir}/${pkgname}"-cub
 131   git config submodule."third_party/eigen".url "${srcdir}/${pkgname}"-eigen
 132   git config submodule."third_party/googletest".url "${srcdir}/${pkgname}"-googletest
 133   #https://github.com/rocm-arch/python-pytorch-rocm/issues/49
 134   #git config submodule."third_party/benchmark".url "${srcdir}/${pkgname}"-benchmark
 135   git config submodule."third_party/protobuf".url "${srcdir}/${pkgname}"-protobuf
 136   git config submodule."third_party/ios-cmake".url "${srcdir}/${pkgname}"-ios-cmake
 137   git config submodule."third_party/NNPACK".url "${srcdir}/${pkgname}"-NNPACK
 138   git config submodule."third_party/gloo".url "${srcdir}/${pkgname}"-gloo
 139   git config submodule."third_party/NNPACK_deps/pthreadpool".url "${srcdir}/${pkgname}"-pthreadpool
 140   git config submodule."third_party/NNPACK_deps/FXdiv".url "${srcdir}/${pkgname}"-FXdiv
 141   git config submodule."third_party/NNPACK_deps/FP16".url "${srcdir}/${pkgname}"-FP16
 142   git config submodule."third_party/NNPACK_deps/psimd".url "${srcdir}/${pkgname}"-psimd
 143   git config submodule."third_party/zstd".url "${srcdir}/${pkgname}"-zstd
 144   git config submodule."third_party/cpuinfo".url "${srcdir}/${pkgname}"-cpuinfo
 145   git config submodule."third_party/python-enum".url "${srcdir}/${pkgname}"-enum34
 146   git config submodule."third_party/python-peachpy".url "${srcdir}/${pkgname}"-PeachPy
 147   git config submodule."third_party/python-six".url "${srcdir}/${pkgname}"-six
 148   git config submodule."third_party/onnx".url "${srcdir}/${pkgname}"-onnx
 149   git config submodule."third_party/onnx-tensorrt".url "${srcdir}/${pkgname}"-onnx-tensorrt
 150   git config submodule."third_party/sleef".url "${srcdir}/${pkgname}"-sleef
 151   git config submodule."third_party/ideep".url "${srcdir}/${pkgname}"-ideep
 152   git config submodule."third_party/gemmlowp/gemmlowp".url "${srcdir}/${pkgname}"-gemmlowp
 153   git config submodule."third_party/QNNPACK".url "${srcdir}/${pkgname}"-QNNPACK
 154   git config submodule."third_party/neon2sse".url "${srcdir}/${pkgname}"-ARM_NEON_2_x86_SSE
 155   git config submodule."third_party/fbgemm".url "${srcdir}/${pkgname}"-fbgemm
 156   git config submodule."third_party/foxi".url "${srcdir}/${pkgname}"-foxi
 157   git config submodule."third_party/tbb".url "${srcdir}/${pkgname}"-tbb
 158   git config submodule."android/libs/fbjni".url "${srcdir}/${pkgname}"-fbjni
 159   git config submodule."third_party/XNNPACK".url "${srcdir}/${pkgname}"-XNNPACK
 160   git config submodule."third_party/fmt".url "${srcdir}/${pkgname}"-fmt
 161   git config submodule."third_party/tensorpipe".url "${srcdir}/${pkgname}"-tensorpipe
 162   git config submodule."third_party/kineto".url "${srcdir}/${pkgname}"-kineto
 163   git config submodule."third_party/pocketfft".url "${srcdir}/${pkgname}"-pocketfft
 164   git config submodule."third_party/ittapi".url "${srcdir}/${pkgname}"-ittapi
 165   git config submodule."third_party/flatbuffers".url "${srcdir}/${pkgname}"-flatbuffers
 166   git config submodule."third_party/nlohmann".url "${srcdir}/${pkgname}"-json
 167
 168   git -c protocol.file.allow=always submodule update --init --recursive
 169   sed "1i#include <mutex>" -i third_party/kineto/libkineto/src/RoctracerActivityApi.h
 170
 171   # https://bugs.archlinux.org/task/64981
 172   patch -N torch/utils/cpp_extension.py "${srcdir}"/fix_include_system.patch
 173
 174   # Use system libuv
 175   patch -Np1 -i "${srcdir}"/use-system-libuv.patch
 176
 177   # fix https://github.com/pytorch/vision/issues/3695
 178   patch -Np1 -i "${srcdir}/fix-building-for-torchvision.patch"
 179
 180   # Fix building against glog 0.6
 181   patch -Np1 -i "${srcdir}/87773.patch"
 182
 183   # build against ffmpeg4.4
 184   patch -Np1 -i "${srcdir}/ffmpeg4.4.patch"
 185
 186   cd "third_party/gloo"
 187   patch -Np1 -i "${srcdir}/gloo-linux-6.patch"
 188
 189   cd "${srcdir}"
 190
 191   cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-rocm"
 192   cp -r "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-opt-rocm"
 193
 194   export VERBOSE=1
 195   export PYTORCH_BUILD_VERSION="${pkgver}"
 196   export PYTORCH_BUILD_NUMBER=1
 197
 198   # Check tools/setup_helpers/cmake.py, setup.py and CMakeLists.txt for a list of flags that can be set via env vars.
 199   export ATEN_NO_TEST=ON # do not build ATen tests
 200   export USE_MKLDNN=ON
 201   export BUILD_CUSTOM_PROTOBUF=OFF
 202   export BUILD_CAFFE2=ON
 203   export BUILD_CAFFE2_OPS=ON
 204   # export BUILD_SHARED_LIBS=OFF
 205   export USE_FFMPEG=ON
 206   export USE_GFLAGS=ON
 207   export USE_GLOG=ON
 208   export BUILD_BINARY=ON
 209   export USE_OBSERVERS=ON
 210   export USE_OPENCV=ON
 211   # export USE_SYSTEM_LIBS=ON  # experimental, not all libs present in repos
 212   export USE_CUPTI_SO=ON  # make sure cupti.so is used as shared lib
 213   export CC=/usr/bin/gcc-11
 214   export CXX=/usr/bin/g++-11
 215   export CUDAHOSTCXX=/usr/bin/g++-11
 216   export CUDA_HOST_COMPILER="${CUDAHOSTCXX}"
 217 }
 218
 219 build() {
 220   export PYTORCH_ROCM_ARCH="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1030"
 221
 222   echo "Building with rocm and without non-x86-64 optimizations"
 223   export USE_CUDA=0
 224   export USE_CUDNN=0
 225   export USE_ROCM=1
 226   cd "${srcdir}/${_pkgname}-${pkgver}-rocm"
 227   echo "add_definitions(-march=x86-64)" >> cmake/MiscCheck.cmake
 228
 229   # Apply changes needed for ROCm
 230   python tools/amd_build/build_amd.py
 231
 232   # this horrible hack is necessary because the current release
 233   # ships inconsistent CMake which tries to build objects before
 234   # thier dependencies, build twice when dependencies are available
 235   python setup.py build || python setup.py build
 236
 237   echo "Building with rocm and with non-x86-64 optimizations"
 238   export USE_CUDA=0
 239   export USE_CUDNN=0
 240   export USE_ROCM=1
 241   cd "${srcdir}/${_pkgname}-${pkgver}-opt-rocm"
 242   echo "add_definitions(-march=haswell)" >> cmake/MiscCheck.cmake
 243
 244   # Apply changes needed for ROCm
 245   python tools/amd_build/build_amd.py
 246
 247   # same horrible hack as above
 248   python setup.py build || python setup.py build
 249 }
 250
 251 _package() {
 252   # Prevent setup.py from re-running CMake and rebuilding
 253   sed -e 's/RUN_BUILD_DEPS = True/RUN_BUILD_DEPS = False/g' -i setup.py
 254
 255   python setup.py install --root="${pkgdir}"/ --optimize=1 --skip-build
 256
 257   install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
 258
 259   pytorchpath="usr/lib/python$(get_pyver)/site-packages/torch"
 260   install -d "${pkgdir}/usr/lib"
 261
 262   # put CMake files in correct place
 263   mv "${pkgdir}/${pytorchpath}/share/cmake" "${pkgdir}/usr/lib/cmake"
 264
 265   # put C++ API in correct place
 266   mv "${pkgdir}/${pytorchpath}/include" "${pkgdir}/usr/include"
 267   find "${pkgdir}/${pytorchpath}"/lib/ -type f,l \( -iname '*.so' -or -iname '*.so*' \) -print0 | while read -rd $'\0' _lib; do
 268     mv "${_lib}" "${pkgdir}"/usr/lib/
 269   done
 270
 271   # clean up duplicates
 272   # TODO: move towards direct shared library dependecy of:
 273   #   c10, caffe2, libcpuinfo, gloo, GTest, Intel MKL,
 274   #   ONNX, protobuf, libthreadpool, QNNPACK
 275   rm -rf "${pkgdir}/usr/include/pybind11"
 276
 277   # python module is hardcoded to look there at runtime
 278   ln -s /usr/include "${pkgdir}/${pytorchpath}/include"
 279   find "${pkgdir}"/usr/lib -maxdepth 1 -type f,l \( -iname '*.so' -or -iname '*.so*' \) -print0 | while read -rd $'\0' _lib; do
 280     ln -s ${_lib#"$pkgdir"} "${pkgdir}/${pytorchpath}/lib/"
 281   done
 282 }
 283
 284 package_python-pytorch-rocm() {
 285   pkgdesc="${_pkgdesc} (with ROCm)"
 286   depends+=(rocm-hip-sdk roctracer miopen hipmagma)
 287   conflicts=(python-pytorch)
 288   provides=(python-pytorch)
 289
 290   cd "${srcdir}/${_pkgname}-${pkgver}-rocm"
 291   _package
 292 }
 293
 294 package_python-pytorch-opt-rocm() {
 295   pkgdesc="${_pkgdesc} (with ROCm and AVX2 CPU optimizations)"
 296   depends+=(rocm-hip-sdk roctracer miopen hipmagma)
 297   conflicts=(python-pytorch)
 298   provides=(python-pytorch python-pytorch-rocm)
 299
 300   cd "${srcdir}/${_pkgname}-${pkgver}-opt-rocm"
 301   _package
 302 }