You are not logged in.

Pages: 1

#1 2018-12-14 06:44:05

leomao
Member
Registered: 2013-04-27
Posts: 18

the git submodules listed in python-pytorch PKGBUILD are not correct

The current PKGBUILD is https://git.archlinux.org/svntogit/comm … on-pytorch.
It seems that the submodules listed in the PKGBUILD are not synced with submodules listed in https://github.com/pytorch/pytorch/tree … hird_party.
Also, according to this commit: https://github.com/pytorch/pytorch/comm … 2da311163d, the upstream team suggests initializing the submodules with the flag `--recursive`.  I am not sure what is the correct way to do this in the PKGBUILD...

Offline

#2 2018-12-14 12:09:32

Lone_Wolf
Member
From: Netherlands, Europe
Registered: 2005-10-04
Posts: 11,913

Re: the git submodules listed in python-pytorch PKGBUILD are not correct

The PKGBUILD uses the method advised to work with submodules as seen at https://wiki.archlinux.org/index.php/VC … Submodules .

Maybe the list of necessary modules changed ?

Using submodules for so many external dependencies makes me wonder what pytorch developers are doing .
Have the developers of those external projects stopped releasing stable versions ?

Disliking systemd intensely, but not satisfied with alternatives so focusing on taming systemd.


(A works at time B)  && (time C > time B ) ≠  (A works at time C)

Online

#3 2018-12-14 14:04:09

loqs
Member
Registered: 2014-03-06
Posts: 17,327

Re: the git submodules listed in python-pytorch PKGBUILD are not correct

cat .git/config 
[core]
	repositoryformatversion = 0
	filemode = true
	bare = false
	logallrefupdates = true
[remote "origin"]
	url = /startdir/pytorch-1.0.0
	fetch = +refs/heads/*:refs/remotes/origin/*
[branch "master"]
	remote = origin
	merge = refs/heads/master
[submodule "third_party/ComputeLibrary"]
	active = true
	url = /build/python-pytorch/src/ComputeLibrary
[submodule "third_party/NNPACK_deps/FP16"]
	active = true
	url = /build/python-pytorch/src/FP16
[submodule "third_party/NNPACK_deps/FXdiv"]
	active = true
	url = /build/python-pytorch/src/FXdiv
[submodule "third_party/NNPACK"]
	active = true
	url = /build/python-pytorch/src/NNPACK
[submodule "third_party/QNNPACK"]
	active = true
	url = https://github.com/pytorch/QNNPACK
[submodule "third_party/benchmark"]
	active = true
	url = /build/python-pytorch/src/benchmark
[submodule "third-party/cpuinfo"]
	active = true
	url = https://github.com/Maratyszcza/cpuinfo.git
[submodule "third_party/cub"]
	active = true
	url = /build/python-pytorch/src/cub
[submodule "third_party/eigen"]
	active = true
	url = /build/python-pytorch/src/eigen-git-mirror
[submodule "third_party/fbgemm"]
	active = true
	url = https://github.com/pytorch/fbgemm
[submodule "third_party/gemmlowp/gemmlowp"]
	active = true
	url = https://github.com/google/gemmlowp.git
[submodule "third_party/gloo"]
	active = true
	url = /build/python-pytorch/src/gloo
[submodule "third_party/googletest"]
	active = true
	url = /build/python-pytorch/src/googletest
[submodule "third_party/ideep"]
	active = true
	url = /build/python-pytorch/src/ideep
[submodule "third_party/ios-cmake"]
	active = true
	url = /build/python-pytorch/src/ios-cmake
[submodule "third_party/nccl/nccl"]
	active = true
	url = https://github.com/NVIDIA/nccl
[submodule "third_party/neon2sse"]
	active = true
	url = https://github.com/intel/ARM_NEON_2_x86_SSE.git
[submodule "third_party/onnx"]
	active = true
	url = /build/python-pytorch/src/onnx
[submodule "third_party/onnx-tensorrt"]
	active = true
	url = /build/python-pytorch/src/onnx-tensorrt
[submodule "third_party/protobuf"]
	active = true
	url = /build/python-pytorch/src/protobuf
[submodule "third_party/NNPACK_deps/psimd"]
	active = true
	url = /build/python-pytorch/src/psimd
[submodule "third_party/NNPACK_deps/pthreadpool"]
	active = true
	url = https://github.com/Maratyszcza/pthreadpool.git
[submodule "third_party/pybind11"]
	active = true
	url = /build/python-pytorch/src/pybind11
[submodule "third_party/python-enum"]
	active = true
	url = /build/python-pytorch/src/enum34
[submodule "third_party/python-peachpy"]
	active = true
	url = /build/python-pytorch/src/PeachPy
[submodule "third_party/python-six"]
	active = true
	url = /build/python-pytorch/src/six
[submodule "third_party/sleef"]
	active = true
	url = /build/python-pytorch/src/sleef
[submodule "third_party/zstd"]
	active = true
	url = /build/python-pytorch/src/zstd
[submodule "third_party/catch"]
	url = /build/python-pytorch/src/Catch2
[submodule "third_party/nervanagpu"]
	url = /build/python-pytorch/src/nervanagpu
[submodule "third_party/NNPACK_deps/pthread_ool"]
	url = /build/python-pytorch/src/pthreadpool
[submodule "third_party/cpuinfo"]
	url = /build/python-pytorch/src/cpuinfo
[submodule "third_party/cereal"]
	url = /build/python-pytorch/src/cereal

From the above:
cereal nervanagpu Catch2 are not used
QNNPACK fbgemm gemmlowp nccl neon2sse are not using local repositories
cpuinfo has a mismatch between third-party/cpuinfo and third_party/cpuinfo
pthreadpool has a mismatch between third_party/NNPACK_deps/pthreadpool and third_party/NNPACK_deps/pthread_ool

During the build

Submodule 'mkl-dnn' (https://github.com/01org/mkl-dnn.git) registered for path 'mkl-dnn'
Cloning into '/build/python-pytorch/src/pytorch-1.0.0/third_party/ideep/mkl-dnn'...
Submodule path 'mkl-dnn': checked out 'c0095ec26c215970b9b94915963b59e76398265e'
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   621    0   621    0     0   1881      0 --:--:-- --:--:-- --:--:--  1881
100 76.6M  100 76.6M    0     0  5299k      0  0:00:14  0:00:14 --:--:-- 6152k
Downloaded and unpacked Intel(R) MKL small libraries to /build/python-pytorch/src/pytorch-1.0.0/third_party/ideep/mkl-dnn/external

Edit:
Attempt to add all used sources to sources array

# Maintainer: Sven-Hendrik Haase <sh@lutzhaase.com>
# Contributor: Stephen Zhang <zsrkmyn at gmail dot com>

pkgbase="python-pytorch"
pkgname=("python-pytorch" "python-pytorch-cuda")
_pkgname="pytorch"
pkgver=1.0.0
pkgrel=2
pkgdesc="Tensors and Dynamic neural networks in Python with strong GPU acceleration"
arch=('x86_64')
url="https://pytorch.org"
license=('BSD')
depends=('python' 'python-yaml' 'python-numpy' 'opencv' 'nccl')
makedepends=('python' 'python-setuptools' 'python-yaml' 'python-numpy' 'cmake' 'cuda' 'cudnn' 'git')
source=("${_pkgname}-${pkgver}::git+https://github.com/pytorch/pytorch.git#tag=v$pkgver"
        "git+https://github.com/pybind/pybind11"
        "git+https://github.com/NVlabs/cub"
        "git+https://github.com/eigenteam/eigen-git-mirror"
        "git+https://github.com/google/googletest"
        "git+https://github.com/google/benchmark"
        "git+https://github.com/google/protobuf"
        "git+https://github.com/Yangqing/ios-cmake"
        "git+https://github.com/Maratyszcza/NNPACK"
        "git+https://github.com/facebookincubator/gloo"
        "git+https://github.com/Maratyszcza/pthreadpool"
        "git+https://github.com/Maratyszcza/FXdiv"
        "git+https://github.com/Maratyszcza/FP16"
        "git+https://github.com/Maratyszcza/psimd"
        "git+https://github.com/facebook/zstd"
        "git+https://github.com/Maratyszcza/cpuinfo"
        "git+https://github.com/PeachPy/enum34"
        "git+https://github.com/Maratyszcza/PeachPy"
        "git+https://github.com/benjaminp/six"
        "git+https://github.com/ARM-software/ComputeLibrary"
        "git+https://github.com/onnx/onnx"
        "git+https://github.com/onnx/onnx-tensorrt"
        "git+https://github.com/shibatch/sleef"
        "git+https://github.com/intel/ideep"
        "git+https://github.com/pytorch/QNNPACK"
        "git+https://github.com/pytorch/fbgemm"
        "git+https://github.com/google/gemmlowp"
        "git+https://github.com/intel/ARM_NEON_2_x86_SSE"
        "git+https://github.com/01org/mkl-dnn"
        https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz)
sha256sums=('SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'f00dc3b142a5be399bdeebd7e7ea369545a35d4fb84c86f98b6b048d72685295')

prepare() {
  cd "${_pkgname}-${pkgver}"

  git submodule init
  git config submodule."third_party/pybind11".url "${srcdir}"/pybind11
  git config submodule."third_party/cub".url "${srcdir}"/cub
  git config submodule."third_party/eigen".url "${srcdir}"/eigen-git-mirror
  git config submodule."third_party/googletest".url "${srcdir}"/googletest
  git config submodule."third_party/benchmark".url "${srcdir}"/benchmark
  git config submodule."third_party/protobuf".url "${srcdir}"/protobuf
  git config submodule."third_party/ios-cmake".url "${srcdir}"/ios-cmake
  git config submodule."third_party/NNPACK".url "${srcdir}"/NNPACK
  git config submodule."third_party/gloo".url "${srcdir}"/gloo
  git config submodule."third_party/NNPACK_deps/pthreadpool".url "${srcdir}"/pthreadpool
  git config submodule."third_party/NNPACK_deps/FXdiv".url "${srcdir}"/FXdiv
  git config submodule."third_party/NNPACK_deps/FP16".url "${srcdir}"/FP16
  git config submodule."third_party/NNPACK_deps/psimd".url "${srcdir}"/psimd
  git config submodule."third_party/zstd".url "${srcdir}"/zstd
  git config submodule."third-party/cpuinfo".url "${srcdir}"/cpuinfo
  git config submodule."third_party/python-enum".url "${srcdir}"/enum34
  git config submodule."third_party/python-peachpy".url "${srcdir}"/PeachPy
  git config submodule."third_party/python-six".url "${srcdir}"/six
  git config submodule."third_party/ComputeLibrary".url "${srcdir}"/ComputeLibrary
  git config submodule."third_party/onnx".url "${srcdir}"/onnx
  git config submodule."third_party/onnx-tensorrt".url "${srcdir}"/onnx-tensorrt
  git config submodule."third_party/sleef".url "${srcdir}"/sleef
  git config submodule."third_party/ideep".url "${srcdir}"/ideep
  git config submodule."third_party/QNNPACK".url "${srcdir}"/QNNPACK
  git config submodule."third_party/fbgemm".url "${srcdir}"/fbgemm
  git config submodule."third_party/gemmlowp/gemmlowp".url "${srcdir}"/gemmlowp
  git config submodule."third_party/neon2sse".url "${srcdir}"/ARM_NEON_2_x86_SSE
  git submodule update
  cd third_party/ideep
  git submodule init mkl-dnn
  git config submodule."mkl-dnn".url "${srcdir}"/mkl-dnn
  git submodule update
  cd ../..
  cp -r "${srcdir}"/mklml_lnx_2019.0.1.20180928 third_party/ideep/mkl-dnn/external

  cd ..

  cp -a "${_pkgname}-${pkgver}" "${_pkgname}-${pkgver}-cuda"
}

build() {
  # Uncomment and modify the following line to enable Intel MKL and magma support
  # export CMAKE_PREFIX_PATH=/opt/intel/mkl/include:/opt/intel/mkl/lib/intel64:/opt/magma \
  export CC=gcc-7
  export CXX=g++-7

  echo "Building without cuda"
  export NO_CUDA=1
  export WITH_CUDNN=0
  export USE_OPENCV=1
  export BUILD_BINARY=1

  cd "$srcdir/${_pkgname}-${pkgver}"
  python setup.py build

  echo "Building with cuda"
  export NO_CUDA=0
  export WITH_CUDNN=1
  export CUDAHOSTCXX=g++-7
  export CUDA_HOME=/opt/cuda
  export CUDNN_LIB_DIR=/opt/cuda/lib64
  export CUDNN_INCLUDE_DIR=/opt/cuda/include
  export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
  export TORCH_CUDA_ARCH_LIST="3.0;3.2;3.5;3.7;5.0;5.2;5.3;6.0;6.1;6.2;7.0;7.2;7.5"

  cd "$srcdir/${_pkgname}-${pkgver}-cuda"
  python setup.py build
}

package_python-pytorch() {
  cd "$srcdir/${_pkgname}-${pkgver}"
  python setup.py install --root="$pkgdir"/ --optimize=1 --skip-build
  install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE.txt"
}

package_python-pytorch-cuda() {
  depends+=('cuda' 'cudnn')
  provides=('python-pytorch')
  conflicts=('python-pytorch')
  cd "$srcdir/${_pkgname}-${pkgver}-cuda"
  python setup.py install --root="$pkgdir"/ --optimize=1 --skip-build
  install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE.txt"
}

# vim:set ts=2 sw=2 et:

Last edited by loqs (2018-12-14 18:47:05)

Offline

Pages: 1

Board footer

Powered by FluxBB