The full install log after running python setup.py clean and then git reset --hard, is here: https://drive.google.com/file/d/196biV5V4zmhK2tDcZIC8z8B3OlrTI7Cc/view
Here are it’s last few lines:
[4754/4978] Building CXX object test_api/CMakeFiles/test_api.dir/tensor_options.cpp.o
[4755/4978] Building CXX object test_api/CMakeFiles/test_api.dir/parallel.cpp.o
[4756/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Layout.cpp.o
[4757/4978] Building CXX object caffe2/lib_c10d/CMakeFiles/c10d.dir/ProcessGroupGloo.cpp.o
[4758/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/MemoryFormat.cpp.o
[4759/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/QScheme.cpp.o
[4760/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/DataLoader.cpp.o
FAILED: caffe2/torch/CMakeFiles/torch_python.dir/csrc/DataLoader.cpp.o
/opt/rh/devtoolset-6/root/usr/bin/c++ -DBUILDING_TESTS -DFMT_HEADER_ONLY=1 -DHAVE_MALLOC_USABLE_SIZE=1 -DHAVE_MMAP=1 -DHAVE_SHM_OPEN=1 -DHAVE_SHM_UNLINK=1 -DIDEEP_USE_MKL -DMAGMA_V2 -DMINIZ_DISABLE_ZIP_READER_CRC32_CHECKS -DONNXIFI_ENABLE_EXT=1 -DONNX_ML=1 -DONNX_NAMESPACE=onnx_torch -DTHP_BUILD_MAIN_LIB -DUSE_C10D -DUSE_C10D_GLOO -DUSE_C10D_NCCL -DUSE_CUDA -DUSE_CUDNN -DUSE_DISTRIBUTED -DUSE_EXTERNAL_MZCRC -DUSE_NCCL -DUSE_NUMPY -DUSE_TENSORPIPE -D_FILE_OFFSET_BITS=64 -Dtorch_python_EXPORTS -Iaten/src -I../aten/src -I. -I../ -I../cmake/../third_party/benchmark/include -Icaffe2/contrib/aten -I../third_party/onnx -Ithird_party/onnx -I../third_party/foxi -Ithird_party/foxi -I../torch/.. -I../torch/../aten/src -I../torch/../aten/src/TH -Icaffe2/aten/src -Ithird_party -I../torch/../third_party/gloo -I../torch/../third_party/onnx -I../torch/csrc -I../torch/csrc/api/include -I../torch/lib -I../torch/lib/libshm -I../torch/csrc/api -I../c10/.. -Ithird_party/ideep/mkl-dnn/include -I../third_party/ideep/mkl-dnn/src/../include -I../c10/cuda/../.. -I../torch/lib/libshm/../../../torch/lib -I../third_party/fmt/include -I../torch/lib/c10d/.. -I../torch/lib/c10d/../../.. -Inccl/include -I../third_party/tensorpipe -Ithird_party/tensorpipe -isystem third_party/gloo -isystem ../cmake/../third_party/gloo -isystem ../cmake/../third_party/googletest/googlemock/include -isystem ../cmake/../third_party/googletest/googletest/include -isystem ../third_party/protobuf/src -isystem /opt/OpenBLAS/include -isystem ../third_party/gemmlowp -isystem ../third_party/neon2sse -isystem ../third_party/XNNPACK/include -isystem ../third_party -isystem ../cmake/../third_party/eigen -isystem /u/drspeech/opt/anaconda3-5.3.0/include/python3.7m -isystem /u/drspeech/opt/anaconda3-5.3.0/lib/python3.7/site-packages/numpy/core/include -isystem ../cmake/../third_party/pybind11/include -isystem ../cmake/../third_party/cub -isystem /homes/3/serai/.conda/envs/speechb/include -isystem ../third_party/ideep/mkl-dnn/include -isystem ../third_party/ideep/include -isystem /usr/local/cuda/include -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DUSE_VULKAN_WRAPPER -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-unused-local-typedefs -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -DHAVE_AVX_CPU_DEFINITION -DHAVE_AVX2_CPU_DEFINITION -O3 -DNDEBUG -DNDEBUG -fPIC -DCAFFE2_USE_GLOO -DCUDA_HAS_FP16=1 -DHAVE_GCC_GET_CPUID -DUSE_AVX -DUSE_AVX2 -DTH_HAVE_THREAD -fno-strict-aliasing -Wno-write-strings -Wno-strict-aliasing -pthread -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -Wno-unknown-pragmas -std=gnu++14 -MD -MT caffe2/torch/CMakeFiles/torch_python.dir/csrc/DataLoader.cpp.o -MF caffe2/torch/CMakeFiles/torch_python.dir/csrc/DataLoader.cpp.o.d -o caffe2/torch/CMakeFiles/torch_python.dir/csrc/DataLoader.cpp.o -c ../torch/csrc/DataLoader.cpp
../torch/csrc/DataLoader.cpp: In function ‘PyObject* THPModule_removeWorkerPIDs(PyObject*, PyObject*)’:
../torch/csrc/DataLoader.cpp:185:21: error: expected primary-expression before ‘(’ token
throw ValueError("Cannot find worker information for _BaseDataLoaderIter with id %" PRId64, key);
^
../torch/csrc/DataLoader.cpp:185:89: error: expected ‘)’ before ‘PRId64’
throw ValueError("Cannot find worker information for _BaseDataLoaderIter with id %" PRId64, key);
^~~~~~
[4761/4978] Linking CXX static library lib/libc10d.a
[4762/4978] Building CXX object dist_autograd/CMakeFiles/test_dist_autograd.dir/test_dist_autograd.cpp.o
[4763/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/PtrWrapper.cpp.o
[4764/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/autograd/generated/python_fft_functions.cpp.o
[4765/4978] Building CXX object test_api/CMakeFiles/test_api.dir/tensor_indexing.cpp.o
[4766/4978] Building CXX object test_api/CMakeFiles/test_api.dir/serialize.cpp.o
[4767/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Dtype.cpp.o
[4768/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/python_dimname.cpp.o
[4769/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/autograd/generated/python_linalg_functions.cpp.o
[4770/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/DynamicTypes.cpp.o
[4771/4978] Building CXX object test_api/CMakeFiles/test_api.dir/optim.cpp.o
[4772/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Device.cpp.o
[4773/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Exceptions.cpp.o
[4774/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Generator.cpp.o
[4775/4978] Building CXX object test_api/CMakeFiles/test_api.dir/tensor.cpp.o
In file included from ../cmake/../third_party/googletest/googletest/include/gtest/gtest.h:59:0,
from ../test/cpp/api/tensor.cpp:1:
../test/cpp/api/tensor.cpp: In function ‘void test_TorchTensorCtorSingleDimFloatingType_expected_dtype(c10::ScalarType)’:
../test/cpp/api/tensor.cpp:427:34: warning: ‘bool at::Tensor::is_variable() const’ is deprecated: Tensor.is_variable() is deprecated; everything is a variable now. (If you want to assert that variable has been appropriately handled already, use at::impl::variable_excluded_from_dispatch()) [-Wdeprecated-declarations]
ASSERT_TRUE(tensor.is_variable());
^
In file included from ../aten/src/ATen/TensorIndexing.h:4:0,
from ../test/cpp/api/support.h:8,
from ../test/cpp/api/tensor.cpp:2:
aten/src/ATen/core/TensorBody.h:300:8: note: declared here
bool is_variable() const noexcept {
^~~~~~~~~~~
In file included from ../cmake/../third_party/googletest/googletest/include/gtest/gtest.h:59:0,
from ../test/cpp/api/tensor.cpp:1:
../test/cpp/api/tensor.cpp:436:34: warning: ‘bool at::Tensor::is_variable() const’ is deprecated: Tensor.is_variable() is deprecated; everything is a variable now. (If you want to assert that variable has been appropriately handled already, use at::impl::variable_excluded_from_dispatch()) [-Wdeprecated-declarations]
ASSERT_TRUE(tensor.is_variable());
^
In file included from ../aten/src/ATen/TensorIndexing.h:4:0,
from ../test/cpp/api/support.h:8,
from ../test/cpp/api/tensor.cpp:2:
aten/src/ATen/core/TensorBody.h:300:8: note: declared here
bool is_variable() const noexcept {
^~~~~~~~~~~
In file included from ../cmake/../third_party/googletest/googletest/include/gtest/gtest.h:59:0,
from ../test/cpp/api/tensor.cpp:1:
../test/cpp/api/tensor.cpp:444:34: warning: ‘bool at::Tensor::is_variable() const’ is deprecated: Tensor.is_variable() is deprecated; everything is a variable now. (If you want to assert that variable has been appropriately handled already, use at::impl::variable_excluded_from_dispatch()) [-Wdeprecated-declarations]
ASSERT_TRUE(tensor.is_variable());
^
In file included from ../aten/src/ATen/TensorIndexing.h:4:0,
from ../test/cpp/api/support.h:8,
from ../test/cpp/api/tensor.cpp:2:
aten/src/ATen/core/TensorBody.h:300:8: note: declared here
bool is_variable() const noexcept {
^~~~~~~~~~~
[4776/4978] Building CXX object test_api/CMakeFiles/test_api.dir/sequential.cpp.o
[4777/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/autograd/generated/python_nn_functions.cpp.o
[4778/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Size.cpp.o
[4779/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Module.cpp.o
[4780/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/Storage.cpp.o
[4781/4978] Building CXX object test_api/CMakeFiles/test_api.dir/modules.cpp.o
[4782/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/autograd/generated/python_variable_methods.cpp.o
[4783/4978] Building CXX object caffe2/torch/CMakeFiles/torch_python.dir/csrc/autograd/generated/python_torch_functions.cpp.o
ninja: build stopped: subcommand failed.
Building wheel torch-1.7.0a0+f02753f
-- Building version 1.7.0a0+f02753f
cmake -GNinja -DBLAS=OpenBLAS -DBUILD_PYTHON=True -DBUILD_TEST=True -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/data/data24/scratch/serai/pytorch/torch -DCMAKE_PREFIX_PATH=/homes/3/serai/.conda/envs/speechb -DNUMPY_INCLUDE_DIR=/u/drspeech/opt/anaconda3-5.3.0/lib/python3.7/site-packages/numpy/core/include -DPYTHON_EXECUTABLE=/u/drspeech/opt/anaconda3-5.3.0/bin/python -DPYTHON_INCLUDE_DIR=/u/drspeech/opt/anaconda3-5.3.0/include/python3.7m -DPYTHON_LIBRARY=/u/drspeech/opt/anaconda3-5.3.0/lib/libpython3.7m.so.1.0 -DTORCH_BUILD_VERSION=1.7.0a0+f02753f -DUSE_NUMPY=True /data/data24/scratch/serai/pytorch
cmake --build . --target install --config Release -- -j 24
Traceback (most recent call last):
File "setup.py", line 737, in <module>
build_deps()
File "setup.py", line 321, in build_deps
cmake=cmake)
File "/data/data24/scratch/serai/pytorch/tools/build_pytorch_libs.py", line 62, in build_caffe2
cmake.build(my_env)
File "/data/data24/scratch/serai/pytorch/tools/setup_helpers/cmake.py", line 345, in build
self.run(build_args, my_env)
File "/data/data24/scratch/serai/pytorch/tools/setup_helpers/cmake.py", line 141, in run
check_call(command, cwd=self.build_dir, env=env)
File "/u/drspeech/opt/anaconda3-5.3.0/lib/python3.7/subprocess.py", line 328, in check_call
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--', '-j', '24']' returned non-zero exit status 1.
AND here is the summary part from the beginning
-- ******** Summary ********
-- General:
-- CMake version : 3.18.2
-- CMake command : /homes/3/serai/.conda/envs/speechb/bin/cmake
-- System : Linux
-- C++ compiler : /opt/rh/devtoolset-6/root/usr/bin/c++
-- C++ compiler id : GNU
-- C++ compiler version : 6.3.1
-- BLAS : OpenBLAS
-- CXX flags : -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DUSE_VULKAN_WRAPPER -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-unused-local-typedefs -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format
-- Build type : Release
-- Compile definitions : ONNX_ML=1;ONNXIFI_ENABLE_EXT=1;ONNX_NAMESPACE=onnx_torch;MAGMA_V2;IDEEP_USE_MKL;HAVE_MMAP=1;_FILE_OFFSET_BITS=64;HAVE_SHM_OPEN=1;HAVE_SHM_UNLINK=1;HAVE_MALLOC_USABLE_SIZE=1;USE_EXTERNAL_MZCRC;MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS
-- CMAKE_PREFIX_PATH : /homes/3/serai/.conda/envs/speechb;/usr/local/cuda
-- CMAKE_INSTALL_PREFIX : /data/data24/scratch/serai/pytorch/torch
--
-- TORCH_VERSION : 1.7.0
-- CAFFE2_VERSION : 1.7.0
-- BUILD_CAFFE2_MOBILE : OFF
-- USE_STATIC_DISPATCH : OFF
-- BUILD_BINARY : OFF
-- BUILD_CUSTOM_PROTOBUF : ON
-- Link local protobuf : ON
-- BUILD_DOCS : OFF
-- BUILD_PYTHON : True
-- Python version : 3.7
-- Python executable : /u/drspeech/opt/anaconda3-5.3.0/bin/python
-- Pythonlibs version : 3.7.0
-- Python library : /u/drspeech/opt/anaconda3-5.3.0/lib/libpython3.7m.so.1.0
-- Python includes : /u/drspeech/opt/anaconda3-5.3.0/include/python3.7m
-- Python site-packages: lib/python3.7/site-packages
-- BUILD_CAFFE2_OPS : ON
-- BUILD_SHARED_LIBS : ON
-- BUILD_TEST : True
-- BUILD_JNI : OFF
-- BUILD_MOBILE_AUTOGRAD : OFF
-- INTERN_BUILD_MOBILE :
-- CODE_COVERAGE : OFF
-- USE_ASAN : OFF
-- USE_CUDA : ON
-- CUDA static link : OFF
-- USE_CUDNN : ON
-- CUDA version : 10.2
-- cuDNN version : 7.6.5
-- CUDA root directory : /usr/local/cuda
-- CUDA library : /usr/local/cuda/lib64/stubs/libcuda.so
-- cudart library : /usr/local/cuda/lib64/libcudart.so
-- cublas library : /homes/3/serai/.conda/envs/speechb/lib/libcublas.so
-- cufft library : /usr/local/cuda/lib64/libcufft.so
-- curand library : /usr/local/cuda/lib64/libcurand.so
-- cuDNN library : /homes/3/serai/.conda/envs/speechb/lib/libcudnn.so
-- nvrtc : /homes/3/serai/.conda/envs/speechb/lib/libnvrtc.so
-- CUDA include path : /usr/local/cuda/include
-- NVCC executable : /usr/local/cuda/bin/nvcc
-- NVCC flags : -Xfatbin;-compress-all;-DONNX_NAMESPACE=onnx_torch;-gencode;arch=compute_61,code=sm_61;-Xcudafe;--diag_suppress=cc_clobber_ignored;-Xcudafe;--diag_suppress=integer_sign_change;-Xcudafe;--diag_suppress=useless_using_declaration;-Xcudafe;--diag_suppress=set_but_not_used;-Xcudafe;--diag_suppress=field_without_dll_interface;-Xcudafe;--diag_suppress=base_class_has_different_dll_interface;-Xcudafe;--diag_suppress=dll_interface_conflict_none_assumed;-Xcudafe;--diag_suppress=dll_interface_conflict_dllexport_assumed;-Xcudafe;--diag_suppress=implicit_return_from_non_void_function;-Xcudafe;--diag_suppress=unsigned_compare_with_zero;-Xcudafe;--diag_suppress=declared_but_not_referenced;-Xcudafe;--diag_suppress=bad_friend_decl;-std=c++14;-Xcompiler;-fPIC;--expt-relaxed-constexpr;--expt-extended-lambda;-Wno-deprecated-gpu-targets;--expt-extended-lambda;-gencode;arch=compute_61,code=sm_61;-Xcompiler;-fPIC;-DCUDA_HAS_FP16=1;-D__CUDA_NO_HALF_OPERATORS__;-D__CUDA_NO_HALF_CONVERSIONS__;-D__CUDA_NO_HALF2_OPERATORS__
-- CUDA host compiler : /opt/rh/devtoolset-6/root/usr/bin/cc
-- NVCC --device-c : OFF
-- USE_TENSORRT : OFF
-- USE_ROCM : OFF
-- USE_EIGEN_FOR_BLAS :
-- USE_FBGEMM : ON
-- USE_FAKELOWP : OFF
-- USE_FFMPEG : OFF
-- USE_GFLAGS : OFF
-- USE_GLOG : OFF
-- USE_LEVELDB : OFF
-- USE_LITE_PROTO : OFF
-- USE_LMDB : OFF
-- USE_METAL : OFF
-- USE_MKL :
-- USE_MKLDNN : ON
-- USE_MKLDNN_CBLAS : OFF
-- USE_NCCL : ON
-- USE_SYSTEM_NCCL : OFF
-- USE_NNPACK : ON
-- USE_NUMPY : ON
-- USE_OBSERVERS : ON
-- USE_OPENCL : OFF
-- USE_OPENCV : OFF
-- USE_OPENMP : ON
-- USE_TBB : OFF
-- USE_VULKAN : OFF
-- USE_PROF : OFF
-- USE_QNNPACK : ON
-- USE_PYTORCH_QNNPACK : ON
-- USE_REDIS : OFF
-- USE_ROCKSDB : OFF
-- USE_ZMQ : OFF
-- USE_DISTRIBUTED : ON
-- USE_MPI : OFF
-- USE_GLOO : ON
-- USE_TENSORPIPE : ON
-- Public Dependencies : Threads::Threads;/usr/lib64/libopenblas.so;caffe2::mkldnn
-- Private Dependencies : pthreadpool;cpuinfo;qnnpack;pytorch_qnnpack;nnpack;XNNPACK;fbgemm;fp16;gloo;tensorpipe;aten_op_header_gen;foxi_loader;rt;fmt::fmt-header-only;gcc_s;gcc;dl
-- Configuring done