Cuda error with Pascal 100?

Lifeng_Jin · March 4, 2017, 8:41am

I am running the same code on two different machines, one with Titan X and CUDA 7.5 and the other with Pascal 100 and CUDA 8.0 and I am seeing this error with the Pascal setup:

  File "./python2.7/site-packages/torch/autograd/variable.py", line 145, in backward
    self._execution_engine.run_backward((self,), (gradient,), retain_variables)
  File "./python2.7/site-packages/torch/nn/_functions/linear.py", line 22, in backward
    grad_input = torch.mm(grad_output, weight)
RuntimeError: cublas runtime error : the GPU program failed to execute at /data/users/soumith/miniconda2/conda-bld/pytorch-0.1.9_1487343590888/work/torch/lib/THC/THCBlas.cu:246

smth · March 4, 2017, 2:06pm

I’m not fully sure why a cublas error occurs. Do other CUDA programs run on the Pascal system?
Also, can you try installing from source: https://github.com/pytorch/pytorch#from-source

Lifeng_Jin · March 5, 2017, 3:13am

I was trying to install from source on Mac and this error happens when compiling:

CMake Warning (dev):
  Policy CMP0042 is not set: MACOSX_RPATH is enabled by default.  Run "cmake
  --help-policy CMP0042" for policy details.  Use the cmake_policy command to
  set the policy and suppress this warning.

  MACOSX_RPATH is not specified for the following targets:

   THPP

This warning is for project developers.  Use -Wno-dev to suppress it.

-- Generating done
-- Build files have been written to: /whatever/pytorch/torch/lib/build/THPP
[ 11%] Linking CXX shared library libTHPP.dylib
Undefined symbols for architecture x86_64:
  "_THCSByteTensor_cadd", referenced from:
      thpp::THCSTensor<unsigned char>::cadd(thpp::Tensor const&, long long, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSByteTensor_cmul", referenced from:
      thpp::THCSTensor<unsigned char>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSByteTensor_free", referenced from:
      thpp::THCSTensor<unsigned char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::free() in THCSTensor.cpp.o
  "_THCSByteTensor_new", referenced from:
      thpp::THCSTensor<unsigned char>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::newTensor() const in THCSTensor.cpp.o
  "_THCSByteTensor_newClone", referenced from:
      thpp::THCSTensor<unsigned char>::clone() const in THCSTensor.cpp.o
  "_THCSByteTensor_newContiguous", referenced from:
      thpp::THCSTensor<unsigned char>::contiguous() const in THCSTensor.cpp.o
  "_THCSByteTensor_retain", referenced from:
      thpp::THCSTensor<unsigned char>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<unsigned char>::retain() in THCSTensor.cpp.o
  "_THCSCharTensor_cadd", referenced from:
      thpp::THCSTensor<char>::cadd(thpp::Tensor const&, long long, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSCharTensor_cmul", referenced from:
      thpp::THCSTensor<char>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSCharTensor_free", referenced from:
      thpp::THCSTensor<char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<char>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<char>::free() in THCSTensor.cpp.o
  "_THCSCharTensor_new", referenced from:
      thpp::THCSTensor<char>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<char>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<char>::newTensor() const in THCSTensor.cpp.o
  "_THCSCharTensor_newClone", referenced from:
      thpp::THCSTensor<char>::clone() const in THCSTensor.cpp.o
  "_THCSCharTensor_newContiguous", referenced from:
      thpp::THCSTensor<char>::contiguous() const in THCSTensor.cpp.o
  "_THCSCharTensor_retain", referenced from:
      thpp::THCSTensor<char>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<char>::retain() in THCSTensor.cpp.o
  "_THCSDoubleTensor_cadd", referenced from:
      thpp::THCSTensor<double>::cadd(thpp::Tensor const&, double, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSDoubleTensor_cmul", referenced from:
      thpp::THCSTensor<double>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSDoubleTensor_free", referenced from:
      thpp::THCSTensor<double>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<double>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<double>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<double>::free() in THCSTensor.cpp.o
  "_THCSDoubleTensor_new", referenced from:
      thpp::THCSTensor<double>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<double>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<double>::newTensor() const in THCSTensor.cpp.o
  "_THCSDoubleTensor_newClone", referenced from:
      thpp::THCSTensor<double>::clone() const in THCSTensor.cpp.o
  "_THCSDoubleTensor_newContiguous", referenced from:
      thpp::THCSTensor<double>::contiguous() const in THCSTensor.cpp.o
  "_THCSDoubleTensor_retain", referenced from:
      thpp::THCSTensor<double>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<double>::retain() in THCSTensor.cpp.o
  "_THCSFloatTensor_cadd", referenced from:
      thpp::THCSTensor<float>::cadd(thpp::Tensor const&, double, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSFloatTensor_cmul", referenced from:
      thpp::THCSTensor<float>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSFloatTensor_free", referenced from:
      thpp::THCSTensor<float>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<float>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<float>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<float>::free() in THCSTensor.cpp.o
  "_THCSFloatTensor_new", referenced from:
      thpp::THCSTensor<float>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<float>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<float>::newTensor() const in THCSTensor.cpp.o
  "_THCSFloatTensor_newClone", referenced from:
      thpp::THCSTensor<float>::clone() const in THCSTensor.cpp.o
  "_THCSFloatTensor_newContiguous", referenced from:
      thpp::THCSTensor<float>::contiguous() const in THCSTensor.cpp.o
  "_THCSFloatTensor_retain", referenced from:
      thpp::THCSTensor<float>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<float>::retain() in THCSTensor.cpp.o
  "_THCSHalfTensor_cadd", referenced from:
      thpp::THCSTensor<__half>::cadd(thpp::Tensor const&, double, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSHalfTensor_cmul", referenced from:
      thpp::THCSTensor<__half>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSHalfTensor_free", referenced from:
      thpp::THCSTensor<__half>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::free() in THCSTensor.cpp.o
  "_THCSHalfTensor_new", referenced from:
      thpp::THCSTensor<__half>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::newTensor() const in THCSTensor.cpp.o
  "_THCSHalfTensor_newClone", referenced from:
      thpp::THCSTensor<__half>::clone() const in THCSTensor.cpp.o
  "_THCSHalfTensor_newContiguous", referenced from:
      thpp::THCSTensor<__half>::contiguous() const in THCSTensor.cpp.o
  "_THCSHalfTensor_retain", referenced from:
      thpp::THCSTensor<__half>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<__half>::retain() in THCSTensor.cpp.o
  "_THCSIntTensor_cadd", referenced from:
      thpp::THCSTensor<int>::cadd(thpp::Tensor const&, long long, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSIntTensor_cmul", referenced from:
      thpp::THCSTensor<int>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSIntTensor_free", referenced from:
      thpp::THCSTensor<int>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<int>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<int>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<int>::free() in THCSTensor.cpp.o
  "_THCSIntTensor_new", referenced from:
      thpp::THCSTensor<int>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<int>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<int>::newTensor() const in THCSTensor.cpp.o
  "_THCSIntTensor_newClone", referenced from:
      thpp::THCSTensor<int>::clone() const in THCSTensor.cpp.o
  "_THCSIntTensor_newContiguous", referenced from:
      thpp::THCSTensor<int>::contiguous() const in THCSTensor.cpp.o
  "_THCSIntTensor_retain", referenced from:
      thpp::THCSTensor<int>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<int>::retain() in THCSTensor.cpp.o
  "_THCSLongTensor_cadd", referenced from:
      thpp::THCSTensor<long>::cadd(thpp::Tensor const&, long long, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSLongTensor_cmul", referenced from:
      thpp::THCSTensor<long>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSLongTensor_free", referenced from:
      thpp::THCSTensor<long>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<long>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<long>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<long>::free() in THCSTensor.cpp.o
  "_THCSLongTensor_new", referenced from:
      thpp::THCSTensor<long>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<long>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<long>::newTensor() const in THCSTensor.cpp.o
  "_THCSLongTensor_newClone", referenced from:
      thpp::THCSTensor<long>::clone() const in THCSTensor.cpp.o
  "_THCSLongTensor_newContiguous", referenced from:
      thpp::THCSTensor<long>::contiguous() const in THCSTensor.cpp.o
  "_THCSLongTensor_retain", referenced from:
      thpp::THCSTensor<long>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<long>::retain() in THCSTensor.cpp.o
  "_THCSShortTensor_cadd", referenced from:
      thpp::THCSTensor<short>::cadd(thpp::Tensor const&, long long, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSShortTensor_cmul", referenced from:
      thpp::THCSTensor<short>::cmul(thpp::Tensor const&, thpp::Tensor const&) in THCSTensor.cpp.o
  "_THCSShortTensor_free", referenced from:
      thpp::THCSTensor<short>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<short>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<short>::~THCSTensor() in THCSTensor.cpp.o
      thpp::THCSTensor<short>::free() in THCSTensor.cpp.o
  "_THCSShortTensor_new", referenced from:
      thpp::THCSTensor<short>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<short>::THCSTensor(THCState*) in THCSTensor.cpp.o
      thpp::THCSTensor<short>::newTensor() const in THCSTensor.cpp.o
  "_THCSShortTensor_newClone", referenced from:
      thpp::THCSTensor<short>::clone() const in THCSTensor.cpp.o
  "_THCSShortTensor_newContiguous", referenced from:
      thpp::THCSTensor<short>::contiguous() const in THCSTensor.cpp.o
  "_THCSShortTensor_retain", referenced from:
      thpp::THCSTensor<short>::clone_shallow() in THCSTensor.cpp.o
      thpp::THCSTensor<short>::retain() in THCSTensor.cpp.o
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
make[2]: *** [libTHPP.1.dylib] Error 1
make[1]: *** [CMakeFiles/THPP.dir/all] Error 2
make: *** [all] Error 2