Firstly this is my environment:
- one RTX2060 card;
- Debian buster;
- CUDA 9.2 driver installed from Debian’s apt repository.
- Python 3.6 and pytorch installed by anaconda
- I have tried a tiny GPU run in pytorch and it works.
The problem is: the network works on CPU, but when I try to put it on GPU, it claims:
Traceback (most recent call last):
File "./do_training.py", line 95, in <module>
batch_output = model(batch_input)
File "/opt/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
result = self.forward(*input, **kwargs)
File "/large/yangxi/MonotoneLearn/train_for_pitch/MyModel.py", line 32, in forward
x = self.full_part.forward(x)
File "/opt/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py", line 92, in forward
input = module(input)
File "/opt/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
result = self.forward(*input, **kwargs)
File "/opt/anaconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 92, in forward
return F.linear(input, self.weight, self.bias)
File "/opt/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py", line 1406, in linear
ret = torch.addmm(bias, input, weight.t())
RuntimeError: cublas runtime error : the GPU program failed to execute at /opt/conda/conda-bld/pytorch_1556653183467/work/aten/src/THC/THCBlas.cu:259
This error can appear when tensor dimension mismatch. But the whole thing works on CPU, so it won’t have unmatched tensor size.
This is my source code:
do_training.py
#!/opt/anaconda3/bin/python3
import pysndfile
import optparse
import sys
import os
import re
import math
import random
import numpy
import fnmatch
import h5py
import torch
import MyModel
# parse options
opt_parser = optparse.OptionParser()
opt_parser.add_option("-d", "--data", dest="f_data", help="HDF5 file contains training data, with \"input\" and \"refout\" dataset.", metavar="FILE")
opt_parser.add_option("-i", "--in", dest="f_in", help="Existing network file to continue training.", metavar="FILE")
opt_parser.add_option("-c", "--conv-layers", dest="n_conv_layers", help="Number of 3-length convolution layers for creating new network.", type="int", metavar="NUM_LAYER")
opt_parser.add_option("-f", "--full-layers", dest="n_full_layers", help="Number of fully connected layers for creating new network.", type="int", metavar="NUM_LAYER")
opt_parser.add_option("-o", "--out", dest="f_out", help="Output file to store trained network.", metavar="FILE")
opt_parser.add_option("-t", "--iter", dest="n_iter", help="Number of training iterations to perform.", type="int", metavar="NUM_ITER", default=1000)
opt_parser.add_option("-b", "--batch", dest="batch_sz", help="Size of each training iteration.", type="int", metavar="BATCH_SIZE", default=10)
opt_parser.add_option("-D", "--device", dest="dev_type", help="Device. Default to CUDA when available, or CPU when no GPU is available.")
options,_ = opt_parser.parse_args()
if options.f_data is None:
print("data file is not specified")
exit(-1)
if options.f_in is None and options.n_full_layers is None:
print("neither input file nor layer number is specified")
exit(-1)
if options.f_out is None:
print("output file is not specified")
exit(-1)
# determine device
if options.dev_type is None:
if torch.cuda.is_available():
device = torch.device('cuda:0')
else:
device = torch.device('cpu')
else:
device = torch.device(options.dev_type)
# read data properties
fh_data = h5py.File(options.f_data, 'r')
if "input" not in fh_data:
print("no input in data file %s" % options.f_data)
exit(-1)
if "refout" not in fh_data:
print("no refout in data file %s" % options.f_data)
exit(-1)
if len(fh_data["input"]) != len(fh_data["refout"]):
print("inequal number of data: input %d, ref output %d" % (len(fh_data["input"]), len(fh_data["refout"])))
exit(-1)
num_data = len(fh_data["input"])
input_size = len(fh_data["input"][0])
whole_input = torch.from_numpy(fh_data["input"].__array__()).to(device)
whole_refout = torch.from_numpy(fh_data["refout"].__array__()).to(device)
print(whole_input)
print(whole_refout)
# create model
model = None
if options.f_in is None:
model = MyModel.MyModel(input_size, options.n_conv_layers, options.n_full_layers).to(device)
else:
model = torch.load(options.f_in).to(device)
print(model)
print(model.parameters())
# do training
#fcri = torch.nn.NLLLoss()
cri = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.SGD(model.parameters(), lr = 0.01)
batch_input = torch.empty( (options.batch_sz, input_size), dtype=torch.float32, device=device )
batch_refout = torch.empty( options.batch_sz, dtype=torch.long, device=device )
for i in range(options.n_iter):
try:
optimizer.zero_grad()
for j in range(options.batch_sz):
i_data = random.randint(0, num_data-1)
batch_input[j] = whole_input[i_data]
batch_refout[j] = whole_refout[i_data]
batch_output = model(batch_input)
loss = cri(batch_output, batch_refout)
if i % 100 == 0:
print( "iter %d: loss %f" % (i, float(loss)) )
loss.backward()
optimizer.step()
except KeyboardInterrupt:
print("early out by int at iter %d" % i)
break
# save result
torch.save(model, options.f_out)
MyModel.py:
import torch
import collections
class MyModel(torch.nn.Module):
def __init__(self, input_sz, n_conv_layer, n_full_layer):
super(MyModel, self).__init__()
self.input_size = input_sz
conv_layers = collections.OrderedDict()
for i in range(n_conv_layer):
conv_name = "conv_%d" % i
conv_layers[conv_name] = torch.nn.Conv1d(1,1,3,padding=1)
self.conv_part = torch.nn.modules.Sequential(conv_layers)
full_layers = collections.OrderedDict()
for i in range(n_full_layer-1):
full_conn_name = "full_conn_%d" % i
non_linear_name = "acti_%d" % i
full_layers[full_conn_name] = torch.nn.Linear(input_sz,input_sz)
full_layers[non_linear_name] = torch.nn.Tanh()
full_layers["last_full_conn"] = torch.nn.Linear(input_sz, 128)
#full_layers["last_acti"] = torch.nn.LogSoftmax()
self.full_part = torch.nn.modules.Sequential(full_layers)
def forward(self, x):
print("input shape ",x.size())
batch_sz = len(x)
x = x.view(batch_sz, 1, self.input_size)
x = self.conv_part.forward(x)
print("convolution part output shape ",x.size())
x = x.view(batch_sz, self.input_size)
print("full connect part input shape ",x.size())
x = self.full_part.forward(x)
return x
I have put some logs on forward function. Before it is dead, I can see outputs:
input shape torch.Size([10, 1025])
convolution part output shape torch.Size([10, 1, 1025])
full connect part input shape torch.Size([10, 1025])
Which is expected.
