My code runs fine on the cpu but when I try to run it on the GPU, I get the following stack trace:
Traceback (most recent call last):
File "run_experiments.py", line 123, in <module>
train_model(model, train_loader, nb_batches, optimizer, criterion, **vars(args))
File "run_experiments.py", line 54, in train_model
predictions, _ = model.forward(inputs, targets[:, :-1, :])
File "/net/if1/ab3cb/grad_stuff/vislang/project/Visual-Story-Telling/source_code/model_zoo/seq2seq.py", line 41, in forward
_, context_vec = self.encoder(inputs, hidden_init)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/nn/modules/module.py", line 206, in __call__
result = self.forward(*input, **kwargs)
File "/net/if1/ab3cb/grad_stuff/vislang/project/Visual-Story-Telling/source_code/model_zoo/encoder.py", line 36, in forward
output, hidden_state = self.gru(inputs, hidden_init)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/nn/modules/module.py", line 206, in __call__
result = self.forward(*input, **kwargs)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/nn/modules/rnn.py", line 91, in forward
output, hidden = func(input, self.all_weights, hx)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/nn/_functions/rnn.py", line 327, in forward
return func(input, *fargs, **fkwargs)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/autograd/function.py", line 202, in _do_forward
flat_output = super(NestedIOFunction, self)._do_forward(*flat_input)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/autograd/function.py", line 224, in forward
result = self.forward_extended(*nested_tensors)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/nn/_functions/rnn.py", line 269, in forward_extended
cudnn.rnn.forward(self, input, hx, weight, output, hy)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/backends/cudnn/rnn.py", line 239, in forward
fn.hx_desc = cudnn.descriptor(hx)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/backends/cudnn/__init__.py", line 304, in descriptor
descriptor.set(tensor)
File "/if1/ab3cb/miniconda3/envs/fast_torch/lib/python3.5/site-packages/torch/backends/cudnn/__init__.py", line 110, in set
self, _typemap[tensor.type()], tensor.dim(),
KeyError: 'torch.FloatTensor'
My training code is as follows:
def train_model(model, train_loader, nb_batches, optimizer, criterion, **kwargs):
running_loss = 0
for epoch in range(kwargs["epochs"]):
iters = 0
for inputs, targets in tqdm(train_loader, total=nb_batches):
#process the inputs from the data loader to make it compatible with
#the pytorch graph
inputs, targets = torch.from_numpy(inputs).float(), torch.from_numpy(targets).float()
#convert to cuda tensors if cuda flag is true
if torch.cuda.is_available:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets)
#clear out the gradients buffer
optimizer.zero_grad()
predictions, _ = model(inputs, targets[:, :-1, :])
loss = criterion(predictions, targets[:, 1:, :])
loss.backward()
optimizer.step()
running_loss += loss.data[0]
'''if iters % 10 == 0:
print("Loss at {} iteration: {}".format(iters+1, running_loss/(iters+1)))'''
if iters > nb_batches:
break
iters += 1
#define the model, optimizer and criterion
model = Seq2Seq(args.embed_size, args.hidden_size)
if torch.cuda.is_available():
model = model.cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr)
criterion = nn.KLDivLoss()
train_model(model, train_loader, nb_batches, optimizer, criterion, **vars(args))
I have the model definition in this gist. Additionally I recently upgraded my pytorch version due to slow loading of the GPU by following this topic. I have just started using pytorch on the gpu so any help in figuring this out would be appreciated.