How to make the Sine Wave example use CUDA?

I’ve changed the sine wave generator to use float32 to try to help compatibility:

import math
import numpy as np
import torch
T = 20
L = 1000
N = 100
np.random.seed(2)
x = np.empty((N, L), 'int64')
x[:] = np.array(range(L)) + np.random.randint(-4*T, 4*T, N).reshape(N, 1)
data = np.sin(x / 1.0 / T).astype('float32')
torch.save(data, open('traindata.pt', 'wb'))

I’ve tried .cuda()-ing everything I can think of but always get a type error. Here’s one of the tries.

import torch
import torch.nn as nn 
from torch import autograd
import torch.optim as optim
import numpy as np
import matplotlib
matplotlib.use('Agg')


class Variable(autograd.Variable):
    def __init__(self, data, *args, **kwargs):
        data = data.cuda()
        super(Variable, self).__init__(data, *args, **kwargs)

class Sequence(nn.Module):
    def __init__(self):
        super(Sequence, self).__init__()
        self.lstm1 = nn.LSTMCell(1, 51).cuda()
        self.lstm2 = nn.LSTMCell(51, 1).cuda()

    def forward(self, input, future = 0):
        input = input.cuda()
        outputs = []
        h_t = Variable(torch.zeros(input.size(0), 51), requires_grad=False)
        c_t = Variable(torch.zeros(input.size(0), 51), requires_grad=False)
        h_t2 = Variable(torch.zeros(input.size(0), 1), requires_grad=False)
        c_t2 = Variable(torch.zeros(input.size(0), 1), requires_grad=False)        
        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(c_t, (h_t2, c_t2))
            outputs += [c_t2]
        for i in range(future):# if we should predict the future
            h_t, c_t = self.lstm1(c_t2, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(c_t, (h_t2, c_t2))
            outputs += [c_t2]
        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs



if __name__ == '__main__':
    # set ramdom seed to 0
    np.random.seed(0)
    torch.manual_seed(0)
    # load data and make training set
    data = torch.load('traindata.pt')
    input = Variable(torch.from_numpy(data[3:, :-1]), requires_grad=False)
    input.cuda()
    print(type(input))
    target = Variable(torch.from_numpy(data[3:, 1:]), requires_grad=False).cuda()
    # build the model
    seq = Sequence()
    #seq.double()
    seq.cuda()
    criterion = nn.MSELoss()
    # use LBFGS as optimizer since we can load the whole data to train
    optimizer = optim.LBFGS(seq.parameters())
    #begin to train
    for i in range(10):
        print('STEP: ', i)
        def closure():
            optimizer.zero_grad()
            out = seq(input).cuda()
            loss = criterion(out, target)
            print('loss:', loss.data.numpy()[0])
            loss.backward()
            return loss
        optimizer.step(closure)
        # begin to predict
        future = 1000
        pred = seq(input[:3], future = future)
        y = pred.data.numpy()

The error is at optimizer.step(closure)

What’s missing?

what’s the exact stack-trace you are getting?

@smth Thanks for following up.

Here’s the trace:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-34-a3ea2c1c18d8> in <module>()
     68             loss.backward()
     69             return loss
---> 70         optimizer.step(closure)
     71         # begin to predict
     72         future = 1000

/usr/local/lib/python3.5/dist-packages/torch/optim/lbfgs.py in step(self, closure)
     98 
     99         # evaluate initial f(x) and df/dx
--> 100         orig_loss = closure()
    101         loss = orig_loss.data[0]
    102         current_evals = 1

<ipython-input-34-a3ea2c1c18d8> in closure()
     63         def closure():
     64             optimizer.zero_grad()
---> 65             out = seq(input).cuda()
     66             loss = criterion(out, target)
     67             print('loss:', loss.data.numpy()[0])

/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    204 
    205     def __call__(self, *input, **kwargs):
--> 206         result = self.forward(*input, **kwargs)
    207         for hook in self._forward_hooks.values():
    208             hook_result = hook(self, input, result)

<ipython-input-34-a3ea2c1c18d8> in forward(self, input, future)
     27 
     28         for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
---> 29             h_t, c_t = self.lstm1(input_t, (h_t, c_t))
     30             h_t2, c_t2 = self.lstm2(c_t, (h_t2, c_t2))
     31             outputs += [c_t2]

/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    204 
    205     def __call__(self, *input, **kwargs):
--> 206         result = self.forward(*input, **kwargs)
    207         for hook in self._forward_hooks.values():
    208             hook_result = hook(self, input, result)

/usr/local/lib/python3.5/dist-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    497             input, hx,
    498             self.weight_ih, self.weight_hh,
--> 499             self.bias_ih, self.bias_hh,
    500         )
    501 

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in LSTMCell(input, hidden, w_ih, w_hh, b_ih, b_hh)
     23     if input.is_cuda:
     24         igates = F.linear(input, w_ih)
---> 25         hgates = F.linear(hidden[0], w_hh)
     26         state = fusedBackend.LSTMFused()
     27         return state(igates, hgates, hidden[1]) if b_ih is None else state(igates, hgates, hidden[1], b_ih, b_hh)

/usr/local/lib/python3.5/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
    447 def linear(input, weight, bias=None):
    448     state = _functions.linear.Linear()
--> 449     return state(input, weight) if bias is None else state(input, weight, bias)
    450 
    451 

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/linear.py in forward(self, input, weight, bias)
      8         self.save_for_backward(input, weight, bias)
      9         output = input.new(input.size(0), weight.size(0))
---> 10         output.addmm_(0, 1, input, weight.t())
     11         if bias is not None:
     12             # cuBLAS doesn't support 0 strides in sger, so we can't use expand

TypeError: addmm_ received an invalid combination of arguments - got (int, int, torch.FloatTensor, torch.cuda.FloatTensor), but expected one of:
 * (torch.FloatTensor mat1, torch.FloatTensor mat2)
 * (torch.SparseFloatTensor mat1, torch.FloatTensor mat2)
 * (float beta, torch.FloatTensor mat1, torch.FloatTensor mat2)
 * (float alpha, torch.FloatTensor mat1, torch.FloatTensor mat2)
 * (float beta, torch.SparseFloatTensor mat1, torch.FloatTensor mat2)
 * (float alpha, torch.SparseFloatTensor mat1, torch.FloatTensor mat2)
 * (float beta, float alpha, torch.FloatTensor mat1, torch.FloatTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, torch.FloatTensor, torch.cuda.FloatTensor)
 * (float beta, float alpha, torch.SparseFloatTensor mat1, torch.FloatTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, torch.FloatTensor, torch.cuda.FloatTensor)

Your model is not transferred to CUDA maybe. Try adding the line: model.cuda()

@smth After seq = Sequence() I’m calling seq.model(). Is that not correct?

I see the exact issue now. Your subclassing of autograd.Variable is wrong. Replace it instead with this if you want:

def Variable(data, *args, **kwargs):
    return autograd.Variable(data.cuda(), *args, **kwargs)

Here’s a working example:

import torch
import torch.nn as nn
from torch import autograd
import torch.optim as optim
import numpy as np
import matplotlib
matplotlib.use('Agg')

def Variable(data, *args, **kwargs):
    return autograd.Variable(data.cuda(), *args, **kwargs)

class Sequence(nn.Module):
    def __init__(self):
        super(Sequence, self).__init__()
        self.lstm1 = nn.LSTMCell(1, 51).cuda()
        self.lstm2 = nn.LSTMCell(51, 1).cuda()

    def forward(self, input, future = 0):
        input = input.cuda()
        outputs = []
        h_t = Variable(torch.zeros(input.size(0), 51), requires_grad=False)
        c_t = Variable(torch.zeros(input.size(0), 51), requires_grad=False)
        h_t2 = Variable(torch.zeros(input.size(0), 1), requires_grad=False)
        c_t2 = Variable(torch.zeros(input.size(0), 1), requires_grad=False)
        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(c_t, (h_t2, c_t2))
            outputs += [c_t2]
        for i in range(future):# if we should predict the future
            h_t, c_t = self.lstm1(c_t2, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(c_t, (h_t2, c_t2))
            outputs += [c_t2]
        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs



if __name__ == '__main__':
    # set ramdom seed to 0
    np.random.seed(0)
    torch.manual_seed(0)
    # load data and make training set
    data = torch.load('traindata.pt')
    input = Variable(torch.from_numpy(data[3:, :-1]), requires_grad=False)
    target = Variable(torch.from_numpy(data[3:, 1:]), requires_grad=False).cuda()
    # build the model
    seq = Sequence()
    #seq.double()
    seq.cuda()
    criterion = nn.MSELoss()
    # use LBFGS as optimizer since we can load the whole data to train
    optimizer = optim.LBFGS(seq.parameters())
    #begin to train
    for i in range(10):
        print('STEP: ', i)
        def closure():
            optimizer.zero_grad()
            out = seq(input).cuda()
            loss = criterion(out, target)
            print('loss:', loss.data.cpu().numpy()[0])
            loss.backward()
            return loss
        optimizer.step(closure)
        # begin to predict
        future = 1000
        pred = seq(input[:3], future = future)
        y = pred.data.numpy()

Thank you so much for your help. Small note, it needed one more .cpu() in the last line

Needs to be:

y = pred.data.cpu().numpy()