I dont quite understand what is the problem with dimensions for linear layer as I had dimension mismatch but I used hidden of last timestamp i.e.20 to get linear classification
import numpy as np
import h5py
import torch
import torch.nn as nn
from torch.autograd import Variable
from keras.utils import to_categorical
from pdb import set_trace
# Here we define our model as a class
class LSTM(nn.Module):
def __init__(self, input_dim, hidden_dim, batch_size, output_dim=1,
num_layers=2,features=200, s_len=20):
super(LSTM, self).__init__()
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.batch_size = batch_size
self.num_layers = num_layers
self.hidden = self.init_hidden()
self.s_len = s_len
self.features=features
# Define the LSTM layer
self.lstm = nn.LSTM(self.input_dim, self.hidden_dim, self.num_layers)
# Define the output layer
self.linear = nn.Linear(self.hidden_dim, output_dim)
def init_hidden(self):
# This is what we'll initialise our hidden state as
return (torch.zeros(self.num_layers, self.batch_size, self.hidden_dim),
torch.zeros(self.num_layers, self.batch_size, self.hidden_dim))
def forward(self, X):
# Forward pass through LSTM layer
# shape of lstm_out: [input_size, batch_size, hidden_dim]
# shape of self.hidden: (a, b), where a and b both
# have shape (num_layers, batch_size, hidden_dim).
# pack_padded_sequence so that padded items in the sequence won't be shown to the LSTM
#input.view(len(input), self.batch_size, -1) i.e X
########for sequance length before padding as i have already padded sequances
zeros = torch.zeros((1, self.features))
lengths = torch.zeros((X.shape[0]))
for i in range(X.shape[0]):
zcount=0
for j in range(self.s_len):
if (X[i,j,:]==zeros).sum()==self.features:
zcount+=1
# print(count)
lengths[i]=self.s_len-zcount
########################
seq_lengths=lengths
X = torch.nn.utils.rnn.pack_padded_sequence(X, seq_lengths, batch_first=True, enforce_sorted= False)
lstm_out, self.hidden = self.lstm(X,self.hidden)
# set_trace()
# undo the packing operation
lstm_out, _ = torch.nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)
# set_trace()
# Only take the output from the final timetep
# Can pass on the entirety of lstm_out to the next layer if it is a seq2seq prediction
#y_pred = self.linear(lstm_out[-1].view(self.batch_size, -1))
y_pred = self.linear(lstm_out[:,-1,:])
return y_pred.view(-1)
file_path1 = 'tr_20.mat'# 2097,20,200 with sample*seq*features
A01T = h5py.File(file_path1,'r')
temp_tr_data = np.asarray(np.copy(A01T['x']),dtype=np.float32)
tr_data = torch.from_numpy(temp_tr_data)
tr_data.shape
file_path2 = 'tr_label.mat'#binary label
A02T = h5py.File(file_path2,'r')
tr_labels= np.copy(A02T['y'])
print(tr_labels)
print(tr_labels.shape)
print(tr_labels.dtype)
tr_labels = tr_labels[0,0:tr_data.shape[0]:1]
print(tr_labels.shape)
print(tr_labels.dtype)
tr_labels = np.asarray(tr_labels, dtype=np.float32) #............
print(tr_labels.shape)
print(tr_labels.dtype)
tr_labels= torch.from_numpy(tr_labels)
n_epochs = 500
batch_size = 64
learning_rate = 0.01
feature=200
max_length = 20
hidden_size = 10
n_layers =2
#h0 = Variable(torch.randn(n_layers, batch_size, hidden_size))#h1
#Training LSTMs
model = LSTM(feature, hidden_size, batch_size=batch_size, num_layers=n_layers)
loss_fn = torch.nn.MSELoss(size_average=False)
optimiser = torch.optim.Adam(model.parameters(), lr=learning_rate)
#####################
# Train model
#####################
hist = np.zeros(n_epochs)
for epoch in range(n_epochs):
model.zero_grad()
# tr_data is a torch Variable
permutation = torch.randperm(tr_data.size()[0])
for i in range(0,tr_data.size()[0], batch_size):
optimiser.zero_grad()
indices = permutation[i:i+batch_size]
batch_x, batch_y = tr_data[indices], tr_labels[indices]
# in case you wanted a semi-full example
y_pred = model.forward(batch_x)
loss = loss_fn(y_pred, batch_y)
if epoch % 100 == 0:
print("Epoch ", epoch, "MSE: ", loss.item())
hist[epoch] = loss.item()
# Zero out gradient, else they will accumulate between epochs
optimiser.zero_grad()
loss.backward(retain_graph=True)# Backward pass
optimiser.step()# Update parameters
Epoch 0 MSE: 24.005014419555664
Epoch 0 MSE: 20.296175003051758
Epoch 0 MSE: 14.667539596557617
/usr/local/lib/python3.6/dist-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction=‘sum’ instead.
warnings.warn(warning.format(ret))
Epoch 0 MSE: 11.551937103271484
Epoch 0 MSE: 9.712754249572754
Epoch 0 MSE: 7.439380645751953
Epoch 0 MSE: 7.079190254211426
Epoch 0 MSE: 13.051213264465332
Epoch 0 MSE: 9.997764587402344
Epoch 0 MSE: 11.738643646240234
Epoch 0 MSE: 12.547919273376465
Epoch 0 MSE: 6.7377543449401855
Epoch 0 MSE: 6.711551666259766
Epoch 0 MSE: 8.105769157409668
Epoch 0 MSE: 6.859475135803223
Epoch 0 MSE: 7.038944244384766
Epoch 0 MSE: 10.451923370361328
Epoch 0 MSE: 9.070074081420898
Epoch 0 MSE: 7.982135772705078
Epoch 0 MSE: 6.470491409301758
Epoch 0 MSE: 6.424145221710205
Epoch 0 MSE: 10.381178855895996
Epoch 0 MSE: 6.682828426361084
Epoch 0 MSE: 7.862491130828857
Epoch 0 MSE: 6.042342185974121
Epoch 0 MSE: 8.889896392822266
Epoch 0 MSE: 7.864719867706299
Epoch 0 MSE: 8.209149360656738
Epoch 0 MSE: 8.037969589233398
Epoch 0 MSE: 4.490922927856445
Epoch 0 MSE: 7.113046169281006
Epoch 0 MSE: 6.170615196228027
Epoch 0 MSE: 4.747644424438477
RuntimeError Traceback (most recent call last)
in ()
24
25 # in case you wanted a semi-full example
—> 26 y_pred = model.forward(batch_x)
27 loss = loss_fn(y_pred, batch_y)
28
6 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py in apply_permutation(tensor, permutation, dim)
19 def apply_permutation(tensor, permutation, dim=1):
20 # type: (Tensor, Tensor, int) -> Tensor
—> 21 return tensor.index_select(dim, permutation)
22
23
RuntimeError: invalid argument 3: out of range at /pytorch/aten/src/TH/generic/THTensor.cpp:318