I have created the following NN using PyTorch API (for NLP Multi-class Classification)
class MultiClassClassifer(nn.Module):
#define all the layers used in model
def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim):
#Constructor
super(MultiClassClassifer, self).__init__()
#embedding layer
self.embedding = nn.Embedding(vocab_size, embedding_dim)
#dense layer
self.hiddenLayer = nn.Linear(embedding_dim, hidden_dim)
#Batch normalization layer
self.batchnorm = nn.BatchNorm1d(hidden_dim)
#output layer
self.output = nn.Linear(hidden_dim, output_dim)
#activation layer
self.act = nn.Softmax(dim=1) #2d-tensor
#initialize weights of embedding layer
self.init_weights()
def init_weights(self):
initrange = 1.0
self.embedding.weight.data.uniform_(-initrange, initrange)
def forward(self, text, text_lengths):
embedded = self.embedding(text)
embedded = torch.mean(embedded, dim=1, keepdim=True)
print(embedded.shape)
#packed sequence
packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True)
tensor, batch_size = packed_embedded[0], packed_embedded[1]
print(tensor.shape)
hidden_1 = self.batchnorm(self.hiddenLayer(tensor))
print(hidden_1.shape)
output = self.act(self.output(hidden_1))
print(output.shape)
return output
Instantiating the model
INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 64
OUTPUT_DIM = 3
model = MultiClassClassifer(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)
To train the model I have created the following train()
method:
def train(model, iterator, optimizer, criterion):
epoch_loss = 0
model.train()
for batch in iterator:
optimizer.zero_grad()
text, text_lengths = batch.text_normalized_tweet
predictions = model(text, text_lengths)
loss = criterion(predictions, batch.label)
print(loss)
print(loss.item())
loss.backward() #exception occurs here
optimizer.step()
epoch_loss += loss.item()
return epoch_loss / len(iterator)
I have created the following NN using PyTorch API (for NLP Multi-class Classification)
class MultiClassClassifer(nn.Module):
#define all the layers used in model
def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim):
#Constructor
super(MultiClassClassifer, self).__init__()
#embedding layer
self.embedding = nn.Embedding(vocab_size, embedding_dim)
#dense layer
self.hiddenLayer = nn.Linear(embedding_dim, hidden_dim)
#Batch normalization layer
self.batchnorm = nn.BatchNorm1d(hidden_dim)
#output layer
self.output = nn.Linear(hidden_dim, output_dim)
#activation layer
self.act = nn.Softmax(dim=1) #2d-tensor
#initialize weights of embedding layer
self.init_weights()
def init_weights(self):
initrange = 1.0
self.embedding.weight.data.uniform_(-initrange, initrange)
def forward(self, text, text_lengths):
embedded = self.embedding(text)
embedded = torch.mean(embedded, dim=1, keepdim=True)
print(embedded.shape)
#packed sequence
packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True)
tensor, batch_size = packed_embedded[0], packed_embedded[1]
print(tensor.shape)
hidden_1 = self.batchnorm(self.hiddenLayer(tensor))
print(hidden_1.shape)
output = self.act(self.output(hidden_1))
print(output.shape)
return output
Instantiating the model
INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 64
OUTPUT_DIM = 3
model = MultiClassClassifer(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)
To train the model I have created the following train()
method:
def train(model, iterator, optimizer, criterion):
epoch_loss = 0
model.train()
for batch in iterator:
optimizer.zero_grad()
text, text_lengths = batch.text_normalized_tweet
predictions = model(text, text_lengths)
loss = criterion(predictions, batch.label)
print(loss)
print(loss.item())
loss.backward() #exception occurs here
optimizer.step()
epoch_loss += loss.item()
return epoch_loss / len(iterator)
However, when I start the first epoch of the training, during the loss.backward()
step I receive the following error
IndexError: select(): index 1 out of range for tensor of size [1, 32, 100] at dimension 0
Here is the output of the print()
functions
torch.Size([32, 1, 100])
torch.Size([32, 100])
torch.Size([32, 64])
torch.Size([32, 3])
tensor(1.1309, grad_fn=<NllLossBackward0>)
1.1309444904327393
Does the error have to do with the nn.Embedding
layer of my network? And if so, how can I fix this exception.
You may find here my Golab Notebook.