Hello All,
I am trying to use LSTMCell to form multiple LSTM layer and not using LSTM. Getting following error and looks like it is due due to nn.utils.rnn.pack_padded_sequence but not able to debug further. Could you please help me on this?
My network definition is:
class RNN(nn.Module):
def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, bidirectional, dropout, pad_idx): super().__init__() self.n_layers = n_layers self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim # Number of time steps self.sequence_len = 3 self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx) # Initialize LSTM Cell for the first layer self.lstm_cell_layer_1 = nn.LSTMCell(self.embedding_dim, self.hidden_dim) # Initialize LSTM Cell for the second layer self.lstm_cell_layer_2 = nn.LSTMCell(self.hidden_dim, self.hidden_dim) # Initialize LSTM Cell for the second layer self.lstm_cell_layer_3 = nn.LSTMCell(self.hidden_dim, self.hidden_dim) ## we would need to initialize the hidden and cell state for each LSTM layer. self.fc = nn.Linear(hidden_dim*2, output_dim) self.dropout = nn.Dropout(dropout) def forward(self, text, text_lengths): #text = [sent len, batch size] embedded = self.dropout(self.embedding(text)) #embedded = [sent len, batch size, emb dim] ## Initialisation #pack sequence packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths) # packed_embedded = embedded print(f'text_lengths{text_lengths}') print(f'text.size(0){text.size(0)}') # out = packed_embedded.view(self.sequence_len, text.size(0), -1) # Creation of cell state and hidden state for layer 1 hidden_state = torch.zeros(self.embedding_dim, self.hidden_dim) cell_state = torch.zeros(self.embedding_dim, self.hidden_dim) # Creation of cell state and hidden state for layer 2 hidden_state_2 = torch.zeros(self.embedding_dim, self.hidden_dim) cell_state_2 = torch.zeros(self.embedding_dim, self.hidden_dim) # Creation of cell state and hidden state for layer 3 hidden_state_3 = torch.zeros(self.embedding_dim, self.hidden_dim) cell_state_3 = torch.zeros(self.embedding_dim, self.hidden_dim) # Weights initialization torch.nn.init.xavier_normal_(hidden_state) torch.nn.init.xavier_normal_(cell_state) torch.nn.init.xavier_normal_(hidden_state_2) torch.nn.init.xavier_normal_(cell_state_2) torch.nn.init.xavier_normal_(hidden_state_3) torch.nn.init.xavier_normal_(cell_state_3) ## End of Initialisation # Unfolding LSTM for input_t in range(self.sequence_len): # print(f'packed_embedded.size(0){len(packed_embedded)}') hidden_state_1, cell_state_1 = self.lstm_cell_layer_1(packed_embedded,(hidden_state, cell_state)) hidden_state_2, cell_state_2 = self.lstm_cell_2(hidden_state_1, (hidden_state_2, cell_state_2)) hidden_state_3, cell_state_3 = self.lstm_cell_3(hidden_state_2, (hidden_state_3, cell_state_3)) #unpack sequence output, output_lengths = nn.utils.rnn.pad_packed_sequence(hidden_state_3) hidden = self.dropout(torch.cat((hidden_state_3[-2,:,:], hidden_state_3[-1,:,:]), dim = 1)) return self.fc(hidden)
Error:
AttributeError Traceback (most recent call last)
in ()
7 start_time = time.time()
8
----> 9 train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
10 valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
115 frames
in train(model, iterator, optimizer, criterion)
13
14 text_lengths = text_lengths.cpu()
—> 15 predictions = model(text, text_lengths).squeeze(1)
16
17 loss = criterion(predictions, batch.label)/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
725 result = self._slow_forward(*input, **kwargs)
726 else:
→ 727 result = self.forward(*input, **kwargs)
728 for hook in itertools.chain(
729 _global_forward_hooks.values(),in forward(self, text, text_lengths)
76 for input_t in range(self.sequence_len):
77 # print(f’packed_embedded.size(0){len(packed_embedded)}')
—> 78 hidden_state_1, cell_state_1 = self.lstm_cell_layer_1(packed_embedded,(hidden_state, cell_state))
79 hidden_state_2, cell_state_2 = self.lstm_cell_2(hidden_state_1, (hidden_state_2, cell_state_2))
80 hidden_state_3, cell_state_3 = self.lstm_cell_3(hidden_state_2, (hidden_state_3, cell_state_3))/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
725 result = self._slow_forward(*input, **kwargs)
726 else:
→ 727 result = self.forward(*input, **kwargs)
728 for hook in itertools.chain(
729 _global_forward_hooks.values(),/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
963
964 def forward(self, input: Tensor, hx: Optional[Tuple[Tensor, Tensor]] = None) → Tuple[Tensor, Tensor]:
→ 965 self.check_forward_input(input)
966 if hx is None:
967 zeros = torch.zeros(input.size(0), self.hidden_size, dtype=input.dtype, device=input.device)/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py in check_forward_input(self, input)
788
789 def check_forward_input(self, input: Tensor) → None:
→ 790 if input.size(1) != self.input_size:
791 raise RuntimeError(
792 “input has inconsistent input_size: got {}, expected {}”.format(AttributeError: ‘PackedSequence’ object has no attribute ‘size’