Hello,

I struggled all the day with an issue I never had before (I used usually images data)

My model performs well on the validation data of the imdb dataset during the training. I save then the weights.

If I want to evaluation again on the same validation data, then the results are worse a lot (I am using model.eval() before the forward function). So I do not really understand what’s going on.

I tried to print the results inside the different layers, and I saw that, the results are different starting from Linear layer. I was wondering if I have done something wrong in my model :

```
class SequenceNet(nn.Module):
def __init__(self, embedding_matrix, num_classes, hidden_sizes=64, padding_value=1):
super(SequenceNet, self).__init__()
embed_size = embedding_matrix.shape[1]
LSTM_UNITS = hidden_sizes
DENSE_HIDDEN_UNITS = LSTM_UNITS * 4
self.embedding = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix), freeze=False, sparse=True, padding_idx=padding_value)
self.norm_embedding = nn.LayerNorm(embed_size)
self.embedding_dropout = SpatialDropout(0.3)
self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
self.norm_seq = nn.LayerNorm(LSTM_UNITS * 2 * 2)
self.fc1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
self.fc2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
self.out = nn.Linear(DENSE_HIDDEN_UNITS, num_classes)
def extract_mean(self, x_pack, lengths):
h_t, _ = pad_packed_sequence(x_pack, batch_first=True, padding_value=0.0)
#print("extract mean", h_t.shape, h_t.sum(dim=1).shape, lengths.shape)
h_mean = h_t.sum(dim=1)/lengths.view(-1,1)
#print(h_mean.shape)
return h_mean
def extract_max(self, x_pack):
h_t, _ = pad_packed_sequence(x_pack, batch_first=True, padding_value=-float("inf"))
h_max = h_t.max(dim=1)[0]
return h_max
def forward(self, x, lengths):
x = self.embedding(x)
x = self.norm_embedding(x)
x = self.embedding_dropout(x)
x_pack = pack_padded_sequence(x, batch_first=True, lengths=lengths)
h_lstm1, _ = self.lstm1(x_pack)
h_lstm2, _ = self.lstm2(h_lstm1)
#print("SHAPE :")
# global average pooling and unpack seq
avg_pool = self.extract_mean(h_lstm2, lengths)
#print(avg_pool.shape)
# global max pooling and unpack seq
max_pool = self.extract_max(h_lstm2)
#print(max_pool.shape)
h_conc = torch.cat((max_pool, avg_pool), 1)
#print(h_conc.shape)
h_conc = self.norm_seq(h_conc)
h_conc_linear1 = F.relu(self.fc1(h_conc))
#h_conc_linear1 = h_conc + h_conc_linear1
h_conc_linear2 = F.relu(self.fc2(h_conc_linear1))
hidden = h_conc_linear2
result = self.out(hidden)
return result
def _forward(self, x, lengths):
x = self.embedding(x)
print("embedding :", x)
x = self.norm_embedding(x)
x = self.embedding_dropout(x)
print("norm_embedding :", x)
x_pack = pack_padded_sequence(x, batch_first=True, lengths=lengths)
h_lstm1, _ = self.lstm1(x_pack)
h_lstm2, _ = self.lstm2(h_lstm1)
#print("SHAPE :")
# global average pooling and unpack seq
avg_pool = self.extract_mean(h_lstm2, lengths)
print("avg pool :", avg_pool)
#print(avg_pool.shape)
# global max pooling and unpack seq
max_pool = self.extract_max(h_lstm2)
print("max_pool" , max_pool)
h_conc = torch.cat((max_pool, avg_pool), 1)
print("h_conc :", h_conc)
h_conc = self.norm_seq(h_conc)
print("h_conc_norm :", h_conc)
h_conc_linear1 = F.relu(self.fc1(h_conc))
print("h_conc_1 :", h_conc_linear1)
#h_conc_linear1 = h_conc + h_conc_linear1
print("h_conc_1_res :", h_conc_linear1)
h_conc_linear2 = F.relu(self.fc2(h_conc_linear1))
print("h_conc_2 :", h_conc_linear2)
hidden = h_conc_linear2
print("hidden :", hidden)
result = self.out(hidden)
print("result:", result)
return result
```

The _forward function is just to display in order to debug.

The entire script is attached are on this address : https://drive.google.com/drive/folders/1-Oq75wH7hi4Zx8duiUWkprH_m_f9Aa_i?usp=sharing

When

debug = True

, it means I just want to evaluate the model on the validation data. (I am using the IMDB dataset)

In the link I have shared, I also saved the two different print of different layers for one forward using :

```
A = next(iter(val_dataloader2))
preds = model._forward(A["text"].to(device), A["lengths"].to(device))
```

The results are different after the Layer Norm “h_conc”. Therefore, the issue seems to come from the linear layer but I have no idea why.

Does anyone have an idea why I got different results after loading the weights ?

I am using pytorch 1.4 on windows 10 with cuda 10.1 . I am with anaconda and using spyder 4 as an IDE.

EDIT : I have done more test :

- removing the linear1 and linear2 seem to solve the issue, but I do not understand why.
- the issue seems to come from apex, if I trained without apex, it works well.