My model is an emotion predictor for 7 classes. The model seems to be working fine. But it isnt learning. What could be the problem?
My training and valid functions.
import torch.optim as optim
optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0, betas=(0.9, 0.98), eps=1e-9)
criterion = nn.CrossEntropyLoss()
def train(train_dataloader,epoch):
running_loss = 0.0
correct=0
total=0
train_loss_list=[]
model.train()
for i, data in enumerate(train_dataloader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
inputs.requires_grad = True
# zero the parameter gradients
optimizer.zero_grad()
labels = torch.argmax(labels,dim =1)
# forward + backward + optimize
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
train_loss_list.append(loss.item())
total += labels.size(0)
_, predicted = torch.max(outputs.data, 1)
correct += (predicted == labels).sum().item()
mean_loss = np.mean(np.asarray(train_loss_list))
# if i%100==0:
# print('Iteration - {} Epoch - {} Total training loss - {}'.format(i,epoch,mean_loss))
print("train accuracy after epoch " +str(epoch) +" is " + str(100 * correct / total))
def validation(valid_dataloader,epoch):
model.eval()
with torch.no_grad():
val_loss_list=[]
correct = 0
total = 0
for i, data in enumerate(valid_dataloader, 0):
inputs, labels = data
labels = torch.argmax(labels,dim =1)
outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
loss = criterion(outputs, labels)
val_loss_list.append(loss.item())
# if i%100==0:
# print('Iteration - {} Epoch - {} Loss - {}'.format(i,epoch,np.mean(np.asarray(val_loss_list))))
print("valid accuracy after epoch "+str(epoch) +" is " + str(100 * correct / total))
mean_loss = np.mean(np.asarray(val_loss_list))
# print('Total validation loss {} after {} epochs'.format(mean_loss,epoch))
model_save_path = os.path.join( 'best_check_point_'+str(epoch)+'_'+str(mean_loss))
state_dict = {'model': model.state_dict(),'optimizer': optimizer.state_dict(),'epoch': epoch}
torch.save(state_dict, model_save_path)
for epoch in range(100):
train(train_dataloader,epoch)
validation(val_dataloader,epoch)
This is my output:
train accuracy after epoch 0 is 55.35390199637023
valid accuracy after epoch 0 is 33.05322128851541
train accuracy after epoch 1 is 56.152450090744104
valid accuracy after epoch 1 is 24.089635854341736
train accuracy after epoch 2 is 56.91470054446461
valid accuracy after epoch 2 is 32.212885154061624
train accuracy after epoch 3 is 56.8421052631579
valid accuracy after epoch 3 is 22.689075630252102
train accuracy after epoch 4 is 57.168784029038115
valid accuracy after epoch 4 is 22.689075630252102
train accuracy after epoch 5 is 57.53176043557169
valid accuracy after epoch 5 is 21.568627450980394
train accuracy after epoch 6 is 57.313974591651544
valid accuracy after epoch 6 is 21.288515406162464
train accuracy after epoch 7 is 58.87477313974592
valid accuracy after epoch 7 is 36.97478991596638
train accuracy after epoch 8 is 56.95099818511797
valid accuracy after epoch 8 is 31.092436974789916
train accuracy after epoch 9 is 55.82577132486389
valid accuracy after epoch 9 is 26.050420168067227
train accuracy after epoch 10 is 56.95099818511797
valid accuracy after epoch 10 is 26.050420168067227
train accuracy after epoch 11 is 57.3502722323049
valid accuracy after epoch 11 is 21.288515406162464
train accuracy after epoch 12 is 58.22141560798548
valid accuracy after epoch 12 is 21.008403361344538
train accuracy after epoch 13 is 56.62431941923775
valid accuracy after epoch 13 is 21.84873949579832
train accuracy after epoch 14 is 57.168784029038115
valid accuracy after epoch 14 is 26.89075630252101
train accuracy after epoch 15 is 57.45916515426497
valid accuracy after epoch 15 is 21.84873949579832
train accuracy after epoch 16 is 56.261343012704174
valid accuracy after epoch 16 is 28.571428571428573
train accuracy after epoch 17 is 57.93103448275862
valid accuracy after epoch 17 is 21.568627450980394
train accuracy after epoch 18 is 58.00362976406534
valid accuracy after epoch 18 is 19.327731092436974
train accuracy after epoch 19 is 56.33393829401089
valid accuracy after epoch 19 is 19.88795518207283
train accuracy after epoch 20 is 57.3502722323049
valid accuracy after epoch 20 is 19.327731092436974
Model:
import torch.nn as nn
import torch.nn.functional as F
class TDNN(nn.Module):
def __init__(
self,
input_dim=23,
output_dim=512,
context_size=5,
stride=1,
dilation=1,
batch_norm=False,
dropout_p=0.2
):
super(TDNN, self).__init__()
self.context_size = context_size
self.stride = stride
self.input_dim = input_dim
self.output_dim = output_dim
self.dilation = dilation
self.dropout_p = dropout_p
self.batch_norm = batch_norm
self.kernel = nn.Linear(input_dim*context_size, output_dim)
self.nonlinearity = nn.ReLU()
if self.batch_norm:
self.bn = nn.BatchNorm1d(output_dim)
if self.dropout_p:
self.drop = nn.Dropout(p=self.dropout_p)
def forward(self, x):
_, _, d = x.shape
assert (d == self.input_dim), 'Input dimension was wrong. Expected ({}), got ({})'.format(self.input_dim, d)
x = x.unsqueeze(1)
# Unfold input into smaller temporal contexts
x = F.unfold(
x,
(self.context_size, self.input_dim),
stride=(1,self.input_dim),
dilation=(self.dilation,1)
)
# N, output_dim*context_size, new_t = x.shape
x = x.transpose(1,2)
x = self.kernel(x.float())
x = self.nonlinearity(x)
if self.dropout_p:
x = self.drop(x)
if self.batch_norm:
x = x.transpose(1,2)
x = self.bn(x)
x = x.transpose(1,2)
return x
import torch.nn as nn
# from models.tdnn import TDNN
import torch
import torch.nn.functional as F
class X_vector(nn.Module):
def __init__(self, input_dim = 20, num_classes=7):
super(X_vector, self).__init__()
self.tdnn1 = TDNN(input_dim=input_dim, output_dim=512, context_size=5, dilation=1,dropout_p=0.5)
self.tdnn2 = TDNN(input_dim=512, output_dim=512, context_size=3, dilation=1,dropout_p=0.5)
self.tdnn3 = TDNN(input_dim=512, output_dim=512, context_size=2, dilation=2,dropout_p=0.5)
self.tdnn4 = TDNN(input_dim=512, output_dim=512, context_size=1, dilation=1,dropout_p=0.5)
self.tdnn5 = TDNN(input_dim=512, output_dim=512, context_size=1, dilation=3,dropout_p=0.5)
#### Frame levelPooling
self.segment6 = nn.Linear(1024, 512)
self.segment7 = nn.Linear(512, 512)
self.output = nn.Linear(512, num_classes)
# self.softmax = nn.Softmax(dim=1)
def forward(self, inputs):
tdnn1_out = self.tdnn1(inputs)
# return tdnn1_out
tdnn2_out = self.tdnn2(tdnn1_out)
tdnn3_out = self.tdnn3(tdnn2_out)
tdnn4_out = self.tdnn4(tdnn3_out)
tdnn5_out = self.tdnn5(tdnn4_out)
### Stat Pool
mean = torch.mean(tdnn5_out,1)
std = torch.std(tdnn5_out,1)
stat_pooling = torch.cat((mean,std),1)
segment6_out = self.segment6(stat_pooling)
x_vec = self.segment7(segment6_out)
predictions = self.output(x_vec)
return predictions
input_feats = [4,100,20]
input = torch.rand(input_feats)
model = X_vector()
out = model(input)
print(out)
tensor([[ 0.0170, -0.0404, -0.0174, -0.0133, 0.0375, 0.0010, 0.0251],
[ 0.0170, -0.0417, -0.0166, -0.0157, 0.0377, -0.0012, 0.0280],
[ 0.0174, -0.0404, -0.0149, -0.0133, 0.0371, -0.0003, 0.0263],
[ 0.0173, -0.0411, -0.0154, -0.0155, 0.0387, -0.0016, 0.0273]],
grad_fn=)