Hello guys,
I was just playing around with a toy example on MNIST, but i was using a conv1D network. Achieved around 96% after 5 epochs. Then i wanted to implement my own normalization function, but had to cast the data to float() first because you cant use mean() on a bytetensor. Just calling .float() on the data caused the network to suddenly drop to 50% for some odd reason. Does anyone know what might be happening here? I have the 2 notebooks on collab:
https://drive.google.com/open?id=1nNzMyf--R-j8K4GxUSHwArF2nzvinxVi
https://drive.google.com/open?id=1PAZ4dCxKD-pR8LRSnGQgbCFDJ143nQRI
EPOCHS = 5
BS = 128
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
transform = transforms.Compose([transforms.ToTensor()])
mnist_train = torchvision.datasets.MNIST('.', download=True, transform=transform,train=True)
mnist_test = torchvision.datasets.MNIST('.', download=True, transform=transform, train=False)
#####
#mnist_train.data = mnist_train.data.float()
#mnist_test.data = mnist_test.data.float()
trainloader = torch.utils.data.DataLoader(mnist_train,batch_size=BS,shuffle=True)
testloader = torch.utils.data.DataLoader(mnist_test,batch_size=10000,shuffle=False)
class conv_net(nn.Module):
def __init__(self, lr):
super(conv_net, self).__init__()
self.layer_1 = nn.Conv1d(1, 32, kernel_size=7, stride=5)
self.layer_2 = nn.Conv1d(32, 32, kernel_size=7, stride=5)
self.out = nn.Linear(960, 10)
self.optimizer = optim.Adam(self.parameters(), lr = lr)
def forward(self, x):
#print(x.dtype)
x = x.to(device)
x = x.view((x.size()[0], 1, -1))
x = F.leaky_relu(self.layer_1(x), negative_slope=0.1)
x = F.relu(self.layer_2(x)).view(x.size(0),-1)
x = self.out(x)
return x
model = conv_net(1e-3).to(device)
criterion = torch.nn.CrossEntropyLoss()
for epoch in range(EPOCHS):
for x_train, y_train in iter(trainloader):
model.train()
y_train = y_train.to(device)
pred = model(x_train)
loss = criterion(pred, y_train)
loss.backward()
model.optimizer.step()
model.optimizer.zero_grad()
with torch.no_grad():
counter = 0
for x_test, y_test in iter(testloader):
y_test = y_test.to(device)
model.eval()
pred = model(x_test)
pred = torch.argmax(F.log_softmax(pred), dim = 1)
correct = (pred==y_test).sum().item()
print(f'Test - Accuracy: {(correct/len(y_test)):.4f}') for epoch in range(EPOCHS):
for x_train, y_train in iter(trainloader):
model.train()
y_train = y_train.to(device)
pred = model(x_train)
loss = criterion(pred, y_train)
loss.backward()
model.optimizer.step()
model.optimizer.zero_grad()
with torch.no_grad():
counter = 0
for x_test, y_test in iter(testloader):
y_test = y_test.to(device)
model.eval()
pred = model(x_test)
pred = torch.argmax(F.log_softmax(pred), dim = 1)
correct = (pred==y_test).sum().item()
print(f'Test - Accuracy: {(correct/len(y_test)):.4f}')