Hi guys,
New to ML and pytorch and have thrown myself in the deep end trying to get a CIFAR network up and running using LeNet Archcitecture. I am confident in my Binarization functions and how I am loading the data set in. What I am having problems with is is when I run, the network it does not seem to improve.
A few notes: Binarization requires I keep a copy of the full precision weights for updates during backprop (code adapted from Courbariaux on binarization).
Here is the code:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from Binarize import *
------------------------------------------------
EnableCUDAExecution = False
Batch_Size = 50
LearningRate = 0.01
Epochs = 3
Momentum = 0.8
WeightDecay = 0
logInterval = -1
seed = 10
------------------------------------------------
class LeNet(nn.Module):
def init(self):
super(LeNet, self).init()
self.conv1 = BinarizeConv2d(3, 6, 5)
self.conv2 = BinarizeConv2d(6, 16, 5)
self.fc1 = BinarizeLinear(16*5*5, 120)
self.fc2 = BinarizeLinear(120, 84)
self.fc3 = BinarizeLinear(84, 10)
torch.nn.init.xavier_uniform(self.fc1.weight)
torch.nn.init.xavier_uniform(self.fc2.weight)
torch.nn.init.xavier_uniform(self.fc3.weight)
def forward(self, x):
out = F.relu(self.conv1(x))
out = F.max_pool2d(out, 2)
out = F.relu(self.conv2(out))
out = F.max_pool2d(out, 2)
out = out.view(out.size(0), -1)
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = self.fc3(out)
return out
def train(epoch):
model.train()
if EnableCUDAExecution is True:
model.cuda()
for BatchIndex, (Inputs, Labels) in enumerate(TrainingSetLoader):
if EnableCUDAExecution is True:
Inputs, Labels = Inputs.cuda(), Labels.cuda()
Inputs, Labels = Variable(Inputs), Variable(Labels)
Optimizer.zero_grad()
Output = model(Inputs)
Loss = LossFunction(Output, Labels)
Optimizer.zero_grad()
Loss.backward()
for p in list(model.parameters()):
if hasattr(p, 'org'):
p.data.copy_(p.org)
Optimizer.step()
for p in list(model.parameters()):
if hasattr(p, 'org'):
p.org.copy_(p.data.clamp_(-1, 1))
if logInterval != -1:
if (BatchIndex) % logInterval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, BatchIndex * len(Inputs),
len(TrainingSetLoader.dataset), 100. *
BatchIndex / len(TrainingSetLoader),
Loss.data[0]))
def test(epoch):
model.eval()
testLoss = 0
Correct = 0
for BatchIndex, (Data, Labels) in enumerate(TestSetLoader):
if EnableCUDAExecution is True:
Data, Labels = Data.cuda(), Labels.cuda()
Data, Labels = Variable(Data, volatile=True), Variable(Labels)
Output = model(Data)
testLoss += LossFunction(Output, Labels)
pred = Output.data.max(1, keepdim=True)[1]
Correct += pred.eq(Labels.data.view_as(pred)).cpu().sum()
testLoss /= len(TestSetLoader.dataset) # Determine the test loss and echo the respective output
print('Epoch ' + str(epoch) + ": ", end='')
# print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
# testLoss, Correct, len(TestSetLoader.dataset),
# 100. * Correct / len(TestSetLoader.dataset)))
validationErrorRate = 100 - (100. * Correct / len(TestSetLoader.dataset))
print('Epoch ' + str(epoch) + ": ", str(validationErrorRate))
return validationErrorRate, testLoss
TrainingSetLoader, TestSetLoader = loadCIFAR(Batch_Size)
model = LeNet()
if EnableCUDAExecution == True:
torch.cuda.set_device(1)
model.cuda()
Optimizer = torch.optim.SGD(model.parameters(), lr=LearningRate, momentum=Momentum)
LossFunction = torch.nn.CrossEntropyLoss()
print(‘Here we go…’)
for Epoch in range(1, Epochs + 1):
train(Epoch)
validationErrorRate, test_loss = test(Epoch)
#print(Epoch, validationErrorRate, test_loss)
Definitely some redundant code in there, but I am mainly focused on simply getting this to network to improve. If anyone thinks they can help but needs more detail, please let me know, happy to post any other necessary code/files. Also, bit of a python noob, so please forgive any ignorance on my part. Cheers guys!