How to test model with unseen data

1

Hi everybody!
I am newbee in machine learning and I have a bit stupid question… i have trained my model and got .pt file. How can i test my model with unseen data? :sweat_smile:
My code for learning is the following:

import torch
import numpy as np
from torchsummary import summary

# check if CUDA is available
train_on_gpu = torch.cuda.is_available()

if not train_on_gpu:
    print('CUDA is not available.  Training on CPU ...')
else:
    print('CUDA is available!  Training on GPU ...')

    ###################В Data loading ###################
import torchvision.datasets
from torchvision import datasets
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torch.utils.data.sampler import SubsetRandomSampler

# Read data from folders
main_path = 'D:/RTU/dataset/ready_dataset_2classes'
train_data_path = main_path + '/train'
test_data_path = main_path + '/test'
weigths_path = 'D:/RTU/dataset/ready_dataset_2classes/weights'

# number of subprocesses to use for data loading
num_workers = 0
# how many samples per batch to load
batch_size = 20 #20
# percentage of training set to use as validation
valid_size = 0.2

# convert data to a normalized torch.FloatTensor
transform = transforms.Compose([
    transforms.Resize(32),
    transforms.CenterCrop(32),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

# choose the training and test datasets
train_data = torchvision.datasets.ImageFolder(root=train_data_path, transform=transform)
train_data_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True,  num_workers=0)
test_data = torchvision.datasets.ImageFolder(root=test_data_path, transform=transform)
test_data_loader  = DataLoader(test_data, batch_size=batch_size, shuffle=True, num_workers=0) 
# obtain training indices that will be used for validation
num_train = len(train_data)
indices = list(range(num_train))
np.random.shuffle(indices)
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]

# define samplers for obtaining training and validation batches
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)

# prepare data loaders (combine dataset and sampler)
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,
    sampler=train_sampler, num_workers=num_workers)
valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, 
    sampler=valid_sampler, num_workers=num_workers)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, 
    num_workers=num_workers)

# specify the image classes
classes = ['Modified', 'Original']

# helper function to un-normalize and display an image
def imshow(img):
    img = img / 2 + 0.5  # unnormalize
    plt.imshow(np.transpose(img, (1, 2, 0)))  # convert from Tensor image

    # creating checkpoints
def savePoint(main_path, model, optimizer, epoch, valid_loss_min):
    torch.save({
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'epoch': epoch,
        'valid_loss_min': valid_loss_min,
    }, main_path)
    
def loadPoint(main_path, model, optimizer, epoch, valid_loss_min):
    checkpoint = torch.load(main_path)
    model.load_state_dict(checkpoint['model_state_dict'])
    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
    epoch = checkpoint['epoch']
    valid_loss_min = checkpoint['valid_loss_min']

    ################### Network architecture definition ###################
import torch.nn as nn
import torch.nn.functional as F

# define the CNN architecture
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # convolutional layer (sees 32x32x3 image tensor)
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        # convolutional layer (sees 16x16x16 tensor)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        # convolutional layer (sees 8x8x32 tensor)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)
        # max pooling layer
        self.pool = nn.MaxPool2d(2, 2)
        # linear layer (64 * 4 * 4 -> 500)
        self.fc1 = nn.Linear(64 * 4 * 4, 500)
        # linear layer (500 -> 10)
        self.fc2 = nn.Linear(500, 250)
        self.fc3 = nn.Linear(250, 2)  
        # dropout layer (p=0.25)
        self.dropout = nn.Dropout(0.25) #0.25

    def forward(self, x):
        # add sequence of convolutional and max pooling layers
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        # flatten image input
        x = x.view(-1, 64 * 4 * 4)
        # add dropout layer
        x = self.dropout(x)
        # add 1st hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        # add dropout layer
        x = self.dropout(x)
        # add 2nd hidden layer, with relu activation function
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.fc3(x)       
        return x

# create a complete CNN
model = Net()
summary(model, input_size=(3, 32, 32))
# print(model)


# move tensors to GPU if CUDA is available
if train_on_gpu:
    model.cuda()

    #Loss and optimization specification
import torch.optim as optim

# specify loss function (categorical cross-entropy)
criterion = nn.CrossEntropyLoss()

# specify optimizer
optimizer = optim.SGD(model.parameters(), lr=0.000001)

################### Network training ###################
# number of epochs to train the model
n_epochs = 8 #5000
print('Starting training!')
valid_loss_min = np.Inf # track change in validation loss
for epoch in range(1, n_epochs+1):
    # keep track of training and validation loss
    train_loss = 0.0
    valid_loss = 0.0
    loadPoint(weigths_path, model, optimizer, epoch, valid_loss_min)
    ###################
    # train the model #
    ###################
    model.train()
    for data, target in train_loader:
        # move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # clear the gradients of all optimized variables
        optimizer.zero_grad()
        # forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # calculate the batch loss
        loss = criterion(output, target)
        # backward pass: compute gradient of the loss with respect to model parameters
        loss.backward()
        # perform a single optimization step (parameter update)
        optimizer.step()
        # update training loss
        train_loss += loss.item()*data.size(0)
        
    ######################    
    # validate the model #
    ######################

    model.eval()
    for data, target in valid_loader:
        # move tensors to GPU if CUDA is available
        if train_on_gpu:
            data, target = data.cuda(), target.cuda()
        # forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # calculate the batch loss
        loss = criterion(output, target)
        # update average validation loss 
        valid_loss += loss.item()*data.size(0)
    
    # calculate average losses
    train_loss = train_loss/len(train_loader.sampler)
    valid_loss = valid_loss/len(valid_loader.sampler)
        
    # print training/validation statistics 
    print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(
        epoch, train_loss, valid_loss))

    #checkpoint
    if valid_loss <= valid_loss_min:
        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
        valid_loss_min,
        valid_loss))
        savePoint(weigths_path, model, optimizer, epoch, valid_loss_min)
        valid_loss_min = valid_loss

# Loading the model with the lowest validation loss
#loadPoint(weigths_path, model, optimizer, epoch, valid_loss_min)
#model.load_state_dict(torch.load('weights.pt'))

################### Neural network testing with UNSEEN data ###################

# track test loss
test_loss = 0.0
class_correct = list(0. for i in range(2))
class_total = list(0. for i in range(2))

model.eval()
# iterate over test data
for data, target in test_loader:
    # move tensors to GPU if CUDA is available
    if train_on_gpu:
        data, target = data.cuda(), target.cuda()
    # forward pass: compute predicted outputs by passing inputs to the model
    output = model(data)
    # calculate the batch loss
    loss = criterion(output, target)
    # update test loss 
    test_loss += loss.item()*data.size(0)
    # convert output probabilities to predicted class
    _, pred = torch.max(output, 1)    
    # compare predictions to true label
    correct_tensor = pred.eq(target.data.view_as(pred))
    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())
    # calculate test accuracy for each object class
    
    for i in range(len(target.data)):
        label = target.data[i]
        class_correct[label] += correct[i].item()
        class_total[label] += 1
        
# average test loss
test_loss = test_loss/len(test_loader.dataset)
print('Test Loss: {:.6f}\n'.format(test_loss))

for i in range(2):
    if class_total[i] > 0:
        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (
            classes[i], 100 * class_correct[i] / class_total[i],
            np.sum(class_correct[i]), np.sum(class_total[i])))
    else:
        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))

print('\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (
    100. * np.sum(class_correct) / np.sum(class_total),
    np.sum(class_correct), np.sum(class_total)))

So the idea is that i will give the image, which the model did not see at all and model will give me an answer for which class does this image fits.
I have tried to cut the part of code where i am testing the model after the learning but i got some errors, so it seems like i am doing something wrong
p.s. sorry for my bad english :stuck_out_tongue:

2

Hi, what you are doing right is to split the data into training and validation.
You are training the model and then validating it already on unseen data (that is the idea of the validation loop).

Your testing loop currently does exact the same, which is not needed in most cases. I think you want to use your model simply on any other, new data right? This would then be called “inference”. You simply pass data to your model and want the output. By doing that, you don’t want to perform the gradient calculation nor the loss calculation which speeds up the process and uses way less ram (this is also recommended at the validation loop since you want to validate and not train.).

If i get you right, you loaded the model from a checkpoint, which is good!
Then you set your model to eval mode, good.
Now you want to deactivate the autograd engine before your loop. Do:

with torch.no_grad():
    <your loop here>

Then you need to do some decision making. Your model ouputs two values so i guess you are doing a binary classification (either one or the other output), am i right?
If so, then you should replace:

self.fc3 = nn.Linear(250, 2)

with

self.fc3 = nn.Linear(250, 1)

In this case your model outputs logits as well but the CrossEntropyLoss would not work. Use:

torch.nn.BCEWithLogitsLoss()

for that (but this is just a hint, your current approach works as well).

Back to the for loop over your test data:
your model outputs logits so you need to process and interpret them. You can do the following:

output_logits = your_model(input_batch)  # shaped like: batch_size, num_classes

if output_logits.size(1) == 1:
    # this is for binary classification: 
    # you have one output, everything below 0.5 relates to class 1, everything above to class 2
    output_probabilities = torch.sigmoid(output_logits)  # out logits are now in between 0 and 1
    outs = (output_probabilities.squeeze(1) > 0.5).float()  # outputs a vector ontaining zeroes and ones
    correct = outs.eq(targets).sum().item()  # number of correct predictions
else:
    # this would fit to your current approch assuming exact one class is correct
    # get the index of the highest output logit for each sample (no need to use sigmoid here):
    values, predictions = torch.max(output_logits, dim=1)
    correct = predictions.eq(targets).sum().item()

And that’s it. You have your predictions as a tensor and the amount of correct predictions if you have the ground truth.
If no ground truth is given, then remove the parts including the targets :slight_smile:

If you are facing errors: please include the errors in your question as well. We as the readers might help better in that case.