Different results with and without dataloader

Hi, everyone,

I am creating a sign language classifier and i got following problem with its evaluation/usage:

I have two different functions which compute an accuracy of my trained model. One of them (with_dl) computes 94% while the other (without_dl) outputs 6.4%
Actually my aim is to use the model on my webcam and after some research i ended up with this problem here.

This is my Code: (having cuda available)

import torch
from net import FingeralphabetNet
from torch.utils.data import DataLoader
import torchvision as tv
import pandas as pd
from PIL import Image
import os
fn = "Nets/11.try/15.pth"
nn = FingeralphabetNet()
if torch.cuda.is_available():
    device = torch.device('cuda')
    nn = nn.to(device)
    device = torch.device('cpu')
    nn.load_state_dict(torch.load(fn, map_location=device))

abc = [chr(i) for i in range(ord("A"), ord("Z")+1)] + ["SCH", "CH", "NOTHING"]
transforms = tv.transforms.Compose([
    tv.transforms.Resize((120, 160)),
    tv.transforms.Normalize([.5]*3, [.5]*3),

def without_dl(nn, img_dir):
    with torch.no_grad():
        acc = 0
        count = 0
        for ind in range(29):
            for file in os.listdir("{}/{}".format(img_dir, ind)):
                img = Image.open("{}/{}/{}".format(img_dir, ind, file))
                img = transforms(img)
                outputs = nn(img.unsqueeze(0))
                _, prediction = torch.max(outputs, 1)
                if prediction[0] == ind:
                    acc += 1
                count += 1
                if count % 500 == 499:
                    print(count + 1)
    return acc/count

def with_dl(nn, img_dir):
    dataset = tv.datasets.ImageFolder(img_dir, transform=transforms)
    dataloader = DataLoader(dataset, 
                    batch_size=32, shuffle=False, num_workers=8, 
                    pin_memory=True, drop_last=True)
    acc = 0
    count = 0
    with torch.no_grad():
        for X, y in dataloader:
            outputs = nn(X)
            _, prediction = torch.max(outputs, 1)
            y = y.to(device=torch.device('cuda'))
            for i in range(y.shape[0]):
                if y[i] == prediction[i]:
                    acc += 1
                count += 1
                if count % 500 == 499:
                    print(count + 1)
        return acc/count

img_dir = "../dataset/test2"
without_res = without_dl(nn, img_dir)
with_res = with_dl(nn, img_dir)

# OUTPUT: 0.06563218390804598
# OUTPUT: 0.9443035055350554

I am looking forward to your help :smile:

Could you check if the current class to index mapping is the same in both approaches?
Based on your code it looks like you are using ind to get the class folder.
You can check if with printing dataset.class_to_idx.
Let me know, if the mapping is the same, and I’ll have another look at your code.