validation loss and training loss is not increasing. I try to imply l2 regularization in form of weight decay as 0.05 but I have removed and tried maybe that could be the reason.
class CustomDataset(Dataset):
def __init__(self, root_folder_path):
self.root_folder_path = root_folder_path
self.image_files = []
self.labels = []
# Collect image paths and corresponding labels
folders = sorted([f for f in os.listdir(root_folder_path) if os.path.isdir(os.path.join(root_folder_path, f))])
self.label_dict = {folder: i for i, folder in enumerate(folders)}
for folder in folders:
folder_path = os.path.join(root_folder_path, folder)
image_files = sorted([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f)) and f.endswith('.jpg')])
self.image_files.extend([os.path.join(folder_path, img) for img in image_files])
self.labels.extend([self.label_dict[folder]] * len(image_files))
self.transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((900, 300)),
transforms.Grayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5], std=[0.5])
])
def __len__(self):
return len(self.image_files)
def __getitem__(self, idx):
image_path = self.image_files[idx]
label = self.labels[idx]
image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
image = self.transform(image)
#print("Image shape:", image.shape) # Print the shape of the image
one_hot_label = torch.zeros(len(self.label_dict))
one_hot_label[label] = 1
return image, one_hot_label
main script
from custom_dataset import CustomDataset
if __name__ == '__main__':
# Instantiate your custom dataset and dataloaders
root_folder_path = r'W:\MASTER_BAGCHI_SCHALDACH\THESIS\code and dataset\image_dataset_300_900_10_classes'
dataset = CustomDataset(root_folder_path)
print("Labels:", sorted(dataset.label_dict.keys()))
print("Total number of labels:", len(dataset.label_dict))
# Display some images from each folder
n_images_to_display = 4
n_folders = len(dataset.label_dict)
fig, ax = plt.subplots(n_images_to_display, n_folders, figsize=(n_folders * 4, n_images_to_display * 4))
for i, (folder, label) in enumerate(dataset.label_dict.items()):
folder_images = [dataset[i][0] for i, lbl in enumerate(dataset.labels) if lbl == label]
indices_to_display = random.sample(range(len(folder_images)), min(n_images_to_display, len(folder_images)))
for j, ind in enumerate(indices_to_display):
ax[j, i].imshow(folder_images[ind].squeeze(), cmap='gray') # Squeeze to remove the channel dimension for grayscale images
ax[j, i].axis('off')
ax[0, i].set_title(folder, fontsize=30)
plt.show()
fig.tight_layout(pad=0, w_pad=0, h_pad=0)
from torch.utils.data import DataLoader, Subset
from sklearn.model_selection import train_test_split
TEST_SIZE = 0.2
BATCH_SIZE = 32
SEED = 42
# Get the labels from the dataset
labels = np.array([label for _, label in dataset])
# generate indices: instead of the actual data we pass in integers instead
train_indices, test_indices, _, _ = train_test_split(
range(len(dataset)),
labels,
stratify=labels,
test_size=TEST_SIZE,
random_state=SEED
)
# generate subset based on indices
train_split = Subset(dataset, train_indices)
test_split = Subset(dataset, test_indices)
print('Length of train_batch:',len(train_split))
print('Length of test_batch:',len(test_split))
# create batches
train_loader = DataLoader(train_split, batch_size=BATCH_SIZE, num_workers=6,shuffle=True,pin_memory=True)
test_loader = DataLoader(test_split, batch_size=BATCH_SIZE,num_workers=6,pin_memory=True)
for batch in train_loader:
images, labels = batch
#print('Train batch size:', images.size())
#print('Shape of labels array:',labels.size())
for batch in test_loader:
images, labels = batch
#print('Test batch size:', images.size())
#print('Shape of labels array:',labels.size())
class ImageClassificationBase(nn.Module):
def training_step(self, batch):
images, labels = batch
out = self(images) # Generate predictions
loss = F.cross_entropy(out, labels) # Calculate loss
return loss
def accuracy(self,outputs, labels):
#_, preds = torch.max(outputs, dim=1)
preds = torch.argmax(outputs, dim=1)
#preds_one_hot = F.one_hot(preds, num_classes=labels.shape[1]) # Convert predictions to one-hot encoding
#print("Shape of preds:", preds.shape) # Check the shape of preds
#correct=(preds_one_hot == labels).float().sum() # Count the number of correct predictions
correct = (preds == torch.argmax(labels, dim=1)).float().sum() # Count the number of correct predictions
total = len(labels) # Total number of samples
acc = correct / total # Calculate accuracy
return acc
#return torch.sum(preds_one_hot == labels).float().mean()
def validation_step(self, batch):
images, labels = batch
out = self(images) # Generate predictions
loss = F.cross_entropy(out, labels) # Calculate loss
acc = self.accuracy(out, labels) # Calculate accuracy
#batch_size = labels.shape[0]
#acc = self.accuracy(out, labels, batch_size)
return {'val_loss': loss.detach(), 'val_acc': acc}
def validation_epoch_end(self, outputs):
batch_losses = [x['val_loss'] for x in outputs]
epoch_loss = torch.stack(batch_losses).mean() # Combine losses
batch_accs = [x['val_acc'] for x in outputs]
epoch_acc = torch.stack(batch_accs).mean() # Combine accuracies
return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}
def epoch_end(self, epoch, result):
print("Epoch [{}], train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}".format(
epoch, result['train_loss'], result['val_loss'], result['val_acc']))
import torch.nn.init as init
class ImageClassification(ImageClassificationBase):
def __init__(self):
super().__init__()
self.network = nn.Sequential(
#image size is [1,900,300] as [channel, height,width]
nn.Conv2d(1, 32, kernel_size = 3, padding = 1),
nn.LeakyReLU(0.01),
nn.BatchNorm2d(32),
nn.AvgPool2d(kernel_size=2, stride=2),
nn.Conv2d(32,32, kernel_size = 3, padding = 1),
nn.LeakyReLU(0.01),
nn.BatchNorm2d(32),
nn.AvgPool2d(kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size = 3, padding = 1),
nn.LeakyReLU(0.01),
nn.BatchNorm2d(64),
nn.AvgPool2d(kernel_size=2, stride=2),
nn.Conv2d(64 ,64, kernel_size = 3, padding = 1),
nn.LeakyReLU(0.01),
nn.BatchNorm2d(64),
nn.AvgPool2d(kernel_size=2, stride=2),
nn.Flatten(),
nn.Dropout(0.3),
nn.Linear(64 * 56 * 18, 64), # Assuming input size after convolutional layers is 64 * 56 * 18
nn.LeakyReLU(0.01),
nn.BatchNorm1d(64),
nn.Dropout(0.2),
nn.Linear(64, 64),
nn.LeakyReLU(0.01),
nn.BatchNorm1d(64),
nn.Dropout(0.2),
nn.Linear(64, 10) # Output layer
)
# Initialize the weights of convolutional layers
self._initialize_weights()
def _initialize_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='leaky_relu')
def forward(self, xb):
return self.network(xb)
def get_default_device():
#Set Device to GPU or CPU
if torch.cuda.is_available():
return torch.device('cuda')
else:
return torch.device('cpu')
def to_device(data, device):
"Move data to the device"
if isinstance(data,(list,tuple)):
return [to_device(x,device) for x in data]
return data.to(device,non_blocking = True)
class DeviceDataLoader():
#Wrap a dataloader to move data to a device
def __init__(self, dl, device):
self.dl = dl
self.device = device
def __iter__(self):
#Yield a batch of data after moving it to device
for b in self.dl:
yield to_device(b,self.device)
def __len__(self):
#Number of batches
return len(self.dl)
device = get_default_device()
device
torch.cuda.empty_cache()
model = ImageClassification()
random_seed = 42
torch.manual_seed(random_seed)
train_loader = DeviceDataLoader(train_loader, device)
test_loader = DeviceDataLoader(test_loader, device)
to_device(model, device)
@torch.no_grad()
def evaluate(model, test_loader):
model.eval()
outputs = [model.validation_step(batch) for batch in test_loader]
return model.validation_epoch_end(outputs)
# Define the RMSprop optimizer
optimizer = torch.optim.RMSprop(model.parameters(), lr=0.001, alpha=0.99, eps=1e-08, momentum=0.9)
from torch.optim.lr_scheduler import LambdaLR
# Define the custom scheduler function
def lr_schedule(epoch, lr):
if epoch < 10:
return lr
else:
return lr * torch.exp(torch.tensor(-0.1))
# Create a LambdaLR scheduler using the custom function
scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: lr_schedule(epoch, lr=0.001))
def fit(epochs, model, train_loader, test_loader, optimizer):
history = []
for epoch in range(epochs):
# Training Phase
model.train()
train_losses = []
correct_train = 0
total_train = 0
for batch in train_loader:
#images,labels = batch
#out = model(images)
#loss = F.cross_entropy(out,labels)
loss = model.training_step(batch)
#train_losses.append(loss.item())
train_losses.append(loss)
# Calculate training accuracy
#preds = torch.argmax(out, dim=1)
#correct_train += (preds == torch.argmax(labels, dim=1)).sum().item()
#total_train += labels.size(0)
loss.backward()
optimizer.step()
optimizer.zero_grad()
scheduler.step()
# Validation phase
result = evaluate(model, test_loader)
result['train_loss'] = torch.stack(train_losses).mean().item()
#result['train_loss'] = torch.tensor(train_losses).mean().item()
#result['train_acc'] = correct_train / total_train
model.epoch_end(epoch, result)
history.append(result)
return history
model=to_device(ImageClassification(),device)
#initial evaluation of the model
#evaluate(model,test_loader)
# Initial evaluation of the model
initial_result = evaluate(model, test_loader)
accuracy_percentage = initial_result['val_acc'] * 100
print('Initial Test Loss: {:.4f}, Initial Test Accuracy: {:.4f}%'.format(initial_result['val_loss'], accuracy_percentage))
#set the no. of epochs, optimizer funtion and learning rate
num_epochs = 10
#fitting the model on training data and record the result after each epoch
history = fit(num_epochs, model, train_loader, test_loader, optimizer)
output
Labels: ['120', '144', '168', '192', '216', '24', '240', '48', '72', '96']
Total number of labels: 10
Length of train_batch: 1835
Length of test_batch: 459
Initial Test Loss: 2.3067, Initial Test Accuracy: 9.5833%
Epoch [0], train_loss: 2.4796, val_loss: 2.3525, val_acc: 0.1229
Epoch [1], train_loss: 2.4644, val_loss: 2.3175, val_acc: 0.1188
Epoch [2], train_loss: 2.4614, val_loss: 2.3247, val_acc: 0.1083
Epoch [3], train_loss: 2.4695, val_loss: 2.3192, val_acc: 0.1167
Epoch [4], train_loss: 2.4771, val_loss: 2.3155, val_acc: 0.1292
Epoch [5], train_loss: 2.4994, val_loss: 2.3175, val_acc: 0.1292
Epoch [6], train_loss: 2.4528, val_loss: 2.3189, val_acc: 0.1125
Epoch [7], train_loss: 2.4887, val_loss: 2.3146, val_acc: 0.1331
Epoch [8], train_loss: 2.4908, val_loss: 2.3149, val_acc: 0.1208
Epoch [9], train_loss: 2.4809, val_loss: 2.3195, val_acc: 0.1208