Hi everyone,
I’m working on calculating the logits for my entire training set to identify specific training parameters. To avoid mismatches between the logits used in training and those used for hyperparameter calculation, I need to keep the augmentation settings per sample fixed during both processes. If the augmentation settings vary, it could lead to inaccuracies in my results.
I have written this simple solution and I would like to know if it will affect any other parts of the training except the augmentation part. Any tips or insights would be greatly appreciated!
import torch
import torchvision.transforms as transforms
from torchvision.datasets import CIFAR10
from torch.utils.data import DataLoader
from PIL import Image
import random
from torchvision import datasets
# Define criterion and optimizer
import torch.optim as optim
# Define the model
import torch.nn as nn
import torch.nn.functional as F
def get_deterministic_transforms(seed):
# Set the random seed for reproducibility
torch.manual_seed(seed)
random.seed(seed)
# Define your augmentation transforms here
transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(10),
transforms.ToTensor(),
])
return transform
class CIFAR10Instance(datasets.CIFAR10):
"""CIFAR100Instance Dataset.
"""
def __init__(self, root, train=True,
transform=None, target_transform=None,
download=False):
super().__init__(root=root, train=train, download=download,
transform=transform, target_transform=target_transform)
self.seed = 10
def __getitem__(self, index):
img, target = self.data[index], self.targets[index]
img = Image.fromarray(img)
# Store current random state
random_state = torch.get_rng_state()
img = get_deterministic_transforms(index+self.seed)(img)
# Restore previous random state
torch.set_rng_state(random_state)
if self.transform is not None:
img = self.transform(img)
if self.target_transform is not None:
target = self.target_transform(target)
return img, target, index
# Create dataset and dataloader
train_dataset = CIFAR10Instance(root='./data', train=True, download=True)
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
class SimpleCNN(nn.Module):
def __init__(self):
super(SimpleCNN, self).__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1)
self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding=1)
self.fc1 = nn.Linear(in_features=32*8*8, out_features=128)
self.fc2 = nn.Linear(in_features=128, out_features=10)
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.max_pool2d(x, 2)
x = F.relu(self.conv2(x))
x = F.max_pool2d(x, 2)
x = x.view(-1, 32*8*8) # Flatten the tensor
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
# Check if GPU is available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = SimpleCNN().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
# Training loop
num_epochs = 5
idx = 1000
for epoch in range(num_epochs):
train_loader.dataset.seed += 1
for batch_idx, (data, target, index) in enumerate(train_loader):
# Move data and target to GPU
data, target = data.to(device), target.to(device)
if idx in index:
print(data[torch.nonzero(index==idx)[0]][0,0,0])
# Forward pass
output = model(data)
loss = criterion(output, target)
# Backward pass and optimization
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch_idx % 100 == 0:
print(f'Epoch [{epoch+1}/{num_epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {loss.item():.4f}')