Computing the mean and std of dataset

26 
import numpy as np
from PIL import ImageStat

class Stats(ImageStat.Stat):
  def __add__(self, other):
    # add self.h and other.h element-wise
    return Stats(list(np.add(self.h, other.h)))
28

If I am training my model on a batch size of 4, should I compute the mean and std according to a batch size of 4? or is it more accurate to compute mean and std based on bigger batches (like 8) and then train my model on a batch size of 4?

Thanks.

29

Finally, do we know a good method to calculate mean and std?

30

Any batch_size should work. Training batch_size isn’t directly related to the batch_size you use for calculating mean and std.
You could choose 4 for both, or choose 4 and 8.

31

Very very late. I think this one is (almost) mathematically correct.

Instead of center crop one could run count a number of pixels, like pixel_count += images.nelement() if the image sizes are different.

dataset = datasets.ImageFolder('train', transform=transforms.Compose([transforms.ToTensor()]))

loader = data.DataLoader(dataset,
                         batch_size=10,
                         num_workers=0,
                         shuffle=False,
                         drop_last=False)

mean = 0.0
for images, _ in loader:
    batch_samples = images.size(0) 
    images = images.view(batch_samples, images.size(1), -1)
    mean += images.mean(2).sum(0)
mean = mean / len(loader.dataset)

var = 0.0
pixel_count = 0
for images, _ in loader:
    batch_samples = images.size(0)
    images = images.view(batch_samples, images.size(1), -1)
    var += ((images - mean.unsqueeze(1))**2).sum([0,2])
    pixel_count += images.nelement()
std = torch.sqrt(var / pixel_count)
32

The code looks good!
But there is an issue when counting num of pixels.
As we count for each channel, we should exclude the channel dimension:

pixel_count += images.nelement() / images.size(1)

The updated version:


loader = data.DataLoader(dataset,
                         batch_size=10,
                         num_workers=0,
                         shuffle=False,
                         drop_last=False)

mean = 0.0
for images, _ in loader:
    batch_samples = images.size(0) 
    images = images.view(batch_samples, images.size(1), -1)
    mean += images.mean(2).sum(0)
mean = mean / len(loader.dataset)

var = 0.0
pixel_count = 0
for images, _ in loader:
    batch_samples = images.size(0)
    images = images.view(batch_samples, images.size(1), -1)
    var += ((images - mean.unsqueeze(1))**2).sum([0,2])
    pixel_count += images.nelement() / images.size(1)
std = torch.sqrt(var / pixel_count)
33

Here is my implementation and also I’ve performed the sanity check

import torch
import torchvision.transforms as transforms
from PIL import Image

def compute_mean_std(image_paths):
    # Initialize variables to store cumulative sum of pixel values
    mean = torch.zeros(3)  # Assuming RGB images
    var = torch.zeros(3)
    
    # Define transformation to convert image to tensor
    to_tensor = transforms.ToTensor()

    # step I: Mean
    for image_path in image_paths:
        # Open image and convert to tensor
        image = Image.open(image_path)
        image_tensor = to_tensor(image)
        mean += torch.mean(image_tensor, dim=(1, 2))

    mean /= len(image_paths)
    
    # step II: Std-dev
    # first we need mean from step I
    
    for image_path in image_paths:
        # Open image and convert to tensor
        image = Image.open(image_path)
        image_tensor = to_tensor(image)
        var += torch.mean((image_tensor - mean.unsqueeze(1).unsqueeze(2))**2, dim=(1, 2))
    
    return mean, torch.sqrt(var / len(image_paths))

mean, std_dev = compute_mean_std(df['image_names'].values.tolist())

print(f"Mean of the data is: {mean}")
print(f"Standard deviation of the data is: {std_dev}")

# references: 
# https://gist.github.com/Huud/8e0823fa7be2dcd1bb9f3c418cb94c19
# https://apcentral.collegeboard.org/courses/ap-statistics/classroom-resources/why-variances-add-and-why-it-matters
# https://stackoverflow.com/questions/60101240/finding-mean-and-standard-deviation-across-image-channels-pytorch/60803379#60803379

import torch
import torchvision.transforms as transforms
from torchvision.datasets import CIFAR10

# Define CIFAR-10 dataset and apply transforms
transform = transforms.Compose([
    transforms.ToTensor(),
])

# Load CIFAR-10 dataset
cifar_dataset = CIFAR10(root="./data", train=True, download=True, transform=transform)

# Initialize variables to store cumulative sum of pixel values
mean = torch.zeros(3)
var_temp = torch.zeros(3)

# Compute mean
num_samples = len(cifar_dataset)
for i in range(num_samples):
    image, _ = cifar_dataset[i]
    # Compute mean for each channel
    mean += torch.mean(image, dim=(1, 2))

mean /= num_samples

# Compute variance
for i in range(num_samples):
    image, _ = cifar_dataset[i]
    # Compute squared difference from mean and sum for each channel
    var_temp += torch.mean((image - mean.unsqueeze(1).unsqueeze(2))**2, dim=(1, 2))

# Compute standard deviation
std_dev = torch.sqrt(var_temp / num_samples)

print(f"Computed Mean of the CIFAR-10 dataset is: {mean}")
print(f"Computed Standard deviation of the CIFAR-10 dataset is: {std_dev}")

# Known mean and standard deviation values for CIFAR-10 dataset
known_mean = torch.tensor([0.4914, 0.4822, 0.4465])
known_std_dev = torch.tensor([0.2470, 0.2435, 0.2616])

# Compare with known mean and standard deviation
print(f"Known mean for CIFAR-10 dataset is: {known_mean}")
print(f"Known standard deviation for CIFAR-10 dataset is: {known_std_dev}")