This is the distribution most of the times. Distribution is pretty random. If I remove the subset part and the labels part, then none of the class has samples as 0 in the batch, but the sampling contains majority from class 2,3 and minority from 1,4 which should not be the case.
Its sampling is random. When I run the code, sometimes it samples really badly, sometimes its better. Why is this so?
Yes train labels contains labels for complete set.
class Dataset(data.Dataset):
def __init__(self, list_IDs, labels, sample_weight, augment=True, shuffle=True):
#'Initialization'
self.list_IDs = list_IDs
self.labels=labels
self.augment = augment
self.shuffle = shuffle
self.sample_weight = sample_weight
# self.on_epoch_end()
#
def __getitem__(self, index):
np.random.seed(index)
# image_path = np.random.choice(a=self.list_IDs)
image_path = self.list_IDs[index]
my_path=image_path.replace("/home/ken.chang@ccds.io/mnt","/data" )
image = np.load(my_path)
if self.augment:
image = self.__preprocess_input(image)
label = (int(image_path[-5])-1)
image = TF.to_tensor(image).float()
label = torch.from_numpy(np.array(label))
return image, label
def __len__(self):
return int(len(self.list_IDs))
def on_epoch_end(self):
print("hello_epoch")
if self.shuffle == True:
c = list(zip(self.list_IDs,self.sample_weight, self.labels))
random.shuffle(c)
self.list_IDs, self.sample_weight, self.labels = zip(*c)
def __preprocess_input(self, image):
if np.random.rand(1)[0]>.5:
image = np.fliplr(image)
if np.random.rand(1)[0]>.5:
image = np.flipud(image)
image = rotate(image, angle=np.random.rand(1)[0]*45, mode='nearest', reshape=False)
return image
#draw from each sample with equal class probability
def cal_sample_weight(files):
print("file length ",len(files))
labels = [int(f[-5])-1 for f in files]
class_count = [labels.count(c) for c in np.unique(labels)]
print("class count is ",class_count)
weights_class = [((1.0/len(np.unique(labels)))/(cc)) for cc in class_count]
# weights_class = [1.0/(cc) for cc in class_count]
print("Weight class is ", weights_class)
weight_array = np.array([weights_class[l] for l in labels])
return torch.from_numpy(weight_array)
# return weight_array
os.chdir('/data/2015P002510/MammoDensity/scripts_preprocessing/splits')
with open("Train_images_DMIST2.txt", "rb") as fp:
Train_files1 = pickle.load(fp)
with open("Train_images_DMIST3.txt", "rb") as fp:
Train_files2 = pickle.load(fp)
with open("Train_images_DMIST4.txt", "rb") as fp:
Train_files3 = pickle.load(fp)
with open("Train_images_MGH.txt", "rb") as fp:
Train_files4 = pickle.load(fp)
Train_files = Train_files1+Train_files2+Train_files3+Train_files4
#
with open("Val_images_DMIST2.txt", "rb") as fp:
Val_files1 = pickle.load(fp)
with open("Val_images_DMIST3.txt", "rb") as fp:
Val_files2 = pickle.load(fp)
with open("Val_images_DMIST4.txt", "rb") as fp:
Val_files3 = pickle.load(fp)
with open("Val_images_MGH.txt", "rb") as fp:
Val_files4 = pickle.load(fp)
Val_files = Val_files1+Val_files2+Val_files3+Val_files4
print([(int(image_path[-5])-1) for image_path in Train_files[0:32]])
random.shuffle(Train_files)
random.shuffle(Val_files)
print([(int(image_path[-5])-1) for image_path in Train_files[0:32]])
Train_labels = [(int(image_path[-5])-1) for image_path in Train_files]
Val_labels = [(int(image_path[-5])-1) for image_path in Val_files]
#
Train_sample_weight = cal_sample_weight(Train_files)
print(Train_sample_weight[0:32])
Val_sample_weight = cal_sample_weight(Val_files)
# LOADING DATASET
training_set = Dataset(list_IDs=Train_files,labels=Train_labels, sample_weight=Train_sample_weight, shuffle=True, augment=True)
print("length of train is ", len(training_set))
sampler_train = torch.utils.data.sampler.WeightedRandomSampler(weights = Train_sample_weight, num_samples=len(training_set))
trainloader = DataLoader(torch.utils.data.Subset(training_set,Train_labels), batch_size=batch_size, sampler=sampler_train,num_workers=24, pin_memory=True)
# trainloader = DataLoader(dataset=training_set, batch_size=batch_size, sampler=sampler_train,shuffle=False, num_workers=24, pin_memory=True)
test_set = Dataset(Val_files,labels=Val_labels, sample_weight=Val_sample_weight, augment=False)
print("lenght of val is ", len(test_set))
sampler_test = torch.utils.data.sampler.WeightedRandomSampler(weights = Val_sample_weight, num_samples=len(test_set))
testloader = DataLoader(dataset=test_set, batch_size=batch_size, sampler=sampler_test, num_workers=24, pin_memory=True)
