I implemented a image classificaton algorithm and there are 10 classes.
I have small dataset with 216 rgb images.
When I train my network, loss value doesn’t decrease.
Here is my code:
class signLanguageDataset(Dataset):
def __init__(self, h5_root_dir, train):
self.h5_root_dir = h5_root_dir
self.h_dataset = None
self.train = train
with h5py.File(self.h5_root_dir, 'r') as file:
self.total = [str(list(file)[i][: len(list(file)[i])-5]) for i in range(len(file)) if(list(file)[i][len(list(file)[i])-4:] == 'flow')]
def __len__(self):
return len(self.total)
def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
if self.h_dataset is None:
self.h_dataset = h5py.File(self.h5_root_dir, 'r')
flow = self.h_dataset[self.total[idx]+'_flow'][()]
spt = self.h_dataset[self.total[idx] + '_spt'][()]
label = self.h_dataset[self.total[idx] + '_label'][()][0]
spt, flow, label = self.transform(spt, flow, label, self.train)
sample = {'flow': flow, 'spt': spt, 'label': label}
return sample
def transform(self, spt, flow, label, train):
flow_parse = [flow[..., i] for i in range(flow.shape[2])]
toPil = transforms.ToPILImage()
jitter = transforms.ColorJitter()
spt = toPil(spt)
flow_parse = [toPil(img) for img in flow_parse]
if train:
i, j, h, w = transforms.RandomCrop.get_params(spt, (224,224))
spt = TF.crop(spt, i, j, h, w)
flow_parse = [TF.crop(img, i, j, h, w) for img in flow_parse]
if np.random.random() > 0.5:
spt = TF.hflip(spt)
flow_parse = [TF.hflip(img) for img in flow_parse]
else:
spt = TF.center_crop(spt, (224,224))
flow_parse = [TF.center_crop(img, (224,224)) for img in flow_parse]
spt = jitter(spt)
spt = TF.to_tensor(spt)
spt = TF.normalize(spt, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
flow_parse = [TF.to_tensor(img) for img in flow_parse]
flow_parse = [TF.normalize(img, [0.5], [0.5]) for img in flow_parse]
flow_parse = [torch.reshape(img,(img.size(1), img.size(2))) for img in flow_parse]
flow = torch.stack([img for img in flow_parse], dim=0)
label = torch.tensor(label).long()
return spt, flow, label
As you can see, the data set contains two different image type which are spt and flow(labels are joint).
Actually I train this separetly.
When I train these spt and flow together(fusion), my loss value is decreasing.
But if I train these separately, my model doesn’t learn anything.
Here is my Network architecture:
class TwoDCNN(nn.Module): # (3 x 224 x 224)
def __init__(self, in_channels, num_classes, drop=0.5):
super(TwoDCNN, self).__init__()
self.pool = nn.MaxPool2d(kernel_size=3, stride=2)
self.relu = nn.ReLU()
self.conv1 = nn.Conv2d(in_channels=in_channels, kernel_size=7, out_channels=96, stride=2)
self.bn1 = nn.BatchNorm2d(num_features=96)
self.conv2 = nn.Conv2d(in_channels=96, kernel_size=5, out_channels=256, stride=2, padding=1)
self.bn2 = nn.BatchNorm2d(num_features=256)
self.conv3 = nn.Conv2d(in_channels=256, kernel_size=3, out_channels=512, stride=1, padding=1)
self.bn3 = nn.BatchNorm2d(num_features=512)
self.conv4 = nn.Conv2d(in_channels=512, kernel_size=3, out_channels=512, stride=1, padding=1)
self.bn4 = nn.BatchNorm2d(num_features=512)
self.conv5 = nn.Conv2d(in_channels=512, kernel_size=3, out_channels=512, stride=1, padding=1)
self.bn5 = nn.BatchNorm2d(num_features=512)
self.drop = nn.Dropout(p=drop)
self.fc6 = nn.Linear(in_features=5 * 5 * 512, out_features=4096) # 5 x 5
self.fc7 = nn.Linear(in_features=4096, out_features=2048)
self.fc8 = nn.Linear(in_features=2048, out_features=num_classes)
def forward(self, x):
x = self.pool(self.relu(self.bn1(self.conv1(x))))
x = self.pool(self.relu(self.bn2(self.conv2(x))))
x = self.relu(self.bn3(self.conv3(x)))
x = self.relu(self.bn4(self.conv4(x)))
x = self.pool(self.relu(self.bn5(self.conv5(x))))
x = x.view(-1, 5 * 5 * 512)
x = self.drop(self.relu(self.fc6(x)))
x = self.drop(self.relu(self.fc7(x)))
x = self.fc8(x)
return x
And here is my train function:
def train_model(.......):
best_model_wts = copy.deepcopy(two_stream_cnn.state_dict())
best_acc = 0.0
for epoch in range(num_epochs):
# Each epoch has a training and validation phase
for phase in ['train', 'validation']:
if phase == 'train':
cnn.train() # Set model to training mode
else:
cnn.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for i, sample in enumerate(dataloaders[phase]):
flow, spt, labels = sample['flow'].to(device), sample['spt'].to(device, dtype=torch.float), sample['label'].to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# Get model outputs and calculate loss
outputs = cnn(spt)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
if lr_sch != None and phase == 'train':
lr_sch.step()
# statistics
running_loss += loss.item() * spt.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(dataloaders[phase].dataset)
epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc * 100))
I tried different hyper parameters like learnig rate and also I train this data in resnet18.
But the result remained the same.
Here is my initial parameters:
num_epochs = 250
batch_size = 2
num_classes = 10
drop_spt = 0.0
CNN = TwoDCNN(3, num_classes, drop_spt)
CNN.to(device)
dataloaders = signLanDataLoader(batch_size)
optimizer_spt = optim.Adam(CNN.parameters())
lr_sch = lr_scheduler.CosineAnnealingLR(optimizer_spt, len(dataloaders['train']))
criterion = nn.CrossEntropyLoss().to(device)
train_model(......)
I also tried StepLR optimizer scheduler.
Here are results:
Epoch 1/250
----------
train Loss: 4.0784 Acc: 8.0460
validation Loss: 2.2950 Acc: 9.1954
Training complete in 0m 31s
Epoch 2/250
----------
train Loss: 2.3268 Acc: 10.3448
validation Loss: 2.3149 Acc: 9.1954
Training complete in 0m 10s
Epoch 3/250
----------
train Loss: 2.3422 Acc: 9.5785
validation Loss: 2.3080 Acc: 8.0460
Training complete in 0m 10s
Epoch 4/250
----------
train Loss: 2.3181 Acc: 9.5785
validation Loss: 2.3934 Acc: 11.4943
Training complete in 0m 10s
Epoch 5/250
----------
train Loss: 2.3862 Acc: 6.5134
validation Loss: 2.3104 Acc: 6.8966
Training complete in 0m 10s
Epoch 6/250
----------
train Loss: 2.3142 Acc: 13.0268
validation Loss: 2.3044 Acc: 11.4943
Training complete in 0m 10s
Epoch 7/250
----------
train Loss: 2.3369 Acc: 10.3448
validation Loss: 2.3203 Acc: 11.4943
Training complete in 0m 10s
Epoch 8/250
----------
train Loss: 2.3093 Acc: 11.4943
validation Loss: 2.3650 Acc: 5.7471
Training complete in 0m 10s
Epoch 9/250
----------
train Loss: 2.3147 Acc: 11.1111
validation Loss: 2.3100 Acc: 10.3448
Training complete in 0m 10s
Epoch 10/250
----------
train Loss: 2.3122 Acc: 12.6437
validation Loss: 2.3171 Acc: 9.1954
Training complete in 0m 10s
Epoch 11/250
----------
train Loss: 2.3077 Acc: 11.4943
validation Loss: 2.3102 Acc: 11.4943
Training complete in 0m 10s
Epoch 12/250
----------
train Loss: 2.3059 Acc: 11.4943
validation Loss: 2.3120 Acc: 5.7471
Training complete in 0m 10s
Epoch 13/250
----------
train Loss: 2.2962 Acc: 12.6437
validation Loss: 2.3100 Acc: 9.1954
Training complete in 0m 10s
Epoch 14/250
----------
train Loss: 2.3012 Acc: 11.8774
validation Loss: 2.3120 Acc: 8.0460
Training complete in 0m 10s
Epoch 15/250
----------
train Loss: 2.3003 Acc: 11.8774
validation Loss: 2.3107 Acc: 3.4483
Training complete in 0m 10s
Epoch 16/250
----------
train Loss: 2.2929 Acc: 10.3448
validation Loss: 2.3147 Acc: 5.7471
Training complete in 0m 10s
Epoch 17/250
----------
train Loss: 2.2957 Acc: 12.2605
validation Loss: 2.3013 Acc: 9.1954
Training complete in 0m 10s
Epoch 18/250
----------
train Loss: 2.3036 Acc: 14.1762
validation Loss: 2.4307 Acc: 5.7471
Training complete in 0m 10s
Epoch 19/250
----------
train Loss: 2.3066 Acc: 13.4100
validation Loss: 2.3023 Acc: 8.0460
Training complete in 0m 10s
Epoch 20/250
----------
train Loss: 2.2789 Acc: 12.6437
validation Loss: 2.6025 Acc: 10.3448
Training complete in 0m 10s
Epoch 21/250
----------
train Loss: 2.3840 Acc: 12.6437
validation Loss: 2.3095 Acc: 11.4943
Training complete in 0m 10s
Epoch 22/250
----------
train Loss: 2.2915 Acc: 11.4943
validation Loss: 2.3127 Acc: 11.4943
Training complete in 0m 10s
Epoch 23/250
----------
train Loss: 2.2920 Acc: 13.4100
validation Loss: 2.3110 Acc: 8.0460
Training complete in 0m 10s
Epoch 24/250
----------
train Loss: 2.2889 Acc: 12.2605
validation Loss: 2.3115 Acc: 8.0460
Training complete in 0m 10s
Epoch 25/250
----------
train Loss: 2.2901 Acc: 14.5594
validation Loss: 2.3141 Acc: 8.0460
Training complete in 0m 10s
Epoch 26/250
----------
train Loss: 2.2885 Acc: 13.4100
validation Loss: 2.3163 Acc: 8.0460
Training complete in 0m 10s
Epoch 27/250
----------
train Loss: 2.2903 Acc: 13.4100
validation Loss: 2.3204 Acc: 8.0460
Training complete in 0m 10s
Epoch 28/250
----------
train Loss: 2.2888 Acc: 13.4100
validation Loss: 2.3243 Acc: 8.0460
Training complete in 0m 10s
Epoch 29/250
----------
train Loss: 2.2905 Acc: 13.4100
validation Loss: 2.3209 Acc: 8.0460
Training complete in 0m 10s
Epoch 30/250
----------
train Loss: 2.2884 Acc: 13.4100
validation Loss: 2.3253 Acc: 8.0460
Training complete in 0m 10s
Epoch 31/250
----------
train Loss: 2.2905 Acc: 13.4100
validation Loss: 2.3187 Acc: 8.0460
Training complete in 0m 10s
Epoch 32/250
----------
train Loss: 2.2874 Acc: 13.4100
validation Loss: 2.3177 Acc: 8.0460
Training complete in 0m 10s
Epoch 33/250
----------
train Loss: 2.2888 Acc: 11.4943
validation Loss: 2.3186 Acc: 8.0460
Training complete in 0m 10s
Epoch 34/250
----------
train Loss: 2.2885 Acc: 11.1111
validation Loss: 2.3169 Acc: 8.0460
Training complete in 0m 10s
Epoch 35/250
----------
train Loss: 2.2891 Acc: 13.4100
validation Loss: 2.3179 Acc: 8.0460
Training complete in 0m 10s
Epoch 36/250
----------
train Loss: 2.2888 Acc: 13.4100
validation Loss: 2.3181 Acc: 8.0460
Training complete in 0m 10s
Epoch 37/250
----------
train Loss: 2.2892 Acc: 13.4100
validation Loss: 2.3195 Acc: 8.0460
Training complete in 0m 10s
Epoch 38/250
----------
train Loss: 2.2887 Acc: 13.4100
validation Loss: 2.3173 Acc: 8.0460
Training complete in 0m 10s
Epoch 39/250
----------
train Loss: 2.2895 Acc: 13.4100
validation Loss: 2.3135 Acc: 8.0460
Training complete in 0m 10s
Epoch 40/250
----------
train Loss: 2.2875 Acc: 13.4100
validation Loss: 2.3155 Acc: 8.0460
Training complete in 0m 10s
Epoch 41/250
----------
train Loss: 2.2895 Acc: 13.4100
validation Loss: 2.3127 Acc: 8.0460
Training complete in 0m 10s
Epoch 42/250
----------
train Loss: 2.2880 Acc: 13.4100
validation Loss: 2.3140 Acc: 8.0460
Training complete in 0m 10s
Epoch 43/250
----------
train Loss: 2.2900 Acc: 13.4100
validation Loss: 2.3183 Acc: 8.0460
Training complete in 0m 10s
Epoch 44/250
----------
train Loss: 2.2875 Acc: 13.4100
validation Loss: 2.3186 Acc: 8.0460
Training complete in 0m 10s
Epoch 45/250
----------
train Loss: 2.2901 Acc: 13.4100
validation Loss: 2.3221 Acc: 8.0460
Training complete in 0m 10s
Epoch 46/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3196 Acc: 8.0460
Training complete in 0m 10s
Epoch 47/250
----------
train Loss: 2.2889 Acc: 13.4100
validation Loss: 2.3198 Acc: 8.0460
Training complete in 0m 10s
Epoch 48/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3182 Acc: 8.0460
Training complete in 0m 10s
Epoch 49/250
----------
train Loss: 2.2899 Acc: 13.4100
validation Loss: 2.3180 Acc: 8.0460
Training complete in 0m 10s
Epoch 50/250
----------
train Loss: 2.2873 Acc: 13.4100
validation Loss: 2.3147 Acc: 8.0460
Training complete in 0m 10s
Epoch 51/250
----------
train Loss: 2.2899 Acc: 13.4100
validation Loss: 2.3127 Acc: 8.0460
Training complete in 0m 10s
Epoch 52/250
----------
train Loss: 2.2888 Acc: 13.4100
validation Loss: 2.3128 Acc: 8.0460
Training complete in 0m 10s
Epoch 53/250
----------
train Loss: 2.2894 Acc: 13.4100
validation Loss: 2.3151 Acc: 8.0460
Training complete in 0m 10s
Epoch 54/250
----------
train Loss: 2.2885 Acc: 13.4100
validation Loss: 2.3148 Acc: 8.0460
Training complete in 0m 10s
Epoch 55/250
----------
train Loss: 2.2886 Acc: 13.4100
validation Loss: 2.3166 Acc: 8.0460
Training complete in 0m 10s
Epoch 56/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3175 Acc: 8.0460
Training complete in 0m 10s
Epoch 57/250
----------
train Loss: 2.2893 Acc: 13.4100
validation Loss: 2.3238 Acc: 8.0460
Training complete in 0m 10s
Epoch 58/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3258 Acc: 8.0460
Training complete in 0m 10s
Epoch 59/250
----------
train Loss: 2.2897 Acc: 13.4100
validation Loss: 2.3202 Acc: 8.0460
Training complete in 0m 10s
Epoch 60/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3223 Acc: 8.0460
Training complete in 0m 10s
Epoch 61/250
----------
train Loss: 2.2891 Acc: 13.4100
validation Loss: 2.3192 Acc: 8.0460
Training complete in 0m 10s
Epoch 62/250
----------
train Loss: 2.2878 Acc: 13.4100
validation Loss: 2.3202 Acc: 8.0460
Training complete in 0m 10s
Epoch 63/250
----------
train Loss: 2.2887 Acc: 13.4100
validation Loss: 2.3199 Acc: 8.0460
Training complete in 0m 10s
Epoch 64/250
----------
train Loss: 2.2872 Acc: 13.4100
validation Loss: 2.3228 Acc: 8.0460
Training complete in 0m 10s
Epoch 65/250
----------
train Loss: 2.2888 Acc: 13.4100
validation Loss: 2.3227 Acc: 8.0460
Training complete in 0m 10s
Epoch 66/250
----------
train Loss: 2.2875 Acc: 13.4100
validation Loss: 2.3229 Acc: 8.0460
Training complete in 0m 10s
Epoch 67/250
----------
train Loss: 2.2893 Acc: 13.4100
validation Loss: 2.3257 Acc: 8.0460
Training complete in 0m 10s
Epoch 68/250
----------
train Loss: 2.2875 Acc: 13.4100
validation Loss: 2.3242 Acc: 8.0460
Training complete in 0m 10s
Epoch 69/250
----------
train Loss: 2.2897 Acc: 10.3448
validation Loss: 2.3213 Acc: 11.4943
What is the problem?
