Hi ! I recently found a strange problem when using the PyTorch training network.
- When the model is trained, the loss does not decrease
- I’m not sure what caused this
Please allow me to show my code below.
This is my train.py where I run the training code.
import torch
import torch.nn as nn
import torch.optim as optim
from make_data import train_dataloader, test_dataloader
from make_net import Net
import time
num_epochs = 900
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
criterion = nn.MSELoss()
def train_model(model, my_criterion):
since = time.time()
train_loader = train_dataloader
criterion = my_criterion
net = model()
opt_Adam = optim.Adam(net.parameters(), lr=0.2, betas=(0.9, 0.99))
if torch.cuda.is_available():
print("Let's use", torch.cuda.device_count(), "GPUs")
net = nn.DataParallel(net)
net.to(device)
for epoch in range(num_epochs):
running_loss = 0.0
for i, sample in enumerate(train_loader, 0):
image, pressure = sample['image'], sample['pressure']
image = image.float()
image = image.to(device)
# image.shape torch.Size([256, 403, 640])
# print("image.shape", image.shape)
pressure = pressure.float()
pressure = pressure.to(device)
opt_Adam.zero_grad()
output = net(pressure)
# output.shape torch.Size([256, 403, 640])
# print("output.shape", output.shape)
loss = criterion(output, image)
loss.backward()
opt_Adam.step()
running_loss += loss.item()
if i % 10 == 9:
# print every 200 mini-batch
print("[%d, %5d], loss: %.3f" % (epoch+1, i+1, running_loss/10))
running_loss = 0.0
print("Finished Training!")
train_model(Net, criterion)
And this is my net.py where I define my network
import torch
import torch.nn as nn
import torch.nn.functional as F
from make_ops import conv_out_size_same
from make_data import batch_size
s_h, s_w = 403, 640
# 403,640
s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
# 202, 320
s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
# 101, 160
s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
# 51, 80
s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)
# 25, 40
s_h32, s_w32 = conv_out_size_same(s_h16, 2), conv_out_size_same(s_w16, 2)
# 12, 20
s_h64, s_w64 = conv_out_size_same(s_h32, 2), conv_out_size_same(s_w32, 2)
# 6, 10
s_h128, s_w128 = conv_out_size_same(s_h64, 2), conv_out_size_same(s_w64, 2)
# 3，5
s_h256, s_w256 = conv_out_size_same(s_h128, 2), conv_out_size_same(s_w128, 2)
# 2, 3
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.CONV1_DEPTH = 2
self.CONV2_DEPTH = 4
self.CONV3_DEPTH = 8
self.CONV4_DEPTH = 16
self.CONV5_DEPTH = 32
self.CONV6_DEPTH = 64
self.CONV7_DEPTH = 128
self.CONV8_DEPTH = 256
self.f_dim = 32
self.channel_dim = 1
self.FC_NODE = 512
self.IMG_HEIGHT = 403
self.IMG_WIDTH = 640
self.batch_size = batch_size
self.fc1 = nn.Linear(in_features=10, out_features=self.f_dim*8)
self.fc2 = nn.Linear(in_features=self.f_dim*8, out_features=self.f_dim*8*s_w256*s_h256)
self.fc3 = nn.Linear(in_features=self.CONV8_DEPTH*s_h256*s_w256, out_features=self.FC_NODE)
self.fc4 = nn.Linear(in_features=self.FC_NODE, out_features=self.IMG_HEIGHT * self.IMG_WIDTH)
self.avg_pool = nn.AdaptiveAvgPool2d((s_h256, s_w256))
self.deconv1 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim*8, out_channels=self.f_dim*4,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim*4),
nn.ELU()
)
self.deconv2 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim * 4, out_channels=self.f_dim * 2,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim * 2),
nn.ELU()
)
self.deconv3 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim * 2, out_channels=self.f_dim,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim),
nn.ELU()
)
self.deconv4 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim, out_channels=self.f_dim//2,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim//2),
nn.ELU()
)
self.deconv5 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim//2, out_channels=self.f_dim//4,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim//4),
nn.ELU()
)
self.deconv6 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim//4, out_channels=self.f_dim//8,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim//8),
nn.ELU()
)
self.deconv7 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim//8, out_channels=self.f_dim//16,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.f_dim//16),
nn.ELU()
)
self.deconv8 = nn.Sequential(
nn.ConvTranspose2d(in_channels=self.f_dim//16, out_channels=self.channel_dim,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.channel_dim),
nn.Tanh()
)
self.conv1 = nn.Sequential(
nn.Conv2d(in_channels=self.channel_dim, out_channels=self.CONV1_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV1_DEPTH),
nn.ELU()
)
self.conv2 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV1_DEPTH, out_channels=self.CONV2_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV2_DEPTH),
nn.ELU()
)
self.conv3 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV2_DEPTH, out_channels=self.CONV3_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV3_DEPTH),
nn.ELU()
)
self.conv4 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV3_DEPTH, out_channels=self.CONV4_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV4_DEPTH),
nn.ELU()
)
self.conv5 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV4_DEPTH, out_channels=self.CONV5_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV5_DEPTH),
nn.ELU()
)
self.conv6 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV5_DEPTH, out_channels=self.CONV6_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV6_DEPTH),
nn.ELU()
)
self.conv7 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV6_DEPTH, out_channels=self.CONV7_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV7_DEPTH),
nn.ELU()
)
self.conv8 = nn.Sequential(
nn.Conv2d(in_channels=self.CONV7_DEPTH, out_channels=self.CONV8_DEPTH,
kernel_size=2, stride=2),
nn.BatchNorm2d(num_features=self.CONV8_DEPTH),
nn.ELU()
)
self.layer = nn.Sequential(
nn.BatchNorm2d(num_features=self.f_dim*8),
nn.ELU()
)
def forward(self, x):
x = self.fc1(x)
x = self.fc2(x)
x = x.view(-1, self.f_dim*8, s_h256, s_w256)
x = self.layer(x)
x = self.deconv1(x)
x = self.deconv2(x)
x = self.deconv3(x)
x = self.deconv4(x)
x = self.deconv5(x)
x = self.deconv6(x)
x = self.deconv7(x)
x = self.deconv8(x)
x = self.conv1(x)
x = self.conv2(x)
x = self.conv3(x)
x = self.conv4(x)
x = self.conv5(x)
x = self.conv6(x)
x = self.conv7(x)
x = self.conv8(x)
x = self.avg_pool(x)
x = x.view(-1, self.num_flat_features(x))
x = F.elu(self.fc3(x))
x = F.elu(self.fc4(x))
x = x.view(-1, self.IMG_HEIGHT, self.IMG_WIDTH)
return x
def num_flat_features(self, x):
size = x.size()[1:]
num_features = 1
for s in size:
num_features *= s
return num_features
When I run the training code, I got the loss values as shown below.
[1, 10], loss: 27806.338
[1, 20], loss: 1088497.401
[1, 30], loss: 2364.557
[1, 40], loss: 2366.722
[1, 50], loss: 2368.215
[1, 60], loss: 2370.851
[1, 70], loss: 2365.583
[1, 80], loss: 2366.041
[2, 10], loss: 2368.178
[2, 20], loss: 2363.056
[2, 30], loss: 2374.572
[2, 40], loss: 2361.862
[2, 50], loss: 2364.390
[2, 60], loss: 2366.633
[2, 70], loss: 2372.771
[2, 80], loss: 2362.416
[3, 10], loss: 2369.942
[3, 20], loss: 2367.277
Sooo confused !!!
I would appreciate it if you could put forward some suggestions on this question.
Thank you very much!!!
Wish you a happy life!!