Los not decreasing

1

I am trying neural style transfer here is my code

class Vgg16(torch.nn.Module):
    def __init__(self):
        super(Vgg16, self).__init__()
        features = list(vgg16(pretrained = True).features)[:23]
        self.features = nn.ModuleList(features).eval() 
        
    def forward(self, x):
        results = []
        for ii,model in enumerate(self.features):
            x = model(x)
            if ii in {3,8,15,22}:
                results.append(x)
        return results
normalization_mean = torch.tensor([0.485, 0.456, 0.406])
normalization_std = torch.tensor([0.229, 0.224, 0.225])
class Normalization(nn.Module):
    def __init__(self, mean, std):
        super(Normalization, self).__init__()
        self.mean = torch.tensor(mean).view(-1, 1, 1)
        self.std = torch.tensor(std).view(-1, 1, 1)
    def forward(self, img):
        return (img - self.mean) / self.std
normalization = Normalization(normalization_mean, normalization_std)
def style_loss(style,actual_style):
    q,w,e,r=style.shape
    s_loss=(gram_matrix(style)-gram_matrix(actual_style)).pow(2).sum()
    return s_loss
def gram_matrix(inp):
    q,w,e,r=inp.shape
    reshape=inp.view(q*w,e*r)
    g_mat=torch.mm(reshape,reshape.t())
    return g_mat/(q*w*e*r)
def content_loss(content,actual_content):
    q,w,e,r=content.shape
    c_loss=(content-actual_content).pow(2).sum()
    return c_loss/(q*w*e*r)
content_im=cv2.imread('/home/jatin/ML_intern/challenge-master/hij.jpeg')
style_im=cv2.imread('/home/jatin/ML_intern/challenge-master/download.jpeg')
s=128
c_im=cv2.resize(content_im,(s,s))
c_im=torch.from_numpy(c_im)
c_im=c_im.view(3,s,s)
c_im=normalization(c_im)
c_im=c_im.view(1,3,s,s)
c_im=c_im.float()
c_im=c_im/255
s_im=cv2.resize(style_im,(s,s))
s_im=torch.from_numpy(s_im)
s_im=s_im.view(3,s,s)
s_im=s_im.float()
s_im=s_im/255
#s_im=(s_im-0.4)/0.2
s_im=normalization(s_im)
s_im=s_im.view(1,3,s,s)
print(s_im)
y_con=net(c_im.float())
y_sty=net(s_im.float())
def loss_fn(y_pred,y_con,y_sty,y):
    s_loss=style_loss(y_pred[1],y_sty[1])
    c_loss=0
    for ij in range(len(y_pred)):
        if ij==1:
            c_loss+=content_loss(y_pred[ij],y_con[ij])
        else:
            s_loss+=style_loss(y_pred[ij],y_sty[ij])
    loss=c_loss+100*s_loss
    print('s_loss=',s_loss)
    print('c_loss=',c_loss)
    return loss
inp=torch.randn(1,3,s,s,requires_grad=True)
optimizer = optim.LBFGS([inp])
or i in range(2000):
    def closure():
        inp.data.clamp_(0,1)
        optimizer.zero_grad()
        y_pred=net(inp.float())
        loss=loss_fn(y_pred,y_con,y_sty,inp)
        loss.backward(retain_graph=True)
        return loss
    optimizer.step(closure)
    print('#########################################################################################',i)
    if i%5==0:
        with torch.no_grad():
            bc=inp.view(s,s,3)
            bc.data.clamp_(0,1)
            bc=bc*255
            bc=bc.detach()
            bc=bc.numpy()    
            bc=bc.astype(np.uint8)
            bc=cv2.cvtColor(bc,cv2.COLOR_BGR2RGB)
        plt.imshow(bc)
        plt.show()

But after I normalised the input images the loss was initially decreasing after some iterations the loss was constant

2

It seems that your training might just be stuck, so that you would have to play around with some hyperparameters or was the model training better without the normalization?
Also, is there a particular reason you are reimplementing the Normalize method instead of using torchvision.transforms.Normalize?

3

The model did better without normalisation and there is no reason for using Normalize instead using the one provided by pytorch. I will try to tune the hyperparameters