Why loss is so small

1

train.py


import math
from numpy.core.fromnumeric import shape
import torch
import matplotlib.pyplot as plt
import json
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms.functional as TF
import numpy as np
import cv2
import time
from PIL import Image
import random
import sys
import os
sys.path.append(os.path.abspath("C:/Users/x065p/Desktop/test/python/PyTorch/blender_vgg16/"))
from blender.gen_material_with_bsdf import blender_render
from vgg16 import Model

numClasses=27
mom_folder="C:/Users/x065p/Desktop/blender/result/image/python/"
losss=[]
epochs = 50
run_sample=1
# random.seed(4)
blender_object=0


device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")



def bsdf_arg(arr):
    out =[]
    for i in arr:
        # print(type(i))
        try:
            for j in i:
                out.append(j)
        except:
            out.append(i)
    return out

    
f = open(mom_folder+"1000/save_value.txt",'r',encoding = 'utf-8')
datas=f.read()
datas=json.loads(datas)
model = Model().to(device)
optimizer = torch.optim.RMSprop(model.parameters(), lr=0.005)
loss_function = torch.nn.MSELoss()
blender_render_object=blender_render()


for epoch in range(epochs):
    input_img_arr=[]
    input_targe_arr=[]
    for j in range(run_sample):
        i=int(random.random()*1000)
        # print(i)
        # input_img = cv2.imread('data/1000/image/{}.png'.format(i))
        input_img = Image.open(mom_folder+'1000/{}.png'.format(i)).convert('RGB')
        input_img= input_img.resize((224,244))
        # input_img.show()
        input_img = TF.to_tensor(input_img)
        input_img=np.array(input_img)
        # print(input_img.shape)

        input_targe=bsdf_arg( datas[i])
        input_targe=np.array(input_targe)

        input_img_arr.append(input_img)
        input_targe_arr.append(input_targe)

    input_img_arr=torch.tensor(input_img_arr,requires_grad=True)
    input_img_arr=input_img_arr.float()
    input_img_arr=input_img_arr.to(device)

    input_targe_arr=torch.tensor(input_targe_arr)
    input_targe_arr=input_targe_arr.float()
    input_targe_arr=input_targe_arr.to(device)

    prediction = model(input_img_arr)
    predict_bsdf=prediction.detach().numpy()[0]

    blender_object=blender_render_object.img_by_bsdf(predict_bsdf,blender_object)
    # time.sleep(0.01)
    blender_image = Image.open(mom_folder+'tem{}.png'.format(j)).convert('RGB')
    blender_image= blender_image.resize((224,244))
    # blender_image.show()
    blender_image = TF.to_tensor(blender_image)
    blender_image = torch.tensor(blender_image,requires_grad=True)
    # blender_image=np.array(blender_image)
    # print(blender_image.shape)
    # print(input_img_arr[0].shape)
    loss = loss_function(blender_image, input_img_arr[0])
    losss.append(loss.item())

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    every=1
    if i % every == every-1:
        print("loss:",loss.item())
        print(epoch+1,"/",epochs,";",j+1,"/",run_sample)
        torch.save(model.state_dict(), mom_folder+"model_save.bin")
        # print("save")

with open(mom_folder+"loss.json","w") as f:
    f.write(json.dumps(losss))
    f.close()
    

#plt.plot(range(epochs), losss)
#plt.show()

vgg16.py

import torch
import torch.nn as nn
import torch.nn.functional as F

nn_net=[200*200*3,20*20*3,100,27]
# nn_net=[120000,27,100,27]
numClasses=27
class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.vgg16 = nn.Sequential(
            nn.Conv2d(3, 64, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 64, 3),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(64, 128, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, 3),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 256, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, 3),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 512, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )
        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))
        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(inplace=True),
            # nn.Dropout(),
            nn.Linear(4096, 4096),
            # nn.ReLU(inplace=True),
            # nn.Dropout(),
            nn.Sigmoid(),
            nn.Linear(4096, numClasses),

        )
    def forward(self, x):
        x = self.vgg16(x)
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

log

C:\Users\x065p\Desktop\blender\blender file\python.blend\Text:94: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
loss: 0.07332869619131088
1
Saved: 'C:\Users\x065p\Desktop\blender\result\image\python\tem1.png'
 Time: 00:02.07 (Saving: 00:00.00)

loss: 0.024082180112600327
2
Saved: 'C:\Users\x065p\Desktop\blender\result\image\python\tem2.png'
 Time: 00:02.31 (Saving: 00:00.00)

loss: 0.08349122852087021
3
Saved: 'C:\Users\x065p\Desktop\blender\result\image\python\tem3.png'
 Time: 00:02.11 (Saving: 00:00.01)

loss: 0.07594556361436844
4
Saved: 'C:\Users\x065p\Desktop\blender\result\image\python\tem4.png'
 Time: 00:02.22 (Saving: 00:00.00)

loss: 0.09595263004302979
5
Saved: 'C:\Users\x065p\Desktop\blender\result\image\python\tem5.png'
 Time: 00:02.28 (Saving: 00:00.00)

#flow

1 model input: image

0

2 model predict out : 27 length bsdf parameter

[0.45234,0.340502,0.9123...........]

3 blender render image by model predict bsdf parameter

https://i.imgur.com/PPaZHiu.png

4 count loss with input image and blender render image

2

I don’t quite understand the training as it seems you are calculating the loss via:

loss = loss_function(blender_image, input_img_arr[0])

where blender_image is loaded via PIL (and you are most ikely adding requires_grad=True to this tensor to avoid an error when calling loss.backward()) while input_img_arr is a tensor containing other images loaded by PIL. The model is not used to create any of these tensors and will thus not be trained, or is this your use case?

3

My ideal is
1 model input material image
2 model out put some material parameter
3 render image via by material parameter(form blender render engine)
4 count loss between input image and image(render form blender) which two image should look similar

but I get error like this
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

so I add requires_grad=True to blender_image

What do I need to do for make the model trained?
Thanks a lot.

4

You would need to make step 3 differentiable, i.e. render the image through PyTorch (or a library built on top of PyTorch such as PyTorch3D which seems to implement a differentiable renderer) as I expect that Blender breaks the computation graph.

1 Like