Error in loss function

Hello, i’m making some changes on a normal CNN, to be compatible with other model, that i create.
I get the following error:

File “C:/Users/user/.spyder-py3/cnn.py”, line 105, in
loss = criterion(output,img)

RuntimeError: The size of tensor a (10) must match the size of tensor b (28) at non-singleton dimension 3

This is the CNN:

import time
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision.datasets as dsets
from torch.autograd import Variable

GPU = 0

def to_img(x):
x = x.view(x.size(0), 1, 28, 28)
return x

train_dataset = dsets.MNIST(root=’./data’,
train=True,
transform=transforms.ToTensor(),
download=True)

test_dataset = dsets.MNIST(root=’./data’,
train=False,
transform=transforms.ToTensor())

batch_size =251
n_iters = 71800

num_epochs = n_iters / (len(train_dataset) / batch_size)
num_epochs = int(num_epochs)

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)

class CNNModel(nn.Module):
def init(self):
super(CNNModel, self).init()

    self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)
    self.relu1 = nn.ReLU()

    self.maxpool1 = nn.MaxPool2d(kernel_size=2)

    self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)
    self.relu2 = nn.ReLU()

    self.maxpool2 = nn.MaxPool2d(kernel_size=2)

    self.fc1 = nn.Linear(32 * 7 * 7, 10)

def forward(self, x):
    out = self.cnn1(x)
    out = self.relu1(out)
   
    out = self.maxpool1(out)
   
    out = self.cnn2(out)
    out = self.relu2(out)

    out = self.maxpool2(out)

    out = out.view(out.size(0), -1)

    out = self.fc1(out)
   
    return out

model = CNNModel()

if torch.cuda.is_available():
torch.cuda.set_device(GPU)
model.cuda()
print(“cuda OK”)

criterion = nn.MSELoss()

learning_rate = 1e-3

optimizer = torch.optim.Adam(
model.parameters(), lr=learning_rate, weight_decay=0)

iter = 0
start = time.time()
for epoch in range(num_epochs):
for data in train_loader:
img, _ = data
img = Variable(img).cuda()
nova=to_img(img)

    st = time.time()

    optimizer.zero_grad()

    output = model(img)

    loss = criterion(output,img)

    loss.backward()

    optimizer.step()

    iter += 1

       
print('epoch [{}/{}], loss:{:.4f}'
              .format(epoch + 1, num_epochs, loss.item()))

Anyone knows what i am doing wrong?

Thank you,
Mike

The criterion takes the output and the corresponding label as it arguments, your code is passing output of shape 10 and input img of (3,28,28)
Try these changes

  1. img, target = data
  2. loss = criterion(target, label)

in the for loop it should be,

from data in train_loader:
image,target = data

data is basically a tuple (image,target)

Try this once

So, how should i define the label variable?

data is a tuple,
so it should be like
target and label are the same thing, I just used a different variable for it.
for data in train_loader:
image, label = data

and then loss = criterion(label,output)

Oh, ok, i didnt understand that they are the same thing, thank you. I already made, your suggestions, then i get the following errors:

File “C:/Users/user/.spyder-py3/my_cnn.py”, line 109, in
loss = criterion(label,output)

File “C:\Users\user\Anaconda3\lib\site-packages\torch\nn\modules\loss.py”, line 431, in forward
return F.mse_loss(input, target, reduction=self.reduction)

File “C:\Users\user\Anaconda3\lib\site-packages\torch\nn\functional.py”, line 2199, in mse_loss
ret = (input - target) ** 2

RuntimeError: The size of tensor a (251) must match the size of tensor b (10) at non-singleton dimension 1

This is the alterations:

criterion = nn.MSELoss()

best= 12311242

learning_rate = 1e-3

optimizer = torch.optim.Adam(
model.parameters(), lr=learning_rate, weight_decay=0)

iter = 0
start = time.time()
for epoch in range(num_epochs):
for data in train_loader:
img, label= data
label = Variable(label).cuda()
img = Variable(img).cuda()
nova=to_img(img)

    st = time.time()

    optimizer.zero_grad()

    output = model(img)

    loss = criterion(label,output)

    loss.backward()

    optimizer.step()

    iter += 1

print('epoch [{}/{}], loss:{:.4f}'
              .format(epoch + 1, num_epochs, loss.item()))

if epoch % 1 == 0:
    pic = to_img(output.cpu().data)
    if loss.item()<best:
        best=loss.item()

Can you print the shapes of label and output and send here?

Of course, its here:

label- torch.Size([251])
output- torch.Size([251, 10])

Okay, I got your problem. Label is giving out the exact classes for each element in the mini batch.
Output consists of the probability over 10 classes for all minibatches. You can take an argmax over output to get the predicted class and then pass it into criterion.

You can make this change:

loss = criterion(label,torch.argmax(output,dim=1))

Yeah it’s saying you are having torch.Longtensors while passing to the MSE function.

So change the type of tensors to float .type(torch.FloatTensor) then pass

Is this, that you are suggesting?

    label=float.type(torch.FloatTensor)
    output=float.type(torch.FloatTensor)

    loss=criterion(label,torch.argmax(output,dim=1))

Now i get this:

File “C:/Users/user/.spyder-py3/original_cnn.py”, line 109, in
label=float.type(torch.FloatTensor)

AttributeError: type object ‘float’ has no attribute ‘type’

label=label.type(torch.FloatTensor)
output=output.type(torch.FloatTensor)

Ok, thank you, i made the alteration that you suggest. I´m stuck here now. I get this errors:

File “C:/Users/user/.spyder-py3/original_cnn.py”, line 112, in
loss = criterion(label,torch.argmax(output,dim=1))

File “C:\Users\user\Anaconda3\lib\site-packages\torch\nn\modules\module.py”, line 541, in call
result = self.forward(*input, **kwargs)

File “C:\Users\user\Anaconda3\lib\site-packages\torch\nn\modules\loss.py”, line 431, in forward
return F.mse_loss(input, target, reduction=self.reduction)

File “C:\Users\user\Anaconda3\lib\site-packages\torch\nn\functional.py”, line 2204, in mse_loss
ret = torch._C._nn.mse_loss(expanded_input, expanded_target, _Reduction.get_enum(reduction))

RuntimeError: Expected object of scalar type Float but got scalar type Long for argument #2 ‘target’ in call to _thnn_mse_loss_forward

It’s saying that this output torch.argmax(output,dim=1) is still long. can you print those tensors at the end you can actually see the type of the tensor or type(torch.argmax(output,dim=1))

If you are saying me to do this:
print(type(torch.argmax(output,dim=1)))
print(type(label))

The output is: <class ‘torch.Tensor’>
<class ‘torch.Tensor’>

Is this what you told me to do?

The below is the best I came up to reproduce your error. Adding .type(torch.FloatTensor) should fix your error.

label=torch.Tensor([0,1,2])
output=torch.Tensor(3,3)
criterion(label,torch.argmax(output,dim=1).type(torch.FloatTensor))

Output : tensor(1.)

If you remove that then it’s your error

Thank you for your patient and time, you are teaching me a lot. After those changes i run my code and then i get the following error:

File “C:/Users/user/.spyder-py3/original_cnn.py”, line 115, in
loss.backward()

File “C:\Users\user\Anaconda3\lib\site-packages\torch\tensor.py”, line 166, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)

File “C:\Users\user\Anaconda3\lib\site-packages\torch\autograd_init_.py”, line 99, in backward
allow_unreachable=True) # allow_unreachable flag

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

Loop code:

for epoch in range(num_epochs):
   for data in train_loader: 
       img, label= data
       label = Variable(label).cuda()
       img = Variable(img).cuda()
       nova=to_img(img)
       
       st = time.time()

       optimizer.zero_grad()

       output = model(img)

       label=torch.Tensor([0,1,2])
       output=torch.Tensor(3,3)
       loss=criterion(label,torch.argmax(output,dim=1).type(torch.FloatTensor))

       loss.backward()

       optimizer.step()

       iter += 1

I searched about this and tried a few changes in my code, like loss.backward(requires_grad=True) but still not working: TypeError: backward() got an unexpected keyword argument ‘requires_grad’
Or this : label.requires_grad=True
(torch.argmax(output,dim=1).type(torch.FloatTensor)).requires_grad=True

But i can´t find a solution, can you help me in this?

I modified your code to this in the for loop:

        label=torch.tensor([[0, 0, 0], [0, 1, 0], [0, 0, 2]])
		output=torch.randn(3, 3, requires_grad=True)
		loss=criterion(output, label.float())

And it works.

If you want to do classification you need to use cross entropy loss. Remember to check shape of tensors you pass into the loss. The complete code for mnist is this:

import time
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision.datasets as dsets
from torch.autograd import Variable
from torchvision.utils import save_image

GPU = 0

def to_img(x):
	x = x.view(x.size(0), 1, 28, 28)
	return x

train_dataset = dsets.MNIST(root='./data',
	train=True,
	transform=transforms.ToTensor(),
	download=True)

test_dataset = dsets.MNIST(root='./data',
	train=False,
	transform=transforms.ToTensor())

batch_size =64
n_iters = 71800

num_epochs = n_iters / (len(train_dataset) / batch_size)
num_epochs = int(num_epochs)

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
		batch_size=batch_size,
		shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
		batch_size=batch_size,
		shuffle=False)

class CNNModel(nn.Module):
	def  __init__(self):
		super().__init__()
		self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)
		self.relu1 = nn.ReLU()
		self.maxpool1 = nn.MaxPool2d(kernel_size=2)
		
		self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)
		self.relu2 = nn.ReLU()
		
		self.maxpool2 = nn.MaxPool2d(kernel_size=2)
		
		self.fc1 = nn.Linear(32 * 7 * 7, 10)

	def forward(self, x):
	    out = self.cnn1(x)
	    out = self.relu1(out)
	   
	    out = self.maxpool1(out)
	   
	    out = self.cnn2(out)
	    out = self.relu2(out)

	    out = self.maxpool2(out)

	    out = out.view(out.size(0), -1)

	    out = self.fc1(out)
	   
	    return out


model = CNNModel()

if torch.cuda.is_available():
	torch.cuda.set_device(GPU)
	model.cuda()
	print("cuda OK")

criterion = nn.CrossEntropyLoss()

best= 12311242

learning_rate = 1e-3

optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0)

iter = 0
start = time.time()
for epoch in range(2):
	for data in train_loader:
		img, label= data
		label = label.cuda()
		img = img.cuda()
		# nova=to_img(img)

    	# st = time.time()
		optimizer.zero_grad()
		output = model(img)
		
		# label=torch.tensor([[0, 0, 0], [0, 1, 0], [0, 0, 2]])
		# output=torch.randn(3, 3, requires_grad=True)

		loss=criterion(output, label.long())
		loss.backward()
    	
		optimizer.step()
		iter += 1

print('done')

Thank you for your time, could you just explain me more, about this:

 label=torch.tensor([[0, 0, 0], [0, 1, 0], [0, 0, 2]])
		output=torch.randn(3, 3, requires_grad=True)
		loss=criterion(output, label.float())

For cross entropy loss you need two tensors with the same shape. I just converted the label to one hot vector.