Weight tensor should be defined either for all or no classes

vision
1

Getting error as :

RuntimeError                              Traceback (most recent call last)
<ipython-input-42-cafff645f983> in <module>()
     13         print("targets[:, 0] size ==> ",len(targets[:, 0]))
     14 
---> 15         loss = criterion(outputs, targets[:, 0])
     16         loss.backward()
     17         optimizer.step()

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

<ipython-input-1-992294f0aa44> in forward(self, outputs, targets)
     11 
     12     def forward(self, outputs, targets):
---> 13         return self.loss(F.log_softmax(outputs), targets)

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    145         _assert_no_grad(target)
    146         return F.nll_loss(input, target, self.weight, self.size_average,
--> 147                           self.ignore_index, self.reduce)
    148 
    149 

/opt/anaconda/lib/python3.6/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1049         return torch._C._nn.nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1050     elif dim == 4:
-> 1051         return torch._C._nn.nll_loss2d(input, target, weight, size_average, ignore_index, reduce)
   1052     else:
   1053         raise ValueError('Expected 2 or 4 dimensions (got {})'.format(dim))

RuntimeError: weight tensor should be defined either for all or no classes at /opt/conda/conda-bld/pytorch_1513368888240/work/torch/lib/THNN/generic/SpatialClassNLLCriterion.c:60

Here is my code :

weight = torch.ones(22)

criterion = CrossEntropyLoss2d(weight)

for epoch in range(1, num_epochs+1):
    epoch_loss = []
    iteration=1
    for step, (images, labels) in enumerate(trainLoader):
        print("Iter:"+str(iteration))
        iteration=iteration+1
        inputs = Variable(images)
        targets = Variable(labels)
        
        outputs = model(inputs)
        optimizer.zero_grad()
        print("outputs size ==> ",len(outputs))
        print("targets[:, 0] size ==> ",len(targets[:, 0]))

        loss = criterion(outputs, targets[:, 0])
        loss.backward()
        optimizer.step()
        epoch_loss.append(loss.data[0])
        
        average = sum(epoch_loss) / len(epoch_loss)
        
        print("loss: "+str(average)+" epoch: "+str(epoch)+", step: "+str(step))

Can you please help me here ?

Thanks in advance

2

What are the sizes of the inputs to criterion? (ie, what is outputs.size(), targets[:,0].size())?

The error implies that the size of the weights (22) isn’t equal to the number of classes in outputs

1 Like
3

Hi Richard , thanks a lot for your help.

I solved that one… I was giving class count wrongly.

My actual class count is 2.

So , I changed the weight size to 2.

But , now I am getting different error :


RuntimeError                              Traceback (most recent call last)
<ipython-input-62-a24d68a5b61a> in <module>()
     25 #                 f'target (epoch: {epoch}, step: {step})')
     26 
---> 27         loss = criterion(outputs, targets[:, 0])
     28         loss.backward()
     29         optimizer.step()

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

<ipython-input-1-62b1bfa509f1> in forward(self, outputs, targets)
     11 
     12     def forward(self, outputs, targets):
---> 13         return self.loss(F.log_softmax(outputs), targets)

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    145         _assert_no_grad(target)
    146         return F.nll_loss(input, target, self.weight, self.size_average,
--> 147                           self.ignore_index, self.reduce)
    148 
    149 

/opt/anaconda/lib/python3.6/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1049         return torch._C._nn.nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1050     elif dim == 4:
-> 1051         return torch._C._nn.nll_loss2d(input, target, weight, size_average, ignore_index, reduce)
   1052     else:
   1053         raise ValueError('Expected 2 or 4 dimensions (got {})'.format(dim))

RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed.  at /opt/conda/conda-bld/pytorch_1513368888240/work/torch/lib/THNN/generic/SpatialClassNLLCriterion.c:111

Can you please help ?

Thanks in advance

4

Now it seems, that your target has some illegal values.
It’s either negative or has values < n_classes.
Could you check this?
If you have two classes, your target should take values [0, 1].

5

Hi, am having the same problem… and my target has 3 classes, 0 and 1 and 255 which is the ignore index. I defined my weights as [0.1,0.9] for 0 and 1, but it didnt work… I suppose it s because of the ignore index 255, but isnt it gonna be ignored anyhow?
How I can work around this? Thanks :wink:

6

This code snippet seems to work:

criterion = nn.CrossEntropyLoss(
    ignore_index=255, weight=torch.tensor([1., 2.]), reduction='none')

x = torch.randn(3, 2)
y = torch.tensor([0, 1, 255])

loss = criterion(x, y)
print(loss)
> tensor([0.8519, 1.1084, 0.0000])
3 Likes
7

Thanks. But I have one further question.
I have target with shape [n, w, h] which n is the batch size; output with shape [n, c, w, h]; and my mask is binary mask with ignore index 255.

I tried with your script and find it only works when the output channel dimension is 2, namely output shape [n,2,w,h] , one channel for 0, and one channel for 1.

It kinda makes sense coz the weight dimension should match the class dimension in the output, but just since it is a binary class problem, I wanna save some memory to output just 1 channel with 0 and 1 in the same channel. output [n,1,w,h]

Then to use the weights, do I stil lhave to reshape it into 2 channel? or am I doing something wrong here.
Thanks :wink:

8

For a binary classification you could either use your setup with two output channels and nn.CrossEntropyLoss, or alternatively you could output a single channel and use nn.BCEWithLogitsLoss.
For former is used for a multi-class classification, while the latter approach is used for a binary or multi-label classification.

1 Like
9

Hi, Thanks for reply…

Yes, am aware of that. am not using BCE loss, am using my own defined focalloss,which I used NLLLOSS inside.

class FocalLossnd(nn.Module):
    def __init__(self, weights=None, gamma=0, reduction='mean', ignore_idx=255):
        super().__init__()
        self.weights = weights
        self.gamma = gamma
        self.reduction= reduction
        self.eps = 1e-6
        self.ignore_idx = ignore_idx

    def forward(self, pred, target):
        target = target.type(torch.cuda.LongTensor)
        pt = torch.softmax(pred, dim=1)
        focal_weights = torch.pow((torch.ones(1).to(pred.device) - pt), self.gamma)
        focal = focal_weights * torch.log(pt + self.eps)

        criterion = nn.NLLLoss(
            weight=self.weights,
            ignore_index=self.ignore_idx,
            reduction=self.reduction)
        loss = criterion(focal, target)

        return loss
10

I am getting the below error if anyone can help with this. Thanks!

RuntimeError Traceback (most recent call last)
in
14 #YOUR CODE HERE
15 predNN = nn(X_train_T)#?? # Forward pass
—> 16 error = loss(predNN, y_train_l.squeeze())#?? # find the loss
17 optimizer.zero_grad() # clear the gradients
18 error.backward() # Send loss backward

~\anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1193 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1194 return forward_call(*input, **kwargs)
1195 # Do not call functions when jit is used
1196 full_backward_hooks, non_full_backward_hooks = ,

~\anaconda3\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
1172
1173 def forward(self, input: Tensor, target: Tensor) → Tensor:
→ 1174 return F.cross_entropy(input, target, weight=self.weight,
1175 ignore_index=self.ignore_index, reduction=self.reduction,
1176 label_smoothing=self.label_smoothing)

~\anaconda3\lib\site-packages\torch\nn\functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
3024 if size_average is not None or reduce is not None:
3025 reduction = _Reduction.legacy_get_string(size_average, reduce)
→ 3026 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
3027
3028

RuntimeError: weight tensor should be defined either for all 20 classes or no classes but got weight tensor of shape: [8]

My code is as mentioned below:

class NeuralNet(N.Module):
    def __init__(self, input_size, Nh1, Nh2, output_size):
        super(NeuralNet, self).__init__()
        self.fc1 = N.Linear(input_size, Nh1)
        self.fc2 = N.Linear(Nh1, Nh2)
        self.fc3 = N.Linear(Nh2, output_size)
        self.relu = N.ReLU()

    def forward(self, x):
        x = self.relu(self.fc1(x))
        x = self.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class LinearNet(N.Module):
    def __init__(self, input_size, output_size):
        super(LinearNet, self).__init__()
        self.fc = N.Linear(input_size, output_size)

    def forward(self, x):
        x = self.fc(x)
        return x

# Scaling features to have a maximum value of 1; grayscale images have value between 0 and 255
featureScale = 255
Nfeatures = Xtrain.shape[1]
X_train_T = Xtrain/featureScale
X_train_T = torch.tensor(X_train_T.astype(np.float32)).to(device)

# Converting to one hot vectors
y_train_l = torch.tensor(ytrain).to(device)
y_train_T = torch.nn.functional.one_hot(y_train_l)

Nclasses = np.max(ytrain)+1

nn = NeuralNet(Nfeatures,Nclasses,20,20).to(device)
sm = N.Softmax(dim=1)

#Weight the cross entropy loss to balance the classes
Nsamples_per_class = y_train_T.sum(axis=0)
Weight = Nsamples_per_class.sum()/Nsamples_per_class
loss = torch.nn.CrossEntropyLoss(weight=Weight)
learning_rate = 0.01
#YOUR CODE HERE
optimizer= optim.SGD(nn.parameters(), lr=learning_rate) # define optimizer

for epoch in range(20000):
    
    #YOUR CODE HERE
    predNN = nn(X_train_T)#??   # Forward pass
    error = loss(predNN, y_train_l.squeeze())#??    # find the loss
    optimizer.zero_grad()              # clear the gradients
    error.backward()              # Send loss backward
    optimizer.step()              # update weights 

    if(np.mod(epoch,5000)==0):
        print("Error =",error.detach().cpu().item())
        fig,ax = plt.subplots(1,2,figsize=(12,4))
        ax[0].plot(y_train_T[0:40].detach().cpu())
        ax[1].plot(sm(predNN[0:40]).detach().cpu())
        plt.show()

Er1
Er11406×829 76.8 KB

11

Based on the posted code snippet you are creating a model returning logits for 20 classes while the error message indicates that the weight argument to nn.CrossEntropyLoss only contains 8 values.

It also seems you might be initializing the model wrong, since this code:

Nclasses = np.max(ytrain)+1
nn = NeuralNet(Nfeatures,Nclasses,20,20).to(device)

passes Nclasses as the second argument, which is then used as Nh1 in:

class NeuralNet(N.Module):
    def __init__(self, input_size, Nh1, Nh2, output_size):

instead of output_size.

Could you double check this line of code, please?