How to use OUSMLoss?

i am trying to use OSUMLoss like this :


def get_loss(name):
    loss_dict = {
        'CrossEntropy': F.cross_entropy,
        'SoftCrossEntropy': soft_cross_entropy_loss,
        'LabelSmoothingCrossEntropy':
        LabelSmoothingCrossEntropy(epsilon=0.1),
        'SCE': SymmetricCrossEntropy(), 
        'BCEWithLogitsLoss': F.binary_cross_entropy_with_logits,
        'Coral': coral_loss, 
        'MSELoss': F.mse_loss,
        'MAELoss': F.l1_loss,
        'huber': F.smooth_l1_loss,
    }
    return loss_dict[name]

class OUSMLoss(nn.Module):
    '''
    Implementation of 
    Loss with Online Uncertainty Sample Mining:
    https://arxiv.org/pdf/1901.07759.pdf
    # Params
    k: num of samples to drop in a mini batch
    loss: loss function name (see get_loss function above)
    trigger: the epoch it starts to train on OUSM (please call `.update(epoch)` each epoch)
    '''

    def __init__(self, k=1,  trigger=5, ousm=True):
        super(OUSMLoss, self).__init__()
        self.k = k
        self.loss_name = 'CrossEntropy'
        self.loss = get_loss(self.loss_name)
        self.trigger = trigger
        self.ousm = ousm

    def forward(self, logits, targets, indices=None):
        logits, targets = _check_input_type(logits, targets, self.loss_name)
        bs = logits.shape[0]
        if self.ousm and bs - self.k > 0:
            losses = self.loss(logits, targets, reduction='none')
            if len(losses.shape) == 2:
                losses = losses.mean(1)
            _, idxs = losses.topk(bs-self.k, largest=False)
            losses = losses.index_select(0, idxs)
            return losses.mean()
        else:
            return self.loss(logits, targets)

    def update(self, current_epoch):
        self.current_epoch = current_epoch
        if current_epoch == self.trigger:
            self.ousm = True
            print('criterion: ousm is True.')

    def __repr__(self):
        return f'OUSM(loss={self.loss_name}, k={self.k}, trigger={self.trigger}, ousm={self.ousm})'

ref : kaggle-panda-challenge-public/noisy_loss.py at 208caf4c83a5ab9d181e66eee447cd2e475d58dc · analokmaus/kaggle-panda-challenge-public · GitHub

when i do :

loss = OUSMLoss(logits, target)
loss.backward()

i get,

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<timed exec> in <module>

<ipython-input-105-6eecc42fef33> in train_epoch(loader, optimizer)
     16         loss = OUSMLoss(logits, target)
     17 
---> 18         loss.backward()
     19         optimizer.step()
     20 

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
    592                 return modules[name]
    593         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 594             type(self).__name__, name))
    595 
    596     def __setattr__(self, name, value):

AttributeError: 'OUSMLoss' object has no attribute 'backward'

how can i solve this issue? @ptrblck

OUSMLoss is defined as an nn.Module, while .backward() is a tensor method.
You would either have to implement the backward() method in this module or call .backward() on the loss tensor (probably the return tensor).

dear @ptrblck

i was trying to use OUSMLoss from this repo : kaggle-panda-challenge-public/noisy_loss.py at master · analokmaus/kaggle-panda-challenge-public · GitHub

for a multi class (5 class) image classification problem.

i was trying to use OUSMLoss like this :

class myOUSMLoss(nn.Module):
    '''
    Implementation of 
    Loss with Online Uncertainty Sample Mining:
    https://arxiv.org/pdf/1901.07759.pdf
    # Params
    k: num of samples to drop in a mini batch
    loss: loss function name (see get_loss function above)
    trigger: the epoch it starts to train on OUSM (please call `.update(epoch)` each epoch)
    '''

    def __init__(self, k=1,  trigger=0, ousm=True):
        super(myOUSMLoss, self).__init__()
        self.k = k
  
        
        self.trigger = trigger
        self.ousm = ousm

    def forward(self, logits, targets, indices=None):
        bs = logits.shape[0]
        print(bs)
        print(bs - self.k)
        if self.ousm and bs - self.k > 0:
            #losses = self.loss(logits, targets, reduction='none')
            losses = bi_tempered_logistic_loss(logits, targets, t1=t1, t2=t2, label_smoothing=smoothing)
            if len(losses.shape) == 2:
                losses = losses.mean(1)
            _, idxs = losses.topk(bs-self.k, largest=False)
            losses = losses.index_select(0, idxs)
            return losses.mean()

and then from train one epoch and valid one epoch function just using loss function like this :

myloss = myOUSMLoss()
loss = myloss(data, target)

and it gives me the following error :


16
14
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-18-d97376da71cb> in <module>()
----> 1 get_ipython().run_cell_magic('time', '', '\nacc_max = 0.\nbest_file = f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/{kernel_type}_best_fold{fold}.pth\'\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), \'Epoch:\', epoch)\n    scheduler.step(epoch-1)\n\n    train_loss = train_epoch(train_loader, optimizer)\n    val_loss, acc = val_epoch(valid_loader)\n\n    content = time.ctime() + \' \' + f\'Epoch {epoch}, lr: {optimizer.param_groups[0]["lr"]:.7f}, train loss: {np.mean(train_loss):.5f}, val loss: {np.mean(val_loss):.5f}, acc: {(acc):.5f}\'\n    print(content)\n    with open(f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/log_{kernel_type}.txt\', \'a\') as appender:\n        appender.write(content + \'\\n\')\n\n    if acc > acc_max:\n        print(\'acc ({:.6f} --> {:.6f}).  Saving model ...\'.format(acc_max, acc))\n        torch.save(model.state_dict(), best_file)\n        acc_max = acc\n\n\n#torch.save(model.state_dict(), os.path.join(f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/{kernel_type}_final_fold{fold}.pth\'))')
5 frames
<decorator-gen-60> in time(self, line, cell, local_ns)
<timed exec> in <module>()
<ipython-input-14-5055b70e47f6>
 in forward(self, logits, targets, indices)
     31             if len(losses.shape) == 2:
     32                 losses = losses.mean(1)
---> 33             _, idxs = losses.topk(bs-self.k, largest=False)
     34             losses = losses.index_select(0, idxs)
     35             return losses.mean()
RuntimeError: invalid argument 5: k not in range for dimension at /pytorch/aten/src/THC/generic/THCTensorTopK.cu:26

how can i fix this issue?

This error might be raised, if you are trying to select the top k values, while the size of the specified dimension is smaller than k.
Check the shape of losses and make sure it contains more than bs-self.k values.

@ptrblck i checked that bs - self.k is 15

but when i tried :

print("losses.shape ",losses.shape)

and

print("losses ",list(losses.size()))

i get this output :

losses.shape  torch.Size([])

losses  []

what am i missing?

@ptrblck fixed that issue and now getting Nan

this is the code i am using :

def soft_cross_entropy_loss(logits, targets, weights=1, reduction='none'):

    if len(targets.shape) == 1 or targets.shape[1] == 1:

        onehot_targets = torch.eye(logits.shape[1])[targets].to(logits.device)

    else:

        onehot_targets = targets

    loss = -torch.sum(onehot_targets * F.log_softmax(logits, 1), 1)

    if reduction == 'none':

        return loss

    elif reduction == 'sum':

        return loss.sum()

    elif reduction == 'mean':

        return loss.mean()
def ousm(logits, targets, indices=None):

    logits, targets = _check_input_type(logits, targets, 'SoftCrossEntropy')

    bs = logits.shape[0]

    k = 1

    print(bs)

    print("bs - k ", bs - k)

    

    if bs - k > 0:

      #losses = self.loss(logits, targets)

      losses = soft_cross_entropy_loss(logits, targets, reduction='none')

      #losses = bi_tempered_logistic_loss(logits, targets, t1=t1, t2=t2, label_smoothing=smoothing)

      print(losses)

      print("losses.shape ",losses.shape)

      print("losses ",list(losses.size()))

      print(len(losses.shape))

      if len(losses.shape) == 2:

          losses = losses.mean(1)

      print("losses.shape test ",losses.shape)

      print("losses test",list(losses.size()))

      _, idxs = losses.topk(bs-k, largest=True)

      losses = losses.index_select(0, idxs)

      return losses.mean()

and i get output like this :


16
bs - k  15
tensor([1.5355, 1.6762, 1.6320, 1.5614, 1.5086, 1.6454, 1.4942, 1.5916, 1.6462,
        1.7642, 1.4607, 1.5639, 1.6708, 1.4343, 1.7360, 1.6173],
       device='cuda:0', grad_fn=<NegBackward>)
losses.shape  torch.Size([16])
losses  [16]
1
losses.shape test  torch.Size([16])
losses test [16]

after some times,

16
bs - k  15
tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
       device='cuda:0', grad_fn=<NegBackward>)
losses.shape  torch.Size([16])
losses  [16]
1
losses.shape test  torch.Size([16])
losses test [16]

I would recommend to check the custom loss function and make sure no invalid operations are used, such as a zero division etc.
Anomaly detection might also be helpful to further isolate the issue.