BCE loss giving very small negative values


(Mihai Calugar) #1

I’m trying to train a model based on segmentation using UNet architecture . I am using code similar to https://github.com/milesial/Pytorch-UNet for training and using the dataset for Carvana Masking Challenge ( specifically the one mentioned on github) the network works just fine and obtain losses of maximum 0.7.

My dataset contains biomedical images of breast scans and corresponding masks, but whenever i run it losses corresponding to each batch give me very small negative values and i can’t figure out why.

import os
import sys
import time
from optparse import OptionParser

import numpy as np
import torch
import torch.nn as nn
from torch import optim

from unet import UNet
from utils import get_ids, split_ids, split_train_val, get_imgs_and_masks, batch


def train_net(net,
              epochs=5,
              batch_size=1,
              lr=0.1,
              val_percent=0.05,
              save_cp=True,
              gpu=True,
              img_scale=0.5):

    dir_img = 'E:/Mihaica/Faculta/An4/Licenta/TrainingIncercare/256-jpg-calcifieri/256-jpg/roi_256_24bp/'
    dir_mask = 'E:/Mihaica/Faculta/An4/Licenta/TrainingIncercare/256-jpg-calcifieri/256-jpg/mask_256/'

    dir_checkpoint = 'checkpoints/'

    ids = get_ids(dir_img)
    ids = split_ids(ids)

    iddataset = split_train_val(ids, val_percent)

    print('''
    Starting training:
        Epochs: {}
        Batch size: {}
        Learning rate: {}
        Training size: {}
        Validation size: {}
        Checkpoints: {}
        CUDA: {}
    '''.format(epochs, batch_size, lr, len(iddataset['train']),
               len(iddataset['val']), str(save_cp), str(gpu)))

    N_train = len(iddataset['train'])

    optimizer = optim.SGD(net.parameters(),
                          lr=lr,
                          momentum=0.9,
                          weight_decay=0.0005)

    criterion = nn.BCELoss()

    # train = get_imgs_and_masks(iddataset['train'], dir_img, dir_mask, img_scale)
    # print(train[0])

    for epoch in range(epochs):
        print('Starting epoch {}/{}.'.format(epoch + 1, epochs))
        net.train()

        # reset the generators
        train = get_imgs_and_masks(iddataset['train'], dir_img, dir_mask, img_scale)
        # val = get_imgs_and_masks(iddataset['val'], dir_img, dir_mask, img_scale)

        epoch_loss = 0

        for i, b in enumerate(batch(train, batch_size)):
            imgs = np.array([i[0] for i in b])
            # imgs = np.array([i[0] for i in b]).astype(np.float32)
            true_masks = np.array([i[1] for i in b])

            imgs = torch.from_numpy(imgs)
            true_masks = torch.from_numpy(true_masks)

            if gpu:
                imgs = imgs.cuda()
                true_masks = true_masks.cuda()

            masks_pred = net(imgs)
            masks_probs_flat = masks_pred.view(-1)

            true_masks_flat = true_masks.view(-1)

            loss = criterion(masks_probs_flat, true_masks_flat)
            epoch_loss += loss.item()

            print('{0:.4f} --- loss: {1:.6f}'.format(i * batch_size / N_train, loss.item()))

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        print('Epoch finished ! Loss: {}'.format(epoch_loss / i))

        # if 1:
        #     val_dice = eval_net(net, val, gpu)
        #     print('Validation Dice Coeff: {}'.format(val_dice))

        if save_cp:
            torch.save(net.state_dict(),
                       dir_checkpoint + 'CP{}.pth'.format(epoch + 1))
            print('Checkpoint {} saved !'.format(epoch + 1))

Utils for loading and preprocessing:

import os
from PIL import Image
from .utils import resize_and_crop, get_square, normalize, hwc_to_chw


def get_ids(dir):
    return (f[:-4] for f in os.listdir(dir))


def split_ids(ids, n=2):
    # Split each id in n, creating n tuples (id, k) for each id
    return ((id, i)  for id in ids for i in range(n))


def to_cropped_imgs(ids, dir, suffix, scale):
    """From a list of tuples, returns the correct cropped img"""
    for id1, pos in ids:
        id = id1.replace(".", "")
        im = resize_and_crop(Image.open(dir + id + suffix), scale=scale)
        yield get_square(im, pos)


def get_imgs_and_masks(ids, dir_img, dir_mask, scale):
    """Return all the couples (img, mask)"""

    imgs = to_cropped_imgs(ids, dir_img, '.jpg', scale)

    # need to transform from HWC to CHW
    imgs_switched = map(hwc_to_chw, imgs)
    imgs_normalized = map(normalize, imgs_switched)

    masks = to_cropped_imgs(ids, dir_mask, '_mask.jpg', scale)

    return zip(imgs_normalized, masks)

Any sort of help is much appreciated. Thanks :smile: