Keypoint rcnn for own dataset

hkwei · April 25, 2020, 3:08pm

I want to train a keypoints detection model for my own dataset based on torchvision KeyPointsRCNN.
My dataset has 3 keypoints, the model is defined as follows:
‘’‘python
def get_model_keypoints(num_keypoints):
# load an instance segmentation model pre-trained pre-trained on COCO
model = torchvision.models.detection.keypointrcnn_resnet50_fpn(pretrained=‘legacy’ )
keypoint_predictor = KeypointRCNNPredictor(512, num_keypoints=num_keypoints)
model.num_keypoints = 3
model.roi_heads.keypoint_predictor = keypoint_predictor
return model
‘’’

Howerever, in train mode, the keypoints loss is always zero,the test code is as follows:
‘’'python
model = Network.get_model_keypoints(3)

use our dataset and defined transformations

dataset = FishKeyDataset.FishKeyDataset(‘label.json’)
dataset_test = FishKeyDataset.FishKeyDataset(‘label.json’)

split the dataset in train and test set

indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])

define training and validation data loaders

data_loader = torch.utils.data.DataLoader(
dataset, batch_size=2, shuffle=True, num_workers=0,collate_fn=utils.collate_fn)

data_loader_test = torch.utils.data.DataLoader(
dataset_test, batch_size=1, shuffle=False, num_workers=0,collate_fn=utils.collate_fn)
model.train()

images,targets = next(iter(data_loader))
print(targets)
images = list(image for image in images)
targets = [{k: v for k, v in t.items()} for t in targets]

for images,targets in data_loader_test:
#print(label)
images = list(image for image in images)
targets = [{k: v for k, v in t.items()} for t in targets]
print(model(images,targets))
‘’’
The output targets is as follows:
‘’‘python
({‘boxes’: tensor([[ 76., 274., 426., 362.]]), ‘labels’: tensor([1]), ‘keypoints’: tensor([[[340., 49., 1.],
[308., 43., 1.],
[ 51., 53., 1.]]]), ‘image_id’: tensor([254]), ‘area’: tensor([30800.]), ‘iscrowd’: tensor([0])}, {‘boxes’: tensor([[181., 250., 443., 351.]]), ‘labels’: tensor([1]), ‘keypoints’: tensor([[[252., 82., 1.],
[217., 71., 1.],
[ 32., 45., 1.]]]), ‘image_id’: tensor([175]), ‘area’: tensor([26462.]), ‘iscrowd’: tensor([0])})
‘’’
The output of the model are:
‘’‘python
{‘loss_classifier’: tensor(0.0788, grad_fn=), ‘loss_box_reg’: tensor(0.0339, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0388, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0104, grad_fn=)}
{‘loss_classifier’: tensor(0.1225, grad_fn=), ‘loss_box_reg’: tensor(0.0478, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0033, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0028, grad_fn=)}
{‘loss_classifier’: tensor(0.0759, grad_fn=), ‘loss_box_reg’: tensor(0.0121, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0199, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0037, grad_fn=)}
{‘loss_classifier’: tensor(0.1153, grad_fn=), ‘loss_box_reg’: tensor(0.0230, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0418, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0059, grad_fn=)}
{‘loss_classifier’: tensor(0.0844, grad_fn=), ‘loss_box_reg’: tensor(0.0264, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0262, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0053, grad_fn=)}
{‘loss_classifier’: tensor(0.1212, grad_fn=), ‘loss_box_reg’: tensor(0.0427, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0105, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0037, grad_fn=)}
{‘loss_classifier’: tensor(0.0760, grad_fn=), ‘loss_box_reg’: tensor(0.0287, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0216, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0050, grad_fn=)}
{‘loss_classifier’: tensor(0.1165, grad_fn=), ‘loss_box_reg’: tensor(0.0323, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0308, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0042, grad_fn=)}
{‘loss_classifier’: tensor(0.1276, grad_fn=), ‘loss_box_reg’: tensor(0.0366, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0118, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0048, grad_fn=)}
{‘loss_classifier’: tensor(0.1045, grad_fn=), ‘loss_box_reg’: tensor(0.0414, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0058, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0051, grad_fn=)}
{‘loss_classifier’: tensor(0.0884, grad_fn=), ‘loss_box_reg’: tensor(0.0172, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0315, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0053, grad_fn=)}
{‘loss_classifier’: tensor(0.1212, grad_fn=), ‘loss_box_reg’: tensor(0.0338, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0492, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0074, grad_fn=)}
{‘loss_classifier’: tensor(0.0504, grad_fn=), ‘loss_box_reg’: tensor(0.0170, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0246, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0186, grad_fn=)}
{‘loss_classifier’: tensor(0.0931, grad_fn=), ‘loss_box_reg’: tensor(0.0332, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0208, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0036, grad_fn=)}
{‘loss_classifier’: tensor(0.0659, grad_fn=), ‘loss_box_reg’: tensor(0.0181, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0130, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0080, grad_fn=)}
{‘loss_classifier’: tensor(0.1727, grad_fn=), ‘loss_box_reg’: tensor(0.0516, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0300, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0063, grad_fn=)}
{‘loss_classifier’: tensor(0.1007, grad_fn=), ‘loss_box_reg’: tensor(0.0521, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0038, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0039, grad_fn=)}
{‘loss_classifier’: tensor(0.1188, grad_fn=), ‘loss_box_reg’: tensor(0.0376, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0499, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0087, grad_fn=)}
{‘loss_classifier’: tensor(0.1187, grad_fn=), ‘loss_box_reg’: tensor(0.0302, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0922, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0141, grad_fn=)}
{‘loss_classifier’: tensor(0.0602, grad_fn=), ‘loss_box_reg’: tensor(0.0303, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0236, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0064, grad_fn=)}
{‘loss_classifier’: tensor(0.0670, grad_fn=), ‘loss_box_reg’: tensor(0.0237, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0293, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0050, grad_fn=)}
{‘loss_classifier’: tensor(0.1475, grad_fn=), ‘loss_box_reg’: tensor(0.0523, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0641, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0121, grad_fn=)}
{‘loss_classifier’: tensor(0.1260, grad_fn=), ‘loss_box_reg’: tensor(0.0388, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0219, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0060, grad_fn=)}
{‘loss_classifier’: tensor(0.1105, grad_fn=), ‘loss_box_reg’: tensor(0.0527, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0131, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0204, grad_fn=)}
{‘loss_classifier’: tensor(0.0820, grad_fn=), ‘loss_box_reg’: tensor(0.0125, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0335, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0060, grad_fn=)}
{‘loss_classifier’: tensor(0.0531, grad_fn=), ‘loss_box_reg’: tensor(0.0186, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0259, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0051, grad_fn=)}
{‘loss_classifier’: tensor(0.0750, grad_fn=), ‘loss_box_reg’: tensor(0.0244, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0289, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0063, grad_fn=)}
{‘loss_classifier’: tensor(0.0695, grad_fn=), ‘loss_box_reg’: tensor(0.0121, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0185, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0056, grad_fn=)}
{‘loss_classifier’: tensor(0.1165, grad_fn=), ‘loss_box_reg’: tensor(0.0370, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0113, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0044, grad_fn=)}
{‘loss_classifier’: tensor(0.1469, grad_fn=), ‘loss_box_reg’: tensor(0.0508, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0254, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0050, grad_fn=)}
{‘loss_classifier’: tensor(0.1019, grad_fn=), ‘loss_box_reg’: tensor(0.0297, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0084, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0062, grad_fn=)}
{‘loss_classifier’: tensor(0.0671, grad_fn=), ‘loss_box_reg’: tensor(0.0159, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0380, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0108, grad_fn=)}
{‘loss_classifier’: tensor(0.1052, grad_fn=), ‘loss_box_reg’: tensor(0.0281, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0118, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0021, grad_fn=)}
{‘loss_classifier’: tensor(0.0766, grad_fn=), ‘loss_box_reg’: tensor(0.0238, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0153, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0092, grad_fn=)}
{‘loss_classifier’: tensor(0.0872, grad_fn=), ‘loss_box_reg’: tensor(0.0325, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0602, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0109, grad_fn=)}
{‘loss_classifier’: tensor(0.1635, grad_fn=), ‘loss_box_reg’: tensor(0.0344, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0380, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0046, grad_fn=)}
{‘loss_classifier’: tensor(0.0795, grad_fn=), ‘loss_box_reg’: tensor(0.0385, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0229, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0045, grad_fn=)}
{‘loss_classifier’: tensor(0.1881, grad_fn=), ‘loss_box_reg’: tensor(0.0694, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0271, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0256, grad_fn=)}
{‘loss_classifier’: tensor(0.0982, grad_fn=), ‘loss_box_reg’: tensor(0.0331, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0446, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0066, grad_fn=)}
{‘loss_classifier’: tensor(0.0563, grad_fn=), ‘loss_box_reg’: tensor(0.0084, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0087, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0049, grad_fn=)}
{‘loss_classifier’: tensor(0.0459, grad_fn=), ‘loss_box_reg’: tensor(0.0232, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0229, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0057, grad_fn=)}
{‘loss_classifier’: tensor(0.1231, grad_fn=), ‘loss_box_reg’: tensor(0.0431, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0214, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0051, grad_fn=)}
{‘loss_classifier’: tensor(0.0843, grad_fn=), ‘loss_box_reg’: tensor(0.0283, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0582, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0118, grad_fn=)}
{‘loss_classifier’: tensor(0.1148, grad_fn=), ‘loss_box_reg’: tensor(0.0393, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0412, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0068, grad_fn=)}
{‘loss_classifier’: tensor(0.1580, grad_fn=), ‘loss_box_reg’: tensor(0.0415, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0311, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0059, grad_fn=)}
{‘loss_classifier’: tensor(0.0983, grad_fn=), ‘loss_box_reg’: tensor(0.0210, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0134, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0057, grad_fn=)}
{‘loss_classifier’: tensor(0.0572, grad_fn=), ‘loss_box_reg’: tensor(0.0104, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0307, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0093, grad_fn=)}
{‘loss_classifier’: tensor(0.0890, grad_fn=), ‘loss_box_reg’: tensor(0.0437, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0063, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0033, grad_fn=)}
{‘loss_classifier’: tensor(0.1362, grad_fn=), ‘loss_box_reg’: tensor(0.0371, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0163, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0039, grad_fn=)}
{‘loss_classifier’: tensor(0.1101, grad_fn=), ‘loss_box_reg’: tensor(0.0270, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0359, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0032, grad_fn=)}
{‘loss_classifier’: tensor(0.1380, grad_fn=), ‘loss_box_reg’: tensor(0.0450, grad_fn=), ‘loss_keypoint’: tensor(0., grad_fn=), ‘loss_objectness’: tensor(0.0053, grad_fn=), ‘loss_rpn_box_reg’: tensor(0.0053, grad_fn=)}
‘’’
The ‘loss_keypoint’ are always zero.
So ,what is wrong with my code?

dude7586 · August 5, 2020, 3:35pm

Check out my code for fine tuning the keypointrcnn_resnet50_fpn model for detecting 2 keypoints.

github.com

sandmantan90/Keypoint-RCNN/blob/master/keypoint_CNN1.1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "keypoint_CNN.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "53320d6fe6eb4cedad0a6d3ff9529650": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {

This file has been truncated. show original

It works!