Traceback (most recent call last):
File “tools/train.py”, line 96, in
main()
File “tools/train.py”, line 92, in main
logger=logger)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/apis/train.py”, line 59, in train_detector
_non_dist_train(model, dataset, cfg, validate=validate)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/apis/train.py”, line 121, in _non_dist_train
runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmcv/runner/runner.py”, line 349, in run
epoch_runner(data_loaders[i], **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmcv/runner/runner.py”, line 255, in train
self.model, data_batch, train_mode=True, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/apis/train.py”, line 37, in batch_processor
losses = model(**data)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py”, line 141, in forward
return self.module(*inputs[0], **kwargs[0])
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/models/detectors/base.py”, line 80, in forward
return self.forward_train(img, img_meta, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/models/detectors/two_stage.py”, line 88, in forward_train
x = self.extract_feat(img)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/models/detectors/two_stage.py”, line 75, in extract_feat
x = self.backbone(img)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/models/backbones/hrnet.py”, line 450, in forward
x = self.layer1(x)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/container.py”, line 92, in forward
input = module(input)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/mmdet-0.6rc0+5da6db5-py3.6.egg/mmdet/models/backbones/hrnet.py”, line 99, in forward
out = self.bn1(out)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py”, line 76, in forward
exponential_average_factor, self.eps)
File “/home/hil/anaconda3/envs/hrnet/lib/python3.6/site-packages/torch/nn/functional.py”, line 1623, in batch_norm
training, momentum, eps, torch.backends.cudnn.enabled
You are running out of memory and need to reduce the memory usage e.g. via lowering the batch size of your inputs.
Hi , Thanks
I have single NVIDIA T600 gpu of 4GB memory.
I am using PE_faster_rcnn_hrnetv2p_w18_1x.py as a custom config file and running the following command: python tools/train.py configs/hrnet/PE_faster_rcnn_hrnetv2p_w18_1x.py
PE_faster_rcnn_hrnetv2p_w18_1x.py file has following content:
model settings
model = dict(
type=‘FasterRCNN’,
pretrained=‘hrnetv2_pretrained/hrnetv2_w18_imagenet_pretrained.pth’,
backbone=dict(
type=‘HighResolutionNet’,
extra=dict(
stage1=dict(
num_modules=1,
num_branches=1,
block=‘BOTTLENECK’,
num_blocks=(4,),
num_channels=(64,),
fuse_method=‘SUM’),
stage2=dict(
num_modules=1,
num_branches=2,
block=‘BASIC’,
num_blocks=(4, 4),
num_channels=(18, 36),
fuse_method=‘SUM’),
stage3=dict(
num_modules=4,
num_branches=3,
block=‘BASIC’,
num_blocks=(4, 4, 4),
num_channels=(18, 36, 72),
fuse_method=‘SUM’),
stage4=dict(
num_modules=3,
num_branches=4,
block=‘BASIC’,
num_blocks=(4, 4, 4, 4),
num_channels=(18, 36, 72, 144),
fuse_method=‘SUM’))),
neck=dict(
type=‘HRFPN’,
in_channels=[18, 36, 72, 144],
out_channels=256),
rpn_head=dict(
type=‘RPNHead’,
in_channels=256,
feat_channels=256,
anchor_scales=[8],
anchor_ratios=[0.5, 1.0, 2.0],
anchor_strides=[4, 8, 16, 32, 64],
target_means=[.0, .0, .0, .0],
target_stds=[1.0, 1.0, 1.0, 1.0],
use_sigmoid_cls=True),
bbox_roi_extractor=dict(
type=‘SingleRoIExtractor’,
roi_layer=dict(type=‘RoIAlign’, out_size=7, sample_num=2),
out_channels=256,
featmap_strides=[4, 8, 16, 32]),
bbox_head=dict(
type=‘SharedFCBBoxHead’,
num_fcs=2,
in_channels=256,
fc_out_channels=1024,
roi_feat_size=7,
num_classes=81,
target_means=[0., 0., 0., 0.],
target_stds=[0.1, 0.1, 0.2, 0.2],
reg_class_agnostic=False))
model training and testing settings
train_cfg = dict(
rpn=dict(
assigner=dict(
type=‘MaxIoUAssigner’,
pos_iou_thr=0.7,
neg_iou_thr=0.3,
min_pos_iou=0.3,
ignore_iof_thr=-1),
sampler=dict(
type=‘RandomSampler’,
num=256,
pos_fraction=0.5,
neg_pos_ub=-1,
add_gt_as_proposals=False),
allowed_border=0,
pos_weight=-1,
smoothl1_beta=1 / 9.0,
debug=False),
rcnn=dict(
assigner=dict(
type=‘MaxIoUAssigner’,
pos_iou_thr=0.5,
neg_iou_thr=0.5,
min_pos_iou=0.5,
ignore_iof_thr=-1),
sampler=dict(
type=‘RandomSampler’,
num=512,
pos_fraction=0.25,
neg_pos_ub=-1,
add_gt_as_proposals=True),
pos_weight=-1,
debug=False))
test_cfg = dict(
rpn=dict(
nms_across_levels=False,
nms_pre=2000,
nms_post=2000,
max_num=2000,
nms_thr=0.7,
min_bbox_size=0),
rcnn=dict(
score_thr=0.05, nms=dict(type=‘nms’, iou_thr=0.5), max_per_img=1)
# soft-nms is also supported for rcnn testing
# e.g., nms=dict(type=‘soft_nms’, iou_thr=0.5, min_score=0.05)
)
NOTE:
dataset settings
if you use zip format to store all images of coco, please use CocoZipDataset
dataset_type = ‘PEDataset’
data_root = ‘…/images/’
img_norm_cfg = dict(
mean=[127.5], std=[127.5], to_rgb=False)
else
dataset_type = ‘CocoDataset’
data_root = ‘data/coco/’
img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
data = dict(
imgs_per_gpu=4,
workers_per_gpu=0,
train=dict(
type=dataset_type,
ann_file=data_root + ‘bbox.json’,
img_prefix=data_root + ‘train/’,
img_scale=(1333, 800),
img_norm_cfg=img_norm_cfg,
size_divisor=32,
flip_ratio=0,
with_mask=False,
with_crowd=False,
with_label=True),
val=dict(
type=dataset_type,
ann_file=data_root + ‘bbox.json’,
img_prefix=data_root + ‘train/’,
img_scale=(1333, 800),
img_norm_cfg=img_norm_cfg,
size_divisor=32,
flip_ratio=0,
with_mask=False,
with_crowd=False,
with_label=True),
test=dict(
type=dataset_type,
ann_file=data_root + ‘test_names.json’,
img_prefix=data_root + ‘test/’,
img_scale=(1333, 800),
img_norm_cfg=img_norm_cfg,
size_divisor=32,
flip_ratio=0,
with_mask=False,
with_label=False,
test_mode=True))
optimizer
if you use 8 GPUs for training, please change lr to 0.02
optimizer = dict(type=‘SGD’, lr=0.001, momentum=0.9, weight_decay=0.0001)
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
learning policy
lr_config = dict(
policy=‘step’,
warmup=‘linear’,
warmup_iters=500,
warmup_ratio=1.0 / 3,
step=[8, 11])
checkpoint_config = dict(interval=1)
yapf:disable
log_config = dict(
interval=50,
hooks=[
dict(type=‘TextLoggerHook’),
dict(type=‘TensorboardLoggerHook’)
])
yapf:enable
runtime settings
total_epochs = 1
dist_params = dict(backend=‘nccl’)
log_level = ‘INFO’
work_dir = ‘./work_dirs/faster_rcnn_hrnetv2p_w18_1x/’
load_from = None
resume_from = None
workflow = [(‘train’, 1)]