I have eight different models in a project in which two are works fine without giving any error on multiple GPUs. When I integrated a new model, I got a replica error and the point is that I made a config file and this file remains unchanged for both models.
Config.py file
import torch, os
import yaml
from IPython import embed
def get_config(args):
configuration = dict(
SEED=1337, # random seed for reproduce results
INPUT_SIZE=[112, 112], # support: [112, 112] and [224, 224]
EMBEDDING_SIZE=512, # feature dimension
)
if args.workers_id == 'cpu' or not torch.cuda.is_available():
configuration['GPU_ID'] = []
print("check", args.workers_id, torch.cuda.is_available())
else:
configuration['GPU_ID'] = [int(i) for i in args.workers_id.split(',')]
if len(configuration['GPU_ID']) == 0:
configuration['DEVICE'] = torch.device('cpu')
configuration['MULTI_GPU'] = False
else:
configuration['DEVICE'] = torch.device('cuda:%d' % configuration['GPU_ID'][0])
if len(configuration['GPU_ID']) == 1:
configuration['MULTI_GPU'] = False
else:
configuration['MULTI_GPU'] = True
configuration['NUM_EPOCH'] = args.epochs
configuration['BATCH_SIZE'] = args.batch_size
if args.data_mode == 'casia':
configuration['DATA_ROOT'] = '/media/khawar/HDD_Khawar/face_datasets/WIDER_train/images/'
elif args.data_mode == "CelebA":
configuration['DATA_ROOT'] = '/media/khawar/HDD_Khawar/face_datasets/CelebA'
elif args.data_mode == "faces":
configuration['DATA_ROOT'] = '/raid/khawar/dataset/faces/'
else:
print('fff')
# raise Exception(args.data_mode)
configuration['EVAL_PATH'] = './eval/'
assert args.net in ['VIT', 'VITs', 'VITs_Eff', 'CAiT', 'DeepViT', 'PiT', 'LeViT','Comb_ViT', 'Dino_VIT', 'CvT','Swin', 'T2TViT']
configuration['BACKBONE_NAME'] = args.net
assert args.head in ['Softmax', 'ArcFace', 'CosFace', 'SFaceLoss']
configuration['HEAD_NAME'] = args.head
configuration['TARGET'] = [i for i in args.target.split(',')]
if args.resume:
configuration['BACKBONE_RESUME_ROOT'] = args.resume
else:
configuration['BACKBONE_RESUME_ROOT'] = '' # the root to resume training from a saved checkpoint
configuration['WORK_PATH'] = args.outdir # the root to buffer your checkpoints
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
return configuration
main.py
GPU_ID = cfg['GPU_ID'] # specify your GPU ids
print('GPU_ID', GPU_ID)
Traceback (most recent call last):
File "train.py", line 367, in <module>
outputs, emb = BACKBONE(inputs.float(), labels)
File "/raid/khawar/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "/raid/khawar/.local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 167, in forward
outputs = self.parallel_apply(replicas, inputs, kwargs)
File "/raid/khawar/.local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 177, in parallel_apply
return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
File "/raid/khawar/.local/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 86, in parallel_apply
output.reraise()
File "/raid/khawar/.local/lib/python3.6/site-packages/torch/_utils.py", line 429, in reraise
raise self.exc_type(msg)
RuntimeError: Caught RuntimeError in replica 0 on device 0.