I set the input as half and also print out the dtype of the input, it did show that the input dtype is torch.cuda.HalfTensor, but I don’t know why I still got the error stated in the title.
Code:
@torch.no_grad()
def inference_on_dir(model,
inference_dir,
output_path='output',
padding_factor=8,
inference_size=None,
paired_data=False, # dir of paired testdata instead of a sequence
save_flo_flow=False, # save as .flo for quantative evaluation
attn_splits_list=None,
corr_radius_list=None,
prop_radius_list=None,
pred_bidir_flow=False,
fwd_bwd_consistency_check=False,
):
""" Inference on a directory """
model.eval().half()
if fwd_bwd_consistency_check:
assert pred_bidir_flow
if not os.path.exists(output_path):
os.makedirs(output_path)
filenames = sorted(glob(inference_dir + '/*'))
print('%d images found' % len(filenames))
stride = 2 if paired_data else 1
if paired_data:
assert len(filenames) % 2 == 0
for test_id in range(0, len(filenames) - 1, stride):
image1 = frame_utils.read_gen(filenames[test_id])
image2 = frame_utils.read_gen(filenames[test_id + 1])
image1 = np.array(image1).astype(np.uint8)
image2 = np.array(image2).astype(np.uint8)
if len(image1.shape) == 2: # gray image, for example, HD1K
image1 = np.tile(image1[..., None], (1, 1, 3))
image2 = np.tile(image2[..., None], (1, 1, 3))
else:
image1 = image1[..., :3]
image2 = image2[..., :3]
image1 = torch.from_numpy(image1).permute(2, 0, 1).float().half()
image2 = torch.from_numpy(image2).permute(2, 0, 1).float().half()
if inference_size is None:
padder = InputPadder(image1.shape, padding_factor=padding_factor)
image1, image2 = padder.pad(image1[None].cuda().half(), image2[None].cuda().half())
else:
image1, image2 = image1[None].cuda().half(), image2[None].cuda().half()
# resize before inference
if inference_size is not None:
assert isinstance(inference_size, list) or isinstance(inference_size, tuple)
ori_size = image1.shape[-2:]
image1 = F.interpolate(image1, size=inference_size, mode='bilinear',
align_corners=True)
image2 = F.interpolate(image2, size=inference_size, mode='bilinear',
align_corners=True)
image1 = image1.type(torch.cuda.HalfTensor)
image2 = image2.type(torch.cuda.HalfTensor)
print(image1.dtype)
print(image2.dtype)
results_dict = model(image1, image2,
attn_splits_list=attn_splits_list,
corr_radius_list=corr_radius_list,
prop_radius_list=prop_radius_list,
pred_bidir_flow=pred_bidir_flow,
)
flow_pr = results_dict['flow_preds'][-1] # [B, 2, H, W]
# resize back
if inference_size is not None:
flow_pr = F.interpolate(flow_pr, size=ori_size, mode='bilinear',
align_corners=True)
flow_pr[:, 0] = flow_pr[:, 0] * ori_size[-1] / inference_size[-1]
flow_pr[:, 1] = flow_pr[:, 1] * ori_size[-2] / inference_size[-2]
if inference_size is None:
flow = padder.unpad(flow_pr[0]).permute(1, 2, 0).cpu().numpy() # [H, W, 2]
else:
flow = flow_pr[0].permute(1, 2, 0).cpu().numpy() # [H, W, 2]
output_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_flow.flo')
save_flo_file(flow, output_file)
output_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_flow.png')
# save vis flow
save_vis_flow_tofile(flow, output_file)
# also predict backward flow
if pred_bidir_flow:
assert flow_pr.size(0) == 2 # [2, H, W, 2]
if inference_size is None:
flow_bwd = padder.unpad(flow_pr[1]).permute(1, 2, 0).cpu().numpy() # [H, W, 2]
else:
flow_bwd = flow_pr[1].permute(1, 2, 0).cpu().numpy() # [H, W, 2]
output_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_flow_bwd.flo')
save_flo_file(flow_bwd, output_file)
output_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_flow_bwd.png')
# save vis flow
save_vis_flow_tofile(flow_bwd, output_file)
# forward-backward consistency check
# occlusion is 1
if fwd_bwd_consistency_check:
if inference_size is None:
fwd_flow = padder.unpad(flow_pr[0]).unsqueeze(0) # [1, 2, H, W]
bwd_flow = padder.unpad(flow_pr[1]).unsqueeze(0) # [1, 2, H, W]
else:
fwd_flow = flow_pr[0].unsqueeze(0)
bwd_flow = flow_pr[1].unsqueeze(0)
fwd_occ, bwd_occ = forward_backward_consistency_check(fwd_flow, bwd_flow) # [1, H, W] float
fwd_occ_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_occ.png')
bwd_occ_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_occ_bwd.png')
Image.fromarray((fwd_occ[0].cpu().numpy() * 255.).astype(np.uint8)).save(fwd_occ_file)
Image.fromarray((bwd_occ[0].cpu().numpy() * 255.).astype(np.uint8)).save(bwd_occ_file)
if save_flo_flow:
output_file = os.path.join(output_path, os.path.basename(filenames[test_id])[:-4] + '_pred.flo')
frame_utils.writeFlow(output_file, flow)
Error:
(gmflow) frank4133@frank4133-System-Product-Name:~/gmflow$ CUDA_VISIBLE_DEVICES=0 python main.py \--inference_dir demo/asus/for_gmflow \--output_path output/asus \--pred_bidir_flow \--resume pretrained/gmflow_sintel-0c07dcb3.pth
Number of params: 4680288
Load checkpoint: pretrained/gmflow_sintel-0c07dcb3.pth
/home/frank4133/gmflow/main.py:198: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
checkpoint = torch.load(args.resume, map_location=loc)
start_epoch: 0, start_step: 0
40 images found
torch.float16
torch.float16
torch.float16
Traceback (most recent call last):
File "/home/frank4133/gmflow/main.py", line 557, in <module>
main(args)
File "/home/frank4133/gmflow/main.py", line 317, in main
inference_on_dir(model_without_ddp,
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/gmflow/evaluate.py", line 636, in inference_on_dir
results_dict = model(image1, image2,
^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/gmflow/gmflow/gmflow.py", line 106, in forward
feature0_list, feature1_list = self.extract_feature(img0, img1) # list of features
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/gmflow/gmflow/gmflow.py", line 52, in extract_feature
features = self.backbone(concat) # list of [2B, C, H, W], resolution from high to low
^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/gmflow/gmflow/backbone.py", line 102, in forward
x = self.conv1(x)
^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/conv.py", line 458, in forward
return self._conv_forward(input, self.weight, self.bias)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/frank4133/anaconda3/lib/python3.11/site-packages/torch/nn/modules/conv.py", line 454, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.cuda.HalfTensor) should be the same