Bug:
Traceback (most recent call last):
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train.py”, line 230, in
main()
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train.py”, line 175, in main
train_model(
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py”, line 180, in train_model
accumulated_iter = train_one_epoch(
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py”, line 56, in train_one_epoch
loss, tb_dict, disp_dict = model_func(model, batch)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/init.py”, line 44, in model_func
ret_dict, tb_dict, disp_dict = model(batch_dict)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/detectors/pv_rcnn.py”, line 11, in forward
batch_dict = cur_module(batch_dict)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/roi_heads/pvrcnn_head.py”, line 151, in forward
pooled_features = self.roi_grid_pool(batch_dict) # (BxN, 6x6x6, C)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/roi_heads/pvrcnn_head.py”, line 97, in roi_grid_pool
pooled_points, pooled_features = self.roi_grid_pool_layer(
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/ops/pointnet2/pointnet2_stack/pointnet2_modules.py”, line 95, in forward
new_features = self.mlpsk # (1, C, M1 + M2 …, nsample)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py”, line 215, in forward
input = module(input)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/activation.py”, line 101, in forward
return F.relu(input, inplace=self.inplace)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py”, line 1471, in relu
result = torch.relu(input)
RuntimeError: handle_0 INTERNAL ASSERT FAILED at “…/c10/cuda/driver_api.cpp”:15, please report a bug to PyTorch.
My environment:
Pytorch version: 2.0.1
CUDA version:12.1
OS: Ubuntu 22.04
Python version:python 3.10
My issue is here.