RuntimeError: handle_0 INTERNAL ASSERT FAILED at "../c10/cuda/driver_api.cpp":15, please report a bug to PyTorch

Iruma_Megomi · October 16, 2023, 2:30pm

Bug:
Traceback (most recent call last):
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train.py”, line 230, in
main()
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train.py”, line 175, in main
train_model(
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py”, line 180, in train_model
accumulated_iter = train_one_epoch(
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py”, line 56, in train_one_epoch
loss, tb_dict, disp_dict = model_func(model, batch)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/init.py”, line 44, in model_func
ret_dict, tb_dict, disp_dict = model(batch_dict)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/detectors/pv_rcnn.py”, line 11, in forward
batch_dict = cur_module(batch_dict)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/roi_heads/pvrcnn_head.py”, line 151, in forward
pooled_features = self.roi_grid_pool(batch_dict) # (BxN, 6x6x6, C)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/models/roi_heads/pvrcnn_head.py”, line 97, in roi_grid_pool
pooled_points, pooled_features = self.roi_grid_pool_layer(
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/media/iruma/Data/AIR/radar/OpenPCDet/tools/…/pcdet/ops/pointnet2/pointnet2_stack/pointnet2_modules.py”, line 95, in forward
new_features = self.mlpsk # (1, C, M1 + M2 …, nsample)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py”, line 215, in forward
input = module(input)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1527, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/activation.py”, line 101, in forward
return F.relu(input, inplace=self.inplace)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py”, line 1471, in relu
result = torch.relu(input)
RuntimeError: handle_0 INTERNAL ASSERT FAILED at “…/c10/cuda/driver_api.cpp”:15, please report a bug to PyTorch.
My environment:
Pytorch version: 2.0.1
CUDA version:12.1
OS: Ubuntu 22.04
Python version:python 3.10

ptrblck · October 16, 2023, 2:47pm

Could you create an issue on GitHub, please?

Iruma_Megomi · October 16, 2023, 3:08pm

github.com/open-mmlab/OpenPCDet

RuntimeError: handle_0 INTERNAL ASSERT FAILED at "../c10/cuda/driver_api.cpp":15, please report a bug to PyTorch.

opened 02:11PM - 16 Oct 23 UTC

IrumaMegumi

Traceback (most recent call last): File "/media/iruma/Data/AIR/radar/OpenPCDe…t/tools/train.py", line 230, in <module> main() File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/train.py", line 175, in main train_model( File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py", line 180, in train_model accumulated_iter = train_one_epoch( File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/train_utils/train_utils.py", line 56, in train_one_epoch loss, tb_dict, disp_dict = model_func(model, batch) File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/../pcdet/models/__init__.py", line 44, in model_func ret_dict, tb_dict, disp_dict = model(batch_dict) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/../pcdet/models/detectors/pv_rcnn.py", line 11, in forward batch_dict = cur_module(batch_dict) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/../pcdet/models/roi_heads/pvrcnn_head.py", line 151, in forward pooled_features = self.roi_grid_pool(batch_dict) # (BxN, 6x6x6, C) File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/../pcdet/models/roi_heads/pvrcnn_head.py", line 97, in roi_grid_pool pooled_points, pooled_features = self.roi_grid_pool_layer( File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/media/iruma/Data/AIR/radar/OpenPCDet/tools/../pcdet/ops/pointnet2/pointnet2_stack/pointnet2_modules.py", line 95, in forward new_features = self.mlps[k](new_features) # (1, C, M1 + M2 ..., nsample) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py", line 215, in forward input = module(input) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/activation.py", line 101, in forward return F.relu(input, inplace=self.inplace) File "/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py", line 1471, in relu result = torch.relu(input) RuntimeError: handle_0 INTERNAL ASSERT FAILED at "../c10/cuda/driver_api.cpp":15, please report a bug to PyTorch. When I use the train.py code, I found the error. I don't know how to solve it. Please help me. Thanks.

My issue is here.