Hi, i’m having a tough time understanding what causes the indicated error. I am trying to develop a token classification model for ABSA with pytorch lightning and BERT, on colab, but got stuck at this error when calling the trainer. Can anybody give me some insight? Thanks in advance!
Here is the error traceback, if it can be useful.
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloaders, val_dataloaders, datamodule, train_dataloader, ckpt_path)
736 train_dataloaders = train_dataloader
737 self._call_and_handle_interrupt(
→ 738 self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
739 )
740
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _call_and_handle_interrupt(self, trainer_fn, *args, **kwargs)
680 “”"
681 try:
→ 682 return trainer_fn(*args, **kwargs)
683 # TODO: treat KeyboardInterrupt as BaseException (delete the code below) in v1.7
684 except KeyboardInterrupt as exception:
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _fit_impl(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
770 # TODO: ckpt_path only in v1.7
771 ckpt_path = ckpt_path or self.resume_from_checkpoint
→ 772 self._run(model, ckpt_path=ckpt_path)
773
774 assert self.state.stopped
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _run(self, model, ckpt_path)
1193
1194 # dispatch start_training or start_evaluating or start_predicting
→ 1195 self._dispatch()
1196
1197 # plugin will finalized fitting (e.g. ddp_spawn will load trained model)
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _dispatch(self)
1272 self.training_type_plugin.start_predicting(self)
1273 else:
→ 1274 self.training_type_plugin.start_training(self)
1275
1276 def run_stage(self):
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
200 def start_training(self, trainer: “pl.Trainer”) → None:
201 # double dispatch to initiate the training loop
→ 202 self._results = trainer.run_stage()
203
204 def start_evaluating(self, trainer: “pl.Trainer”) → None:
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
1282 if self.predicting:
1283 return self._run_predict()
→ 1284 return self._run_train()
1285
1286 def _pre_training_routine(self):
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _run_train(self)
1304 self.progress_bar_callback.disable()
1305
→ 1306 self._run_sanity_check(self.lightning_module)
1307
1308 # enable train mode
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _run_sanity_check(self, ref_model)
1368 # run eval step
1369 with torch.no_grad():
→ 1370 self._evaluation_loop.run()
1371
1372 self.call_hook(“on_sanity_check_end”)
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
143 try:
144 self.on_advance_start(*args, **kwargs)
→ 145 self.advance(*args, **kwargs)
146 self.on_advance_end()
147 self.restarting = False
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py in advance(self, *args, **kwargs)
107 dl_max_batches = self._max_batches[dataloader_idx]
108
→ 109 dl_outputs = self.epoch_loop.run(dataloader, dataloader_idx, dl_max_batches, self.num_dataloaders)
110
111 # store batch level output per dataloader
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
143 try:
144 self.on_advance_start(*args, **kwargs)
→ 145 self.advance(*args, **kwargs)
146 self.on_advance_end()
147 self.restarting = False
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in advance(self, data_fetcher, dataloader_idx, dl_max_batches, num_dataloaders)
120 # lightning module methods
121 with self.trainer.profiler.profile(“evaluation_step_and_end”):
→ 122 output = self._evaluation_step(batch, batch_idx, dataloader_idx)
123 output = self._evaluation_step_end(output)
124
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in _evaluation_step(self, batch, batch_idx, dataloader_idx)
215 self.trainer.lightning_module._current_fx_name = “validation_step”
216 with self.trainer.profiler.profile(“validation_step”):
→ 217 output = self.trainer.accelerator.validation_step(step_kwargs)
218
219 return output
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/accelerators/accelerator.py in validation_step(self, step_kwargs)
234 “”"
235 with self.precision_plugin.val_step_context():
→ 236 return self.training_type_plugin.validation_step(*step_kwargs.values())
237
238 def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) → Optional[STEP_OUTPUT]:
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in validation_step(self, *args, **kwargs)
217
218 def validation_step(self, *args, **kwargs):
→ 219 return self.model.validation_step(*args, **kwargs)
220
221 def test_step(self, *args, **kwargs):
in validation_step(self, batch, batch_idx)
46 attention_mask = batch[“attention_mask”]
47 labels = batch[“labels”]
—> 48 loss, outputs = self(input_ids, attention_mask, labels)
49 #self.log(“val_loss”, loss, prog_bar=True, logger=True)
50 return loss
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
in forward(self, input_ids, attention_mask, y)
12 def forward(self, input_ids, attention_mask, y=None):
13 print(‘input:’, input_ids.size(), ‘att’, attention_mask.size(), ‘y’, y.size())
—> 14 output = self.bert(input_ids, attention_mask, y)
15 logits = output.logits
16
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1118 input = bw_hook.setup_input_hook(input)
1119
→ 1120 result = forward_call(input, **kwargs)
1121 if _global_forward_hooks or self._forward_hooks:
1122 for hook in (_global_forward_hooks.values(), *self._forward_hooks.values()):
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
1733 output_attentions=output_attentions,
1734 output_hidden_states=output_hidden_states,
→ 1735 return_dict=return_dict,
1736 )
1737
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1118 input = bw_hook.setup_input_hook(input)
1119
→ 1120 result = forward_call(input, **kwargs)
1121 if _global_forward_hooks or self._forward_hooks:
1122 for hook in (_global_forward_hooks.values(), *self._forward_hooks.values()):
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
1004 output_attentions=output_attentions,
1005 output_hidden_states=output_hidden_states,
→ 1006 return_dict=return_dict,
1007 )
1008 sequence_output = encoder_outputs[0]
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
588 encoder_attention_mask,
589 past_key_value,
→ 590 output_attentions,
591 )
592
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
473 head_mask,
474 output_attentions=output_attentions,
→ 475 past_key_value=self_attn_past_key_value,
476 )
477 attention_output = self_attention_outputs[0]
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
405 encoder_attention_mask,
406 past_key_value,
→ 407 output_attentions,
408 )
409 attention_output = self.output(self_outputs[0], hidden_states)
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
264 output_attentions=False,
265 ):
→ 266 mixed_query_layer = self.query(hidden_states)
267
268 # If this is instantiated as a cross-attention module, the keys
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/linear.py in forward(self, input)
101
102 def forward(self, input: Tensor) → Tensor:
→ 103 return F.linear(input, self.weight, self.bias)
104
105 def extra_repr(self) → str:
/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
1846 if has_torch_function_variadic(input, weight, bias):
1847 return handle_torch_function(linear, (input, weight, bias), input, weight, bias=bias)
→ 1848 return torch._C._nn.linear(input, weight, bias)
1849
1850
RuntimeError: CUDA error: CUBLAS_STATUS_NOT_INITIALIZED when calling cublasCreate(handle)
