Here are the detailed trace back, if needed:
File "/data/disk1/private/zhangjuexiao/allennlp/allennlp/training/trainer.py", line 434, in _train_epoch
loss = self._batch_loss(batch, for_training=True) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/training/trainer.py", line 371, in _batch_loss
output_dict = self._model(**batch) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/models/reading_comprehension/bidaf.py", line
174, in forward embedded_question = self._highway_layer(self._text_field_embedder(question)) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/modules/text_field_embedders/basic_text_field_e
mbedder.py", line 52, in forward token_vectors = embedder(tensor) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/modules/token_embedders/token_characters_en
coder.py", line 36, in forward return self._dropout(self._encoder(self._embedding(token_characters),
mask)) File "/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/modules/time_distributed.py", line 35, in forward
reshaped_outputs = self._module(*reshaped_inputs) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/allennlp/allennlp/modules/seq2vec_encoders/cnn_encoder.py",
line 106, in forward self._activation(convolution_layer(tokens)).max(dim=2)[0] File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/module.py", line 357, in call result = self.forward(*input, **kwargs) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/modules/conv.py", line 168, in forward self.padding, self.dilation, self.groups) File
"/data/disk1/private/zhangjuexiao/anaconda3/envs/test_cudnn/lib/python3.6/site-
packages/torch/nn/functional.py", line 54, in conv1d return f(input, weight, bias) RuntimeError:
CUDNN_STATUS_NOT_INITIALIZED