<PyTorch tutorial> cuDNN error: CUDNN_STATUS_EXECUTION_FAILED

Hello, I’m a beginner of PyTorch.
Now, I’m studying with the tutorial of PyTorch (Transfer learning tutorial).
When I ran following code, I got an error, ‘RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED’.
Please let me know what is wrong in the code or in my running environment.
My setting is
GPU: Geforce RTX 2070
CUDA ver: 9.0
cuDNN ver: 7.5 (7005)

code:

from future import print_function, division

import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.autograd import Variable
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy

plt.ion() # interactive mode

data_transforms = {
‘train’: transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
‘val’: transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}

data_dir = ‘hymenoptera_data’
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
data_transforms[x])
for x in [‘train’, ‘val’]}
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,
shuffle=True, num_workers=4)
for x in [‘train’, ‘val’]}
dataset_sizes = {x: len(image_datasets[x]) for x in [‘train’, ‘val’]}
class_names = image_datasets[‘train’].classes

use_gpu = torch.cuda.is_available()

def imshow(inp, title=None):
“”“Imshow for Tensor.”""
inp = inp.numpy().transpose((1, 2, 0))
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
inp = std * inp + mean
inp = np.clip(inp, 0, 1)
plt.imshow(inp)
if title is not None:
plt.title(title)
plt.pause(0.1) # pause a bit so that plots are updated

def train_model(model, criterion, optimizer, scheduler, num_epochs = 25):
since = time.time()

best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0

for epoch in range(num_epochs):
    print('Epoch {}/{}'.format(epoch, num_epochs - 1))
    print('-' * 10)

    # Each epoch has a training and validation phase
    for phase in ['train', 'val']:
        if phase == 'train':
            scheduler.step()
            model.train(True)  # Set model to training mode
        else:
            model.train(False)  # Set model to evaluate mode

        running_loss = 0.0
        running_corrects = 0

        # Iterate over data.
        for data in dataloaders[phase]:
            # get the inputs
            inputs, labels = data

            # wrap them in Variable
            if use_gpu:
                inputs = Variable(inputs.cuda())
                labels = Variable(labels.cuda())
            else:
                inputs, labels = Variable(inputs), Variable(labels)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward
            outputs = model(inputs)
            _, preds = torch.max(outputs.data, 1)
            loss = criterion(outputs, labels)

            # backward + optimize only if in training phase
            if phase == 'train':
                loss.backward()
                optimizer.step()

            # statistics
            running_loss += loss.data[0] * inputs.size(0)
            running_corrects += torch.sum(preds == labels.data)

        epoch_loss = running_loss / dataset_sizes[phase]
        epoch_acc = running_corrects / dataset_sizes[phase]

        print('{} Loss: {:.4f} Acc: {:.4f}'.format(
            phase, epoch_loss, epoch_acc))

        # deep copy the model
        if phase == 'val' and epoch_acc > best_acc:
            best_acc = epoch_acc
            best_model_wts = copy.deepcopy(model.state_dict())

    print()

time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))

# load best model weights
model.load_state_dict(best_model_wts)
return model

def visualize_model(model, num_images=6):
was_training = model.training
model.eval()
images_so_far = 0
fig = plt.figure()

for i, data in enumerate(dataloaders['val']):
    inputs, labels = data
    if use_gpu:
        inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
    else:
        inputs, labels = Variable(inputs), Variable(labels)

    outputs = model(inputs)
    _, preds = torch.max(outputs.data, 1)

    for j in range(inputs.size()[0]):
        images_so_far += 1
        ax = plt.subplot(num_images//2, 2, images_so_far)
        ax.axis('off')
        ax.set_title('predicted: {}'.format(class_names[preds[j]]))
        imshow(inputs.cpu().data[j])

        if images_so_far == num_images:
            model.train(mode=was_training)
            return
model.train(mode=was_training)

def main():
# Get a batch of training data
inputs, classes = next(iter(dataloaders[‘train’]))

# Make a grid from batch
out = torchvision.utils.make_grid(inputs)

# imshow(out, title=[class_names[x] for x in classes])

model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 2)

if use_gpu:
    model_ft = model_ft.cuda()

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=25)

if name == ‘main’:
main()

error:

Traceback (most recent call last):
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 194, in
main()
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 190, in main
num_epochs=25)
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 100, in train_model
outputs = model(inputs)
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torchvision\models\resnet.py”, line 150, in forward
x = self.conv1(x)
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\module.py”, line 489, in call
result = self.forward(*input, **kwargs)
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\conv.py”, line 320, in forward
self.padding, self.dilation, self.groups)
RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED

Is your code running fine of the CPU?
If so, could you disable cuDNN and try to run your code again using torch.backends.cudnn.enabled = False?

I have just solved this problem before, but I don’t know if it is suitable for you. Please check your following aspects:
1 The type of data you entered, LongTensor or something else
2 Is the shape of your neural network the same as the shape of your data?

Thank you all, I solved the problem.
All of your advice were helpful~!