I have a model that I apply to 3D data. To provide an example I created a random dataset that always return the same tensor of the wanted size.
When executing the code below, the .cuda() command is extremely slow (4s) except the first call (0.008s).
If I comment loss.backward() it becomes much faster (0.008s).
I also put below the reports of torch.utils.bottleneck with and without loss.backward().
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from time import time
class RandomDataset(Dataset):
def __init__(self, size=(1, 169, 208, 179), length=18):
self.item = torch.rand(size)
self.length = length
def __len__(self):
return self.length
def __getitem__(self, idx):
return self.item
class Test(nn.Module):
"""
Classifier for a multi-class classification task
"""
def __init__(self):
super(Test, self).__init__()
self.features = nn.Sequential(
nn.Conv3d(1, 8, 3),
nn.BatchNorm3d(8),
nn.ReLU(),
nn.MaxPool3d(2, 2),
nn.Conv3d(8, 16, 3),
nn.BatchNorm3d(16),
nn.ReLU(),
nn.MaxPool3d(2, 2),
nn.Conv3d(16, 32, 3),
nn.BatchNorm3d(32),
nn.ReLU(),
nn.MaxPool3d(2, 2),
nn.Conv3d(32, 64, 3),
nn.BatchNorm3d(64),
nn.ReLU(),
nn.MaxPool3d(2, 2)
)
self.classifier = nn.Sequential(
nn.Linear(64 * 792, 1000),
nn.ReLU(),
nn.Linear(1000, 100),
nn.ReLU(),
nn.Linear(100, 2)
)
def forward(self, x):
x = self.features(x)
x = x.view(x.size(0), -1)
x = self.classifier(x)
return x
if __name__ == "__main__":
batch_size = 3
dataset = RandomDataset()
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True)
model = Test()
model = model.cuda()
total_time=0
criterion = nn.CrossEntropyLoss()
for i, data in enumerate(dataloader):
t0 = time()
data_gpu = data.cuda()
t1 = time()
total_time += t1 - t0
print("Loading data on GPU", t1 - t0)
output = model(data_gpu)
labels = torch.Tensor([0] * batch_size).long().cuda()
loss = criterion(output, labels)
loss.backward()
print("Mean time on loading data on GPU:", total_time / (len(dataset) / batch_size))
With loss.backward()
--------------------------------------------------------------------------------
Environment Summary
--------------------------------------------------------------------------------
PyTorch 1.0.0 compiled w/ CUDA 8.0.61
Running with Python 3.6 and
`pip3 list` truncated output:
numpy (1.14.3)
--------------------------------------------------------------------------------
cProfile output
--------------------------------------------------------------------------------
8396 function calls (7632 primitive calls) in 23.731 seconds
Ordered by: internal time
List reduced from 268 to 15 due to restriction <15>
ncalls tottime percall cumtime percall filename:lineno(function)
46 22.976 0.499 22.976 0.499 {method 'cuda' of 'torch._C._TensorBase' objects}
18 0.317 0.018 0.317 0.018 {method 'uniform_' of 'torch._C._TensorBase' objects}
18 0.259 0.014 0.259 0.014 {built-in method addmm}
6 0.086 0.014 0.086 0.014 {built-in method stack}
1 0.042 0.042 0.042 0.042 {built-in method rand}
6 0.018 0.003 0.018 0.003 {method 'run_backward' of 'torch._C._EngineBase' objects}
24 0.005 0.000 0.005 0.000 {built-in method conv3d}
24 0.002 0.000 0.002 0.000 {built-in method batch_norm}
36 0.002 0.000 0.002 0.000 {built-in method threshold}
1 0.002 0.002 23.731 23.731 script.py:1(<module>)
404 0.002 0.000 0.004 0.000 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py:537(__setattr__)
24 0.002 0.000 0.002 0.000 {built-in method torch._C._nn.max_pool3d_with_indices}
24 0.001 0.000 0.004 0.000 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py:58(forward)
150/12 0.001 0.000 0.276 0.023 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py:483(__call__)
1224/1003 0.001 0.000 0.004 0.000 {built-in method builtins.isinstance}
without loss.backward()
--------------------------------------------------------------------------------
Environment Summary
--------------------------------------------------------------------------------
PyTorch 1.0.0 compiled w/ CUDA 8.0.61
Running with Python 3.6 and
`pip3 list` truncated output:
numpy (1.14.3)
--------------------------------------------------------------------------------
cProfile output
--------------------------------------------------------------------------------
8336 function calls (7572 primitive calls) in 4.048 seconds
Ordered by: internal time
List reduced from 263 to 15 due to restriction <15>
ncalls tottime percall cumtime percall filename:lineno(function)
46 3.318 0.072 3.318 0.072 {method 'cuda' of 'torch._C._TensorBase' objects}
18 0.312 0.017 0.312 0.017 {method 'uniform_' of 'torch._C._TensorBase' objects}
18 0.258 0.014 0.258 0.014 {built-in method addmm}
6 0.082 0.014 0.082 0.014 {built-in method stack}
1 0.040 0.040 0.040 0.040 {built-in method rand}
24 0.006 0.000 0.006 0.000 {built-in method conv3d}
36 0.004 0.000 0.004 0.000 {built-in method threshold}
24 0.003 0.000 0.003 0.000 {built-in method batch_norm}
24 0.003 0.000 0.003 0.000 {built-in method torch._C._nn.max_pool3d_with_indices}
404 0.002 0.000 0.004 0.000 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py:537(__setattr__)
1 0.002 0.002 4.048 4.048 script.py:1(<module>)
150/12 0.001 0.000 0.281 0.023 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py:483(__call__)
24 0.001 0.000 0.005 0.000 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py:58(forward)
100/1 0.001 0.000 0.003 0.003 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/abc.py:196(__subclasscheck__)
24/1 0.001 0.000 1.952 1.952 /home/elina.thibeausutre/miniconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py:185(_apply)