Hi ptrblck.It’s me again.I hope you are okay.Okay i just learned that there is a parameter torch.backends.cudnn.benchmark=True.It will increase speed of training.But i didn’t found any example on this even in pytorch documentation.So here is my training code.Can you tell me where to use this parameter.Thank you.

```
class Train:
def __init__(self, model, dataloader, optimizer, stats, scheduler=None, L1lambda = 0,criterion=None,use_amp=True):
self.model = model
self.dataloader = dataloader
self.optimizer = optimizer
self.scheduler = scheduler
self.stats = stats
self.L1lambda = L1lambda
self.criterion=criterion
self.loss1=0.0
self.loss2=0.0
self.loss=0.0
self.use_amp=True
def run(self):
self.model.train()
torch.cuda.empty_cache()
pbar = tqdm_notebook(self.dataloader)
for data1,data2,target1,target2 in pbar:
# get samples
data1,data2 = data1.to(self.model.device), data2.to(self.model.device)
target1, target2 = target2.to(self.model.device), target2.to(self.model.device)
self.optimizer.zero_grad()
output1,output2 = self.model(data1,data2)
#print(target.shape)
target1=target1.unsqueeze(1)
target2=target2.unsqueeze(1)
self.loss1=self.criterion[0](output1.float(), target1.half())
self.loss2=self.criterion[1](output2.float(), target2.half())
#print("loss1 {}".format(self.loss1))
#print("loss2 {}".format(self.loss2))
self.loss=(self.loss1+self.loss2)/2.0
# In PyTorch, we need to set the gradients to zero before starting to do backpropragation because PyTorch accumulates the gradients on subsequent backward passes.
# Because of this, when you start your training loop, ideally you should zero out the gradients so that you do the parameter update correctly.
# Predict
#Implementing L1 regularization
#print(self.loss)
if self.L1lambda > 0:
reg_loss = 0.
for param in self.model.parameters():
reg_loss += torch.sum(param.abs())
self.loss += self.L1lambda * reg_loss
#print(self.loss)
# Backpropagation
if self.use_amp:
with amp.scale_loss(self.loss, self.optimizer) as scaled_loss:
scaled_loss.backward()
else:
self.loss.backward()
#self.loss.backward()
#self.optimizer.step()
# Update pbar-tqdm
#pred = y_pred.argmax(dim=1, keepdim=True) # get the index of the max log-probability
'''n_digits=1
output = (output * 10**n_digits).round() / (10**n_digits)
target = (target * 10**n_digits).round() / (10**n_digits)
correct = output.eq(target).sum().item()
print("correct is {}".format(correct))'''
correct=0
lr = 0
if self.scheduler:
lr = self.scheduler.get_last_lr()[0]
else:
# not recalling why i used sekf.optimizer.lr_scheduler.get_last_lr[0]
lr = self.optimizer.param_groups[0]['lr']
#lr = if self.scheduler else (self.optimizer.lr_scheduler.get_last_lr()[0] if self.optimizer.lr_scheduler else self.optimizer.param_groups[0]['lr'])
self.stats.add_batch_train_stats(self.loss.item(),correct, len(data1), lr)
pbar.set_description(self.stats.get_latest_batch_desc())
if self.scheduler:
self.scheduler.step()
```

My calling function of train module

```
import torch.optim as optim
import torch.nn as nn
import time
EPOCHS =1
#max_lr=1
optimizer = optim.SGD(model.parameters(), lr=0.01)
model, optimizer = amp.initialize(
model, optimizer, opt_level="O2",
keep_batchnorm_fp32=True, loss_scale="dynamic"
)
#criterion=nn.L1Loss()
criterion=[nn.BCEWithLogitsLoss(),nn.L1Loss()]
#criterion = nn.BCEWithLogitsLoss()
start=time.time()
model.gotrain(optimizer, train_loader, test_loader, EPOCHS, "/content/gdrive/My Drive",criterion,use_amp=True)
print(time.time()-start)
```

Thank you sir