Hi all,
I’m dealing with a problem which is similar to the multi-task learning with one different:
in the multi-task problems the mean of the losses related to each branches will be backpropagated, (loss1 + loss2 + …+ lossN)/N. However, in my problem each branches should backpropagate its loss to its related part of model, i.e., loss1 will backpropagate into the part of the model related to the branch1 which I named that part paral_1 and in the same manner the paral_2, …, paral_N supposed to dedicate to its related branches. How I handled the different backprogation related to each branch ? I defined N=10 different dataloaders corresponding to each tasks. So whenever a specific dataloader gets activated only the related part of the model will be activated, requires_grad= True, and the not-related parts of the model will be freezed, requires_grad= False. in this way I can trained all branches, in my case N= 10, simultaneously.
having said that, when there is ONLY one dataloader, N=1, I observed that the model is trained(high accuracy) ,However when I increase the number of dataloader, which means the number of branches increases correspondingly, the model is not trained at all(almost zero accuracy for each branch). It makes sense that when the number of branches increase each branch should train as in
each branch trained separately, because, the dataloaders are independent from each other and with the freezing technique I explained above each branches is in charge of its own part of the model, NOTED non-related parts get freezed at training phase , so parameters of each branches gets updated at the training phase independently. So why the model is not trained(almost zero accuracy) when the parameters of the branches get updated in parallel, at backpropation time. But when the number of branches set to one the model get trained(high accuracy).
I hope I could convey what is the problem. and the following piece I code is what I suspect causing this problem but I could not see where is issue:
def train_kd(model, optimizer, dataloader, ...):
"""Train the model on `num_steps` batches
# set model to training mode
model.train()
# dataloader_list = [dataloader[i] for i in range(len(dataloader))]
dataloader_list = [dataloader[6]]
batches = []
for i, batches in enumerate(zip(*dataloader_list)):
loss_functions = [
net.loss_fn_kd0, # loss0
net.loss_fn_kd1, # loss1
net.loss_fn_kd2, # loss2
net.loss_fn_kd3, # loss3
net.loss_fn_kd4, # loss4
net.loss_fn_kd5, # loss5
net.loss_fn_kd6, # loss6
net.loss_fn_kd7, # loss7
net.loss_fn_kd8, # loss8
net.loss_fn_kd9 # loss9
]
loss0 =torch.tensor(0.0)
loss1 =torch.tensor(0.0)
loss2 =torch.tensor(0.0)
loss3 =torch.tensor(0.0)
loss4 =torch.tensor(0.0)
loss5 =torch.tensor(0.0)
loss6 =torch.tensor(0.0)
loss7 =torch.tensor(0.0)
loss8 =torch.tensor(0.0)
loss9 =torch.tensor(0.0)
loss_score = [
Variable(loss0, requires_grad = True),
Variable(loss1, requires_grad = True),
Variable(loss2, requires_grad = True),
Variable(loss3, requires_grad = True),
Variable(loss4, requires_grad = True),
Variable(loss5, requires_grad = True),
Variable(loss6, requires_grad = True),
Variable(loss7, requires_grad = True),
Variable(loss8, requires_grad = True),
Variable(loss9, requires_grad = True)
]
for j, (train_batch, labels_batch) in enumerate(batches, 6):
require_grad_list = [False] * 10
require_grad_list[j] = True
# move to GPU if available
if params.cuda:
# convert to torch Variables
train_batch, labels_batch = Variable(train_batch), Variable(labels_batch)
train_batch, labels_batch = train_batch.cuda(async=True), labels_batch.cuda(async=True)
output_batch = model(train_batch)
# get one batch output from teacher_outputs list
if params.cuda:
output_batch = output_batch.cuda(async=True)
output_teacher_batch = torch.from_numpy(teacher_outputs[i])
if params.cuda:
output_teacher_batch = output_teacher_batch.cuda(async=True)
output_teacher_batch = Variable(output_teacher_batch, requires_grad=False)
#############################################
#############multi-task learning#############
#############################################
## layer4 freezing parallel 0
for child in model.layer4.paral_0.children():
for param in child.parameters():
param.requires_grad = require_grad_list[0]
for param in model.layer4.fc0.parameters():
param.requires_grad = require_grad_list[0]
## layer4 freezing parallel 1
for child in model.layer4.paral_1.children():
for param in child.parameters():
param.requires_grad = require_grad_list[1]
for param in model.layer4.fc1.parameters():
param.requires_grad = require_grad_list[1]
## layer4 freezing parallel 2
for child in model.layer4.paral_2.children():
for param in child.parameters():
param.requires_grad = require_grad_list[2]
for param in model.layer4.fc2.parameters():
param.requires_grad = require_grad_list[2]
## layer4 freezing parallel 3
for child in model.layer4.paral_3.children():
for param in child.parameters():
param.requires_grad = require_grad_list[3]
for param in model.layer4.fc3.parameters():
param.requires_grad = require_grad_list[3]
## layer4 freezing parallel 4
for child in model.layer4.paral_4.children():
for param in child.parameters():
param.requires_grad = require_grad_list[4]
for param in model.layer4.fc4.parameters():
param.requires_grad = require_grad_list[4]
## layer4 freezing parallel 5
for child in model.layer4.paral_5.children():
for param in child.parameters():
param.requires_grad = require_grad_list[5]
for param in model.layer4.fc5.parameters():
param.requires_grad = require_grad_list[5]
## layer4 freezing parallel 6
for child in model.layer4.paral_6.children():
for param in child.parameters():
param.requires_grad = require_grad_list[6]
for param in model.layer4.fc6.parameters():
param.requires_grad = require_grad_list[6]
## layer4 freezing parallel 7
for child in model.layer4.paral_7.children():
for param in child.parameters():
param.requires_grad = require_grad_list[7]
for param in model.layer4.fc7.parameters():
param.requires_grad = require_grad_list[7]
## layer4 freezing parallel 8
for child in model.layer4.paral_8.children():
for param in child.parameters():
param.requires_grad = require_grad_list[8]
for param in model.layer4.fc8.parameters():
param.requires_grad = require_grad_list[8]
## layer4 freezing parallel 9
for child in model.layer4.paral_9.children():
for param in child.parameters():
param.requires_grad = require_grad_list[9]
for param in model.layer4.fc9.parameters():
param.requires_grad = require_grad_list[9]
optimizer.zero_grad()
# print('one pass')
loss_score[j].backward()
# print('two pass')
optimizer.step()
# print('---------------------------------')
# Evaluate summaries only once in a while
if i % params.save_summary_steps == 0:
# extract data from torch Variable, move to cpu, convert to numpy arrays
output_batch = output_batch.data.cpu().numpy()
labels_batch = labels_batch.data.cpu().numpy()
.....
# compute all metrics on this batch
summary_batch = {metric:metrics[metric](output_batch, labels_batch, params)
for metric in metrics}
I used also nn.CrossEntropyLoss()(outputs, labels) for calculating the loss for each branches.