RuntimeError: invalid argument 4: expecting vector of indices at /pytorch/aten/src/THC/generic/

I have two mask tensors which both have size of [b,c,w,h] (c=63*63)
weight_1 and weight_2 are their corresponding label [b,1,w,h] ] indicating which position in mask should I pick
Because mask_1 and mask_2 are paired (mask_1[i] is paired with mask_2[i] ), I want to select common min numbers of mask for each pair
below is my code

def select_mask(mask_1, weight_1, mask_2, weight_2, o_sz=63, g_sz=127):
    mask_1 = mask_1.permute(0, 2, 3, 1).contiguous()
    mask_2 = mask_2.permute(0, 2, 3, 1).contiguous()

    sel_pm_1 = torch.tensor([])
    sel_pm_2 = torch.tensor([])
    sel_pm_1 = torch.autograd.Variable(sel_pm_1).cuda()
    sel_pm_2 = torch.autograd.Variable(sel_pm_2).cuda()
    for i in range(weight_1.size(0)):
        tmp = weight_1[i][0].view(-1) #[w*h] (w=25,h=25)
        tmp2 = weight_2[i][0].view(-1) #[w*h]
        tmp = Variable(  
        tmp2 = Variable(
        n_tmp = tmp.nelement()
        n_tmp2 = tmp2.nelement()
        mi = min(n_tmp, n_tmp2)
        tmp = tmp[:mi].squeeze()
        tmp2 = tmp2[:mi].squeeze()

        T = mask_1[i].view(25*25,63*63)
        T2 = mask_2[i].view(25*25,63*63)
        tmp_pm_1 = torch.index_select(T,0,tmp)
        tmp_pm_2 = torch.index_select(T2,0,tmp2)
        sel_pm_1 =[tmp_pm_1,sel_pm_1])
        sel_pm_2 =[tmp_pm_2,sel_pm_2])

    if sel_pm_1.nelement()==0 or sel_pm_2.nelement()==0: return None,None,False

    #sel_pm_1 = [n,63*63]
    sel_pm_1 = sel_pm_1.view(-1, 1, o_sz, o_sz)
    sel_pm_1 = nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(sel_pm_1)
    sel_pm_1 = sel_pm_1.view(-1, g_sz , g_sz)

    sel_pm_2 = sel_pm_2.view(-1, 1, o_sz, o_sz)
    sel_pm_2 = nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(sel_pm_2)
    sel_pm_2 = sel_pm_2.view(-1, g_sz , g_sz)
    return sel_pm_1, sel_pm_2, True

Forwarding seems no problem , however , I got the following error message
Where could possibly go wrong?

Traceback (most recent call last):
  File "/home/teresa/SiamMask/tools/", line 301, in <module>
  File "/home/teresa/SiamMask/tools/", line 169, in main
    train(train_loader, dist_model, optimizer, lr_scheduler, args.start_epoch, cfg)
  File "/home/teresa/SiamMask/tools/", line 255, in train
  File "/home/teresa/anaconda3/envs/siammask/lib/python3.6/site-packages/torch/", line 93, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/teresa/anaconda3/envs/siammask/lib/python3.6/site-packages/torch/autograd/", line 90, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: invalid argument 4: expecting vector of indices at /pytorch/aten/src/THC/generic/