Why I can't use torch.index_select in forward function?

1 
    def forward(self, title, content):
        title = self.encoder_tit(title)
        print(title.size())
#         print(title)
        title_rehape = torch.index_select(title, 2, torch.LongTensor([0,2,1]))

<ipython-input-260-15e690fe555f> in forward(self, title, content)
     53         print(title.size())
     54         print(title)
---> 55         title_rehape = torch.index_select(title, 2, torch.LongTensor([0,2,1]))
     56         title_out_1 = self.title_conv_1(title_rehape)
     57         title_out_2 = self.title_conv_2(title_rehape)

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/autograd/variable.py in index_select(self, dim, index)
    679 
    680     def index_select(self, dim, index):
--> 681         return IndexSelect.apply(self, dim, index)
    682 
    683     def gather(self, dim, index):

RuntimeError: save_for_backward can only save input or output tensors, but argument 0 doesn't satisfy this condition

Why I can’t use torch.index_select in forward function?I’m very confused.

1 Like
2

Wrap the index in a torch.autograd.Variable:

title_rehape = torch.index_select(title, 2, Variable(torch.LongTensor([0,2,1])))

The error message is not terribly intuitive, but usually passing Tensors vs. Variables is an all-or-nothing thing.

Best regards

Thomas

1 Like
3 
 def forward(self, title, content):
        title = self.encoder_tit(title)
        print(title.size())
        title_rehape = torch.index_select(title, 2, Variable(torch.LongTensor([0,2,1])))
        print("title_reshape size is {}".format(title_rehape.size()))

The result is:

torch.Size([1, 3, 64])
title_reshape size is torch.Size([1, 3, 3])

The expected result is [1, 64, 3].Why it’s [1,3,3].I’m confused.
And there is another question emerging.

class Text(nn.Module): 
    def __init__(self):
        super(CNNText, self).__init__()
        self.encoder_tit = nn.Embedding(3281, 64)
        self.encoder_con = nn.Embedding(496037, 512)
        
        self.title_conv_1 = nn.Sequential(
            nn.Conv1d(in_channels = 64,
                      out_channels = 1,
                      kernel_size = (1, 64)),
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
        )            
        self.fc = nn.Linear(5, 9)

    def forward(self, title, content):
        title = self.encoder_tit(title)
        print(title.size())
        title_rehape = torch.index_select(title, 2, Variable(torch.LongTensor([0,2,1])))
        print("title_reshape size is {}".format(title_rehape.size()))
        title_out_1 = self.title_conv_1(title_rehape)

The error is:

torch.Size([1, 3, 64])
title_reshape size is torch.Size([1, 3, 3])
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-277-dffba3242fa0> in <module>()
     15     optimizer.zero_grad()
     16 
---> 17     out = cnnt(T, C)
     18     target = cla[epoch]
     19     loss += Loss(out, target)

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
--> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
    226             hook_result = hook(self, input, result)

<ipython-input-276-f39b5fc5f928> in forward(self, title, content)
     54         title_rehape = torch.index_select(title, 2, Variable(torch.LongTensor([0,2,1])))
     55         print("title_reshape size is {}".format(title_rehape.size()))
---> 56         title_out_1 = self.title_conv_1(title_rehape)
     57         title_out_2 = self.title_conv_2(title_rehape)
     58         content = self.encoder_con(content)

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
--> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
    226             hook_result = hook(self, input, result)

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     65     def forward(self, input):
     66         for module in self._modules.values():
---> 67             input = module(input)
     68         return input
     69 

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
--> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
    226             hook_result = hook(self, input, result)

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    152     def forward(self, input):
    153         return F.conv1d(input, self.weight, self.bias, self.stride,
--> 154                         self.padding, self.dilation, self.groups)
    155 
    156 

/home/quoniammm/anaconda3/envs/py3Tfgpu/lib/python3.6/site-packages/torch/nn/functional.py in conv1d(input, weight, bias, stride, padding, dilation, groups)
     81     f = ConvNd(_single(stride), _single(padding), _single(dilation), False,
     82                _single(0), groups, torch.backends.cudnn.benchmark, torch.backends.cudnn.enabled)
---> 83     return f(input, weight, bias)
     84 
     85 

RuntimeError: expected 3D tensor

The title_shape has been a 3D tensor.The error makes me very confused.Can you tell me why?
Thanks

4

Are you by any chance confusing permute with index select?

http://pytorch.org/docs/master/tensors.html#torch.Tensor.permute

permute(*dims)
Permute the dimensions of this tensor.

Parameters: *dims (int…) – The desired ordering of dimensions

#Example
>>> x = torch.randn(2, 3, 5)
>>> x.size()
torch.Size([2, 3, 5])
>>> x.permute(2, 0, 1).size()
torch.Size([5, 2, 3])
5

Thanks.Now,
title_reshape size is torch.Size([1, 64, 3])

But the Error is still existed:
RuntimeError: expected 3D tensor

title_reshape is a 3D tensor.

6

The title itself is a 3D tensor - The output of nn.Embedding will be a 3D tensor

Few concerns

  • I really dont know if MaxPool1d with kernel size 1 is useful - Won’t it be the same output as input?
  • Conv1d with an input of (1, 64, 3) and a 2D kernel of size (1, 64) seems odd. Maybe you meant a kernel of size 1. The you will get an output of size (1, 1, 3) at the end of the operation.

The error is due to the fact that the kernel size for your Conv1D is 2 dimension. It is not due to input being a 3D tensor.

7

I use Conv1d here.If input_size is [1, 64, 3].
layer of Conv1d is:

self.title_conv_1 = nn.Sequential(
            nn.Conv1d(in_channels = 64,
                      out_channels = 1,
                      kernel_size = (1, 64)),
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
        )

am i right?

8

If you use a kernel_size of (1, 64), the associated weight for the kernel will be 4 Dimension (1, 64, 1, 64). This cannot be used to do a 1D convolution. Conv1D weights must be of dimension (out_channel x in_channel x kernel_size) - A 3D tensor.

Which is why you got the error.

Try changing the conv1d layer to

nn.Conv1d(in_channels = 64, out_channels = 1, kernel_size = 1)

This will produce a kernel of size (64 x 1 x 1). When convolved with an input of (1 x 64 x 3) will produce an output of (1x1x3)

1 Like