Found at least two devices 'cuda:0' and 'cpu'

I have been trying to run a code but everytime i am getting the following error ‘Found at least two devices ‘cuda:0’ and ‘cpu’’. After following the error trace i tried to put the function on ‘cuda’ but its still not working. This is my error trace:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.8/site-packages/torchinfo/torchinfo.py", line 287, in forward_pass
    _ = model.to(device)(*x, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1212, in _call_impl
    result = forward_call(*input, **kwargs)
  File "/mt/S2S/Transformer_model.py", line 181, in forward
    encoder_out = self.encoder(src, src_lengths)
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1212, in _call_impl
    result = forward_call(*input, **kwargs)
  File "/mt/S2S/Transformer_model.py", line 79, in forward
    x = self._forward(
  File "/mt/S2S/Transformer_model.py", line 56, in _forward
    positions = self.embed_positions(encoder_padding_mask).transpose(0, 1).cuda()
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1212, in _call_impl
    result = forward_call(*input, **kwargs)
  File "/mt/S2S/sinusoidal_positional_embedding.py", line 102, in forward
    self.weights.index_select(0, positions.view(-1))
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument index in method wrapper__index_select)

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "S2S/train.py", line 44, in <module>
    summary(model=model,
  File "/opt/conda/lib/python3.8/site-packages/torchinfo/torchinfo.py", line 217, in summary
    summary_list = forward_pass(
  File "/opt/conda/lib/python3.8/site-packages/torchinfo/torchinfo.py", line 296, in forward_pass
    raise RuntimeError(
RuntimeError: Failed to run torchinfo. See above stack traces for more details. Executed layers up to: [Conv1dSubsampler: 2, Conv1d: 4, Conv1d: 4]

This is the main code where i am getting the error:

class S2TTransformerEncoder(Encoder):

    def __init__(self):
        super().__init__(None)

        self.encoder_freezing_updates = 0
        self.num_updates = 0

        self.dropout_module = FairseqDropout(
            p=0.1, module_name=self.__class__.__name__
        )
        self.embed_scale = math.sqrt(512)
        self.no_scale_embedding=False
        if self.no_scale_embedding:
            self.embed_scale = 1.0
        self.padding_idx = 1

        self.subsample = Conv1dSubsampler()

        self.embed_positions = PositionalEmbedding(
            666, 512, self.padding_idx
        )

        self.transformer_layers = nn.ModuleList(
            [TransformerEncoderLayer(6) for _ in range(6)]
        )
        self.encoder_normalize_before=True
        if self.encoder_normalize_before:
            self.layer_norm = nn.LayerNorm(512)
        else:
            self.layer_norm = None

        self.ctc_proj = None

    def _forward(self, src_tokens, src_lengths, return_all_hiddens=False):
        x, input_lengths = self.subsample(src_tokens, src_lengths)
        x = self.embed_scale * x

        encoder_padding_mask = lengths_to_padding_mask(input_lengths)
        positions = self.embed_positions(encoder_padding_mask).transpose(0, 1).cuda()
        x += positions
        x = self.dropout_module(x)

        encoder_states = []

        for layer in self.transformer_layers:
            x = layer(x, encoder_padding_mask.expand(src_tokens.shape[0],-1))
            if return_all_hiddens:
                encoder_states.append(x)

        if self.layer_norm is not None:
            x = self.layer_norm(x)

        return x

    def forward(self, src_tokens, src_lengths, return_all_hiddens=False):
        if self.num_updates < self.encoder_freezing_updates:
            with torch.no_grad():
                x = self._forward(
                    src_tokens, src_lengths, return_all_hiddens=return_all_hiddens
                )
        else:
            x = self._forward(
                src_tokens, src_lengths, return_all_hiddens=return_all_hiddens
            )
        return x

    def reorder_encoder_out(self, encoder_out, new_order):
        new_encoder_out = (
            []
            if len(encoder_out["encoder_out"]) == 0
            else [x.index_select(1, new_order) for x in encoder_out["encoder_out"]]
        )

        new_encoder_padding_mask = (
            []
            if len(encoder_out["encoder_padding_mask"]) == 0
            else [
                x.index_select(0, new_order)
                for x in encoder_out["encoder_padding_mask"]
            ]
        )

        new_encoder_embedding = (
            []
            if len(encoder_out["encoder_embedding"]) == 0
            else [
                x.index_select(0, new_order) for x in encoder_out["encoder_embedding"]
            ]
        )

        encoder_states = encoder_out["encoder_states"]
        if len(encoder_states) > 0:
            for idx, state in enumerate(encoder_states):
                encoder_states[idx] = state.index_select(1, new_order)

        return new_encoder_out

Using .cuda() inside a forward function is likely to make the error you met.
Before doing forward, make sure that your model and the input tensor are located in the same device.

1 Like

Hi @thecho7 Is there any way in which i can check which tensor is on Cuda and which is on cpu

print(tensor.device) should work