Runtime error in torch.cuda.FloatTensor and weight type torch.Float.Tensor should be same

vision
1

hey, guys

Actually I’m facing the this problem, i go through the previous solution but is could not resolve, I implementing the device but is still showing the error

here my full error report
{
“name”: “RuntimeError”,
“message”: “Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same”,
“stack”: "---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[35], line 25
22 parameter = Parameter(budget=budget)
24 # Run ZOOpt
—> 25 solution = Opt.min(objective, parameter)
26 best_params = solution.get_x() # Extract the best parameters
27 best_learning_rate = best_params[0]

File ~/.local/lib/python3.11/site-packages/zoopt/opt.py:50, in Opt.min(objective, parameter)
48 result = sre.opt()
49 else:
—> 50 result = optimizer.opt(objective, parameter)
51 result.print_solution()
52 return result

File ~/.local/lib/python3.11/site-packages/zoopt/algos/opt_algorithms/racos/racos_optimization.py:57, in RacosOptimization.opt(self, objective, parameter, strategy)
55 else:
56 self.__algorithm = SRacos()
—> 57 self.__best_solution = self.__algorithm.opt(
58 objective, parameter, strategy, ub)
59 else:
60 self.__algorithm = Racos()

File ~/.local/lib/python3.11/site-packages/zoopt/algos/opt_algorithms/racos/sracos.py:47, in SRacos.opt(self, objective, parameter, strategy, ub)
45 self.set_objective(objective)
46 self.set_parameters(parameter)
—> 47 self.init_attribute()
48 stopping_criterion = self._parameter.get_stopping_criterion()
49 i = 0

File ~/.local/lib/python3.11/site-packages/zoopt/algos/opt_algorithms/racos/racos_common.py:92, in RacosCommon.init_attribute(self)
90 break
91 if distinct_flag:
—> 92 self._objective.eval(x)
93 self._data.append(x)
94 i += 1

File ~/.local/lib/python3.11/site-packages/zoopt/objective.py:83, in Objective.eval(self, solution)
81 for i in range(self.__resample_times):
82 if self.__reducedim is False:
—> 83 val = self.__func(solution)
84 else:
85 x = solution.get_x()

Cell In[35], line 9, in objective_function(solution)
6 num_layers = int(params[1])
7 dropout_rate = params[2]
----> 9 train_losses, val_losses, val_accuracies, _ = train_model(num_layers, dropout_rate, learning_rate)
10 return -val_accuracies[-1]

Cell In[33], line 3, in train_model(num_layers, dropout_rate, learning_rate)
2 def train_model(num_layers, dropout_rate, learning_rate):
----> 3 model = CNN(num_layers, dropout_rate).to(device)
5 # Print the model summary
6 print("Model Summary:")

Cell In[30], line 18, in CNN.init(self, num_layers, dropout_rate)
15 out_channels *= 2
17 # Initialize flatten_size dynamically
—> 18 self.flatten_size = self._calculate_flatten_size()
19 self.fc1 = nn.Linear(self.flatten_size, 256)
20 self.fc2 = nn.Linear(256, len(dataset.classes)) # Ensure dataset.classes is defined

Cell In[30], line 28, in CNN._calculate_flatten_size(self)
26 x = dummy_input
27 for layer in self.layers:
—> 28 x = layer(x)
29 return int(torch.prod(torch.tensor(x.size())))

File ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1531 else:
→ 1532 return self._call_impl(*args, **kwargs)

File ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs)
1536 # If we don’t have any hooks, we want to skip the rest of the logic in
1537 # this function, and just call forward.
1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1539 or _global_backward_pre_hooks or _global_backward_hooks
1540 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1541 return forward_call(*args, **kwargs)
1543 try:
1544 result = None

File ~/.local/lib/python3.11/site-packages/torch/nn/modules/conv.py:460, in Conv2d.forward(self, input)
459 def forward(self, input: Tensor) → Tensor:
→ 460 return self._conv_forward(input, self.weight, self.bias)

File ~/.local/lib/python3.11/site-packages/torch/nn/modules/conv.py:456, in Conv2d._conv_forward(self, input, weight, bias)
452 if self.padding_mode != ‘zeros’:
453 return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
454 weight, bias, self.stride,
455 _pair(0), self.dilation, self.groups)
→ 456 return F.conv2d(input, weight, bias, self.stride,
457 self.padding, self.dilation, self.groups)

RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same"
}

2

You would need to share some code as to understand where this is happening.

3

class CNN(nn.Module):
def init(self, num_layers, dropout_rate):
super(CNN, self).init()
self.layers = nn.ModuleList()
in_channels = 3
out_channels = 32
kernel_size = 3
padding = 1

    for _ in range(num_layers):
        self.layers.append(nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=padding))
        self.layers.append(nn.ReLU())
        self.layers.append(nn.MaxPool2d(2, 2))
        in_channels = out_channels
        out_channels *= 2
    
    # Initialize flatten_size dynamically
    self.flatten_size = self._calculate_flatten_size()
    self.fc1 = nn.Linear(self.flatten_size, 256)
    self.fc2 = nn.Linear(256, len(dataset.classes))  # Ensure dataset.classes is defined
    self.dropout = nn.Dropout(p=dropout_rate)

def _calculate_flatten_size(self):
    # Dummy input to calculate flatten_size
    dummy_input = torch.zeros(1, 3, 32, 32).to(device)
    x = dummy_input
    for layer in self.layers:
        x = layer(x)
    return int(torch.prod(torch.tensor(x.size())))

def forward(self, x):
    for layer in self.layers:
        x = layer(x)
    x = x.view(-1, self.flatten_size)
    x = F.relu(self.fc1(x))
    x = self.dropout(x)
    x = self.fc2(x)
    return x

Training function

def train_model(num_layers, dropout_rate, learning_rate):
model = CNN(num_layers, dropout_rate).to(device)

# Print the model summary
print("Model Summary:")
summary(model, input_size=(3, 32, 32))

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
num_epochs = 20

train_losses = []
val_losses = []
val_accuracies = []

for epoch in range(num_epochs):
    model.train()
    running_loss = 0.0
    correct = 0
    total = 0
    
    for images, labels in train_loaders:
        # Move images and labels to the correct device
        images, labels = images.to(device), labels.to(device)
        
        optimizer.zero_grad()
        outputs = model(images)  # Forward pass

        # Ensure the shapes are correct
        if outputs.size(0) != labels.size(0):
            raise ValueError(f'Batch size mismatch: outputs.size(0)={outputs.size(0)}, labels.size(0)={labels.size(0)}')

        loss = criterion(outputs, labels)  # Calculate loss
        loss.backward()  # Backpropagation
        optimizer.step()  # Update weights
        
        running_loss += loss.item()
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    train_losses.append(running_loss / len(train_loaders))

    # Validation phase
    model.eval()
    val_loss = 0.0
    correct = 0
    total = 0
    with torch.no_grad():
        for images, labels in val_loaders:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)

            # Ensure the shapes are correct
            if outputs.size(0) != labels.size(0):
                raise ValueError(f'Batch size mismatch: outputs.size(0)={outputs.size(0)}, labels.size(0)={labels.size(0)}')

            loss = criterion(outputs, labels)
            val_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    val_losses.append(val_loss / len(val_loaders))
    val_accuracies.append(correct / total)

return train_losses, val_losses, val_accuracies, model

ZOOpt objective function

def objective_function(solution):
# Convert ZOOpt Solution object to list of parameters
params = solution.get_x()
learning_rate = params[0]
num_layers = int(params[1])
dropout_rate = params[2]

train_losses, val_losses, val_accuracies, _ = train_model(num_layers, dropout_rate, learning_rate)
return -val_accuracies[-1]  # Return negative accuracy for minimization

ZOOpt configuration

dim = Dimension(
3,
[[1e-5, 1e-2], # learning_rate
[1, 5], # num_layers
[0.1, 0.5]], # dropout_rate
[True, True, True]
)
objective = Objective(objective_function, dim)
budget = 20 # Number of function evaluations
parameter = Parameter(budget=budget)

Run ZOOpt

solution = Opt.min(objective, parameter)
best_params = solution.get_x() # Extract the best parameters
best_learning_rate = best_params[0]
best_num_layers = int(best_params[1])
best_dropout_rate = best_params[2]

print(“Best parameters found by ZOOpt:”)
print(“Learning rate:”, best_learning_rate)
print(“Number of layers:”, best_num_layers)
print(“Dropout rate:”, best_dropout_rate)

#Final training with the best parameters found by ZOOpt
train_losses, val_losses, val_accuracies, final_model = train_model(best_num_layers, best_dropout_rate, best_learning_rate)

“here i try to implementing derivative free optimization called zoopt why run the script is showing the error” device = ‘cuda’

4

Haven’t gone through your code in depth, but have you tried going through each line where it needs to be sent to a device?