ValueError: Expected input batch_size (324) to match target batch_size (4)

110

Hi Sir, I Have a similar error but the .py files are different:
the code has been taken from GitHub: GitHub - Leo-Q-316/ImGAGN: Imbalanced Network Embedding vi aGenerative Adversarial Graph Networks
and the data had been transformed as it says in GitHub. The only problem I faces in the features.cora part.

so I have this error in the following code: ValueError: Expected input batch_size (18) to match target batch_size (15).

from __future__ import division
from __future__ import print_function

import time
import argparse
import numpy as np
import scipy.sparse as sp
import torch
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable

from utils import load_data, accuracy, add_edges
from models import GCN
from models import Generator

# Training settings
parser = argparse.ArgumentParser()
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='Disables CUDA training.')
parser.add_argument('--fastmode', action='store_true', default=False,
                    help='Validate during training pass.')
parser.add_argument('--seed', type=int, default=42, help='Random seed.')
parser.add_argument('--epochs', type=int, default=100,
                    help='Number of epochs to train.')
parser.add_argument('--hidden', type=int, default=128,
                    help='Number of hidden units.')
parser.add_argument('--dropout', type=float, default=0.5,
                    help='Dropout rate (1 - keep probability).')
parser.add_argument('--epochs_gen', type=int, default=10,
                    help='Number of epochs to train for gen.')
parser.add_argument('--ratio_generated', type=float, default=1,
                    help='ratio of generated nodes.')
parser.add_argument('--dataset', choices=['cora', 'citeseer','pubmed', 'dblp', 'wiki'], default='cora')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()


np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

dataset = args.dataset
path = "../Dataset/" + dataset+"/"

if dataset=='wiki':
    num = 3
else:
    num = 10

# Specfic Parameters to get the best result
if dataset=='wiki':
    lr=0.001
elif dataset=='dblp':
    lr=0.0009
else:
    lr=0.01

if dataset == 'cora':
    weight_decay = 0.0008
elif dataset == 'citeseer':
    weight_decay = 0.0005
elif dataset == 'pubmed':
    weight_decay = 0.00008
elif dataset == 'dblp':
    weight_decay = 0.003
elif dataset == 'wiki':
    weight_decay = 0.0005



def train(features, adj):
    global max_recall, test_recall, test_f1, test_AUC, test_acc, test_pre
    model.train()
    optimizer.zero_grad()
    output, output_gen, output_AUC = model(features, adj)
    labels_true = torch.cat((torch.LongTensor(num_real).fill_(0), torch.LongTensor(num_false).fill_(1)))

    if args.cuda:
        labels_true=labels_true.cuda()

    loss_dis = - euclidean_dist(features[minority], features[majority]).mean()
    loss_train = F.nll_loss(output[idx_train], labels[idx_train]) \
                 + F.nll_loss(output_gen[idx_train], labels_true) \
                +loss_dis

    loss_train.backward()
    optimizer.step()


    if not args.fastmode:
        model.eval()
        output, output_gen, output_AUC = model(features, adj)


    recall_val, f1_val, AUC_val, acc_val, pre_val = accuracy(output[idx_val], labels[idx_val], output_AUC[idx_val])
    recall_train, f1_train, AUC_train, acc_train, pre_train = accuracy(output[idx_val], labels[idx_val], output_AUC[idx_val])

    if max_recall < (recall_val + acc_val)/2:
        output, output_gen, output_AUC = model(features, adj)
        recall_tmp, f1_tmp, AUC_tmp, acc_tmp, pre_tmp = accuracy(output[idx_test], labels[idx_test], output_AUC[idx_test])
        test_recall = recall_tmp
        test_f1 = f1_tmp
        test_AUC = AUC_tmp
        test_acc = acc_tmp
        test_pre = pre_tmp
        max_recall = (recall_val + acc_val)/2

    return recall_val, f1_val, acc_val, recall_train, f1_train, acc_train


def euclidean_dist(x, y):
    m, n = x.size(0), y.size(0)
    xx = torch.pow(x, 2).sum(1, keepdim=True).expand(m, n)
    yy = torch.pow(y, 2).sum(1, keepdim=True).expand(n, m).t()
    dist = xx + yy
    dist.addmm_(1, -2, x, y.t())
    dist = dist.clamp(min=1e-12).sqrt()  # for numerical stability
    return dist


# ratio_arr = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
# for ratio in ratio_arr:
adj, adj_real, features, labels, idx_temp, idx_test, generate_node, minority, majority, minority_all = load_data(args.ratio_generated, path=path, dataset=dataset)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
    nhid=args.hidden,
    nclass=labels.max().item() + 1,
    dropout=args.dropout,
    generate_node= generate_node,
    min_node = minority)
optimizer = optim.Adam(model.parameters(),lr=lr, weight_decay=weight_decay)

# num_real = features.shape[0]
num_false = labels.shape[0]- features.shape[0] #diff bw lengths of first row

print(num_false)
model_generator = Generator(minority_all.shape[0])
optimizer_G = torch.optim.Adam(model_generator.parameters(),
                       lr=lr, weight_decay=weight_decay)

max_recall = 0
test_recall = 0
test_f1 = 0
test_AUC = 0
test_acc=0
test_pre =0

if args.cuda:
    model.cuda()
    features = features.cuda()
    adj = adj.cuda()
    labels = labels.cuda()
    idx_temp = idx_temp.cuda()
    idx_test = idx_test.cuda()
    model_generator.cuda()


for epoch_gen in range(args.epochs_gen):
    part = epoch_gen % num
    range_val_maj = range(int(part*len(majority)/num), int((part+1)*len(majority)/num))
    range_val_min = range(int(part * len(minority) / num), int((part + 1) * len(minority) / num))

    range_train_maj = list(range(0,int(part*len(majority)/num)))+ list(range(int((part+1)*len(majority)/num),len(majority)))
    range_train_min = list(range(0,int(part*len(minority)/num)))+ list(range(int((part+1)*len(minority)/num),len(minority)))

    idx_val = torch.cat((majority[range_val_maj], minority[range_val_min]))
    idx_train = torch.cat((majority[range_train_maj], minority[range_train_min]))
    idx_train = torch.cat((idx_train, generate_node))
    num_real = features.shape[0] - len(idx_test) -len(idx_val)

    # Train model
    model_generator.train()
    optimizer_G.zero_grad()
    z = Variable(torch.FloatTensor(np.random.normal(0, 1, (generate_node.shape[0], 100))))
    if args.cuda:
        z=z.cuda()

    adj_min = model_generator(z)
    gen_imgs1 = torch.mm(F.softmax(adj_min[:,0:minority.shape[0]], dim=1), features[minority])
    gen_imgs1_all = torch.mm(F.softmax(adj_min, dim=1), features[minority_all])

    matr = F.softmax(adj_min[:,0:minority.shape[0]], dim =1).data.cpu().numpy()
    pos=np.where(matr>1/matr.shape[1])
    adj_temp = sp.coo_matrix((np.ones(pos[0].shape[0]),(generate_node[pos[0]].numpy(), minority_all[pos[1]].numpy())),
                             shape=(labels.shape[0], labels.shape[0]),
                             dtype=np.float32)

    adj_new = add_edges(adj_real, adj_temp)
    if args.cuda:
        adj_new=adj_new.cuda()

    t_total = time.time()
    # model.eval()
    output, output_gen, output_AUC = model(torch.cat((features, gen_imgs1.data),0), adj)

    labels_true = torch.LongTensor(num_false).fill_(0)
   
    labels_min = torch.LongTensor(num_false).fill_(1)
    if args.cuda:
        labels_true = labels_true.cuda()
        labels_min = labels_min.cuda()

    g_loss = F.nll_loss(output_gen[generate_node], labels_true) \
             + F.nll_loss(output[generate_node], labels_min) \
             + euclidean_dist(features[minority], gen_imgs1).mean()

    print(output[idx_train].size())
    print(labels[idx_train].size())
    print(output_gen[idx_train].size())
    print(labels_true.size()) 
    print(num_false)
    print(num_real)   
         
    g_loss.backward()
    optimizer_G.step()

    for epoch in range(args.epochs):
        recall_val, f1_val, acc_val, recall_train, f1_train, acc_train = train(torch.cat((features, gen_imgs1.data.detach()),0), adj_new)
    print("Epoch:", '%04d' % (epoch_gen + 1),
          "train_recall=", "{:.5f}".format(recall_train), "train_f1=", "{:.5f}".format(f1_train),"train_acc=", "{:.5f}".format(acc_train),
          "val_recall=", "{:.5f}".format(recall_val), "val_f1=", "{:.5f}".format(f1_val),"val_acc=", "{:.5f}".format(acc_val))



print("Test Recall: ", test_recall)
print("Test Accuracy: ", test_acc)
print("Test F1: ", test_f1)
print("Test precision: ", test_pre)
print("Test AUC: ", test_AUC)

I can see that the target_batch size = num_Real+num_false where :
features = sp.csr_matrix(idx_features_labels[:, 0:-1], dtype=np.float32) #reverse order of features cora ndarray
labels = idx_features_labels[:, -1]

Ps this is required for a group project which is due on 24th so I would be grateful if u could help out

111

Check which line of code is raising the shape mismatch as your current script calculates the loss in a few places. Once isolated, check if the input tensor matches the output tensor in its batch size. If not, then check the model implementation and in particular its forward method to narrow down where the batch size changes. On the other hand, if the input matches the output in the batch size, make sure the target also has the same batch size. If not, check how the target is created and why the batch size is different.

112

Hi, @ptrblck! I am kinda in the same situation. I am trying a very basic model:

class Conv2DModel(nn.Module):
    def __init__(self, n_input=1, n_output=35, stride=16, n_channel=32):
        super(Conv2DModel, self).__init__()
        self.conv1 = nn.Conv2d(n_input, n_channel, kernel_size=(1,80), stride=stride)
        self.conv2 = nn.Conv2d(n_channel, n_channel, kernel_size=(1,3))
        self.fc1 = nn.Linear(1976, n_channel)
        self.fc2 = nn.Linear(n_channel, n_output)

    def forward(self, x):
        x = self.conv1(x)
        print(x.shape)
        x = F.relu(x)
        print(x.shape)
        x = self.conv2(x)
        print(x.shape)
        x = F.relu(x)
        print(x.shape)
        x = F.max_pool2d(x, 2)
        x = torch.flatten(x, 1)
        print(x.shape)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output

The error:

ValueError                                Traceback (most recent call last)
Input In [174], in <cell line: 9>()
      9 with tqdm(total=n_epoch) as pbar:
     10     for epoch in range(1, n_epoch + 1):
---> 11         train(model, epoch, log_interval)
     12         test(model, epoch)
     13         scheduler.step()

Input In [173], in train(model, epoch, log_interval)
     15 output = model(data)
     17 # negative log-likelihood for a tensor of size (batch x 1 x n_output)
---> 18 loss = F.nll_loss(output.squeeze(), target)
     20 optimizer.zero_grad()
     21 loss.backward()

File ~/anaconda3/envs/user/lib/python3.8/site-packages/torch/nn/functional.py:2671, in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2669 if size_average is not None or reduce is not None:
   2670     reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2671 return torch._C._nn.nll_loss_nd(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (32) to match target batch_size (256).
113

I don’t know what your input shapes are so could you post a minimal, executable code snippet please?

114

input is torch.Size([1, 256, 8000]).

optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) 

def train(model, epoch, log_interval):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):

        data = data.to(device)
        target = target.to(device)
        # apply transform and model on whole batch directly on device
        data = transform(data)
        data = torch.transpose(data, 1, 0)
        print(data.shape)


        output = model(data)

        # negative log-likelihood for a tensor of size (batch x 1 x n_output)
        loss = F.nll_loss(output.squeeze(), target)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # print training stats
        if batch_idx % log_interval == 0:
            print(f"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}")

        # update progress bar
        pbar.update(pbar_update)
        # record loss
        losses.append(loss.item())

log_interval = 20
n_epoch = 2

pbar_update = 1 / (len(train_loader) + len(test_loader))
losses = []

# The transform needs to live on the same device as the model and the data.
transform = transform.to(device)
with tqdm(total=n_epoch) as pbar:
    for epoch in range(1, n_epoch + 1):
        train(model, epoch, log_interval)
        test(model, epoch)
        scheduler.step()
115

This doesn’t seem to be the case, as this input shape fails with:

model = Conv2DModel()
x = torch.randn(1, 256, 8000)
out = model(x)
> RuntimeError: Expected 4-dimensional input for 4-dimensional weight [32, 1, 1, 80], but got 3-dimensional input of size [1, 256, 8000] instead

which is expected, since you are using nn.Conv2d as the first layer while the input is 3-dimensional.
Adding the channel dimension as 1 fails with:

x = torch.randn(1, 1, 256, 8000)
out = model(x)
> RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x63232 and 1976x32)
116

I am working on a notebook, seems like I was loading another model with a similar name. My bad. In that case, I am getting the “mat1 and mat2” error now.
Seems like I have to set the 63232 to the fc1 like this: self.fc1 = nn.Linear(63232, n_channel).
How do I compute this 63232 without the need to having to set it manually?

117

Greetings!! Sorry, I feel like I’m asking something largely discussed, but can’t fix it on my own.
I get the error mat1 and mat2 shapes cannot be multiplied (16x441 and 7056x64)
This is my implementation:

class CNN(nn.Module):
	def __init__(self, history_length=0, n_classes=3):
        super(CNN, self).__init__()
        self.convo1 = nn.Conv2d(1, 6, kernel_size=5)
        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
        self.convo2 = nn.Conv2d(6, 16, kernel_size=5)
        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
        self.linear1 = nn.Linear(21 * 21 * 16, 64)
        self.linear2 = nn.Linear(64, n_classes)

    def forward(self, x):
        x = self.convo1(x)
        x = F.relu(x)
        x = self.pool1(x)
        x = self.convo2(x)
        x = F.relu(x)
        x = self.pool2(x)

        print(x.shape) # Returns torch.Size([16, 21, 21])
        x = x.view(x.size(0), -1)
		print(x.shape) # Returns torch.Size([16, 441])
        x = self.linear1(x)
        x = self.linear2(x)
        x = x.softmax(dim=1)

        return x

How can I fix it? Any help would be appreciated :slight_smile:

118

Set the in_features of self.linear to 441 and it should work:

self.linear1 = nn.Linear(441, 64)

since the incoming activation has a shape of [batch_size=16, features=441] as given in your code.

PS: I’m not familiar with your use case, but be careful about the usage of .softmax.
If you are working on a multi-class classification and are using nn.CrossEntropyLoss, remove the .softmax call as raw logits are expected.

1 Like
119

Just after posting, I noticed about the softmax issue. Changed the features and now it works! Thanks!

120

Hi @ptrblck, I have a problem with a transformer model with batch_size

from datasets import load_dataset, load_metric
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from torch.utils.data import DataLoader
from transformers import get_scheduler, AdamW
import torch
from tqdm.auto import tqdm
from collections import defaultdict
import torch.nn as nn

n_classes = 39

dataset = load_dataset('csv', data_files={'train': 'train.csv', 'val': 'val.csv', 'test': 'test.csv'})

tokenizer = AutoTokenizer.from_pretrained("sismetanin/rubert-ru-sentiment-rusentiment")

def tokenize_function(examples):
  max_length = len(max(data.text, key=len))
  return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512) # обрезаем все сообщения до 512 символов

tokenized_datasets = dataset.map(tokenize_function, batched=True)
tokenized_datasets = tokenized_datasets.remove_columns(["text"])
tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
tokenized_datasets.set_format("torch")

batch_size = 1

def my_collate(batch):
  data = defaultdict(list)
  {data[key].append(sub[key]) for sub in batch for key in sub}
  data = {key: torch.vstack(value) for key, value in data.items()}
  data['labels'] = torch.nn.functional.one_hot(data['labels'].to(torch.int64)-1, n_classes).view(-1)
  data.pop('text_len')
  return data

train_dataloader = DataLoader(tokenized_datasets["train"], shuffle=True, batch_size=batch_size, collate_fn=my_collate)
eval_dataloader = DataLoader(tokenized_datasets["test"], batch_size=batch_size, collate_fn=my_collate)
model = AutoModelForSequenceClassification.from_pretrained("sismetanin/rubert-ru-sentiment-rusentiment", num_labels=n_classes, ignore_mismatched_sizes=True)

model.classifier = nn.Sequential(nn.Linear(in_features=768, out_features=78),
                                 nn.Tanh(),
                                 nn.Linear(in_features=78, out_features=n_classes))

for param in list(model.bert.embeddings.parameters())[:-1]:
  param.requires_grad = False

optimizer = AdamW(model.parameters(), lr=5e-5)

num_epochs = 3
num_training_steps = num_epochs * len(train_dataloader)
lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps
)


device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)

model.train()
for epoch in tqdm(range(num_epochs)):
  for batch in tqdm(train_dataloader):
    outputs = model(**batch)
    loss = outputs.loss
    loss.backward()

    optimizer.step()
    lr_scheduler.step()
    optimizer.zero_grad()

the full error message:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-52-4f0f72f0c5ef> in <module>
     76 for epoch in tqdm(range(num_epochs)):
     77   for batch in tqdm(train_dataloader):
---> 78     outputs = model(**batch)
     79     loss = outputs.loss
     80     loss.backward()

4 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
   1589             elif self.config.problem_type == "single_label_classification":
   1590                 loss_fct = CrossEntropyLoss()
-> 1591                 loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
   1592             elif self.config.problem_type == "multi_label_classification":
   1593                 loss_fct = BCEWithLogitsLoss()

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)
   1164         return F.cross_entropy(input, target, weight=self.weight,
   1165                                ignore_index=self.ignore_index, reduction=self.reduction,
-> 1166                                label_smoothing=self.label_smoothing)
   1167 
   1168 

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
   3012     if size_average is not None or reduce is not None:
   3013         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 3014     return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
   3015 
   3016 

ValueError: Expected input batch_size (1) to match target batch_size (39).
121

Could you explain what batch contains and check the shapes?
The error is raised due to a mismatch in the shape of the model output and the target, but I can’t see how these shapes are defined and what might be causing the issue.
Often users are flattening an activation tensor in a wrong way and change the batch size by accident, ut I also don’t see the model’s forward method.

122

Hi, @ptrblck I have a problem related to the batch size in my fusion model. I followed your suggestion on flattening the feature vector before the linear layer, but I still get the same error.

I have a fusion model and it works fine when I set the batch size to 1, but when I change it to 2 or any other number, it gives this error:

ile ~\anaconda3\lib\site-packages\torch\nn\functional.py:2996 in cross_entropy
    return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)

ValueError: Expected input batch_size (4) to match target batch_size (2).

Here is my fusion model:



from typing import Union, List, Dict, Any, cast

import torch
import torch.nn as nn

from torch.utils.model_zoo import load_url as load_state_dict_from_url
from torchinfo import summary
from snn_classification import model_SNN
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"



################################# VGG model ##################################################

__all__ = [
    "VGG",   
    "vgg11_bn",

]


model_urls = {
    "vgg11": "https://download.pytorch.org/models/vgg11-8a719046.pth",
    "vgg16": "https://download.pytorch.org/models/vgg16-397923af.pth",
    "vgg19": "https://download.pytorch.org/models/vgg19-dcbb9e9d.pth",
    "vgg11_bn": "https://download.pytorch.org/models/vgg11_bn-6002323d.pth",

}


class VGG(nn.Module):
    def __init__(
        self, features: nn.Module, num_classes: int = 32, init_weights: bool = True, dropout: float = 0.5
    ) -> None:
        super().__init__()
        #_log_api_usage_once(self)
        self.features = features
        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))
        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 1024),
            nn.ReLU(True),
            nn.Dropout(p=dropout),
            nn.Linear(1024, 512),
            nn.ReLU(True),
            nn.Dropout(p=dropout),
            nn.Linear(512, num_classes),
        )
        if init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
                    if m.bias is not None:
                        nn.init.constant_(m.bias, 0)
                elif isinstance(m, nn.BatchNorm2d):
                    nn.init.constant_(m.weight, 1)
                    nn.init.constant_(m.bias, 0)
                elif isinstance(m, nn.Linear):
                    nn.init.normal_(m.weight, 0, 0.01)
                    nn.init.constant_(m.bias, 0)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.features(x)
        x = self.avgpool(x)
        x = torch.flatten(x, 1) 
        #x = x.view(x.size(0), -1)
        #print('the feature aftr flatten',x.shape)
        x = self.classifier(x)
        #print('the classifier ',x.shape)

        return x


def make_layers(cfg: List[Union[str, int]], batch_norm: bool = False) -> nn.Sequential:
    layers: List[nn.Module] = []
    in_channels = 3
    for v in cfg:
        if v == "M":
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            v = cast(int, v)
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    return nn.Sequential(*layers)


cfgs: Dict[str, List[Union[str, int]]] = {
    "A": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
    "B": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
    "D": [64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512, "M", 512, 512, 512, "M"],
    "E": [64, 64, "M", 128, 128, "M", 256, 256, 256, 256, "M", 512, 512, 512, 512, "M", 512, 512, 512, 512, "M"],
}


def _vgg(arch: str, cfg: str, batch_norm: bool, pretrained: bool, progress: bool, **kwargs: Any) -> VGG:
    if pretrained:
        kwargs["init_weights"] = False
    model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs)
    if pretrained:
        state_dict = load_state_dict_from_url(model_urls[arch], progress=progress)
        model.load_state_dict(state_dict)
    return model




def vgg11_bn(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> VGG:
    r"""VGG 11-layer model (configuration "A") with batch normalization
    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_.
    The required minimum input size of the model is 32x32.

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
        progress (bool): If True, displays a progress bar of the download to stderr
    """
    return _vgg("vgg11_bn", "A", True, pretrained, progress, **kwargs)






##################### MLP model#####################################

class MulticlassClassification(nn.Module):
    def __init__(self, num_class=3):
        super(MulticlassClassification, self).__init__()
        
        self.layer_1 = nn.Linear(81, 80)
        self.layer_2 = nn.Linear(80, 30)
        self.layer_3 = nn.Linear(30, 30)      
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(p=0.1)

    def forward(self, x):
        x = self.layer_1(x)
        x = self.relu(x)
        
        x = self.layer_2(x)
        x = self.relu(x)
        x = self.dropout(x)
        
        x = self.layer_3(x)
        x = self.relu(x)
        x = self.dropout(x)
        
        
        return x
    
def model_gens() -> MulticlassClassification:
    model = MulticlassClassification()
    return model
################################################################


net_CNN = vgg11_bn()
net_MLP = model_gens()


############################## fusion model #######################


class MyEnsemble(nn.Module):
    def __init__(self, nb_classes=3):
        super(MyEnsemble, self).__init__()
        self.model_image =  net_CNN
        self.model_EHR = net_MLP     

        # Create new classifier
        self.layer_out = nn.Linear(960, nb_classes)

    
    def forward(self, x1,x3):
        x1 = self.model_image(x1)       
        
		x3 = self.model_EHR(x3)
        x3 = x3.view(x3.size(0), -1) 

        x = torch.kron(x1,x3)
        x = self.layer_out(x)

        return x
    
    
def model_snn_vgg() -> MyEnsemble:
    model = MyEnsemble()
    return model



# model = model_snn_vgg()

# model.to(device=DEVICE,dtype=torch.float)
# print(summary(model,[(1,3, 224, 224),(1,81)]))

# print(model)
123

torch.kron(x1, x3) is changing the batch size as described in the docs.

124

Oh no :frowning: ,that’s correct, it changes the batch size when I print the shape of the kronker product.

Thanks a lot for your answer. Do you think there is a way to keep the batch size and maintain the nature of the krnoker product operation?

125

import cv2
import imghdr
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from sklearn.preprocessing import LabelEncoder
from keras.initializers import RandomNormal, GlorotUniform

Set the path to the ISIC dataset

dataset_path = ‘data/train’

Load the images and labels

image_paths = []
labels = []
label_df = pd.read_csv(‘data/ISIC_2019_Training_GroundTruth.csv’)
for label in os.listdir(dataset_path):
label_path = os.path.join(dataset_path, label)
for image_name in os.listdir(label_path):
if imghdr.what(os.path.join(label_path, image_name)) is not None:
image_paths.append(os.path.join(label_path, image_name))
labels.append(label_df[label_df.image == image_name])

Convert the labels to integers using LabelEncoder

le = LabelEncoder()
labels = le.fit_transform(labels)

Split the data into train and test sets

X_train, X_test, y_train, y_test = train_test_split(image_paths, labels, test_size=0.2)

Create a data generator for data augmentation

datagen = ImageDataGenerator(rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, zoom_range=0.2, horizontal_flip=True)

Read and preprocess the images

X_train_array = []
for img_path in X_train:
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (128, 128))
X_train_array.append(img)

X_test_array = []
for img_path in X_test:
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (128, 128))
X_test_array.append(img)

Convert the data to numpy arrays and normalize it

X_train = np.array(X_train_array) / 255.0
X_test = np.array(X_test_array) / 255.0

ValueError Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_10236\3795273597.py in
27 # Convert the labels to integers using LabelEncoder
28 le = LabelEncoder()
—> 29 labels = le.fit_transform(labels)
30
31 # Split the data into train and test sets

~\anaconda3\lib\site-packages\sklearn\preprocessing_label.py in fit_transform(self, y)
113 Encoded labels.
114 “”"
→ 115 y = column_or_1d(y, warn=True)
116 self.classes_, y = _unique(y, return_inverse=True)
117 return y

~\anaconda3\lib\site-packages\sklearn\utils\validation.py in column_or_1d(y, warn)
1036 return np.ravel(y)
1037
→ 1038 raise ValueError(
1039 “y should be a 1d array, got an array of shape {} instead.”.format(shape)
1040 )

ValueError: y should be a 1d array, got an array of shape (2239, 0, 10) instead.

126

hi
I have the same problem but when i use print(x.shape) , code shows nothing!

127

This shouldn’t be the case so could you post a minimal and executable code snippet to reproduce the issue?

128

Screenshot (9)
Screenshot (9)1907×1008 147 KB

129

Screenshot (10)
Screenshot (10)1907×1007 158 KB