I want to do Federated learning Quantization using PyTorch. I followed some tutorial for centralized learning and tried to apply torch.quantization.quantize_fx to federated learning, but I got this error:
ValueError: optimizer got an empty parameter list
Here is the code:
import torch.quantization.quantize_fx as quantize_fx
import copy
quantized_models_all=[ ] # quantize all clients models
for i in range(3):
model= client_models[i]
model.train()
model_to_quantize = copy.deepcopy(model)
qconfig_dict = {"": torch.quantization.get_default_qat_qconfig(‘qnnpack’)}
model_prepared = quantize_fx.prepare_qat_fx(model_to_quantize, qconfig_dict)
model_quantized = quantize_fx.convert_fx(model_prepared)
quantized_models_all.append(model_quantized)
quantize global model
model_fp = global_model
quantization aware training for static quantization
model_to_quantize = copy.deepcopy(model_fp)
qconfig_dict = {"": torch.quantization.get_default_qat_qconfig(‘qnnpack’)}
model_to_quantize.train()
prepare
model_prepared = quantize_fx.prepare_qat_fx(model_to_quantize, qconfig_dict)
quantize
gmodel_quantized = quantize_fx.convert_fx(model_prepared)
def server_aggregate(gmodel_quantized, quantized_models_all):
global_dict = gmodel_quantized.state_dict()
for k in global_dict.keys():
global_dict[k] = torch.stack([quantized_models_all[i].state_dict()[k].float() for i in range(len(quantized_models_all))], 0).mean(0)
gmodel_quantized.load_state_dict(global_dict)
for model in quantized_models_all:
model.load_state_dict(gmodel_quantized.state_dict())
def test(gmodel_quantized, test_loader):
model.eval()
test_loss = 0
correct = 0
with torch.no_grad():
for data, target in test_loader:
data, target = data.cuda(), target.cuda()
output = gmodel_quantized(data)
test_loss += F.nll_loss(output, target, reduction=‘sum’).item()
pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
test_loss /= len(test_loader.dataset)
acc = correct / len(test_loader.dataset)
return test_loss, acc
train the model
global model
gmodel_quantized = gmodel_quantized.cuda()
client models
for i in range(3):
cmodel= quantized_models_all[i]
cmodel.cuda()
quantized_models_all = [cmodel.cuda() for _ in range(num_selected)]
for model in quantized_models_all:
model.load_state_dict(gmodel_quantized.state_dict()) # initial synchronizing with global model
opt = [optim.SGD(model.parameters(), lr=0.1) for model in quantized_models_all]
ValueError Traceback (most recent call last)
in () 14 model.load_state_dict(gmodel_quantized.state_dict()) # initial synchronizing with global model 15 —> 16 opt = [optim.SGD(model.parameters(), lr=0.1) for model in quantized_models_all]
2 frames
/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py in init(self, params, defaults) 47 param_groups = list(params) 48 if len(param_groups) == 0: —> 49 raise ValueError(“optimizer got an empty parameter list”) 50 if not isinstance(param_groups[0], dict): 51 param_groups = [{‘params’: param_groups}]
ValueError: optimizer got an empty parameter list
I want to quantize models when clients send their model to the central server, and I want to print all loos and accuracy values for each rounds.
I’m using CIFAR100 dataset, and VGG19 model for this training.
This the model architecture:
cfg = {
‘VGG11’: [64, ‘M’, 128, ‘M’, 256, 256, ‘M’, 512, 512, ‘M’, 512, 512, ‘M’],
‘VGG13’: [64, 64, ‘M’, 128, 128, ‘M’, 256, 256, ‘M’, 512, 512, ‘M’, 512, 512, ‘M’],
‘VGG16’: [64, 64, ‘M’, 128, 128, ‘M’, 256, 256, 256, ‘M’, 512, 512, 512, ‘M’, 512, 512, 512, ‘M’],
‘VGG19’: [64, 64, ‘M’, 128, 128, ‘M’, 256, 256, 256, 256, ‘M’, 512, 512, 512, 512, ‘M’, 512, 512, 512, 512, ‘M’],
}
class VGG(nn.Module):
def init(self, vgg_name):
super(VGG, self).init()
self.features = self._make_layers(cfg[vgg_name])
self.classifier = nn.Sequential(
nn.Linear(512, 512),
nn.ReLU(True),
nn.Linear(512, 512),
nn.ReLU(True),
nn.Linear(512, 100)
)
def forward(self, x):
out = self.features(x)
out = out.view(out.size(0), -1)
out = self.classifier(out)
output = F.log_softmax(out, dim=1)
return output
def _make_layers(self, cfg):
layers = []
in_channels = 3
for x in cfg:
if x == ‘M’:
layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
else:
layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1),
nn.BatchNorm2d(x),
nn.ReLU(inplace=True)]
in_channels = x
layers += [nn.AvgPool2d(kernel_size=1, stride=1)]
return nn.Sequential(*layers)
Appreciate any help!
NOTE: there is no problem with the model when train it without quantization!