Pytorch network quntization aware training problem

Hi:

I am trying to use quantization aware training for my CNN network. I used the lastest 1.90 pytorch nightly version. What I want to do is I load a pretrained RestNet18 and finetune it with other dataset. I run into some problem to quantitize the network during training.

I followed the tutorial .

However, I run into an NotImplementedError: “Could not run ‘aten::thnn_conv2d_forward’ with arguments from the ‘QuantizedCPU’ backend.” when I want to use the converted int8 model for testing. In addition, I found the saved converted int8 model is not quantitized properly, it is still represented as floating points number.

Could you help me understand this error?

Many thanks
Shixian Wen

Error meesage

Traceback (most recent call last):
File “attetion_delta_quantization_tutorial.py”, line 902, in
acc = test_cpu(testLoader_mix10,model_one_int8)
File “attetion_delta_quantization_tutorial.py”, line 708, in test_cpu
output = model(data)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 1015, in _call_impl
return forward_call(*input, **kwargs)
File “attetion_delta_quantization_tutorial.py”, line 552, in forward
x = self.model_fe_features(x)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 1015, in _call_impl
return forward_call(*input, **kwargs)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/container.py”, line 139, in forward
input = module(input)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 1015, in _call_impl
return forward_call(*input, **kwargs)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/container.py”, line 139, in forward
input = module(input)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/module.py”, line 1015, in _call_impl
return forward_call(*input, **kwargs)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/conv.py”, line 433, in forward
return self._conv_forward(input, self.weight, self.bias)
File “/home/shixian/anaconda3/envs/pytorchtest/lib/python3.6/site-packages/torch/nn/modules/conv.py”, line 430, in _conv_forward
self.padding, self.dilation, self.groups)
NotImplementedError: Could not run ‘aten::thnn_conv2d_forward’ with arguments from the ‘QuantizedCPU’ backend. This could be because the operator doesn’t exist for this backend, or was omitted during the selective/custom build process (if using custom build). If you are a Facebook employee using PyTorch on mobile, please visit https://fburl.com/ptmfixes for possible resolutions. ‘aten::thnn_conv2d_forward’ is only available for these backends: [CPU, CUDA, BackendSelect, Named, InplaceOrView, AutogradOther, AutogradCPU, AutogradCUDA, AutogradXLA, UNKNOWN_TENSOR_TYPE_ID, AutogradMLC, AutogradNestedTensor, AutogradPrivateUse1, AutogradPrivateUse2, AutogradPrivateUse3, Tracer, Autocast, Batched, VmapMode].

CPU: registered at /pytorch/build/aten/src/ATen/RegisterCPU.cpp:11727 [kernel]
CUDA: registered at /pytorch/build/aten/src/ATen/RegisterCUDA.cpp:13375 [kernel]
BackendSelect: fallthrough registered at /pytorch/aten/src/ATen/core/BackendSelectFallbackKernel.cpp:3 [backend fallback]
Named: registered at /pytorch/aten/src/ATen/core/NamedRegistrations.cpp:7 [backend fallback]
InplaceOrView: fallthrough registered at /pytorch/aten/src/ATen/core/VariableFallbackKernel.cpp:60 [backend fallback]
AutogradOther: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradCPU: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradCUDA: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradXLA: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
UNKNOWN_TENSOR_TYPE_ID: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradMLC: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradNestedTensor: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradPrivateUse1: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradPrivateUse2: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
AutogradPrivateUse3: registered at /pytorch/torch/csrc/autograd/generated/VariableType_0.cpp:9539 [autograd kernel]
Tracer: registered at /pytorch/torch/csrc/autograd/generated/TraceType_0.cpp:9373 [kernel]
Autocast: fallthrough registered at /pytorch/aten/src/ATen/autocast_mode.cpp:250 [backend fallback]
Batched: registered at /pytorch/aten/src/ATen/BatchingRegistrations.cpp:1016 [backend fallback]
VmapMode: fallthrough registered at /pytorch/aten/src/ATen/VmapModeRegistrations.cpp:33 [backend fallback]

Here is my model:

model_fe = models.resnet18(pretrained=True, progress=True, quantize=False)
num_ftrs = model_fe.fc.in_features
model_fe.train()
model_fe.fuse_model()

class Resnet18_ONE(nn.Module):

def __init__(self,model_fe=model_fe):

    super(Resnet18_ONE,self).__init__()
    #self.loss = loss
    self.model_fe_features = nn.Sequential(model_fe.quant,model_fe.conv1,model_fe.bn1,model_fe.relu,model_fe.maxpool,model_fe.layer1,model_fe.layer2,model_fe.layer3,model_fe.layer4,model_fe.avgpool,model_fe.dequant)
    #print(self.base)
    self.linear_sub = nn.Linear(num_ftrs, superclasses)
    self.linear_bird = nn.Linear(num_ftrs, classes_bird)
    self.linear_boat = nn.Linear(num_ftrs, classes_boat)
    self.linear_car = nn.Linear(num_ftrs, classes_car)
    self.linear_cat = nn.Linear(num_ftrs, classes_cat)
    self.linear_fungus = nn.Linear(num_ftrs, classes_fungus)
    self.linear_insect = nn.Linear(num_ftrs, classes_insect)
    self.linear_monkey = nn.Linear(num_ftrs, classes_monkey)
    self.linear_truck = nn.Linear(num_ftrs, classes_truck)
    self.linear_dog = nn.Linear(num_ftrs, classes_dog)
    self.linear_fruit = nn.Linear(num_ftrs, classes_fruit)
    
def forward(self,x):

    x = self.model_fe_features(x)
    x = torch.flatten(x, 1)
    if task == 'SUB':
        #print("I am in sub")
        x = self.linear_sub(x)
    elif task == 'BIRD':
        #print("I am in bird")
        x = self.linear_bird(x)
    elif task == 'BOAT':
        x = self.linear_boat(x)
    elif task == 'CAR':
        x = self.linear_car(x)
    elif task == 'CAT':
        x = self.linear_cat(x)
    elif task == 'FUNGUS':
        x = self.linear_fungus(x)
    elif task == 'INSECT':
        x = self.linear_insect(x)
    elif task == 'MONKEY':
        x = self.linear_monkey(x)
    elif task == 'TRUCK':
        x = self.linear_truck(x)
    elif task == 'DOG':
        x = self.linear_dog(x)
    else:
        #print("enter fruit")
        x = self.linear_fruit(x)
               
    return x

Here is how I set and train the quantization model:

model_one = Resnet18_ONE(model_fe=model_fe)
model_one.qconfig = torch.quantization.get_default_qat_qconfig(‘fbgemm’)
model_one_prepared = torch.quantization.prepare(model_one,inplace=True)
optimizer_sub = optim.SGD(filter(lambda p: p.requires_grad, model_one_prepared.parameters()), lr=args.lr, momentum=args.momentum)
train(epoch,trainLoader_mix10,optimizer_sub,model_one_prepared)

After training the model, Here is how I save and covert the model to int8:

model_one_prepared.to(‘cpu’)
model_one_int8 = torch.quantization.convert(model_one_prepared.eval(),inplace=False)
model_one_prepared.eval()
model_one_int8.eval()
acc = test_cpu(testLoader_mix10,model_one_int8)
torch.save({‘modelONE_state_dict’: model_one_int8.state_dict(),}, save_path_quantize)