I’m trying to use the new torch.backends._nnapi.prepare.convert_model_to_nnapi() function to prepare a PyTorch CNN model for NNAPI on an Android device. I’m basically following this tutorial. Here’s my code:
#Prepare the trained model for NNAPI on Android device
import sys
import os
import torch
import torch.utils.bundled_inputs
import torch.utils.mobile_optimizer
import torch.backends._nnapi.prepare
import torchvision.models.quantization.mobilenet
from pathlib import Path
#import a function from a different module that returns the model in eval mode, loading the state dict from a checkpoint file (pretrained)
from common.model import trace_model
# This script supports 3 modes of quantization:
# - "none": Fully floating-point model.
# - "core": Quantize the core of the model, but wrap it a
# quantizer/dequantizer pair, so the interface uses floating point.
# - "full": Quantize the model, and use quantized tensors
# for input and output.
#
# "none" maintains maximum accuracy
# "core" sacrifices some accuracy for performance,
# but maintains the same interface.
# "full" maximized performance (with the same accuracy as "core"),
# but requires the application to use quantized tensors.
#
# There is a fourth option, not supported by this script,
# where we include the quant/dequant steps as NNAPI operators.
def make_mymodel_nnapi(output_dir_path, quantize_mode):
quantize_core, quantize_iface = {
"none": (False, False),
"core": (True, False),
"full": (True, True),
}[quantize_mode]
#get the model. Ignore layers
model, layers = trace_model()
#Commented this section out because model I'm using doesn't have fuse_model() fxn.
# Fuse BatchNorm operators in the floating point model.
# (Quantized models already have this done.)
# Remove dropout for this inference-only use case.
#if not quantize_core:
# model.fuse_model()
#assert type(model.classifier[0]) == torch.nn.Dropout
#model.classifier[0] = torch.nn.Identity()
#create some input based on my CNN's shape
input_float = torch.zeros(2, 272, 17, 2)
input_tensor = input_float
# If we're doing a quantized model, we need to trace only the quantized core.
# So capture the quantizer and dequantizer, use them to prepare the input,
# and replace them with identity modules so we can trace without them.
if quantize_core:
quantizer = model.quant
dequantizer = model.dequant
model.quant = torch.nn.Identity()
model.dequant = torch.nn.Identity()
input_tensor = quantizer(input_float)
# Many NNAPI backends prefer NHWC tensors, so convert our input to channels_last,
# and set the "nnapi_nhwc" attribute for the converter.
input_tensor = input_tensor.contiguous(memory_format=torch.channels_last)
input_tensor.nnapi_nhwc = True
# Trace the model. NNAPI conversion only works with TorchScript models,
# and traced models are more likely to convert successfully than scripted.
with torch.no_grad():
traced = torch.jit.trace(model, input_tensor)
#ERROR IN THIS FUNCTION
#Prepare model for NNAPI
nnapi_model = torch.backends._nnapi.prepare.convert_model_to_nnapi(traced, input_tensor)
if __name__ == "__main__":
for quantize_mode in ["none", "core", "full"]:
make_mymodel_nnapi(Path(os.environ["HOME"]) / "mobilenetv2-nnapi", quantize_mode)
What I get is an error in the convert_model_to_nnapi() function:
Traceback (most recent call last):
File "trace_model_nnapi.py", line 131, in <module>
make_mymodel_nnapi(Path(os.environ["HOME"]) / "mobilenetv2-nnapi", quantize_mode)
File "trace_model_nnapi.py", line 89, in make_mymodel_nnapi
nnapi_model = torch.backends._nnapi.prepare.convert_model_to_nnapi(traced, input_tensor)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/prepare.py", line 169, in convert_model_to_nnapi
ser_model, used_weights, inp_mem_fmts, out_mem_fmts = serialize_model(model, inputs)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 1363, in serialize_model
return _NnapiSerializer(config).serialize_model(module, inputs)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 520, in serialize_model
self.add_node(node)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 641, in add_node
raise Exception("Unsupported node kind (%r) in node %r" % (node.kind(), node))
Exception: Unsupported node kind ('aten::contiguous') in node %x.2 : Tensor = aten::contiguous(%x.1, %39) # /home/nodog/docs/asp/mymodel/common/model.py:267:0
And if I comment out the “Many NNAPI backends prefer NHWC tensors” part of the code:
Traceback (most recent call last):
File "trace_model_nnapi.py", line 131, in <module>
make_mymodel_nnapi(Path(os.environ["HOME"]) / "mobilenetv2-nnapi", quantize_mode)
File "trace_model_nnapi.py", line 89, in make_mynmodel_nnapi
nnapi_model = torch.backends._nnapi.prepare.convert_model_to_nnapi(traced, input_tensor)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/prepare.py", line 169, in convert_model_to_nnapi
ser_model, used_weights, inp_mem_fmts, out_mem_fmts = serialize_model(model, inputs)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 1363, in serialize_model
return _NnapiSerializer(config).serialize_model(module, inputs)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 520, in serialize_model
self.add_node(node)
File "/home/nodog/.local/lib/python3.8/site-packages/torch/backends/_nnapi/serializer.py", line 641, in add_node
raise Exception("Unsupported node kind (%r) in node %r" % (node.kind(), node))
Exception: Unsupported node kind ('aten::size') in node %40 : int = aten::size(%x.1, %39) # /home/nodog/docs/asp/mymodel/common/model.py:269:0
It appears that ‘aten::size’ and ‘aten::contiguous’ are not included in ADDER_MAP (what looks to be a list of node types) in serializer.py. Does anyone know if I’m doing something wrong, or if some functionality just hasn’t been added to the NNAPI system yet?
By the way, the CNN model I use has Conv1d, Batchnorm, Dropout, and Relu layers.
Thanks!