I am working on a react-native project that was bootstrapped through expo-cli. It is a very basic app that only has a camera feature for now. I have my own trained PyTorch model on some dataset and my goal is to test the model on-device through the app. I followed the steps here to add the react-native-pytorch-core
library and also implemented these steps https://pytorch.org/live/docs/tutorials/prepare-custom-model/ to bundle the model with the app. However, when trying to run the model on the captured photo imported, I am always getting null is not an object
. I have my model under assets/models
which should bundle with the app. I guess I am missing something but I couldn’t figure it out since the documentation is scarce. I have attached the code below and a screenshot of the stacktrace.
This is the TakePhoto component:
import { StatusBar } from 'expo-status-bar';
import { SafeAreaProvider } from 'react-native-safe-area-context';
import { Camera } from 'expo-camera';
import React, { useState, useEffect, useRef } from 'react';
import { Text, View, TouchableOpacity, StyleSheet, Image, Dimensions, Button } from 'react-native';
import { MobileModel, ModelResult, Image as img, ImageUtil } from 'react-native-pytorch-core';
const model = require('../assets/models/model_best_resnet.ptl');
type ImageClassificationResult = {
maxIdx: number;
confidence: number;
};
export default function TakePhoto(props) {
const [hasPermission, setHasPermission] = useState(null);
const [cameraRef, setCameraRef] = useState(null);
const [type, setType] = useState(Camera.Constants.Type.back);
const [photo, setPhoto] = useState(null);
useEffect(() => {
(async () => {
const { status } = await Camera.requestCameraPermissionsAsync();
setHasPermission(status === 'granted');
})();
}, []);
async function classifyImage() {
try {
const {metrics, result} = await MobileModel.execute<ImageClassificationResult>(model, {photo,});
console.log(metrics);
console.log(result.confidence);
} catch(exception) {
console.log(exception);
}
}
if (hasPermission === null) {
return <View />;
}
if (hasPermission === false) {
return <Text>No access to camera</Text>;
}
return (
<View style={{ flex: 1 }}>
{ (photo == null) ?
<Camera style={{ flex: 1 }} type={type} ref={ref => {
setCameraRef(ref);
}}>
<View
style={styles.pictureView}>
<TouchableOpacity
style={styles.textCommand}
onPress={() => {
setType(
type === Camera.Constants.Type.back
? Camera.Constants.Type.front
: Camera.Constants.Type.back
);
}}>
<Text style={{ fontSize: 18, marginBottom: 10, color: 'white' }}> Flip </Text>
</TouchableOpacity>
<TouchableOpacity style={styles.textCommand} onPress={props.goBackHandler}>
<Text style={{ fontSize: 18, marginBottom: 10, color: 'white' }}> Close </Text>
</TouchableOpacity>
<TouchableOpacity style={{alignSelf: 'center'}} onPress={async() => {
if(cameraRef){
let p = await cameraRef.takePictureAsync();
setPhoto(p);
}
}}>
<View style={styles.snapButtonOut}>
<View style={styles.snapButtonIn} >
</View>
</View>
</TouchableOpacity>
</View>
</Camera>
: null }
{
(photo != null) ?
<View style={styles.photo}>
<Image style={{flex:1, width: '100%', height: '100%'}} resizeMode={'contain'} source={photo}></Image>
<Button onPress={() => setPhoto(null)} title="Retake"></Button>
<Button onPress={classifyImage} title="Submit"></Button>
</View> : null
}
</View>
);
}
const styles = StyleSheet.create({
textCommand: {
alignSelf: 'flex-end'
},
pictureView: {
flex: 1,
backgroundColor: 'transparent',
justifyContent: 'flex-end'
},
snapButtonOut: {
marginBottom: 20,
borderWidth: 2,
borderRadius:50,
borderColor: 'white',
height: 50,
width:50,
display: 'flex',
justifyContent: 'center',
alignItems: 'center'
},
snapButtonIn: {
borderWidth: 2,
borderRadius:50,
borderColor: 'white',
height: 40,
width:40,
backgroundColor: 'white'
},
photo: {
flex: 1,
borderRadius: 50,
}
});