Hello all,
I am fairly new to libtorch and trying to recreate a model that I had once written in python. It consists of multiple blocks, something like that of a resnet.
Following an example, I create a custom class and register each of its members. However, I am not able to register my custom Block. My code is as follows -
#include "PredictFrame.h"
#include <cmath>
using namespace std;
int calcInterim(int in_channels, int out_channels, ExpandingArray<3UL> kernel){
return int(floor((kernel->at(0)*kernel->at(1)*kernel->at(2)*in_channels*out_channels)/
((kernel->at(1)*kernel->at(2)*in_channels)+(kernel->at(0)*out_channels))));
};
Block::Block(int in_channels, int out_channels, ExpandingArray<3UL> kernel,
ExpandingArray<3UL> stride, ExpandingArray<3UL> padding):
spatialConv(torch::nn::Conv3dOptions(in_channels, calcInterim(in_channels, out_channels, kernel), {1, kernel->at(1), kernel->at(2)})
.stride({1, stride->at(1), stride->at(2)}).padding({0, padding->at(1), padding->at(2)})),
temporalConv(torch::nn::Conv3dOptions(calcInterim(in_channels, out_channels, kernel), out_channels, {kernel->at(0),1,1})
.stride({stride->at(0), 1,1}).padding({padding->at(0),1,1})),
norm3D(torch::nn::BatchNorm3dOptions(calcInterim(in_channels, out_channels, kernel))){
register_module("block_spatial_conv", spatialConv);
register_module("block_temporal_conv", temporalConv);
register_module("block_norm", norm3D);
}
torch::Tensor Block::forward(const torch::Tensor& input) {
auto output = spatialConv(input);
output = norm3D(output);
output = torch::nn::functional::relu(output);
output = temporalConv(output);
return output;
}
ResidualBlock::ResidualBlock(int in_channels, int out_channels, ExpandingArray<3UL> kernel, ExpandingArray<3UL> stride,
ExpandingArray<3UL> padding):
conv1(in_channels, out_channels, kernel, stride, padding),
conv2(out_channels, out_channels, kernel, stride, padding),
norm3D(out_channels){
register_module<Block>("conv1_res", torch::nn::ModuleHolder<Block>(make_shared<Block>(conv1)));
register_module<Block>("conv2_res", torch::nn::ModuleHolder<Block>(make_shared<Block>(conv2)));
register_module("norm3D_res", norm3D);
}
torch::Tensor ResidualBlock::forward(const torch::Tensor& input) {
auto out= conv1.forward(input);
out = norm3D(out);
auto out1 = torch::nn::functional::relu(out);
out = conv2.forward(out);
out = norm3D(out);
return (out1+out);
}
R2plus1D::R2plus1D(int in_channels, int out_channels, ExpandingArray<3UL> kernel,
ExpandingArray<3UL> stride, ExpandingArray<3UL> padding, int depth):
layer1(in_channels, out_channels, kernel, stride, padding) {
register_module<ResidualBlock>("resBlock1", torch::nn::ModuleHolder<ResidualBlock>(make_shared<ResidualBlock>(layer1)));
for (int i = 0; i < depth; ++i)
moduleList->push_back(ResidualBlock(out_channels, out_channels, kernel, stride, padding));
layers(moduleList);
}
torch::Tensor R2plus1D::forward(const Tensor &input) {
auto out = layer1.forward(input);
out = layers->forward(out);
return out;
}
PredictFrame::PredictFrame():
conv1(3, 64, ExpandingArray<3ul>{3, 5, 5}, ExpandingArray<3UL>{1, 2, 2}, ExpandingArray<3UL>{1, 3, 3})
{
register_module("PredictFrame_block", torch::nn::ModuleHolder<Block>(make_shared<Block>(conv1)));
moduleList->push_back(R2plus1D(64, 64, {3,3,3}, {1,1,1}, {1,1,1}, 1));
moduleList->push_back(R2plus1D(64, 128, {3,3,3}, {1,1,1}, {1,1,1}, 1));
moduleList->push_back(R2plus1D(128, 256, {3, 3, 3}, {1, 1, 1}, {1, 1, 1}, 1));
moduleList->push_back(R2plus1D(256, 512, {3, 3, 3}, {1, 1, 1}, {1, 1, 1}, 1));
sequential(moduleList);
}
torch::Tensor PredictFrame::forward(const torch::Tensor& input) {
auto out = conv1.forward(input);
return sequential->forward(out);
}
The header file is as follows -
//
// Created by atharva on 02/10/20.
//
#ifndef SUPERTUX_RL_PREDICTFRAME_H
#define SUPERTUX_RL_PREDICTFRAME_H
#include <torch/torch.h>
#include <cmath>
using namespace torch;
using namespace std;
class Block : public torch::nn::Module{
public:
Block(int in_channels, int out_channels, ExpandingArray<3UL> kernel,
ExpandingArray<3UL> stride, ExpandingArray<3UL> padding);
torch::Tensor forward(const torch::Tensor& input);
private:
torch::nn::Conv3d spatialConv{nullptr};
torch::nn::Conv3d temporalConv{nullptr};
torch::nn::BatchNorm3d norm3D{nullptr};
};
class ResidualBlock : public torch::nn::Module{
public:
ResidualBlock(int in_channels, int out_channels, ExpandingArray<3UL> kernel, ExpandingArray<3UL> stride,
ExpandingArray<3UL> padding);
torch::Tensor forward(const torch::Tensor& input);
private:
Block conv1;
Block conv2;
torch::nn::BatchNorm3d norm3D{nullptr};
};
class R2plus1D : public torch::nn::Module{
public:
R2plus1D(int in_channels, int out_channels, ExpandingArray<3UL> kernel, ExpandingArray<3UL> stride, ExpandingArray<3UL> padding,
int depth);
torch::Tensor forward(const torch::Tensor& input);
private:
ResidualBlock layer1;
torch::nn::ModuleList moduleList;
torch::nn::Sequential layers;
};
class PredictFrame : public torch::nn::Module{
public:
PredictFrame();
torch::Tensor forward(const torch::Tensor& input);
private:
Block conv1;
torch::nn::ModuleList moduleList;
torch::nn::Sequential sequential;
};
#endif //SUPERTUX_RL_PREDICTFRAME_H
As in the code above, I have used torch::nn::ModuleHolder(make_shared<Block>(conv1)) to not get an error. But I am not understanding what I am doing over here, and am I on the correct path. Any help would be appreciated.
TIA
Edit - It does not work, on compilation I get the following error -
[ 33%] Building CXX object CMakeFiles/SuperTux_RL.dir/main.cpp.o
[ 66%] Building CXX object CMakeFiles/SuperTux_RL.dir/PredictFrame.cpp.o
In file included from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/modules/container/any_value.h:5,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/modules/container/any_module_holder.h:3,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/module.h:3,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/cloneable.h:3,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn.h:3,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/all.h:13,
from /home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/torch.h:3,
from /home/atharva/CLionProjects/SuperTux_RL/PredictFrame.h:8,
from /home/atharva/CLionProjects/SuperTux_RL/PredictFrame.cpp:5:
/home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/pimpl.h: In instantiation of ‘torch::detail::return_type_of_forward_t<Contained, Args ...> torch::nn::ModuleHolder<Contained>::operator()(Args&& ...) [with Args = {torch::nn::ModuleList&}; Contained = torch::nn::SequentialImpl; torch::detail::return_type_of_forward_t<Contained, Args ...> = void]’:
/home/atharva/CLionProjects/SuperTux_RL/PredictFrame.cpp:61:22: required from here
/home/atharva/libtorch/libtorch/include/torch/csrc/api/include/torch/nn/pimpl.h:125:56: error: return-statement with a value, in function returning ‘torch::detail::return_type_of_forward_t<torch::nn::SequentialImpl, torch::nn::ModuleList&>’ {aka ‘void’} [-fpermissive]
return impl_->forward(::std::forward<Args>(args)...);
^
make[2]: *** [CMakeFiles/SuperTux_RL.dir/build.make:76: CMakeFiles/SuperTux_RL.dir/PredictFrame.cpp.o] Error 1
make[1]: *** [CMakeFiles/Makefile2:76: CMakeFiles/SuperTux_RL.dir/all] Error 2
make: *** [Makefile:84: all] Error 2
Could anyone please tell me the correct way of doing this?