My model is a really basic one, I made it just to learn, but it learns really slow. I ran the model tens thousands of times but the loss only decreased from 640 to 420. How would I increase the rate at which my model learns? I use visual studio 2022 and am on windows 11.
my code:
#include <torch/torch.h>
#include
#include
#include
struct Net : torch::nn::Module {
//constructor initializes a random set of weights which have a size of n rows and m columns, and a bias tensor with a size of m rows
Net()
{
layer1 = register_module(“layer1”, torch::nn::Linear(1, 10));
layer2 = register_module(“layer2”, torch::nn::Linear(10, 1));
}
//redifine the virtual function forward from tensor
torch::Tensor forward(torch::Tensor var1)
{
//goes through input layer and gets changed
var1 = layer1->forward(var1);
//relu removes all the negative numbers and turns them into 0’s
var1 = torch::relu(var1);
//goes through output layer
var1 = layer2->forward(var1);
return var1;
}
//initializes the variables(does not matter what order)
torch::nn::Linear layer1{ nullptr }, layer2{ nullptr };
};
//had to add save and load models because for some reason the torch::load did not work
void save_model(std::string path, const Net& model) {
torch::serialize::OutputArchive output_archive;
model.save(output_archive);
output_archive.save_to(path);
}
void load_model(std::string path, Net& model) {
torch::serialize::InputArchive input_archive;
input_archive.load_from(path);
model.load(input_archive);
}
int main() {
torch::manual_seed(0);
Net net;
float avg = 0;
std::vector<float> average_group;
torch::Tensor specific_values_tensor = torch::tensor({ 5.0 }, torch::kFloat32).view({ 1, 1 });
try {
load_model("test_model01.pt", net);
std::cout << "Model loaded successfully." << std::endl;
}
catch (const std::exception& e) {
std::cerr << "Failed to load model: " << e.what() << std::endl;
}
std::cout << net << std::endl;
//auto output = net.forward(specific_values_tensor);
//std::cout << output << std::endl;
torch::optim::Adam optimizer(net.parameters(), torch::optim::AdamOptions(0.005));
auto scheduler = torch::optim::StepLR(optimizer, /*step_size=*/100, /*gamma=*/0.1);
std::cout << "enter the number of iterations that should be done (each iteration amount will be done 3 times)\n";
size_t iter_num = 5;
try
{
std::cin >> iter_num;
if (std::cin.fail())
throw("wrong input type");
}
catch (const char* msg)
{
std::cout << msg << std::endl;
std::cin.clear();
std::cin.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
std::cin >> iter_num;
}
//a 3 time loop that starts with a 1 because of later logic reasons
for(int op = 0; op < 3; op++)
{
avg = 0;
for (size_t a = 0; a < iter_num; a++)
{
auto input = torch::rand({ 1, 1 }) * 5;
auto target = input * input;
optimizer.zero_grad();
auto output = net.forward(input);
auto loss = torch::mse_loss(output, target);
//calculates the negative gradient
loss.backward();
torch::nn::utils::clip_grad_norm_(net.parameters(), /max_norm=/0.5);
//changes values based off the negative gradient calculated by loss.backward()
optimizer.step();
scheduler.step();
if (a % 100 == 0)
{
auto prev_loss = loss;
std::cout << "\niteration number: " << a;
std::cout << "\nLoss: " << loss.item() << std::endl;
std::cout << "Prediction: " << output.item();
std::cout << "\nTarget: " << target.item();
}
avg += loss.item();
}
//the average is divided by the iter_num*op because the cycle loops 3 times
average_group.push_back(avg / (iter_num));
}
for (auto iterator : average_group)
{
std::cout << "\naverage: ";
std::cout << iterator;
}
save_model(“test_model01.pt”, net);
//potential: save test_model01.pt, had an average last loss of 420. split it into 3 different models(or more if needed) and change activation function: ReLU(control)
//sigmoid and tanh. Tweak some parameters, add batch sizes.
}