Hello,
I get this error: RuntimeWarning: invalid value encountered in double_scalars
related to this line of code: FILE.write("\nEpoch “+str(epoch)+”: Training Loss = " +str(np.sum(trg_loss).max()/len(trg_loss)))
The output looks as follows with NaN: Epoch : 0
Epoch 0: Training Loss = nan
The entire code is posted below:
>
> model = MyModel()
>
> print(model)
>
> ## name of text file to save training stats
> TRAINING_LOG = "log.txt"
> FILE = open(TRAINING_LOG, 'w')
>
>
> #%% Here we train and evaluate the model
>
> saved_models_path = os.getcwd() + '/models/'
> if(not os.path.isdir(saved_models_path)):
> os.makedirs(saved_models_path)
>
>
> # Check if CUDA is available
> use_cuda = torch.cuda.is_available()
> # print("\n\n USE CUDA: ", use_cuda)
> # print("\n\n")
>
>
> ## Setting cuda as False for stability. Please comment out if using GPU
> use_cuda = False
>
> device = torch.device("cuda:0" if use_cuda else "cpu")
> # cudnn.benchmark = True
>
>
>
>
> # Set training parameters
> params = {'batch_size': 64,
> 'shuffle': True,
> 'num_workers': 10}
> max_epochs = 100
> core_num = 20
> tune_every = 10 # test and validate the model every x epochs
>
> data_library = {}
> prior = '100MHz'
> target = '250MHz'
>
> ## Amount of dataset to use for the experiment
> data_frac = 1 ## What fraction of data to use for training (please set to 1 for final experiment )
>
>
>
> # Load all the data from the txt file
> file_IDs = open('ID_list.txt','r').read().split('\n')
> file_IDs = file_IDs[:-1] # remove last line
> file_IDs = file_IDs[:int(len(file_IDs)*data_frac)]
> complete_dataset = Dataset(file_IDs)
>
>
> # create your optimizer
> optimizer = optim.SGD(model.parameters(), lr=0.005, momentum = 0.1)
> scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.95)
>
> #%% Here we train the network
>
> # Divide the dataset into the training and validation set
> lengths = [int(np.ceil(len(complete_dataset)*0.8)), int(np.floor(len(complete_dataset)*0.1)), int(np.floor(len(complete_dataset)*0.1))]
> training_set, validation_set, evaluation_set = torch.utils.data.random_split(complete_dataset, lengths)
> training_generator = data.DataLoader(training_set, **params)
> validation_generator = data.DataLoader(validation_set, **params)
> evaluation_generator = data.DataLoader(evaluation_set, **params)
>
>
> # instantiate the model to make it a double tensor
> forward_model = model.double()
>
> # # check if model works for random instance of the data
> # t100 ,t250 = next(iter(training_generator))
> # one_prediction = forward_model(t100)
>
>
> loss_function = nn.MSELoss()
>
>
> for param in forward_model.parameters():
> param.requires_grad = True
>
> def train(low_res, high_res):
> # Loop over epochs
> global forward_model, optimizer
> low_res, high_res = low_res.to(device), high_res.to(device)
> optimizer.zero_grad()
> prediction_training = forward_model(low_res)
> loss = loss_function(prediction_training, high_res) #Here compute tha value that estimates how far the outut is from the target
> #print("\nTraining Loss: ", loss)
> loss.backward() #The whole graph is differentiatted wrt the loss and all ensors in the graph that have 'required_gard = true' will have their .grad tensot accumulaed ith the gradient.
> optimizer.step()
> return loss
>
>
> def val(low_res, high_res):
> global forward_model, optimizer
> low_res, high_res = low_res.to(device), high_res.to(device)
> optimizer.zero_grad()
> prediction_training = forward_model(low_res)
> loss = loss_function(prediction_training, high_res) #Here compute tha value that estimates how far the outut is from the target
> #print("\nValidation Loss: ", loss)
> #scheduler.step()
> return loss
>
>
>
> for epoch in range(max_epochs):
> print('\nEpoch : ' + str(epoch))
> FILE.write('\nEpoch : ' + str(epoch))
>
> # Training
> trg_loss = []
> forward_model.train()
> with torch.set_grad_enabled(True):
> for data_library['100MHz'], data_library['250MHz'] in training_generator:
> if(core_num>1):
> Parallel(n_jobs=core_num)
> #print(data_library['20MHz'].shape)
> delayed(train)(data_library[prior],data_library[target])
> else:
> new_loss = train(data_library[prior],data_library[target])
> trg_loss.append(new_loss)
> # Parallel(n_jobs=core_num)
> # print(data_library['20MHz'].shape)
> # delayed(train)(data_library[prior],data_library[target])
> #print("\nTraining Loss: ", sum(trg_loss).detach().numpy().max()/len(trg_loss))
> FILE.write("\nEpoch "+str(epoch)+": Training Loss = " +str(np.sum(trg_loss).max()/len(trg_loss)))
>
>
> if epoch % tune_every == 0 and epoch > 0:
> # validation
> val_loss = []
> forward_model.eval()
> with torch.set_grad_enabled(False):
> for data_library['100MHz'], data_library['250MHz'] in validation_generator:
> if(core_num>1):
> Parallel(n_jobs=core_num)
> #print(data_library['20MHz'].shape)
> delayed(train)(data_library[prior],data_library[target])
> else:
> new_loss = val(data_library[prior],data_library[target])
> val_loss.append(new_loss)
>
> #print("\nValidation Loss: ", sum(val_loss).numpy().max()/len(val_loss))
> FILE.write("\nEpoch "+str(epoch)+": Validation Loss = " +str(np.sum(val_loss).max()/len(val_loss)))
> # HYPERPARAMETER TUNING GOES IN HERE
>
> FILE.close()
