i am using google colab to run this repository DIPforSRofNoisyImages and i am facing this error
baby
24-10-03 19:59:07.274 - INFO: save_dir : results
24-10-03 19:59:07.274 - INFO: keyword : test
24-10-03 19:59:07.274 - INFO: data_dir : data
24-10-03 19:59:07.274 - INFO: dataset : set5
24-10-03 19:59:07.274 - INFO: factor : 2
24-10-03 19:59:07.274 - INFO: noise_level : 25
24-10-03 19:59:07.274 - INFO: input_type : noise
24-10-03 19:59:07.274 - INFO: reg_std : 0.03
24-10-03 19:59:07.274 - INFO: input_channels : 32
24-10-03 19:59:07.274 - INFO: dtype : <class ‘torch.cuda.FloatTensor’>
24-10-03 19:59:07.274 - INFO: content_noise_var : 0.01
24-10-03 19:59:07.274 - INFO: recon_noise_var : 0.002
24-10-03 19:59:07.275 - INFO: cpu : False
24-10-03 19:59:07.275 - INFO: Generator : {‘input_depth’: 32, ‘NET_TYPE’: ‘skip’, ‘pad’: ‘reflection’, ‘skip_n33d’: [8, 16, 32, 64, 128], ‘skip_n33u’: [8, 16, 32, 64, 128], ‘skip_n11’: [0, 0, 0, 4, 4], ‘upsample_mode’: ‘bilinear’, ‘n_channels’: 3}
24-10-03 19:59:07.275 - INFO: Downsampler : {‘n_planes’: 3, ‘factor’: 2, ‘kernel_type’: ‘lanczos2’, ‘phase’: 0.5, ‘preserve_size’: True}
24-10-03 19:59:07.275 - INFO: Discriminator : {‘nc’: 3, ‘nfc’: 32, ‘min_nfc’: 32, ‘ker_size’: 3, ‘num_layer’: 5, ‘stride’: 1, ‘pad_size’: 0}
24-10-03 19:59:07.275 - INFO: resume : False
24-10-03 19:59:07.275 - INFO: optimizer_dip : {‘type’: ‘adam’, ‘LR’: 0.01, ‘recon_weight’: 1, ‘adv_weight’: 1.2, ‘ssl_weight’: 1}
24-10-03 19:59:07.275 - INFO: optimizer_disc : {‘type’: ‘adam’, ‘LR’: 0.0001}
24-10-03 19:59:07.275 - INFO: num_iter : 2000
24-10-03 19:59:07.275 - INFO: show_every : 200
baby
HR and LR resolutions: (512, 512), (256, 256)
n_channels : 3
tensor([[[[0.7431, 0.7431, 0.7434, …, 0.5904, 0.5902, 0.5905],
[0.7431, 0.7431, 0.7433, …, 0.5900, 0.5898, 0.5901],
[0.7428, 0.7429, 0.7431, …, 0.5884, 0.5883, 0.5885],
…,
[0.5599, 0.5598, 0.5598, …, 0.4218, 0.4216, 0.4215],
[0.5597, 0.5596, 0.5595, …, 0.4211, 0.4209, 0.4209],
[0.5600, 0.5599, 0.5598, …, 0.4206, 0.4204, 0.4204]],[[0.6069, 0.6073, 0.6080, ..., 0.2347, 0.2351, 0.2346], [0.6067, 0.6071, 0.6077, ..., 0.2342, 0.2346, 0.2341], [0.6061, 0.6065, 0.6071, ..., 0.2337, 0.2341, 0.2336], ..., [0.3083, 0.3079, 0.3075, ..., 0.4578, 0.4576, 0.4575], [0.3083, 0.3079, 0.3075, ..., 0.4576, 0.4574, 0.4573], [0.3083, 0.3080, 0.3075, ..., 0.4575, 0.4573, 0.4571]], [[0.8156, 0.8155, 0.8155, ..., 0.7914, 0.7913, 0.7918], [0.8156, 0.8155, 0.8156, ..., 0.7913, 0.7912, 0.7917], [0.8156, 0.8155, 0.8155, ..., 0.7896, 0.7896, 0.7900], ..., [0.6957, 0.6958, 0.6958, ..., 0.5363, 0.5357, 0.5356], [0.6957, 0.6958, 0.6958, ..., 0.5352, 0.5346, 0.5344], [0.6960, 0.6961, 0.6961, ..., 0.5342, 0.5337, 0.5335]]]], device='cuda:0', grad_fn=<SigmoidBackward0>)tensor([[[[0.6239, 0.6223, 0.6246, …, 0.4963, 0.5026, 0.4943],
[0.6320, 0.6296, 0.6308, …, 0.5029, 0.5121, 0.5029],
[0.6356, 0.6324, 0.6330, …, 0.5095, 0.5197, 0.5109],
…,
[0.1477, 0.1461, 0.1440, …, 0.3105, 0.3122, 0.3160],
[0.1473, 0.1458, 0.1436, …, 0.3088, 0.3109, 0.3147],
[0.1480, 0.1465, 0.1443, …, 0.3085, 0.3105, 0.3145]],[[0.5729, 0.5732, 0.5741, ..., 0.4741, 0.4801, 0.4815], [0.5740, 0.5743, 0.5749, ..., 0.4809, 0.4856, 0.4874], [0.5769, 0.5772, 0.5777, ..., 0.4830, 0.4875, 0.4892], ..., [0.3848, 0.3914, 0.3984, ..., 0.3717, 0.3680, 0.3673], [0.3879, 0.3944, 0.4015, ..., 0.3694, 0.3662, 0.3657], [0.3891, 0.3957, 0.4029, ..., 0.3670, 0.3646, 0.3648]], [[0.6823, 0.6788, 0.6805, ..., 0.4864, 0.4941, 0.4833], [0.6928, 0.6882, 0.6885, ..., 0.4951, 0.5082, 0.4957], [0.6956, 0.6900, 0.6895, ..., 0.5065, 0.5211, 0.5091], ..., [0.1427, 0.1437, 0.1443, ..., 0.2995, 0.2983, 0.3016], [0.1436, 0.1447, 0.1452, ..., 0.2964, 0.2960, 0.2994], [0.1448, 0.1460, 0.1465, ..., 0.2946, 0.2946, 0.2986]]]], device='cuda:0', grad_fn=<CloneBackward0>)/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py:769: UserWarning: Error detected in ConvolutionBackward0. Traceback of forward call that caused the error:
File “/content/train.py”, line 17, in
trainer.train()
File “/content/solver.py”, line 105, in train
self.optimizers[‘dip’].step(self.closure_dip)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py”, line 484, in wrapper
out = func(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py”, line 89, in _use_grad
ret = func(self, *args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/adam.py”, line 205, in step
loss = closure()
File “/content/solver.py”, line 128, in closure_dip
self.image_out_HR = self.net’dip’
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1562, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py”, line 219, in forward
input = module(input)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1562, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py”, line 219, in forward
input = module(input)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1553, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1562, in _call_impl
return forward_call(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/conv.py”, line 458, in forward
return self._conv_forward(input, self.weight, self.bias)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/conv.py”, line 454, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
(Triggered internally at …/torch/csrc/autograd/python_anomaly_mode.cpp:111.)
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
Traceback (most recent call last):
File “/content/train.py”, line 17, in
trainer.train()
File “/content/solver.py”, line 105, in train
self.optimizers[‘dip’].step(self.closure_dip)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py”, line 484, in wrapper
out = func(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py”, line 89, in _use_grad
ret = func(self, *args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/optim/adam.py”, line 205, in step
loss = closure()
File “/content/solver.py”, line 155, in closure_dip
self.ssl_HR.backward(retain_graph=True)
File “/usr/local/lib/python3.10/dist-packages/torch/_tensor.py”, line 521, in backward
torch.autograd.backward(
File “/usr/local/lib/python3.10/dist-packages/torch/autograd/init.py”, line 289, in backward
_engine_run_backward(
File “/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py”, line 769, in _engine_run_backward
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [3, 8, 1, 1]] is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
i have modified the solve.py script and i add some .clone() and still facing the same error here is the solve.py file:
import logging
import time
import numpy as np
import os
import shutil
from collections import OrderedDict
import random
import torch
torch.autograd.set_detect_anomaly(True)
import torch.nn as nn
import torch.nn.functional as F
from torchvision.utils import save_imagefrom skimage.metrics import peak_signal_noise_ratio
from collections import namedtuple
from skimage import iofrom configs import getConfigs
from utils.common_utils import *
from utils.loss import VGG19_ContentLoss
from models import *
from models.downsampler import Downsampler
from models.discriminator import UNetD, Discriminator, WDiscriminator
DenoizeResult = namedtuple(“DenoizeResult”, [‘learned_LR’, ‘learned_HR’, ‘n’, ‘psnr’, ‘step’])class Solver(object) :
def init(self, cfg, img_name) :
self.image_name = img_name
self.cfg = cfg############################## # 1. SETTINGS ############################## # set configs self._setConfigs(cfg) # set random seeds set_random_seed(is_gpu=not self.cpu) # set dirs self._setDirs() # copy the configs.py if os.path.isfile('./configs.py') : shutil.copy('./configs.py', cfg.save_dir) # set loggers self._setLoggers() # save and log configs for k, v in cfg.items() : log = '{} : {}'.format(k, v) self.config_logger.info(log) # set GPUs self._setGPUs() self.eps=1e-8 # set images self.image_name = img_name self._setImages(img_name) ############################## # 2. NETWORK and INPUT NOISE ############################## # set models self.net = OrderedDict() # 1. DIP self.net['dip'] = get_net(**self.Generator).to(self.device) self.net['downsampler'] = Downsampler(**self.Downsampler).to(self.device) # 2. discriminator self.net['netD'] = WDiscriminator(**self.Discriminator).to(self.device) # set input noise self.net_input = get_noise(self.input_channels, self.input_type, self.img_HR_size).type(self.dtype).detach() self.noise = self.net_input.detach().clone() self.real_noise_input = get_noise(self.n_channels, self.input_type, self.img_LR_size, 'n', self.noise_level/255.).type(self.dtype).detach() ############################## # 3. OPTIMIZERS and LOSS ############################## # optimizers self._setOptimizers() # loss self.mse_loss = nn.MSELoss().type(self.dtype) self.start_epoch = 0 self.curr_epoch = 0 self.psnr_best = 0 def train(self) : self.best_result = None self.save_image_out_HR = None self.save_image_out_LR = None self.pseudo_HR = None self.pseudo_LR = None self.prev_out_HR = None self.prev_out_LR = None for i in range(self.num_iter) : self.netDIP_loss = 0 self.netD_loss = 0 for k, v in self.optimizers.items() : v.zero_grad() self.optimizers['netD'].step(self.closure_disc) self.optimizers['dip'].step(self.closure_dip) self.get_currentResults(i) if i % self.show_every == 1 : self.save_images(i) self.plot_closure(i) #-------------------------------------------------------------- def closure_dip(self) : if self.reg_std : self.net_input = self.net_input + (self.noise.normal_() * self.reg_std) self.requires_grad(self.net['netD'], True) self.requires_grad(self.net['dip'], True) self.image_out_HR = self.net['dip'](self.net_input.detach()) #self.image_out_HR = self.net['dip'](self.net_input.clone()) ################# self.image_out_LR = self.net['downsampler'](self.image_out_HR.clone()) #################### # calc reconstruction loss and backpropagate self.noise_out = self.image_out_LR - self.images['LR_noisy'] self.recon_input = self.image_out_LR.clone() ######################## self.recon_loss = self.mse_loss(self.recon_input, self.images['LR_noisy'].type(self.dtype)) * self.optimizer_dip['recon_weight'] self.recon_loss.backward(retain_graph=True) # calc adversarial loss and backpropagate self.noise_out = self.image_out_LR.clone() - self.images['LR_noisy']########################### output = self.net['netD'](self.noise_out.detach()) errG = -output.mean() self.adv_loss = errG * self.optimizer_dip['adv_weight'] self.adv_loss.backward(retain_graph=True) # calc self-supervised loss and backpropagate if self.optimizer_dip['ssl_weight'] > 0 and self.pseudo_HR is not None: self.ssl_HR = self.mse_loss(self.image_out_HR.clone(), self.pseudo_HR.clone())*self.optimizer_dip['ssl_weight'] ################# print(self.image_out_HR) print(self.pseudo_HR) self.ssl_HR.backward(retain_graph=True) else : self.ssl_HR = 0 if self.optimizer_dip['ssl_weight'] > 0 and self.pseudo_LR is not None: self.ssl_LR = self.mse_loss(self.image_out_LR.clone(), self.pseudo_LR.clone())*self.optimizer_dip['ssl_weight'] self.ssl_LR.backward(retain_graph=True) else : self.ssl_LR = 0 # calc netDIP loss self.netDIP_loss = self.recon_loss + self.adv_loss + self.ssl_LR + self.ssl_HR # set self-supervised loss input if self.prev_out_HR is None : self.prev_out_HR = self.image_out_HR.clone() ########################### self.prev_out_LR = self.image_out_LR.clone() ########################### else : self.prev_out_HR = self.save_image_out_HR.clone() ########################### self.prev_out_LR = self.save_image_out_LR.clone() ########################### self.save_image_out_HR = self.image_out_HR.clone() ########################### self.save_image_out_LR = self.image_out_LR.clone() ########################### self.pseudo_HR = self.save_image_out_HR self.pseudo_LR = self.save_image_out_LR return self.netDIP_loss def closure_disc(self) : self.requires_grad(self.net['netD'], True) self.requires_grad(self.net['dip'], False) ########################################### # 1. train with real noise ########################################### output = self.net['netD'](self.real_noise_input.clone())############################# self.errD_real = -output.mean() self.errD_real.backward(retain_graph=True) ########################################### # 2. train with fake noise ########################################### # update dip input if self.reg_std > 0 : self.net_input = self.net_input + (self.noise.normal_() * self.reg_std) # get fake noise and loss self.image_out_HR = self.net['dip'](self.net_input.clone()) ################################## self.image_out_LR = self.net['downsampler'](self.image_out_HR.clone())###################################### self.noise_out = self.image_out_LR - self.images['LR_noisy'] output = self.net['netD'](self.noise_out.detach()) self.errD_fake = output.mean() self.errD_fake.backward(retain_graph=True) # Add gradients from the all-real and all-fake self.netD_loss = -self.errD_real + self.errD_fake return self.netD_loss def save_images(self, step) : # get numpy RGB results final_image_HR = np.clip(self.current_result.learned_HR, 0, 1).transpose(1,2,0).copy()*255 # save clean HR image filename = os.path.join(self.result_dir, self.image_name+"_{:04d}_cleaned.png".format(step)) io.imsave(filename, final_image_HR.astype(np.uint8)) def get_currentResults(self, step) : image_out_HR_np = np.clip(torch_to_np(self.image_out_HR), 0, 1) image_out_LR_np = np.clip(torch_to_np(self.image_out_LR), 0, 1) noise_out_np = np.clip(torch_to_np(self.noise_out), 0, 1) psnr_HR = peak_signal_noise_ratio(torch_to_np(self.images['HR']), image_out_HR_np) self.current_result = DenoizeResult(learned_HR=image_out_HR_np, learned_LR=image_out_LR_np, n=noise_out_np, psnr=psnr_HR, step=step) if self.best_result is None or self.best_result.psnr < self.current_result.psnr: self.best_result = self.current_result def plot_closure(self, step) : print('step : %d netDIP[rec : %f adv : %f sslHR : %f sslLR : %f ] netD[fake : %f real : %f ] psnr[curr : %f best : %f ]' % (step, self.recon_loss, self.adv_loss, self.ssl_HR, self.ssl_LR, self.errD_fake, self.errD_real, self.current_result.psnr, self.best_result.psnr)) print('step : %d netDIP[rec : %f adv : %f sslHR : %f sslLR : %f ] netD[fake : %f real : %f ] psnr[curr : %f best : %f ]' % (step, self.recon_loss, self.adv_loss, self.ssl_HR, self.ssl_LR, self.errD_fake, self.errD_real, self.current_result.psnr, self.best_result.psnr), file=self.file) def requires_grad(self, model, flag=True): for p in model.parameters(): p.requires_grad = flag def finalize(self) : final_image_HR = np.clip(self.best_result.learned_HR, 0, 1).transpose(1,2,0).copy()*255 filename = os.path.join(self.valid_dir, self.image_name+"_cleaned_best.png") io.imsave(filename, final_image_HR.astype(np.uint8)) final_image_HR = np.clip(self.current_result.learned_HR, 0, 1).transpose(1,2,0).copy()*255 filename = os.path.join(self.valid_dir, self.image_name+"_cleaned_final.png") io.imsave(filename, final_image_HR.astype(np.uint8)) print('best result info : [max psnr] : %f [iteration] : %d' % (self.best_result.psnr, self.best_result.step) , file=self.file ) print('best result info : [max psnr] : %f [iteration] : %d' % (self.best_result.psnr, self.best_result.step) ) self.file.close() #----------------------------------------------------------------------------------------------------------------------------------------------- def _setConfigs(self, cfg) : for key in cfg : setattr(self, key, cfg[key]) def _setDirs(self) : self.valid_dir = os.path.join(self.save_dir, 'valid', self.dataset) self.result_dir = os.path.join(self.save_dir, self.dataset) make_dir(self.save_dir) make_dir(self.result_dir) make_dir(self.valid_dir) #tensorboard self.log_dir = os.path.join(self.save_dir, 'tensorboard') make_dir(self.log_dir) # self.board_writer = SummaryWriter(self.log_dir) self.file = open(os.path.join(self.result_dir, self.image_name+'_psnr.txt'), 'w') def _setLoggers(self) : setup_logger('configs', self.save_dir, 'configs', level=logging.INFO, screen=True) setup_logger('valid', self.save_dir, 'valid', level=logging.INFO, is_formatter=True, screen=False) self.config_logger = logging.getLogger('configs') #training logger self.valid_logger = logging.getLogger('valid') #validation logger def _setGPUs(self) : if self.cpu : self.device = torch.device('cpu') else : torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True self.device = torch.device('cuda') def _setImages(self, image_name) : print(image_name) image_path = os.path.join(self.data_dir, self.dataset) img_orig_pil, img_orig_np = get_image(image_path+"/clean/"+image_name+".png", -1) # get rescaled clean HR image new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 8, img_orig_pil.size[1] - img_orig_pil.size[1] % 8) bbox = [ (img_orig_pil.size[0] - new_size[0])/2, (img_orig_pil.size[1] - new_size[1])/2, (img_orig_pil.size[0] + new_size[0])/2, (img_orig_pil.size[1] + new_size[1])/2, ] img_HR_pil = img_orig_pil.crop(bbox) img_HR_np = pil_to_np(img_HR_pil) # get LR images img_LR_clean_pil, img_LR_clean_np = get_image(image_path+"/x"+str(self.factor)+"_"+str(self.noise_level)+"/"+image_name+"_clean.png") img_LR_noisy_pil, img_LR_noisy_np = get_image(image_path+"/x"+str(self.factor)+"_"+str(self.noise_level)+"/"+image_name+"_noisy.png") _, img_gt_np = get_image(image_path+"/x"+str(self.factor)+"_"+str(self.noise_level)+"/"+image_name+"_gt.png") self.img_HR_size = (img_HR_pil.size[1], img_HR_pil.size[0]) self.img_LR_size = (img_LR_noisy_pil.size[1], img_LR_noisy_pil.size[0]) print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str(img_LR_noisy_pil.size))) self.images = { 'orig': np_to_torch(img_orig_np).type(self.dtype), 'LR_clean': np_to_torch(img_LR_clean_np).type(self.dtype), 'LR_noisy': np_to_torch(img_LR_noisy_np).type(self.dtype), 'noise_gt' : np_to_torch(img_gt_np).type(self.dtype), 'HR': np_to_torch(img_HR_np).type(self.dtype), } self.n_channels = self.images['HR'].shape[1] self.Discriminator['nc'] = self.n_channels self.Generator['n_channels'] = self.n_channels self.Downsampler['n_planes'] = self.n_channels print('n_channels : ', self.n_channels) def _setOptimizers(self) : optimizers = OrderedDict() for net_type, _ in self.net.items() : # 1. optimizer of discriminator if net_type == 'netD' : if self.optimizer_disc['type'] == 'adam' : optimizers[net_type] = torch.optim.Adam( self.net['netD'].parameters(), lr=self.optimizer_disc['LR'] ) else : assert False # 2. optimizer of DIP if net_type == 'dip' : if self.optimizer_dip['type'] == 'adam' : optimizers[net_type] = torch.optim.Adam( self.net['dip'].parameters(), lr = self.optimizer_dip['LR'] ) else : assert False self.optimizers = optimizers