I have a dataset that has been preprocessed, and this error continues to occur.
I’ve tried the if__name___==‘main _’: method from several communities, but I keep getting errors if I used it incorrectly.
Is torch.utils.data not working properly?
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "D:\anaconda\envs\mj\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "D:\anaconda\envs\mj\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'video_dataset' on <module '__main__' (built-in)>
This is how it is defined.
class video_dataset():
def __init__(self,frame_list,sequence_length = 16,transform = None):
self.frame_list = frame_list
self.transform = transform
self.sequence_length = sequence_length
def __len__(self):
return len(self.frame_list)
def __getitem__(self,idx):
label,path = self.frame_list[idx]
img = cv2.imread(path)
seq_img = list()
for i in range(16):
img1 = img[:,128*i:128*(i+1),:]
if(self.transform):
img1 = self.transform(img1)
seq_img.append(img1)
seq_image = torch.stack(seq_img)
seq_image = seq_image.reshape(3,16,im_size,im_size)
return seq_image,decoder[label]
This is where the final error occurs.
import torchvision
import torch
from torch import nn
import torch.nn.functional as F
import torchvision.models as models
import torch.optim as optim
import copy
import os
from tqdm.autonotebook import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from torchvision import transforms
from torch.utils.data import DataLoader
import numpy as np
from torch.utils.data.sampler import SubsetRandomSampler
import cv2
import sys
import import_ipynb
from torch.autograd import Variable
iteration = 0
acc_all = list()
loss_all = list()
for epoch in range(num_epochs):
print('')
print(f"--- Epoch {epoch} ---")
phase1 = dataloaders.keys()
for phase in phase1:
print('')
print(f"--- Phase {phase} ---")
epoch_metrics = {"loss": [], "acc": []}
for batch_i, (X, y) in enumerate(dataloaders[phase]):
#iteration = iteration+1
image_sequences = Variable(X.to(device), requires_grad=True)
labels = Variable(y.to(device), requires_grad=False)
optimizer.zero_grad()
#model.lstm.reset_hidden_state()
predictions = model(image_sequences)
loss = cls_criterion(predictions, labels)
acc = 100 * (predictions.detach().argmax(1) == labels).cpu().numpy().mean()
loss.backward()
optimizer.step()
epoch_metrics["loss"].append(loss.item())
epoch_metrics["acc"].append(acc)
if(phase=='train'):
lr,mom = onecyc.calc()
update_lr(optimizer, lr)
update_mom(optimizer, mom)
batches_done = epoch * len(dataloaders[phase]) + batch_i
batches_left = num_epochs * len(dataloaders[phase]) - batches_done
sys.stdout.write(
"\r[Epoch %d/%d] [Batch %d/%d] [Loss: %f (%f), Acc: %.2f%% (%.2f%%)]"
% (
epoch,
num_epochs,
batch_i,
len(dataloaders[phase]),
loss.item(),
np.mean(epoch_metrics["loss"]),
acc,
np.mean(epoch_metrics["acc"]),
)
)
# Empty cache
if torch.cuda.is_available():
torch.cuda.empty_cache()
print('')
print('{} , acc: {}'.format(phase,np.mean(epoch_metrics["acc"])))
torch.save(model.state_dict(),'weights_crime/c3d_{}.h5'.format(epoch))
if(phase=='train'):
acc_all.append(np.mean(epoch_metrics["acc"]))
loss_all.append(np.mean(epoch_metrics["loss"]))
I have a dataset that has been preprocessed, and this error continues to occur. I’ve tried the if__name___==‘main_’: method from several communities, but I keep getting errors if I used it incorrectly.
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "D:\anaconda\envs\mj\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "D:\anaconda\envs\mj\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'video_dataset' on <module '__main__' (built-in)>
This is how it is defined.
class video_dataset():
def __init__(self,frame_list,sequence_length = 16,transform = None):
self.frame_list = frame_list
self.transform = transform
self.sequence_length = sequence_length
def __len__(self):
return len(self.frame_list)
def __getitem__(self,idx):
label,path = self.frame_list[idx]
img = cv2.imread(path)
seq_img = list()
for i in range(16):
img1 = img[:,128*i:128*(i+1),:]
if(self.transform):
img1 = self.transform(img1)
seq_img.append(img1)
seq_image = torch.stack(seq_img)
seq_image = seq_image.reshape(3,16,im_size,im_size)
return seq_image,decoder[label]
import torchvision
import torch
from torch import nn
import torch.nn.functional as F
import torchvision.models as models
import torch.optim as optim
import copy
import os
from tqdm.autonotebook import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from torchvision import transforms
from torch.utils.data import DataLoader
import numpy as np
from torch.utils.data.sampler import SubsetRandomSampler
import cv2
import sys
import import_ipynb
This is where the final error occurs.
from torch.autograd import Variable
iteration = 0
acc_all = list()
loss_all = list()
for epoch in range(num_epochs):
print('')
print(f"--- Epoch {epoch} ---")
phase1 = dataloaders.keys()
for phase in phase1:
print('')
print(f"--- Phase {phase} ---")
epoch_metrics = {"loss": [], "acc": []}
for batch_i, (X, y) in enumerate(dataloaders[phase]):
#iteration = iteration+1
image_sequences = Variable(X.to(device), requires_grad=True)
labels = Variable(y.to(device), requires_grad=False)
optimizer.zero_grad()
#model.lstm.reset_hidden_state()
predictions = model(image_sequences)
loss = cls_criterion(predictions, labels)
acc = 100 * (predictions.detach().argmax(1) == labels).cpu().numpy().mean()
loss.backward()
optimizer.step()
epoch_metrics["loss"].append(loss.item())
epoch_metrics["acc"].append(acc)
if(phase=='train'):
lr,mom = onecyc.calc()
update_lr(optimizer, lr)
update_mom(optimizer, mom)
batches_done = epoch * len(dataloaders[phase]) + batch_i
batches_left = num_epochs * len(dataloaders[phase]) - batches_done
sys.stdout.write(
"\r[Epoch %d/%d] [Batch %d/%d] [Loss: %f (%f), Acc: %.2f%% (%.2f%%)]"
% (
epoch,
num_epochs,
batch_i,
len(dataloaders[phase]),
loss.item(),
np.mean(epoch_metrics["loss"]),
acc,
np.mean(epoch_metrics["acc"]),
)
)
# Empty cache
if torch.cuda.is_available():
torch.cuda.empty_cache()
print('')
print('{} , acc: {}'.format(phase,np.mean(epoch_metrics["acc"])))
torch.save(model.state_dict(),'weights_crime/c3d_{}.h5'.format(epoch))
if(phase=='train'):
acc_all.append(np.mean(epoch_metrics["acc"]))
loss_all.append(np.mean(epoch_metrics["loss"]))
original code is here Videoclassification/train.ipynb at master · sanchit2843/Videoclassification · GitHub