I am trying to load data into my Dataset class in the get_item function rather than init function because of the data is very large and it cannot be loaded all at once to memory. Since the index of DataLoader keeps I am keeping a record of length of previously loaded part of data but this length goes to 0 which was initiated in init every time a new batch is loaded. Is there a way to not call init function of DataSet?
` def __init__(self, data_path, graph_args={}, train_val_test='train'):
'''
train_val_test: (train, val, test)
'''
self.data_path = data_path
self.path_list = sorted(glob.glob(os.path.join(self.data_path,'*.txt')))
self.all_feature=[]
self.all_adjacency=[]
self.all_mean_xy=[]
self.it = 0
#self.load_data()
#total_num = len(self.all_feature)
# equally choose validation set
self.feature_num=0
self.prev=0
def __getitem__(self, idx):
# C = 11: [frame_id, object_id, object_type, position_x, position_y, position_z, object_length, pbject_width, pbject_height, heading] + [mask]
#if(idx>=self.feature_num):
try:
now_feature = self.all_feature[idx-self.prev].copy()
except:
path = self.path_list[self.it]
self.it = self.it+1
self.all_feature, self.all_adjacency, self.all_mean_xy = generate_data(path)
self.prev = self.feature_num
self.feature_num = self.feature_num+len(self.all_feature)
now_feature = self.all_feature[idx-self.prev].copy()`