I’m not sure if this is a PyTorch question but I want to save the 2nd last fc outputs from a pretrained vgg into an hdf5 array to load later on. The issue is I would need to save all tensor outputs as one chunk to use an hdf5 dataset (below) however I cannot seem to append tensors to h5 dataset without creating chunks. Does anyone know of an efficient way to save torch tensors into one chunk in an hdf5 file?
Any help appreciated!
# https://www.tinymind.com/learn/terms/hdf5 import h5py import torch import torch.utils.data as data class H5Dataset(data.Dataset): def __init__(self, file_path): super(H5Dataset, self).__init__() h5_file = h5py.File(file_path) self.data = h5_file.get('data') self.target = h5_file.get('label') def __getitem__(self, index): return (torch.from_numpy(self.data[index,:,:,:]).float(), torch.from_numpy(self.target[index,:,:,:]).float()) def __len__(self): return self.data.shape