Hi everyone, I created my own data set and it’s size=[22806,1,3]. But when I tried to put this dataset to training. I got this error=train x size: torch.Size([22806, 1, 3])
1
Traceback (most recent call last):
File “fev.py”, line 210, in
for inputs, labels in tr:
ValueError: too many values to unpack (expected 2)
tr=training loader
I tried different numbers of batch size but i cannot figure out
Hi,
Could you show the code where you create the dataset and dataloader please?
If you dataset contains a single Tensor, then you cannot do for inputs, labels in tr:, as there is no second element, you should do for inputs in tr:.
cycles = ['LDCC', 'LDR', 'LRP']
variables = ['iBatt_10_s', 'uBatt_10_s', 'tCell_10_s', 'SOC_Cell_10_s']
save_var = ['i', 'u', 'T', 'SOC']
DATA = {}
for cycle in cycles:
print('* Loading cycle in memory: ', cycle)
for i, variable in enumerate(variables):
print(' Loading variable in memory: ', variable)
key_name = cycle + '_' + variable
target_csv = r'/Users/dreamer/Desktop/pytorchh/csv/MiL_sim_' + key_name + '.txt'
f = open(target_csv, 'r')
fresh_data = f.read()
f.close()
fresh_data = fresh_data.split(',')
save_name = cycle + '_' + save_var[i]
DATA[save_name] = np.array(fresh_data).astype(float)
# set their shapes right
for key in DATA:
DATA[key] = DATA[key].reshape(DATA[key].shape[0], 1)
LDCC_i = []
LDCC_u = []
LDCC_T = []
LDCC_SOC = []
k = 0
limit_of_jump = 20
wrt = DATA['LDCC_u']
for i in range(len(wrt) - 1):
if wrt[i] == wrt[i + 1]:
k = k + 1
if k == limit_of_jump or wrt[i] != wrt[i + 1]:
k = 0
LDCC_SOC.append(DATA['LDCC_SOC'][i])
LDCC_i.append(DATA['LDCC_i'][i])
LDCC_u.append(DATA['LDCC_u'][i])
LDCC_T.append(DATA['LDCC_T'][i])
LDR_i = []
LDR_u = []
LDR_T = []
LDR_SOC = []
k = 0
limit_of_jump = 20
wrt = DATA['LDR_u']
for i in range(len(wrt) - 1):
if wrt[i] == wrt[i + 1]:
k = k + 1
if k == limit_of_jump or wrt[i] != wrt[i + 1]:
k = 0
LDR_SOC.append(DATA['LDR_SOC'][i])
LDR_i.append(DATA['LDR_i'][i])
LDR_u.append(DATA['LDR_u'][i])
LDR_T.append(DATA['LDR_T'][i])
def scaler(tensor):
for ch in tensor:
scale = 1.0 / (ch.max(dim=0)[0] - ch.min(dim=0)[0])
ch.mul_(scale).sub_(ch.min(dim=0)[0])
return tensor
filtered_LDCC_u = savgol_filter(np.ravel(LDCC_u), 501, 3) # 51, 3
train_i = np.concatenate((LDCC_i, LDR_i))
train_u = np.concatenate((LDCC_u, LDR_u))
train_T = np.concatenate((LDCC_T ,LDR_T))
train_inputs = np.concatenate((train_i, train_u, train_T), axis=1)
train_outputs = np.concatenate((LDCC_SOC, LDR_SOC))
input_scaler = MinMaxScaler(feature_range=(-1,1))
input_scaler.fit(train_inputs)
input_scaler_outputs=MinMaxScaler(feature_range=(-1,1))
input_scaler_outputs.fit(train_outputs)
inputs_LDCC = input_scaler.transform(np.concatenate((LDCC_i, LDCC_u, LDCC_T), axis=1))
#input_scaler.fit(train_inputs)
#inputs_LDCC1 = input_scaler.fit_transform(np.concatenate((LDCC_i, LDCC_u, LDCC_T), axis=1))
outputs_LDCC=input_scaler_outputs.transform(np.array(LDCC_SOC))
#train_inputs=torch.Tensor(np.concatenate((LDCC_i, LDCC_u, LDCC_T), axis=1))
#inputs_LDCC1=scaler(train_inputs)
inputs_LDR = input_scaler.transform(np.concatenate((DATA['LDR_i'], DATA['LDR_u'], DATA['LDR_T']), axis=1))
outputs_LDR = input_scaler_outputs.transform(DATA['LDR_SOC'])
print(inputs_LDR.shape)
inputs_LRP = input_scaler.transform(np.concatenate((DATA['LRP_i'], DATA['LRP_u'], DATA['LRP_T']), axis=1))
outputs_LRP = input_scaler_outputs.transform(DATA['LRP_SOC'])
#inputs_LDCC1=inputs_LDCC1.numpy()
data_x = np.concatenate((inputs_LDCC, inputs_LDR), axis=0)
data_x = data_x.reshape(data_x.shape[0], 1, data_x.shape[1])
data_y = np.concatenate((outputs_LDCC, outputs_LDR), axis=0)
print("data y: ",data_y.shape)
val_range = int(data_x.shape[0]/100) * 15
val_x = data_x[0:val_range, :,:]
train_x = data_x[val_range:None, :,:]
val_y = data_y[0:val_range, :]
train_y = data_y[val_range:None, :]
'''
for axis in range(0, data_x.shape[1]):
plt.figure()
plt.plot(data_x[:, axis])
plt.show()
'''
train_x=torch.Tensor(train_x)
val_x=torch.Tensor(val_x)
print("train x size:",train_x.size())
class Model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv1d(1, 20, 1, 1)
self.conv2 = nn.Conv1d(20, 50, 1, 1)
self.fc1 = nn.Linear(1*3*50, 500)
#self.dropout1 = nn.Dropout(0.5)
self.fc2 = nn.Linear(500, 10)
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.max_pool1d(x, 1, 1)
x = F.relu(self.conv2(x))
x = F.max_pool1d(x, 1, 1)
x = x.view(-1, 1*3*50)
x = F.relu(self.fc1(x))
#x = self.dropout1(x)
x = self.fc2(x)
return x
'''
class Model(nn.Module):
def __init__(self,i,h1,out):
super().__init__()
self.linear=nn.Linear(i,h1)
self.linear2=nn.Linear(h1,out)
def forward(self,x):
x=F.tanh(self.linear(x))
x=F.tanh(self.linear2(x))
return x
'''
'''
data_x = inputs_LDCC.reshape(inputs_LDCC.shape[0], 1, inputs_LDCC.shape[1])
data_y = outputs_LDCC
val_range = int(data_x.shape[0]/100) * 15
val_x = data_x[0:val_range, :, :]
train_x = data_x[val_range:None, :, :]
val_y = data_y[0:val_range, :]
train_y = data_y[val_range:None, :]
train_x=torch.Tensor(train_x)
train_x=train_x.view(train_x.size(0),-1)
print(train_x.size(0))
val_x=torch.Tensor(val_x)
'''
batch_size=1000
epochs=15
device=torch.device("cuda:0"if torch.cuda.is_available() else "cpu")
tr=torch.utils.data.DataLoader(train_x,batch_size=100,shuffle=True,drop_last=True)
val=torch.utils.data.DataLoader(val_x,batch_size=100,shuffle=False)
model=Model().to(device)
print(tr.__len__())
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
next(iter(tr))
for e in range(epochs):
running_loss = 0.0
running_corrects = 0.0
val_running_loss = 0.0
val_running_corrects = 0.0
for inputs in tr:
inputs = inputs.to(device)
outputs = model(inputs)
loss = criterion(outputs, inputs)
optimizer.zero_grad()
loss.backward()
optimizer.step()
Also, if you want, i can upload data.txt file to any upload site
Hi,
If you have an x and y for every point, you can use a TensorDataset() that you then give to the Dataloader.
That way you will be able to do for x, y in tr.
I solved that problem so thank u so much but I got another problem. I try to explain my problem briefly. Here is my simple nn:
class Model(nn.Module):
def __init__(self, D_in, H1, H2, D_out):
super().__init__()
self.linear1 = nn.Linear(D_in, H1)
self.linear2 = nn.Linear(H1, H2)
self.linear3 = nn.Linear(H2, D_out)
def forward(self, x):
x = F.relu(self.linear1(x))
x = F.relu(self.linear2(x))
x = self.linear3(x)
return x
and my datasets names are train_x train y validation_x validation_y
I build these system on keras and it works fine and my keras code:
model = Sequential()
model.add(LSTM(16, input_shape=(1, 3), return_sequences=False))
model.add(Activation('relu'))
model.add(Dense(12))
model.add(Activation('relu'))
model.add(Dense(1))
model.compile(loss='mse',
optimizer='adam',
metrics=['mse'])
model.fit(train_x, train_y,
batch_size=batch_size,
epochs=500,
validation_data=(val_x, val_y),
callbacks = [tensorboard_CB])
net_outputs_LDCC = model.predict(inputs_LDCC_ready, batch_size=batch_size)
so my question is that how can I get .predict() result at PyTorch? Because I want to compare real result and nn results.
Also all of the datasets are 2D
Anyone can help me please?
To get your validation / test predictions, you should set your model to evaluation mode using model.eval(), wrap your code in a with torch.no_grad() block, and iterate your validation or test DataLoader. Have a look at the ImageNet example to see, how the validate method was created.
Depending on your model, you might get the predicted class using:
output = model(data)
preds = torch.argmax(output, 1)
e.g. if you are dealing with a multi-class classification use case and your last layer is a linear layer with nb_classes output neurons.
Thanx u for giving me respond. Firstly, when I look your example code, I just only have array values so I cannot iterate x,y over my loader so how can I implement your example to my code?
As @albanD explained, you could transform the numpy arrays to tensors using:
data = torch.from_numpy(x)
target = torch.from_numpy(y)
and pass it to a TensorDataset:
dataset = TensorDataset(data, target)
loader = DataLoader(
Dataset,
batch_size=1,
shuffle=False,
num_workers=2
)
@ptrblck thx for respond my question. I fixed that problem but now I got this error
Traceback (most recent call last):
File "fev.py", line 227, in <module>
outputs = model(inputs)
File "/usr/local/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__
result = self.forward(*input, **kwargs)
File "fev.py", line 183, in forward
x = F.relu(self.linear1(x))
File "/usr/local/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__
result = self.forward(*input, **kwargs)
File "/usr/local/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 92, in forward
return F.linear(input, self.weight, self.bias)
File "/usr/local/lib/python3.7/site-packages/torch/nn/functional.py", line 1404, in linear
if input.dim() == 2 and bias is not None:
AttributeError: 'list' object has no attribute 'dim'
my nn is:
class Modele(nn.Module):
def __init__(self, D_in, H1, H2, D_out):
super().__init__()
self.linear1 = nn.Linear(D_in, H1)
self.linear2 = nn.Linear(H1, H2)
self.linear3 = nn.Linear(H2, D_out)
def forward(self, x):
x = F.relu(self.linear1(x))
x = F.relu(self.linear2(x))
x = self.linear3(x)
return x
and my calculations are:
batch_size=1000
epochs=5
device=torch.device("cuda:0"if torch.cuda.is_available() else "cpu")
train_x=train_x.view(-1,1,3)
dataset=TensorDataset(train_x,train_y)
tr=torch.utils.data.DataLoader(dataset,batch_size=100,shuffle=False)
val=torch.utils.data.DataLoader(val_x,batch_size=100,shuffle=False)
model=Modele(1,100,65,1).to(device)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
for e in range(epochs):
running_loss = 0.0
running_corrects = 0.0
val_running_loss = 0.0
val_running_corrects = 0.0
for inputs in tr:
outputs = model(inputs)
loss = criterion(outputs, inputs)
_,preds=torch.max(outputs,1)
outputss.append(preds.max().detach().numpy())
losses.append(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
#outputss.append(outputs.detach().numpy())
#print(loss.item())
Is anyone has an idea about my problem?
Your TensorDataset will return the data and target batch, so you could unwrap inputs or assign the return values to separate tensors:
for inputs in loader:
data, target = inputs
outputs = model(data)
loss = criterion(outputs, target)
# or
for data, target in loader:
...
I appreciate you to give me respond. I already did that changes but I got list error which I mentioned above. Do you have any idea of that problem?
Can you check the type of input that you are using in model(input).
Type of input is torch.tensor
I have a new question about the sizes. I do not have dataset like MNIST format [100,1,28,28]. My dataset shape is [23k,1,3] so 23k is not batch size. I just read data from .txt file to array then transform to tensor. My question is that is the size of my dataset well or should i change dimension of it? Because I want to train it efficiently
Essentially, the first dimension should be the batch size, if you are using Dataloader. For image dataset like MNIST, it is of the form [N X C X H X W]
N = batch size
C = channel (R,G,B)
H, W = height and width
If you are not using Dataloader and manually iterating over dataset, you don’t need it.
oh thank you so much for clarification about my problem