Multivariate Input data LSTM training(val_loss is too low)

Hello friend. I’m training multivariate input data using LSTM network.

You can see my data type below.

  • I’m training 64 Input size data.
  • The ‘Evaluation data’ includes Attack data labeled with ‘T’ containing intermediate random bit values.
  • I trained only normal data(only labeled as ‘R’) and evaluate data which has random bit and labeled as ‘T’ as shown in the data example below.
'''
                            Evaluation Data                     ↓Label
0111100000000000000000000010000100000000000000000000000000000000R
0111100000000000000000000010000100000000000000000000000000000000R
1000010011111101001011001000100100000000000000000000000000000000T  #<--- Attack
0111100000000000000000001010000100000000000000000000000000000000R
0111100000000000000000000100000100000000000000000000000000000000R
...
0111100000000000000000001000000100000000000000000000000000000000R
0111100000000000000000000011000100000000000000000000000000000000R
0010101100100010100010100011010100000000000000000000000000000000T  #<--- Attack
0111100000000000000000000100000100000000000000000000000000000000R
0111100000000000000000001100000100000000000000000000000000000000R
...
0111100100000000000000000011000000000000000000000000000000000000R
0111100100000000000000000010000000000000000000000000000000000000R
0101101001000010100100000000110100000000000000000000000000000000T #<--- Attack
0111100100000000000000001011000000000000000000000000000000000000R
0111100100000000000000001010000000000000000000000000000000000000R
...
0111011000000000000000000010111100000000000000000000000000000000R
0110110111000111100000010010011100000000000000000000000000000000T  #<--- Attack
0111011000000000000000000001111100000000000000000000000000000000R
'''
def sliding_windows(data, seq_length):
    '''
        Separate 64bit by 1bit and make it as list
    '''
    x = []
    y = []
    L = len(data)
    for i in range(L - seq_length-1):
        _x = [list(map(int,e[:-1])) for e in data[i:(i+seq_length)]]
        _y = list(map(int, data[i+seq_length][:-1]))
        x.append(_x)
        y.append(_y)

    return np.array(x),np.array(y)

def sliding_windows_eval(data, seq_length):
    '''
        Separate 64bit by 1bit and make it as list
    '''
    x = []
    y = []
    gt = []
    L = len(data)
    for i in range(L - seq_length):
        _x = [list(map(int,e[:-2])) for e in data[i:(i+seq_length)]]
        _y = list(map(int, data[i+seq_length][:-2]))
        x.append(_x)
        y.append(_y)
        gt.append(data[i + seq_length][-2])

    return np.array(x),np.array(y), np.array(gt)

class LSTM(nn.Module):
    def __init__(self):
        super(LSTM, self).__init__()
        self.num_classes = 64  
        self.num_layers = 1    
        self.input_size = 64   
        self.hidden_size = 512 

        self.lstm = nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size,
                            num_layers=self.num_layers, batch_first=True)

        self.fc = nn.Linear(self.hidden_size, self.num_classes)

    def forward(self, x):
        h_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #([1, batch, 512])
        c_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #([1, batch, 512])

        # Propagate input through LSTM
        output, (h_out, _) = self.lstm(x, (h_0, c_0)) # x : [batch, seq, 64], output : [batch, seq, 64]
                                                      # h_out : [1, batch, 512]
        h_out = h_out.view(-1, self.hidden_size)      
        out = self.fc(h_out)
        return out

# Train ------------------------------------------------------------------------------------------ #
with open('./log/%s_normal.log' % ID, 'r') as f:
    training_data = f.readlines()

    x, y = sliding_windows(training_data, seq_length)
    trainX = torch.Tensor(np.array(x))
    trainY = torch.Tensor(np.array(y))
    dataset = TensorDataset(trainX, trainY)  

    dataloader = DataLoader(dataset, batch_size=1000, shuffle=False)

    # optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate)
    optimizer = torch.optim.SGD(lstm.parameters(), lr=learning_rate)

    for epoch in range(num_epochs):
        for batch_idx, samples in enumerate(dataloader):
            x_samples, y_samples = samples

            outputs = lstm(x_samples)
            optimizer.zero_grad()

            # obtain the loss function
            loss = criterion(outputs, y_samples)

            loss.backward()

            optimizer.step()
        if epoch % 10 == 0:
            print("Epoch: %d, loss: %1.5f" % (epoch, loss.item()))

    PATH = 'D:/002_5_OTIDS/log/{}_normal.pt'.format(ID)
    torch.save(lstm.state_dict(), PATH)
# ----------------------------------------------------------------------------------------------- #

# Test ------------------------------------------------------------------------------------------ #
PATH = 'D:/002_5_OTIDS/log/{}_normal.pt'.format(ID)

lstm.load_state_dict(torch.load(PATH))

val_losses = []
with open('./log_fuzzy/%s_fuzzy.log' % ID, 'r') as f:
    test_data = f.readlines()
    x, y, gt = sliding_windows_eval(test_data, seq_length)
    testX = torch.Tensor(np.array(x))
    testY = torch.Tensor(np.array(y))
    dataset = TensorDataset(testX, testY)  
    dataloader = DataLoader(dataset, batch_size=1, shuffle=False)

    with torch.no_grad(): # [21.05.11]  check one by one
        for idx, samples in enumerate(dataloader):
            x_samples, y_samples = samples
            lstm.eval()
            outputs = lstm(x_samples)
            val_loss = criterion(outputs, y_samples)
            val_losses.append(val_loss)
            if gt[idx] == 'T':
                print(val_loss)
# ----------------------------------------------------------------------------------------------- #

My question is

  1. I got val_loss 0.2xxxx which has label ‘T’ and 0.1xxxx which has label ‘R’
    This doesn’t seem to have big difference. Is this due to the nature of my data?
    or My training is wrong??

  2. Can you check my code a little bit? Is there a wrong?

Thanks ahead.
Have a good day.