LSTM Regression (Many to one)

Hello.
I am trying to fit an lstm model to my data. The training set has 102400 samples, each having 1568 features:

The shape of X_train is: torch.Size([10240, 1, 1568])
The shape of Y_train is: torch.Size([10240, 1])

The target variable for each sequence is a scalar. I break the whole sequence into sequences of length 8, then I use the last output from lstm as the model’s output after passing through a linear layer.

Here is my code:

class RNN(nn.Module):

  def __init__(self,input_size,hidden_size,output_size,num_layers):
    super(RNN, self).__init__()
    self.input_size=input_size
    self.hidden_size=hidden_size
    self.output_size=output_size
    self.num_layers=num_layers

    self.lstm=nn.LSTM(input_size,hidden_size,num_layers)
    self.fc=nn.Linear(hidden_size, output_size)


  def forward(self,sequence):
    #Input shape is (seq_len,batch=1,input_size).
    h_0=torch.zeros(self.num_layers,1,self.hidden_size)
    c_0=torch.zeros(self.num_layers,1,self.hidden_size)
    lstm_out,_=self.lstm(sequence,(h_0,c_0))

    #input shape is (seq_len,batch=1,hidden_size)
    #out shape is (1)
    output=self.fc(lstm_out[-1,:,:]).view(1)

    return output
def train(model,X_train,Y_train,X_test,Y_test,criterion,optimizer,epochs=100):
  preds=torch.zeros([X_test.shape[0],1])
  obs=torch.zeros([X_test.shape[0],1])


  for epoch in range(epochs):
    train_loss=0
    test_loss=0

    for i in range(X_train.shape[0]-batch_size):
      x=Variable(X_train[i:i+seq_len,:,:])
      y=Variable(Y_train[i+seq_len-1,:])
      model.zero_grad()
      yhat=model(x)
      loss=criterion(yhat,y)
      loss.backward()
      optimizer.step()
      train_loss+=loss.item()

    print('Train_'+str(epoch+1)+':  '+str(train_loss))

    for i in range(X_test.shape[0]-batch_size):
      x=X_test[i:i+seq_len,:,:]
      y=Y_test[i+seq_len-1,:]
      yhat=model(x)
      loss=criterion(yhat,y)
      test_loss+=loss.item()

      if epoch==epochs-1:
        preds[i,:]=yhat
        obs[i,:]=y

    print('Test_'+str(epoch+1)+':  '+str(test_loss)+'\n\n')
input_size=X_train.shape[2]    # features
hidden_size=256
output_size=1
num_layers=2
epochs=100

lr=0.001
criterion=nn.MSELoss()
model=RNN(input_size,hidden_size,output_size,num_layers)
optimizer=optim.RMSprop(model.parameters(),lr=lr)

train(model,X_train,Y_train,X_test,Y_test,criterion,optimizer,epochs)

The problem is no matter how complex I define the model, the predictions will not improve (not even for the training set). I have tried increasing lstm layers, changing lr, increasing sequence length, increasing hidden_size, but none worked. I assume there is some problem with my code and if not, I appreciate if you guid me how I can improve it.

Thank you.