LSTM outputs flat line

Question

I've been trying to make a simple LSTM network to predict S&P500 next 5 values % change. My NN however outputs almost a completely flat line.

5% future change and the red is the "prediction":

I know I should never check my model in the train set, but this is just a sanity check to find out if it works at all.

sc = MinMaxScaler(feature_range=(0,1))
dataset = dataset[5:-1]
dataset = dataset.dropna()

close = sc.fit_transform(dataset['Close'].values.reshape(-1,1))
volume = sc.fit_transform(dataset['Volume'].values.reshape(-1,1))
pct = sc.fit_transform(dataset['pct5'].values.reshape(-1,1))

close_train = []
volume_train = []
y = []
pc = []
leng = 60

for i in range(leng, len(close)):
  close_train.append(close[i - 60 : i, 0])
  volume_train.append(volume[i - 60 : i, 0])
  y.append(close[i, 0])
  pc.append(pct[i, 0])


close_train = np.array(close_train, dtype=np.float64)
volume_train = np.array(volume_train, dtype=np.float64)
y = np.array(y)
pc = np.array(pc, dtype=np.float64) #This is just adjusted pct in case you got lost

close_train = np.reshape(close_train, (close_train.shape[0], close_train.shape[1], 1))
volume_train = np.reshape(volume_train, (volume_train.shape[0], volume_train.shape[1], 1))


def buildModel(dataLength, labelLength):
  price = Input(shape=(dataLength, 1), name='price')
  volumen = Input(shape=(dataLength, 1), name='volumen')

  priceLayers1 = LSTM(60, return_sequences=True)(price)
  volumeLayers1 = LSTM(60, return_sequences=True)(volumen)

  priceLayers2 = LSTM(60, return_sequences=True)(price)
  volumeLayers2 = LSTM(60, return_sequences=True)(volumen)

  priceLayers3 = LSTM(60, return_sequences=False)(price)
  volumeLayers3 = LSTM(60, return_sequences=False)(volumen)

  output = concatenate(
      [
       price,
       volumen
      ]
  )

  output = Dense(1, activation='linear', name='dense')(output)

  model = Model(
      [
       price,
       volumen
      ],
      [
       output
      ]
  )
  opt = tf.keras.optimizers.Adam(learning_rate=0.001)
  model.compile(optimizer=opt, loss='mse')
  print(output)
  return model


rnn = buildModel(60, 4)
hist = rnn.fit(
    [
     close_train,
     volume_train
    ],
    [
     pc
    ],
    epochs = 100,
    batch_size=50
)

nsamples, nx, ny = close_train.shape
test_close = close_train.reshape((nsamples,nx*ny))

test_vol = volume_train.reshape((nsamples,nx*ny))

pred = rnn.predict([test_close[0, :60], test_vol[0, :60]])
print(pred_dim)
pred1 = sc.inverse_transform(pred_dim)


final = []
for i in range(0, len(pred1)+60):
  if i <60:
    final.append(None)
    continue
  final.append(pred1[i-60, 0])

plt.figure(figsize=(30,20))
plt.plot(dataset['pct5'])
plt.plot(final, c='r')
plt.axvline(60, c='r')

print(final)

PS: I don't expect it to work accurately since it's practically impossible, but I'd like it to work at all so I can move on :)

Answer 1

The model you show in your question at the moment is a linear regression of the inputs.

i.e.

output = Dense(1, activation='linear', name='dense')(concatenate[price, volume])

I doubt that it is possible to have the model do anything better that predict an average... it probably set the weights as close to 0 as possible and the bias to the average of the signal.

You probably meant to write something like:

def buildModel(dataLength, labelLength):
  price = Input(shape=(dataLength, 1), name='price')
  volumen = Input(shape=(dataLength, 1), name='volumen')

  priceLayers1 = LSTM(60, return_sequences=True)(price)
  volumeLayers1 = LSTM(60, return_sequences=True)(volumen)

  priceLayers2 = LSTM(60, return_sequences=True)(priceLayers1)
  volumeLayers2 = LSTM(60, return_sequences=True)(volumeLayers1)

  priceLayers3 = LSTM(60, return_sequences=False)(priceLayers2)
  volumeLayers3 = LSTM(60, return_sequences=False)(volumeLayers2)

  x = Concatenate()([priceLayers3, volumeLayers3])
  output = Dense(1, activation='linear', name='dense')(x)
  model = tf.keras.Model([price, volumen], output)
  model.compile(optimizer='adam', loss='mse')
  return model