How to get weight on each layers

Question

I'm trying to get the input weight on each layer, including the lstm 1, lstm 2, and weight after the attention layer, and want to display them using a heatmap. But when I run the code, the following error appears. What happened? Because the layer exists. Here is the code:

model.add(LSTM(32, input_shape=(n_timesteps,n_features), return_sequences=True))
#print weights
print(model.get_layer(LSTM).get_weights()[0])
model.add(LSTM(32, input_shape=(n_timesteps,n_features), return_sequences=True))
model.add(Dropout(0.1))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(Dense(n_outputs, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit network
model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=verbose)
# evaluate model
_, accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=0)

Attention layer:

class attention(Layer):
def __init__(self, return_sequences=True):
    self.return_sequences = return_sequences
    super(attention,self).__init__()
def build(self, input_shape):
    self.W=self.add_weight(name="att_weight", shape=(input_shape[-1],1),
                           initializer="normal")
    self.b=self.add_weight(name="att_bias", shape=(input_shape[1],1),
                           initializer="zeros")
    super(attention,self).build(input_shape)
def call(self, x):
    e = K.tanh(K.dot(x,self.W)+self.b)
    a = K.softmax(e, axis=1)
    output = x*a
    if self.return_sequences:
        return output
    return K.sum(output, axis=1)

And this is the error that appears:

ValueError: No such layer: <class 'keras.layers.recurrent_v2.LSTM'>. Existing layers are [<keras.layers.recurrent_v2.LSTM object at 0x7f7b5c215910>].

Answer 1

You can get certain layer weights using model.layers after defining your whole model:

import tensorflow as tf
import seaborn as sb
import matplotlib.pyplot as plt

class attention(tf.keras.layers.Layer):
  def __init__(self, return_sequences=True):
      self.return_sequences = return_sequences
      super(attention,self).__init__()
  def build(self, input_shape):
      self.W=self.add_weight(name="att_weight", shape=(input_shape[-1],1),
                            initializer="normal")
      self.b=self.add_weight(name="att_bias", shape=(input_shape[1],1),
                            initializer="zeros")
      super(attention,self).build(input_shape)
  def call(self, x):
      e = tf.keras.backend.tanh(tf.keras.backend.dot(x,self.W)+self.b)
      a = tf.keras.backend.softmax(e, axis=1)
      output = x*a
      if self.return_sequences:
          return output
      return tf.keras.backend.sum(output, axis=1)

model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(32, input_shape=(5,10), return_sequences=True))
model.add(tf.keras.layers.LSTM(32, return_sequences=True))
model.add(tf.keras.layers.Dropout(0.1))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(tf.keras.layers.Dense(3, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

trainx = tf.random.normal((25, 5, 10))
trainy = tf.random.uniform((25, 3), maxval=3)
model.fit(trainx, trainy, epochs=5, batch_size=4)


lstm1_weights = model.layers[0].get_weights()[0]
lstm2_weights = model.layers[1].get_weights()[0]
attention_weights = model.layers[3].get_weights()[0]

heat_map = sb.heatmap(lstm1_weights)
plt.show()

Model: "sequential_16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 lstm_24 (LSTM)              (None, 5, 32)             5504      
                                                                 
 lstm_25 (LSTM)              (None, 5, 32)             8320      
                                                                 
 dropout_12 (Dropout)        (None, 5, 32)             0         
                                                                 
 attention_12 (attention)    (None, 32)                37        
                                                                 
 dense_12 (Dense)            (None, 3)                 99        
                                                                 
=================================================================
Total params: 13,960
Trainable params: 13,960
Non-trainable params: 0
_________________________________________________________________
Epoch 1/5
7/7 [==============================] - 4s 10ms/step - loss: 5.5033 - accuracy: 0.4400
Epoch 2/5
7/7 [==============================] - 0s 8ms/step - loss: 5.4899 - accuracy: 0.5200
Epoch 3/5
7/7 [==============================] - 0s 9ms/step - loss: 5.4771 - accuracy: 0.4800
Epoch 4/5
7/7 [==============================] - 0s 9ms/step - loss: 5.4701 - accuracy: 0.5200
Epoch 5/5
7/7 [==============================] - 0s 8ms/step - loss: 5.4569 - accuracy: 0.5200

                              

If you want to see how the weights of your layers change during training, you should define a callback as shown in this post.