Search code examples
pythontensorflowmachine-learningkeraskeras-layer

trainable_variables attribute of the custom layer in Keras returns empty list


I tried to build my own custom layer in tensorflow/keras that enforces the layer to be symmetric and what I ended up with is the following:

import tensorflow as tf
from tensorflow.python.framework.ops import enable_eager_execution
enable_eager_execution()

class MyDenseLayer(tf.keras.layers.Layer):
    def __init__(self, num_outputs):
        super(MyDenseLayer, self).__init__()
        self.num_outputs = num_outputs

    def build(self, input_shape):
        X = tf.random.uniform([int(input_shape[-1]),self.num_outputs],minval=0,maxval=1,dtype=tf.dtypes.float32,)
        k = tf.Variable(X, name="kernel")
        self.kernel = 0.5 * (k+tf.transpose(k))

    def call(self, input):
        return tf.matmul(input, self.kernel)

layer = MyDenseLayer(5)
print(layer(tf.ones([3, 5])))
print(layer.trainable_variables)

So far, so good. What I don't understand this: why does the last line

print(layer.trainable_variables)

give me an empty list:

[]

I thought that layer.trainable_variables would show me what my matrix looks like so that I could check whether it is symmetric or not.


Solution

  • You need to add variables using add_weight and then call build() method to create this variable. Alternatively, instead of calling build() directly you can pass an input (as you do in your question) and it will call implicitly the build() method.

    import tensorflow as tf
    from tensorflow.python.framework.ops import enable_eager_execution
    enable_eager_execution()
    
    class MyDenseLayer(tf.keras.layers.Layer):
        def __init__(self, num_outputs):
            super(MyDenseLayer, self).__init__()
            self.num_outputs = num_outputs
        def build(self, input_shape):
            def initializer(*args, **kwargs):
                X = tf.random.uniform([int(input_shape[-1]),self.num_outputs],minval=0,maxval=1,dtype=tf.dtypes.float32,)
                kernel = 0.5 * (X+tf.transpose(X))
                return kernel
            self.kernel = self.add_weight(name='kernel',
                                          shape=(input_shape[-1], self.num_outputs),
                                          initializer=initializer,
                                          trainable=True)
            super(MyDenseLayer, self).build(input_shape)
    
        def call(self, input_):
            return tf.matmul(input_, self.kernel)
    
    layer = MyDenseLayer(5)
    layer.build((5, )) # <-- example of input shape
    print(layer.trainable_variables)
    # [<tf.Variable 'kernel:0' shape=(5, 5) dtype=float32, numpy=
    # array([[0.04476559, 0.8396935 , 0.42732996, 0.75126845, 0.7109113 ],
    #        [0.8396935 , 0.46617424, 0.71654373, 0.5770991 , 0.38461512],
    #        [0.42732996, 0.71654373, 0.75249636, 0.28733748, 0.6064501 ],
    #        [0.75126845, 0.5770991 , 0.28733748, 0.9417101 , 0.61572695],
    #        [0.7109113 , 0.38461512, 0.6064501 , 0.61572695, 0.6960379 ]],
    #       dtype=float32)>]