I am using Keras Functional API to introduce a custom layer, GaussianLayer, that returns a list of two elements (see call method below):
import tensorflow as tf
from keras.layers import Input, Dense
from keras.models import Model
from tensorflow.keras.layers import Layer
def gaussian_loss(y_true, y_pred):
"""
y_true is a scalar (float)
y_pred is a tensor [mu, sigma]
"""
print(y_pred.shape)
return tf.reduce_mean(0.5*tf.log(y_pred[1]) + 0.5*tf.div(tf.square(y_true - y_pred[0]), y_pred[0])) + 1
class GaussianLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(GaussianLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.kernel_1 = self.add_weight(name='kernel',
shape=(30, self.output_dim),
initializer='uniform',
trainable=True)
self.kernel_2 = self.add_weight(name='kernel',
shape=(30, self.output_dim),
initializer='uniform',
trainable=True)
self.bias_1 = self.add_weight(name='bias',
shape=(self.output_dim),
initializer='zeros',
trainable=True)
self.bias_2 = self.add_weight(name='bias',
shape=(self.output_dim),
initializer='zeros',
trainable=True)
super(GaussianLayer, self).build(input_shape)
def call(self, x):
output_mu = K.dot(x, self.kernel_1) + self.bias_1
output_sig = K.dot(x, self.kernel_2) + self.bias_2
output_sig_pos = K.log(1 + K.exp(output_sig)) + 1e-06
return [output_mu, output_sig_pos]
def compute_output_shape(self, input_shape):
return (input_shape[0], self.output_dim)
# This returns a tensor
inputs = Input(shape=(1,))
x = Dense(24, activation='relu')(inputs)
x = Dense(30, activation='relu')(x)
predictions = GaussianLayer(1)(x)
model = Model(inputs, predictions)
model.compile(loss=gaussian_loss, optimizer='adam')
model.fit(train_x, train_y, epochs=600)
Unfortunately this code fails with an AttributeError:
----------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-200-e93218491799> in <module>()
49 x = Dense(24, activation='relu')(inputs)
50 x = Dense(30, activation='relu')(x)
---> 51 predictions = GaussianLayer(1)(x)
52
53 model = Model(inputs, predictions)
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
783 if in_deferred_mode or build_graph and have_all_keras_metadata(inputs):
784 inputs, outputs = self._set_connectivity_metadata_(
--> 785 inputs, outputs, args, kwargs)
786 if context.executing_eagerly():
787 return outputs
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in _set_connectivity_metadata_(self, inputs, outputs, args, kwargs)
895 kwargs.pop('mask', None) # `mask` should not be serialized.
896 self._add_inbound_node(
--> 897 input_tensors=inputs, output_tensors=outputs, arguments=kwargs)
898 return inputs, outputs
899
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in _add_inbound_node(self, input_tensors, output_tensors, arguments)
1076 input_tensors=input_tensors,
1077 output_tensors=output_tensors,
-> 1078 arguments=arguments)
1079
1080 # Update tensor history metadata.
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __init__(self, outbound_layer, inbound_layers, node_indices, tensor_indices, input_tensors, output_tensors, arguments)
1747 # For compatibility with external Keras, we use the deprecated
1748 # accessor here.
-> 1749 layer.outbound_nodes.append(self)
1750 # For compatibility with external Keras, we use the deprecated
1751 # accessor here.
AttributeError: 'Dense' object has no attribute 'outbound_nodes'
I know I need to use the Functional API in order to return a list to be passed to gaussian_loss, but unfortunately this does not seem to work either.
First, don't mix keras and tensorflow.keras modules. Only use one of them in your code.
Second, fix the shape of both bias variables by adding a , at the end to make them a tuple, i.e. shape=(self.output_dim,).
Third, if your custom layer returns a list of two tensors as output then the output shape must be consistent with this, i.e.:
def compute_output_shape(self, input_shape):
return [(input_shape[0], self.output_dim), (input_shape[0], self.output_dim)]