Search code examples
pythontensorflowneural-network

Computing the Jacobian matrix in TensorFlow

I am trying to compute the Jacobian matrix in TensorFlow for the following neural network (but it didn't work with my neural network!):

I found Jacobian matrix code on Computing the Jacobian matrix of a neural network in Python.

Unfortunately, it doesn't work with my neural network... The problem message is:

ValueError: Cannot feed value of shape (1, 51000) for Tensor 'dense_1_input:0', which has shape '(?, 6)'

I think that is the problem in the loop function inside jacobian_tensorflow function.

# Importing some Libraries
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
import numpy as np
import statsmodels.api as sm
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
from tqdm import tqdm
import tensorflow as tf

# Simulation of some data
np.random.seed (245)
nobs = 10000

# Definition normal distributed features
x1 = np.random.normal(size=nobs ,scale=1)
x2 = np.random.normal(size=nobs ,scale=1)
x3 = np.random.normal(size=nobs ,scale=1)
x4 = np.random.normal(size=nobs ,scale=1)
x5 = np.random.normal(size=nobs ,scale=1)

# Features
X = np.c_[np.ones((nobs, 1)), x1, x2, x3, x4, x5]


y = np.cos(x1) + np.sin(x2) + 2*x3 + x4 + 0.01*x5 + np.random.normal(size=nobs, scale=0.01)

# Learning rate
LR = 0.05


# Number of neurons
Neuron_Out = 1
Neuron_Hidden1 = 64
Neuron_Hidden2 = 32

# The activation function
Activate_output = 'linear' # For the last layer, I use linear
Activate_hidden = 'relu' # The difference is Hidden-Layer-Neurons were not linearly transformed


# The optimizer
Optimizer = SGD(lr=LR)


# The loss function
loss = 'mean_squared_error'

# Splitting Data
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=77)

## Neural network
from tensorflow import set_random_seed
set_random_seed (245)
# As in Medium Essa
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())

# Initialize the ANN
model_ANN = Sequential()

# Hidden layer: The hidden layer is defined here -> the number of neurons here is 64, 32
# 'input' is 6 (also 1, x1, x2, x3, x4, x5)-> one is the first column in X Matrix
model_ANN.add(Dense(Neuron_Hidden1, activation=Activate_hidden, input_shape=(6,), use_bias=True))
model_ANN.add(Dense(Neuron_Hidden2, activation=Activate_hidden, use_bias=True))

# Output layer: The output layer defnined here
model_ANN.add(Dense(Neuron_Out, activation=Activate_output, use_bias=True))
model_ANN.summary()

# Fit the model
history_ANN=model_ANN.fit(
    x_train, # Training data
    y_train, # Training targets
    epochs=125)


def jacobian_tensorflow(x):
    jacobian_matrix = []
    for m in range(Neuron_Out):
        # We iterate over the M elements of the output vector
        grad_func = tf.gradients(model_ANN.output[:, m], model_ANN.input)
        gradients = sess.run(grad_func, feed_dict={model_ANN.input: x.reshape((1, x.size))})
        jacobian_matrix.append(gradients[0][0, :])

    return np.array(jacobian_matrix)

# Jacobian matrix computation
def jacobian_tensorflow(x):
    jacobian_matrix = []
    for m in range(Neuron_Out):
        # We iterate over the M elements of the output vector
        grad_func = tf.gradients(model_ANN.output[:, m], model_ANN.input)
        gradients = sess.run(grad_func, feed_dict={model_ANN.input: x.reshape((1, x.size))})
        jacobian_matrix.append(gradients[0][0, :])

    return np.array(jacobian_matrix)

jacobian_tensorflow(x_train)

How could I use the Jacobian computation function for my neural network?

Solution

  • I have modified your code to fix the error and now it’s working. There were few errors like a compile statement missing, a function defined twice and forcing the reshape of input for the Dense layer, even though the shape was good for feed_dict in jacobian_tensorflow function. I have added comments in the code for changes.

    Fixed Code -

    %tensorflow_version 1.x
# Importing some Libraries
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
import numpy as np
import statsmodels.api as sm
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
from tqdm import tqdm
import tensorflow as tf
# Simulation of some data
np.random.seed (245)
nobs =10000

# Definition normalverteilte Features
x1= np.random.normal(size=nobs ,scale=1)
x2= np.random.normal(size=nobs ,scale=1)
x3= np.random.normal(size=nobs ,scale=1)
x4= np.random.normal(size=nobs ,scale=1)
x5= np.random.normal(size=nobs ,scale=1)

# Features
X= np.c_[np.ones((nobs ,1)),x1,x2,x3,x4,x5]


y= np.cos(x1) + np.sin(x2) + 2*x3 + x4 + 0.01*x5 + np.random.normal(size=nobs , scale=0.01)

#Learningrate
LR=0.05


# Number of Neurons
Neuron_Out=1
Neuron_Hidden1=64
Neuron_Hidden2=32

#The Activation function
Activate_output='linear' # für letzte Schicht verwende ich linear
Activate_hidden='relu' # unterschied ist Hidden-Layer-Neuronen werden nicht linear transformiert


#The Optimizer
Optimizer= SGD(lr=LR)


# The loss function
loss='mean_squared_error'

# Splitting Data
from sklearn.model_selection import train_test_split
x_train , x_test , y_train , y_test = train_test_split(X, y, test_size =0.15, random_state =77)

## Neural Network
from tensorflow import set_random_seed
set_random_seed (245)
# As in Medium Essa
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())

#Initialize the ANN
model_ANN= Sequential()

# Hidden Layer-> hier wird Hidden Layer definiert-> Anzahl der Neuronen hier sind 64, 32
# input ist 6 (also 1,x1,x2,x3,x4,x5)-> one is the first column in X Matrix
model_ANN.add(Dense(Neuron_Hidden1, activation=Activate_hidden, input_shape=(6,), use_bias=True))
model_ANN.add(Dense(Neuron_Hidden2, activation=Activate_hidden, use_bias=True))

#Output Layer-> hier wird Output-Layer defniniert
model_ANN.add(Dense(Neuron_Out, activation=Activate_output,use_bias=True))
model_ANN.summary()

# Added the compile statement
model_ANN.compile(loss=loss, optimizer=Optimizer, metrics=['accuracy'])

#Fit the model
history_ANN=model_ANN.fit(
x_train, # training data
y_train, # training targets
epochs=125)

#Jacobian matrix computation
def jacobian_tensorflow(x):
    jacobian_matrix = []
    for m in range(Neuron_Out):
        # We iterate over the M elements of the output vector
        grad_func = tf.gradients(model_ANN.output[:, m],model_ANN.input)
        gradients = sess.run(grad_func, feed_dict={model_ANN.input: x})  # Removed x.reshape((1, x.size)) as reshape is not required bcoz dense accepts the shape
        jacobian_matrix.append(gradients[0][0,:])

    return np.array(jacobian_matrix)

jacobian_tensorflow(x_train)

    Output -

    /tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).
  warnings.warn('An interactive session is already active. This can '
Model: "sequential_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
dense_10 (Dense)             (None, 64)                448
_________________________________________________________________
dense_11 (Dense)             (None, 32)                2080
_________________________________________________________________
dense_12 (Dense)             (None, 1)                 33
=================================================================
Total params: 2,561
Trainable params: 2,561
Non-trainable params: 0
_________________________________________________________________
Epoch 1/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.1999 - accuracy: 0.0000e+00
Epoch 2/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0501 - accuracy: 0.0000e+00
Epoch 3/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0277 - accuracy: 0.0000e+00
Epoch 4/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0219 - accuracy: 0.0000e+00
Epoch 5/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0190 - accuracy: 0.0000e+00
Epoch 6/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0114 - accuracy: 0.0000e+00
Epoch 7/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0138 - accuracy: 0.0000e+00
Epoch 8/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0092 - accuracy: 0.0000e+00
Epoch 9/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0072 - accuracy: 0.0000e+00
Epoch 10/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0073 - accuracy: 0.0000e+00
Epoch 11/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0065 - accuracy: 0.0000e+00
Epoch 12/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0070 - accuracy: 0.0000e+00
Epoch 13/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0055 - accuracy: 0.0000e+00
Epoch 14/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0055 - accuracy: 0.0000e+00
Epoch 15/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0055 - accuracy: 0.0000e+00
Epoch 16/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0057 - accuracy: 0.0000e+00
Epoch 17/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0045 - accuracy: 0.0000e+00
Epoch 18/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0052 - accuracy: 0.0000e+00
Epoch 19/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0055 - accuracy: 0.0000e+00
Epoch 20/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0040 - accuracy: 0.0000e+00
Epoch 21/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0045 - accuracy: 0.0000e+00
Epoch 22/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0040 - accuracy: 0.0000e+00
Epoch 23/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0041 - accuracy: 0.0000e+00
Epoch 24/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0044 - accuracy: 0.0000e+00
Epoch 25/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0037 - accuracy: 0.0000e+00
Epoch 26/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0046 - accuracy: 0.0000e+00
Epoch 27/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0042 - accuracy: 0.0000e+00
Epoch 28/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0039 - accuracy: 0.0000e+00
Epoch 29/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0037 - accuracy: 0.0000e+00
Epoch 30/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0042 - accuracy: 0.0000e+00
Epoch 31/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0033 - accuracy: 0.0000e+00
Epoch 32/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 33/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0033 - accuracy: 0.0000e+00
Epoch 34/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0038 - accuracy: 0.0000e+00
Epoch 35/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0035 - accuracy: 0.0000e+00
Epoch 36/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 37/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 38/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 39/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 40/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0028 - accuracy: 0.0000e+00
Epoch 41/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0032 - accuracy: 0.0000e+00
Epoch 42/125
8500/8500 [==============================] - 1s 77us/step - loss: 0.0029 - accuracy: 0.0000e+00
Epoch 43/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0029 - accuracy: 0.0000e+00
Epoch 44/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0029 - accuracy: 0.0000e+00
Epoch 45/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0028 - accuracy: 0.0000e+00
Epoch 46/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0031 - accuracy: 0.0000e+00
Epoch 47/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0025 - accuracy: 0.0000e+00
Epoch 48/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0022 - accuracy: 0.0000e+00
Epoch 49/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0027 - accuracy: 0.0000e+00
Epoch 50/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0028 - accuracy: 0.0000e+00
Epoch 51/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0024 - accuracy: 0.0000e+00
Epoch 52/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0024 - accuracy: 0.0000e+00
Epoch 53/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0023 - accuracy: 0.0000e+00
Epoch 54/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0026 - accuracy: 0.0000e+00
Epoch 55/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0023 - accuracy: 0.0000e+00
Epoch 56/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0020 - accuracy: 0.0000e+00
Epoch 57/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0024 - accuracy: 0.0000e+00
Epoch 58/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0022 - accuracy: 0.0000e+00
Epoch 59/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0021 - accuracy: 0.0000e+00
Epoch 60/125
8500/8500 [==============================] - 1s 78us/step - loss: 0.0022 - accuracy: 0.0000e+00
Epoch 61/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0020 - accuracy: 0.0000e+00
Epoch 62/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0022 - accuracy: 0.0000e+00
Epoch 63/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0022 - accuracy: 0.0000e+00
Epoch 64/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0023 - accuracy: 0.0000e+00
Epoch 65/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0023 - accuracy: 0.0000e+00
Epoch 66/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 67/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0021 - accuracy: 0.0000e+00
Epoch 68/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0020 - accuracy: 0.0000e+00
Epoch 69/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0020 - accuracy: 0.0000e+00
Epoch 70/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 71/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0019 - accuracy: 0.0000e+00
Epoch 72/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 73/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 74/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 75/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0019 - accuracy: 0.0000e+00
Epoch 76/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 77/125
8500/8500 [==============================] - 1s 83us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 78/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0019 - accuracy: 0.0000e+00
Epoch 79/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0019 - accuracy: 0.0000e+00
Epoch 80/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 81/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 82/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0017 - accuracy: 0.0000e+00
Epoch 83/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0017 - accuracy: 0.0000e+00
Epoch 84/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 85/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0017 - accuracy: 0.0000e+00
Epoch 86/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0017 - accuracy: 0.0000e+00
Epoch 87/125
8500/8500 [==============================] - 1s 83us/step - loss: 0.0017 - accuracy: 0.0000e+00
Epoch 88/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 89/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 90/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0018 - accuracy: 0.0000e+00
Epoch 91/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0015 - accuracy: 0.0000e+00
Epoch 92/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 93/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 94/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 95/125
8500/8500 [==============================] - 1s 83us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 96/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 97/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0016 - accuracy: 0.0000e+00
Epoch 98/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0015 - accuracy: 0.0000e+00
Epoch 99/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 100/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0015 - accuracy: 0.0000e+00
Epoch 101/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 102/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0015 - accuracy: 0.0000e+00
Epoch 103/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0013 - accuracy: 0.0000e+00
Epoch 104/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 105/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0013 - accuracy: 0.0000e+00
Epoch 106/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 107/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 108/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 109/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0013 - accuracy: 0.0000e+00
Epoch 110/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 111/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 112/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 113/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0014 - accuracy: 0.0000e+00
Epoch 114/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 115/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 116/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0013 - accuracy: 0.0000e+00
Epoch 117/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0013 - accuracy: 0.0000e+00
Epoch 118/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 119/125
8500/8500 [==============================] - 1s 79us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 120/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 121/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 122/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 123/125
8500/8500 [==============================] - 1s 82us/step - loss: 0.0012 - accuracy: 0.0000e+00
Epoch 124/125
8500/8500 [==============================] - 1s 81us/step - loss: 0.0011 - accuracy: 0.0000e+00
Epoch 125/125
8500/8500 [==============================] - 1s 80us/step - loss: 0.0018 - accuracy: 0.0000e+00
array([[ 0.6434634 , -0.09752402,  0.8342059 ,  1.6331654 ,  0.82901144,
        -0.00917255]], dtype=float32)