Why does a penalty not change the predictions of a Keras model?

Question

I recently came across this when trying to implement a custom loss function. The following two loss functions produce exactly the same results even though in the second one a large random value is added to what the loss function returns and reproducibility in the jupyter notebook is ensured. Any ideas why that is?

def customLoss1():

  def binary_crossentropy1(y_true, y_pred): 

    bin_cross = tf.keras.losses.BinaryCrossentropy()
    bce = K.mean(bin_cross(y_true, y_pred))

    return bce

  return binary_crossentropy1

def customLoss2():

  def binary_crossentropy2(y_true, y_pred): 

    bin_cross = tf.keras.losses.BinaryCrossentropy()
    bce = K.mean(bin_cross(y_true, y_pred)) + tf.random.normal([], mean=0.0, stddev=10.0)

    return bce

  return binary_crossentropy2

Answer 1

Your error must be somewhere else, because the loss functions you posted do generate different results:

import tensorflow as tf
tf.random.set_seed(11)

def binary_crossentropy1(y_true, y_pred): 

  bin_cross = tf.keras.losses.BinaryCrossentropy(from_logits=True)
  bce = tf.keras.backend.mean(bin_cross(y_true, y_pred))
  return bce

def binary_crossentropy2(y_true, y_pred): 

  bin_cross = tf.keras.losses.BinaryCrossentropy(from_logits=True)
  bce = tf.keras.backend.mean(bin_cross(y_true, y_pred)) + tf.random.normal([], mean=0.0, stddev=10.0)
  return bce

y_true = tf.constant([0, 1, 0, 0])
y_pred = tf.constant([-18.6, 0.51, 2.94, -12.8])
print(binary_crossentropy1(y_true, y_pred))
print(binary_crossentropy2(y_true, y_pred))

tf.Tensor(0.865458, shape=(), dtype=float32)
tf.Tensor(-14.364014, shape=(), dtype=float32)