Tensorboard does not show any scalar summary from estimator

Question

Following the instructions on tf custom estimator

I have created a cnn estimator and tried to train it. While training, i initialized tensorboard and was hoping to see some visualizations about training steps. However, tensorboard only showed the graph of my custom estimator but none of the scalar values i have defined.

Here's roughly what I have in code

def model_fn(features, labels, mode, params=None):
    tf.logging.set_verbosity(tf.logging.INFO)
    n_classes = params['n_classes']
    base_learning_rate = params['learning_rate']
    decay_rate = params['decay_rate']
    embedding_dim = params['embedding_dim']

    x = VGG_block1(features, (3, 3), 64, name='block1_1')
    x = VGG_block1(x, (3, 3), 128, name='block1_2')
    x = VGG_block1(x, (3, 3), 256, name='block1_3', regularizer=tf.contrib.layers.l1_regularizer(.1))
    x = VGG_block2(x, (3, 3), 512, name='block2_4')
    x = VGG_block2(x, (3, 3), 1024, name='block2_5')
    x = conv2d(x, 512, (5, 5), padding='valid', normalizer_fn=batch_norm, activation_fn=tf.nn.leaky_relu,
               weights_initializer=he_uniform())
    x = flatten(x)
    embedding = fully_connected(x, embedding_dim)
    logits = fully_connected(embedding, n_classes)

    # make predictions
    predictions = {
        'classes': tf.argmax(logits, axis=1, name='classes'),
        'probabilities': tf.nn.softmax(logits, name='softmax'),
        'embeddings':embedding
    }

    # if we are in prediction mode
    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

    # otherwise define losses for training
    c_loss, center = center_loss(embedding, labels, .9, n_classes)
    xent_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))
    total_loss = xent_loss + 0.5 * c_loss

    # evaluation methods
    accuracy, update_op = tf.metrics.accuracy(labels=labels, predictions=predictions['classes'], name='accuracy')
    batch_acc = tf.reduce_mean(tf.cast(tf.equal(tf.cast(labels, tf.int64), predictions['classes']), tf.float32))
    tf.summary.scalar('batch_acc', batch_acc)
    tf.summary.scalar('streaming_acc', update_op)
    tf.summary.scalar('total_loss', total_loss)
    tf.summary.scalar('center_loss', c_loss)
    tf.summary.scalar('xent_loss', xent_loss)

    # training mode
    if mode  == tf.estimator.ModeKeys.TRAIN:
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        global_step = tf.Variable(0, trainable=False)
        global_step_op = tf.assign(global_step, global_step + 1)
        learning_rate = tf.train.exponential_decay(base_learning_rate, global_step, 8000, decay_rate, staircase=True)
        optimizer = tf.train.AdamOptimizer(learning_rate)
        with tf.control_dependencies(update_ops+[global_step_op]):
            objective = optimizer.minimize(total_loss)

        return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=objective)

    eval_metric_ops = {
        'accuracy': (accuracy, update_op)
    }
    return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, eval_metric_ops=eval_metric_ops)

X_train, X_test, y_train, y_test = load_data()

epochs = 10
batch_size = 64
n_classes = len(classes)

model_params = {'n_classes':n_classes,
                'learning_rate':0.0001,
                'decay_rate':0.5,
                 'embedding_dim':128}
model_dir = 'output'
face_classifier = tf.estimator.Estimator(model_fn=model_fn, params=model_params, model_dir=model_dir)

My Tensorflow version is 1.12.0

Edit Forgot to mention I was using eager execution for this exercise, for unknown reasons that was the cause of this bug

Answer 1

as was mentioned in the edit, disabling eager execution solved the problem