Tensorflow tf.metrics.mean_iou returns 0

I want to use the function tf.metrics.mean_iou for an FCN for semantic segmentation. It only works, if the confusion matrix is calculated before the IoU, otherwise it returns 0.

Here my examples:

This example returns the correct value 0.66071427

import tensorflow as tf
import numpy as np

y_pred0 = np.array([   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ],   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ]    ])
y_pred1 = tf.constant(y_pred0)
y_pred2 = tf.argmax(y_pred1, axis=3)

y_label = np.array([[[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]], [[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]]])
y_label2 = tf.constant(y_label)

iou, conf_mat = tf.metrics.mean_iou(y_label2, y_pred2, num_classes=2)

sess = tf.Session()
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())

sess.run(conf_mat)
res = sess.run(iou)

print(res)

This example returns 0

import tensorflow as tf
import numpy as np

def intersection_over_union(prediction, labels):
    pred = tf.argmax(prediction, axis=3)
    labl = tf.constant(labels)
    iou, conf_mat = tf.metrics.mean_iou(labl, pred, num_classes=2)
    return iou

y_pred0 = np.array([   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ],   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ]    ])
y_pred1 = tf.constant(y_pred0)

y_label = np.array([[[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]], [[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]]])

mean__iou = intersection_over_union(y_pred1, y_label)

sess = tf.Session()
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())

res = sess.run(mean__iou)

print(res)

It would be very nice to have a function calculating mean IoU without initializing all variables in it. Is there any way to fix the second example? I think that the problem is in calculating IoU and confusion matrix simultanously and i didn't found another way for it as by running them separately by Session().

Thanks

Solution

You need to run the update operation that tf.metrics.mean_iou returns before getting the iou value from the tensor.

Here is the fixed code:

import tensorflow as tf
import numpy as np

def intersection_over_union(prediction, labels):
    pred = tf.argmax(prediction, axis=3)
    labl = tf.constant(labels)
    iou, conf_mat = tf.metrics.mean_iou(labl, pred, num_classes=2)
    return iou, conf_mat

y_pred0 = np.array([   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ],   [ [[0.9,0.1],[0.9,0.1],[0.9,0.1],[0.9,0.1]], [[0.2,0.8],[0.2,0.8],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]], [[0.9,0.1],[0.9,0.1],[0.2,0.8],[0.9,0.1]] ]    ])
y_pred1 = tf.constant(y_pred0)

y_label = np.array([[[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]], [[1,0,1,0],[1,0,1,0],[0,0,1,0],[0,0,1,0]]])

mean__iou, conf_mat = intersection_over_union(y_pred1, y_label)

sess = tf.Session()
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())

sess.run([conf_mat])
res = sess.run(mean__iou)

print(res)

Which returns the correct value: 0.66071427