How to convert Tensor to numpy array inside a map function using tf.py_function

Question

I am trying to create an image augmentation pipeline for an object detection network, where my training examples are augmented as they go into the network. The images and the bounding boxes need to be augmented, but the standard tf.image methods don't work with bounding box data.

All the easy augmentation libraries that work with bounding boxes need numpy arrays but I don't know how to convert my Tensors into numpy arrays inside my .map() function. Even when I wrap my augment function in a tf.py_function call I still get the error AttributeError: 'Tensor' object has no attribute 'numpy' when I try to convert my image via image = image.numpy().

my dataset is loaded via this:

def load_tfrecord_dataset(file_pattern, class_file, size=416):
    LINE_NUMBER = -1
    class_table = tf.lookup.StaticHashTable(tf.lookup.TextFileInitializer(
        class_file, tf.string, 0, tf.int64, LINE_NUMBER, delimiter="\n"), -1)

    files = tf.data.Dataset.list_files(file_pattern)
    dataset = files.flat_map(tf.data.TFRecordDataset)
    return dataset.map(lambda x: tf.py_function(parse_tfrecord(x, class_table, size), [x], tf.float32))
    # return dataset.map(lambda x: parse_tfrecord(x, class_table, size))

this calls my parsing function:

def parse_tfrecord(tfrecord, class_table, size):
    x = tf.io.parse_single_example(tfrecord, IMAGE_FEATURE_MAP)
    x_train = tf.image.decode_jpeg(x['image/encoded'], channels=3)
    x_train = tf.image.resize(x_train, (size, size))

    class_text = tf.sparse.to_dense(
        x['image/object/class/text'], default_value='')
    labels = tf.cast(class_table.lookup(class_text), tf.float32)

    y_train = tf.stack([tf.sparse.to_dense(x['image/object/bbox/xmin']),
                        tf.sparse.to_dense(x['image/object/bbox/ymin']),
                        tf.sparse.to_dense(x['image/object/bbox/xmax']),
                        tf.sparse.to_dense(x['image/object/bbox/ymax']),
                        labels], axis=1)

    x_train, y_train = tf.py_function(augment_images(x_train, y_train), [], tf.uint8)

    paddings = [[0, FLAGS.yolo_max_boxes - tf.shape(y_train)[0]], [0, 0]]
    y_train = tf.pad(y_train, paddings)

    return x_train, y_train

which calls my augment function:

def augment_images(image, boxes):

    image = image.numpy()

    seq = iaa.Sequential([
        iaa.Fliplr(0.5),
        iaa.Flipud(0.5)
    ])

    image, label = seq(image=image, bounding_boxes=boxes)

    return image, label

But no matter which parts of the code I wrap in a tf.py_function or where I try to convert to a numpy array, I always get the same error.

What am I doing wrong?

Answer 1

I was able to recreate your error. This happens only when I try to use numpy() on an image and the error thrown is AttributeError: 'Image' object has no attribute 'numpy', but the numpy() function works perfectly fine in other cases.

In the below simple program, I am reading an image(.jpg file) of bird and later doing tf.image.central_crop to crop the central part of the image. The numpy() worked perfectly fine when we tried to extract string(file path) from the tensor using bytes.decode(path.numpy()), but threw error AttributeError: 'Image' object has no attribute 'numpy' when we tried to use numpy() on image type.

Code to recreate the issue -

%tensorflow_version 2.x
import tensorflow as tf
print(tf.__version__)
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.preprocessing.image import img_to_array, array_to_img
from matplotlib import pyplot as plt
import numpy as np

def load_file_and_process(path):
    image = load_img(bytes.decode(path.numpy()), target_size=(224, 224))
    image = image.numpy()
    image = tf.image.central_crop(image, np.random.uniform(0.50, 1.00))
    return image

train_dataset = tf.data.Dataset.list_files('/content/bird.jpg')
train_dataset = train_dataset.map(lambda x: tf.py_function(load_file_and_process, [x], [tf.float32]))

for f in train_dataset:
  for l in f:
    image = np.array(array_to_img(l))
    plt.imshow(image)

Output -

2.2.0
---------------------------------------------------------------------------
UnknownError                              Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/context.py in execution_mode(mode)
   1985       ctx.executor = executor_new
-> 1986       yield
   1987     finally:

10 frames
UnknownError: AttributeError: 'Image' object has no attribute 'numpy'
Traceback (most recent call last):

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/script_ops.py", line 241, in __call__
    return func(device, token, args)

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/script_ops.py", line 130, in __call__
    ret = self._func(*args)

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py", line 309, in wrapper
    return func(*args, **kwargs)

  File "<ipython-input-22-2aab1a57781b>", line 11, in load_file_and_process
    image = image.numpy()

AttributeError: 'Image' object has no attribute 'numpy'


     [[{{node EagerPyFunc}}]] [Op:IteratorGetNext]

During handling of the above exception, another exception occurred:

UnknownError                              Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/executor.py in wait(self)
     65   def wait(self):
     66     """Waits for ops dispatched in this executor to finish."""
---> 67     pywrap_tfe.TFE_ExecutorWaitForAllPendingNodes(self._handle)
     68 
     69   def clear_error(self):

UnknownError: AttributeError: 'Image' object has no attribute 'numpy'
Traceback (most recent call last):

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/script_ops.py", line 241, in __call__
    return func(device, token, args)

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/script_ops.py", line 130, in __call__
    ret = self._func(*args)

  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py", line 309, in wrapper
    return func(*args, **kwargs)

  File "<ipython-input-22-2aab1a57781b>", line 11, in load_file_and_process
    image = image.numpy()

AttributeError: 'Image' object has no attribute 'numpy'


     [[{{node EagerPyFunc}}]]

Solution - Use img_to_array and array_to_img to convert the image to array and array to image respectively. You need to include from tensorflow.keras.preprocessing.image import img_to_array, array_to_img in your program.

Fixed Code -

%tensorflow_version 2.x
import tensorflow as tf
print(tf.__version__)
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.preprocessing.image import img_to_array, array_to_img
from matplotlib import pyplot as plt
import numpy as np

def load_file_and_process(path):
    image = load_img(bytes.decode(path.numpy()), target_size=(224, 224))
    image = img_to_array(image)
    image = tf.image.central_crop(image, np.random.uniform(0.50, 1.00))
    return image

train_dataset = tf.data.Dataset.list_files('/content/bird.jpg')
train_dataset = train_dataset.map(lambda x: tf.py_function(load_file_and_process, [x], [tf.float32]))

for f in train_dataset:
  for l in f:
    image = np.array(array_to_img(l))
    plt.imshow(image)

Output - 2.2.0

Hope this answers your question. Happy Learning.