I am following the tutorial 'Federated Learning for Image Classification', but using my own dataset and resnet50. I got this error, when running iterative_process.next.
I believe it was caused by tf.data.Dataset.from_generator
here is my code:
par1_train_data_dir = './par1/train'
par2_train_data_dir = './par2/train'
input_shape = (img_height, img_width, 3)
img_gen = ImageDataGenerator(preprocessing_function=preprocess_input)
ds_par1 = tf.data.Dataset.from_generator(
img_gen.flow_from_directory, args=[par1_train_data_dir,(img_height,img_width)],
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size,img_height,img_width,3], [batch_size,num_classes])
)
ds_par2 = tf.data.Dataset.from_generator(
img_gen.flow_from_directory, args=[par2_train_data_dir,(img_height,img_width)],
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size,img_height,img_width,3], [batch_size,num_classes])
)
dataset_dict={}
dataset_dict['1'] = ds_par1
dataset_dict['2'] = ds_par2
def create_tf_dataset_for_client_fn(client_id):
return dataset_dict[client_id]
train_data = tff.simulation.ClientData.from_clients_and_fn(['1','2'],create_tf_dataset_for_client_fn)
def make_federated_data(client_data, client_ids):
return [client_data.create_tf_dataset_for_client(x)
for x in client_ids]
federated_train_data = make_federated_data(train_data, train_data.client_ids)
images, labels = next(img_gen.flow_from_directory(par1_train_data_dir,batch_size=batch_size,target_size=(img_height,img_width)))
sample_batch = (images,labels)
def create_compiled_keras_model():
pretrain_model = tf.keras.applications.resnet.ResNet50(include_top=False, weights='imagenet',
input_tensor=tf.keras.layers.Input(shape=(img_height,
img_width, 3)), pooling=None)
Inp = Input((img_height, img_width, 3))
x = pretrain_model(Inp)
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=Inp, outputs=predictions,name='resnet50_transfer')
model.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer=tf.keras.optimizers.SGD(learning_rate=0.02))
return model
def model_fn():
keras_model = create_compiled_keras_model()
return tff.learning.from_compiled_keras_model(keras_model, sample_batch)
iterative_process = tff.learning.build_federated_averaging_process(model_fn)
state = iterative_process.initialize()
NUM_ROUNDS = 11
for round_num in range(2, NUM_ROUNDS):
state, metrics = iterative_process.next(state, federated_train_data)
print('round {:2d}, metrics={}'.format(round_num, metrics))
I got the error
InvalidArgumentError: TypeError: endswith first arg must be bytes or a tuple of bytes, not str
here is more information
InvalidArgumentError Traceback (most recent call last)
<ipython-input-48-b01b66dc0dcd> in <module>
1 NUM_ROUNDS = 11
2 for round_num in range(2, NUM_ROUNDS):
----> 3 state, metrics = iterative_process.next(state, federated_train_data)
4 print('round {:2d}, metrics={}'.format(round_num, metrics))
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/utils/function_utils.py in __call__(self, *args, **kwargs)
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/utils/function_utils.py in pack_args(parameter_type, args, kwargs, context)
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/utils/function_utils.py in pack_args_into_anonymous_tuple(args, kwargs, type_spec, context)
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/reference_executor.py in ingest(self, arg, type_spec)
627 intrinsic_defs.FEDERATED_MEAN.uri:
628 self._federated_mean,
--> 629 intrinsic_defs.FEDERATED_BROADCAST.uri:
630 self._federated_broadcast,
631 intrinsic_defs.FEDERATED_COLLECT.uri:
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/reference_executor.py in to_representation_for_type(value, type_spec, callable_handler)
239 else:
240 return [
--> 241 to_representation_for_type(v, type_spec.member, callable_handler)
242 for v in value
243 ]
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/reference_executor.py in <listcomp>(.0)
239 else:
240 return [
--> 241 to_representation_for_type(v, type_spec.member, callable_handler)
242 for v in value
243 ]
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/reference_executor.py in to_representation_for_type(value, type_spec, callable_handler)
198 if tf.executing_eagerly():
199 return [
--> 200 to_representation_for_type(v, type_spec.element, callable_handler)
201 for v in value
202 ]
~/miniconda3/lib/python3.6/site-packages/tensorflow_federated/python/core/impl/reference_executor.py in <listcomp>(.0)
197 if isinstance(value, tf.data.Dataset):
198 if tf.executing_eagerly():
--> 199 return [
200 to_representation_for_type(v, type_spec.element, callable_handler)
201 for v in value
~/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/iterator_ops.py in __next__(self)
620
621 def __next__(self): # For Python 3 compatibility
--> 622 return self.next()
623
624 def _next_internal(self):
~/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/iterator_ops.py in next(self)
664 """Returns a nested structure of `Tensor`s containing the next element."""
665 try:
--> 666 return self._next_internal()
667 except errors.OutOfRangeError:
668 raise StopIteration
~/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/iterator_ops.py in _next_internal(self)
649 self._iterator_resource,
650 output_types=self._flat_output_types,
--> 651 output_shapes=self._flat_output_shapes)
652
653 try:
~/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_dataset_ops.py in iterator_get_next_sync(iterator, output_types, output_shapes, name)
2671 _ctx = _context._context or _context.context()
2672 if _ctx is not None and _ctx._thread_local_data.is_eager:
-> 2673 try:
2674 _result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
2675 _ctx._context_handle, _ctx._thread_local_data.device_name,
~/miniconda3/lib/python3.6/site-packages/six.py in raise_from(value, from_value)
InvalidArgumentError: TypeError: endswith first arg must be bytes or a tuple of bytes, not str
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 464, in get_iterator
self._next_id += 1
KeyError: 2
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/ops/script_ops.py", line 221, in __call__
"""
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 585, in generator_py_func
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 466, in get_iterator
# NOTE(mrry): Explicitly create an array of `np.int64` because implicit
File "/root/miniconda3/lib/python3.6/site-packages/keras_preprocessing/image/image_data_generator.py", line 540, in flow_from_directory
interpolation=interpolation
File "/root/miniconda3/lib/python3.6/site-packages/keras_preprocessing/image/directory_iterator.py", line 126, in __init__
classes, filenames = res.get()
File "/root/miniconda3/lib/python3.6/multiprocessing/pool.py", line 644, in get
raise self._value
File "/root/miniconda3/lib/python3.6/multiprocessing/pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "/root/miniconda3/lib/python3.6/site-packages/keras_preprocessing/image/utils.py", line 216, in _list_valid_filenames_in_directory
for root, fname in valid_files:
File "/root/miniconda3/lib/python3.6/site-packages/keras_preprocessing/image/utils.py", line 172, in _iter_valid_files
if fname.lower().endswith('.tiff'):
TypeError: endswith first arg must be bytes or a tuple of bytes, not str
[[{{node PyFunc}}]] [Op:IteratorGetNextSync]
my environment
tensorboard==1.15.0
tensorcache==0.4.2
tensorflow==1.15.2
tensorflow-addons==0.6.0
tensorflow-estimator==1.15.1
tensorflow-federated==0.4.0
UPDATE
I've upgraded tf==2.1.0 and tff==0.12.0, the error disappeared, but I got another error.
It seems that the generator reaches the last batch and does not match the input shape.
But ImageDataGenerator does not need to set drop_remainder.Is there anything wrong with my code?
InvalidArgumentError: ValueError: `generator` yielded an element of shape (50, 224, 224, 3) where an element of shape (64, 224, 224, 3) was expected.
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/ops/script_ops.py", line 236, in __call__
ret = func(*args)
File "/root/miniconda3/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 825, in generator_py_func
"of shape %s was expected." % (ret_array.shape, expected_shape))
ValueError: `generator` yielded an element of shape (50, 224, 224, 3) where an element of shape (64, 224, 224, 3) was expected.
[[{{node PyFunc}}]]
[[import/StatefulPartitionedCall_1/ReduceDataset]] [Op:__inference_wrapped_function_277930]
Function call stack:
wrapped_function
TensorFlow Federated version 0.4.0 has only been tested to work with TensorFlow version 1.13.1 (see the TensorFlow Federated Compatibility table). Is it possible to upgrade to the newest 0.12.0 version of TensorFlow Federated?
I believe your analysis is correct, the code is setting up a dataset from a generator that expects to produce exactly batch_size batches, but is receiving a batch of different size from the img_gen.flow_from_directory generator.
During setup of the dataset, passing None for the batchsize to indicate that the batch size may be variable could possibly work.
Concretely, change these lines:
... = tf.data.Dataset.from_generator(
img_gen.flow_from_directory, args=[par1_train_data_dir,(img_height,img_width)],
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size,img_height,img_width,3], [batch_size,num_classes])
)
To:
... = tf.data.Dataset.from_generator(
img_gen.flow_from_directory, args=[par1_train_data_dir,(img_height,img_width)],
output_types=(tf.float32, tf.float32),
output_shapes=([None,img_height,img_width,3], [None,num_classes])
)