I am trying to do feature visualisation/optimisation with yolo v4. As implementation in Python I use this Github repo and add a layer, which is identical to the input layer, right after the input layer. For inserting this layer I use a function that was posted here (I use the function described in the accepted answer). Sadly this function did not work out of the box as some arguments are not handed over to some layers and I had to explicitly hand them over. Hence I changed the last else-statement and have this code now:
def main(argv):
source_model = helper.get_model()
model = insert_layer_nonseq(source_model, "conv2d", opt_layer_factory, position="before", regex=False)
model.summary()
def opt_layer_factory():
return Conv2D(3, 1, activation='relu', input_shape=(416, 416, 3), name="opt_layer")
def insert_layer_nonseq(model, layer_regex, insert_layer_factory, insert_layer_name=None, position='after', regex=True):
# Auxiliary dictionary to describe the network graph
network_dict = {'input_layers_of': {}, 'new_output_tensor_of': {}}
# Set the input layers of each layer
for layer in model.layers:
for node in layer._outbound_nodes:
layer_name = node.outbound_layer.name
if layer_name not in network_dict['input_layers_of']:
network_dict['input_layers_of'].update(
{layer_name: [layer.name]})
else:
network_dict['input_layers_of'][layer_name].append(layer.name)
# Set the output tensor of the input layer
network_dict['new_output_tensor_of'].update(
{model.layers[0].name: model.input})
# Iterate over all layers after the input
model_outputs = []
for layer in model.layers[1:]:
# Determine input tensors
layer_input = [network_dict['new_output_tensor_of'][layer_aux]
for layer_aux in network_dict['input_layers_of'][layer.name]]
if len(layer_input) == 1:
layer_input = layer_input[0]
# Insert layer if name matches the regular expression or if names match
regex_or_name = re.match(layer_regex, layer.name) if regex else (layer_regex == layer.name)
if regex_or_name:
if position == 'replace':
x = layer_input
elif position == 'after':
x = layer(layer_input)
elif position == 'before':
x = layer_input
else:
raise ValueError('position must be: before, after or replace')
new_layer = insert_layer_factory()
x = new_layer(x)
print('New layer: {} Old layer: {} Type: {}'.format(new_layer.name,
layer.name, position))
if position == 'before':
x = layer(x)
else:
if bool(re.match(r"tf.concat*", layer.name)):
x = layer(layer_input, -1)
elif bool(re.match(r"tf.__operators__.getitem*", layer.name)):
x = layer(layer_input)
elif layer.name == "tf.nn.max_pool2d":
x = layer(layer_input, ksize=13, padding="SAME", strides=1)
elif layer.name == "tf.nn.max_pool2d_1":
x = layer(layer_input, ksize=9, padding="SAME", strides=1)
elif layer.name == "tf.nn.max_pool2d_2":
x = layer(layer_input, ksize=5, padding="SAME", strides=1)
elif layer.name == "tf.image.resize":
x = layer(layer_input, size=(26, 26), method="bilinear")
elif layer.name == "tf.image.resize_1":
x = layer(layer_input, size=(52, 52), method="bilinear")
elif isinstance(layer_input, list):
x = layer(*layer_input)
else:
x = layer(layer_input)
# Set new output tensor (the original one, or the one of the inserted
# layer)
network_dict['new_output_tensor_of'].update({layer.name: x})
# Save tensor in output list if it is output in initial model
if layer_name in model.output_names:
model_outputs.append(x)
return Model(inputs=model.inputs, outputs=model_outputs)
Now I am stuck at a layer called "tf.operators.getitem()", which seems to need additional arguments but I cannot find out which ones.
Any help would be appreciated :)
I found a workaround to not call each layer again with the correct input. Instead of using the original model or layers I took the configuration, added my layer with the needed specifications to the configuration and created a tf model from this configuration. Next I iterated through the layers of the new model and set the weights to the ones from the original model (expect for the added layer of course).
This does not solve the error of my original question but suffices for my project. And overall I think this is a nicer solution when adding a layer to an existing model.