I'm trying to train a multi-output Keras model using a custom data generator. The model suppose to classify the mammography images by BIRADS categories(1-5) and density(A-D). I expect model to return two labels: birads category and density. The model should use two features. While running the model.fit() method, I'm encountering a ValueError related to incompatible shapes between my model's output and the ground truth labels. Specifically, the error is: ValueError: Shapes (None, 5) and (None, 1) are incompatible. Please find the error details and my code below.
I have verified that the shapes of the inputs, outputs, and weights are what I expect them to be within the generator. I have checked the model summary to ensure that the output shapes of my model match the shapes of the target labels. I have tried to convert batch_labels_density into Int in order to overcome the error. Looks I have 1D dimensions arrays (batch_labels_birads, batch_labels_density). Moreover, I cannot use to_categorical() and one-hot encode my labels as I have multi-classificational labels with sparse_categorical_crossentropy.
Why am I getting this error and how to resolve it?
ValueError Traceback (most recent call last)
<ipython-input-5-e97964b44762> in <cell line: 132>()
130 model.summary()
131
--> 132 history = model.fit(
133 my_data_generator(
134 training_data,
1 frames
/usr/local/lib/python3.10/dist-packages/keras/engine/training.py in tf__train_function(iterator)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
16 except:
17 do_return = False
ValueError: in user code:
File "/usr/local/lib/python3.10/dist-packages/keras/engine/training.py", line 1284, in train_function *
return step_function(self, iterator)
File "/usr/local/lib/python3.10/dist-packages/keras/engine/training.py", line 1268, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.10/dist-packages/keras/engine/training.py", line 1249, in run_step **
outputs = model.train_step(data)
File "/usr/local/lib/python3.10/dist-packages/keras/engine/training.py", line 1055, in train_step
return self.compute_metrics(x, y, y_pred, sample_weight)
File "/usr/local/lib/python3.10/dist-packages/keras/engine/training.py", line 1149, in compute_metrics
self.compiled_metrics.update_state(y, y_pred, sample_weight)
File "/usr/local/lib/python3.10/dist-packages/keras/engine/compile_utils.py", line 605, in update_state
metric_obj.update_state(y_t, y_p, sample_weight=mask)
File "/usr/local/lib/python3.10/dist-packages/keras/utils/metrics_utils.py", line 77, in decorated
update_op = update_state_fn(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/keras/metrics/base_metric.py", line 140, in update_state_fn
return ag_update_state(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/keras/metrics/confusion_metrics.py", line 1485, in update_state **
return metrics_utils.update_confusion_matrix_variables(
File "/usr/local/lib/python3.10/dist-packages/keras/utils/metrics_utils.py", line 674, in update_confusion_matrix_variables
y_pred.shape.assert_is_compatible_with(y_true.shape)
!pip install opencv-python
!pip install pydicom
import numpy as np
import pydicom
import cv2
def my_data_generator(df, sample_weights_birads, sample_weights_density, batch_size=150):
# img_gen = ImageDataGenerator(rescale=1. / 255.)
mapping_dict_density = {'A': 0, 'B': 1, 'C': 2, 'D': 3}
while True:
# Select files (IDs) and labels for the batch
batch_indices = np.random.choice(a=len(df), size=batch_size)
batch_df = df.iloc[batch_indices]
# Initialize batch arrays
batch_images = []
batch_labels_birads = []
batch_labels_density = []
batch_features = []
batch_weights_birads = []
batch_weights_density = []
for i, original_idx in enumerate(batch_indices):
row = batch_df.iloc[i]
img_id = row['image_id']
study_id = row['study_id']
img_path = f".....dicom"
img = pydicom.dcmread(img_path).pixel_array
# Resize image based on the height and width from CSV if needed
def conditional_resize(img, target_height=3518, target_width=2800):
pass # image resizing is happening here
img = conditional_resize(img)
# Append to batch
batch_images.append(img)
batch_labels_birads.append(row['breast_birads'])
batch_labels_density.append(mapping_dict_density.get(row['breast_density'], -1))
batch_features.append([row['laterality'], row['view_position']])
batch_weights_birads.append(sample_weights_birads[original_idx])
batch_weights_density.append(sample_weights_density[original_idx])
batch_images = np.array(batch_images)
batch_labels_birads = np.squeeze(np.array(batch_labels_birads))
batch_labels_density = np.squeeze(np.array(batch_labels_density))
batch_features = np.array(batch_features)
batch_weights_birads = np.array(batch_weights_birads)
batch_weights_density = np.array(batch_weights_density)
print('\n birads_output', batch_labels_birads, '\n density_output', batch_labels_density)
print("\n Shape of training image_input:", batch_images.shape,
"\n Shape of training features:", batch_features.shape,
"\n Shape of training birads labels:", batch_labels_birads.shape,
"\n Shape of training birads weights:", batch_weights_birads.shape,
"\n Shape of training density labels:", batch_labels_density.shape,
"\n Shape of training density weights:", batch_weights_density.shape,
'\n batch_labels_birads', type(batch_labels_birads), 'batch_weights_density', type(batch_weights_density),
'\n batch_labels_birads.shape', type(batch_labels_birads.shape), 'batch_weights_density.shape', type(batch_weights_density.shape))
yield {'image_input': batch_images, 'feature_input': batch_features}, \
{'birads_output': batch_labels_birads, 'density_output': batch_labels_density}, \
{'birads_output': batch_weights_birads, 'density_output': batch_weights_density}
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Concatenate
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.utils.class_weight import compute_class_weight
from sklearn.utils import class_weight
import collections
from collections import Counter
num_channels = 1 # Assuming grayscale images
standard_height, standard_width = 3518, 2800
image_input = Input(shape=(standard_height, standard_width, num_channels), name="image_input")
# Define CNN layers
x = Conv2D(32, (3, 3), activation='relu')(image_input)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
# Flatten and dense layers
flatten = Flatten()(x)
# Define tabular data input layer for two features: 'laterality' and 'view_position'
feature_input = Input(shape=(2,), name='feature_input') # Two features
# Concatenate flattened image data and feature data
merged = Concatenate()([flatten, feature_input])
new_layer = Dense(12, activation='relu')(merged) # or any number of units you want
birads_output = Dense(5, activation='softmax', name='birads_output')(new_layer)
density_output = Dense(4, activation='softmax', name='density_output')(new_layer)
# Complete the model
model = Model(inputs=[image_input, feature_input], outputs=[birads_output, density_output])
model.compile(optimizer='adam',
loss={'birads_output': 'sparse_categorical_crossentropy',
'density_output': 'sparse_categorical_crossentropy'},
metrics={'birads_output': ['accuracy', tf.keras.metrics.AUC(name='auc_birads')],
'density_output': ['accuracy', tf.keras.metrics.AUC(name='auc_density')]
})
checkpoint = ModelCheckpoint(filepath='/content/drive/MyDrive/Colab/Callbacks/First_Attempt_Batches',
save_weights_only=True,
save_freq=150) # Save after every 150 images (i.e., one batch)
def get_sample_weights(y):
counter = Counter(y)
max_val = float(max(counter.values()))
sample_weights = np.array([max_val / counter[i] for i in y])
return sample_weights
y_train_birads = training_data['breast_birads'].values
y_train_density = training_data['breast_density'].values
y_val_birads = validation_data['breast_birads'].values
y_val_density = validation_data['breast_density'].values
# Assuming y_train_birads and y_train_density contain your actual training labels
sample_weights_birads_train = get_sample_weights(y_train_birads)
sample_weights_density_train = get_sample_weights(y_train_density)
sample_weights_birads_val = get_sample_weights(y_val_birads)
sample_weights_density_val = get_sample_weights(y_val_density)
# Map the class weights to the individual samples in the dataset
sample_weights_train = {'birads_output': sample_weights_birads_train,
'density_output': sample_weights_density_train}
sample_weights_val = {'birads_output': sample_weights_birads_val,
'density_output': sample_weights_density_val}
print('\n sample_weights_val', sample_weights_val,'\n sample_weights_train', sample_weights_train )
print("\n Shape of model output:", model.output_shape,
"\n Shape of image_input:", image_input.shape,
"\n Shape of feature_input:", feature_input.shape,
"\n Shape of birads label:", y_train_birads.shape,
"\n Shape of birads weight:", sample_weights_birads_train.shape,
"\n Shape of density label:", y_train_density.shape,
"\n Shape of density weight:", sample_weights_density_train.shape,
)
model.summary()
history = model.fit(
my_data_generator(
training_data,
batch_size=150,
# birads_output =
sample_weights_birads=sample_weights_birads_train,
sample_weights_density=sample_weights_density_train,
),
# steps_per_epoch=len(training_data) // 150,
epochs=2,
validation_data=my_data_generator(
validation_data,
batch_size=32,
sample_weights_birads=sample_weights_birads_val,
sample_weights_density=sample_weights_density_val,
),
# validation_steps=len(validation_data) // 32,
callbacks=[checkpoint]
)
While execution I see these printings:
sample_weights_val {'birads_output': array([14.56060606, 14.56060606, 2.53562005, ..., 1. ,
1. , 1. ]), 'density_output': array([1., 1., 1., ..., 1., 1., 1.])}
sample_weights_train {'birads_output': array([ 2.94959552, 2.94959552, 2.94959552, ..., 18.66141732,
2.94959552, 2.94959552]), 'density_output': array([1., 1., 1., ..., 1., 1., 1.])}
Shape of model output: [(None, 5), (None, 4)]
Shape of image_input: (None, 3518, 2800, 1)
Shape of feature_input: (None, 2)
Shape of birads label: (14000,)
Shape of birads weight: (14000,)
Shape of density label: (14000,)
Shape of density weight: (14000,)
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
image_input (InputLayer) [(None, 3518, 2800, 0 []
1)]
conv2d (Conv2D) (None, 3516, 2798, 320 ['image_input[0][0]']
32)
max_pooling2d (MaxPooling2D) (None, 1758, 1399, 0 ['conv2d[0][0]']
32)
conv2d_1 (Conv2D) (None, 1756, 1397, 18496 ['max_pooling2d[0][0]']
64)
max_pooling2d_1 (MaxPooling2D) (None, 878, 698, 64 0 ['conv2d_1[0][0]']
)
flatten (Flatten) (None, 39222016) 0 ['max_pooling2d_1[0][0]']
feature_input (InputLayer) [(None, 2)] 0 []
concatenate (Concatenate) (None, 39222018) 0 ['flatten[0][0]',
'feature_input[0][0]']
dense (Dense) (None, 12) 470664228 ['concatenate[0][0]']
birads_output (Dense) (None, 5) 65 ['dense[0][0]']
density_output (Dense) (None, 4) 52 ['dense[0][0]']
==================================================================================================
Total params: 470,683,161
Trainable params: 470,683,161
Non-trainable params: 0
birads_output [1 1 1 2 1 1 2 1 2 2 1 1 1 1 2 1 2 1 1 1 1 2 1 1 1 1 2 1 2 1 2 1 1 1 1 1 2
2 1 2 2 1 2 1 1 1 2 4 1 2 1 1 1 1 1 4 2 1 3 1 1 2 1 2 1 1 1 1 1 2 1 1 4 3
2 1 1 3 2 1 2 1 2 1 1 1 1 2 1 2 2 2 1 1 1 2 1 1 2 1 2 1 2 1 2 1 2 1 1 1 1
2 1 3 1 3 4 1 1 1 1 1 1 1 2 1 1 1 2 2 3 1 1 2 2 1 1 1 1 3 1 1 1 1 2 1 1 1
5 2]
density_output [3 2 2 2 2 2 3 2 2 2 1 2 2 2 2 1 2 2 2 1 2 2 2 2 2 2 2 2 2 2 3 2 2 2 2 3 3
2 2 2 2 2 2 2 2 2 3 2 3 2 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 2 3 2 2 3 2 2 2 2
2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 2 2 2 1 2 2 2 2 2 2 2 2 2 3
2 2 2 2 3 2 3 2 2 2 2 2 3 2 2 2 1 2 2 2 2 2 2 1 2 3 1 2 2 3 2 2 2 2 2 2 2
2 2]
Shape of training image_input: (150, 3518, 2800)
Shape of training features: (150, 2)
Shape of training birads labels: (150,)
Shape of training birads weights: (150,)
Shape of training density labels: (150,)
Shape of training density weights: (150,)
batch_labels_birads <class 'numpy.ndarray'> batch_weights_density <class 'numpy.ndarray'>
batch_labels_birads.shape <class 'tuple'> batch_weights_density.shape <class 'tuple'>
Epoch 1/2
Eventually, I've reshaped two of my output labels to the shape (batch_size, 1).
batch_labels_birads = batch_labels_birads.reshape(-1, 1)
batch_labels_density = batch_labels_density.reshape(-1, 1)