Improving Multi-Class classification model?

Question

I'm creating a CNN for skin lesion classification. I recently added a weighter loss function to my model to try and improve its accuracy, but even with the new weighted losses, my model still only achieved around 65% accuracy. There aren't any errors. How can I improve on my model?

(hopefully) all the relevant code is below:

My model code:

#classification

def classi(input_shape):
    inputs = layers.Input(shape=input_shape)
    x = layers.Conv2D(64, 3, padding="same")(inputs)
    x = layers.Activation("relu")(x)
    x = layers.BatchNormalization()(x)
    #classi layers
    for filters in [96, 128, 256, 320, 512]:#, 1024, 2048]: #change # of filters??
        x = layers.Conv2D(filters, 3, padding="same")(x)
        x = layers.Activation("relu")(x)
        x = layers.BatchNormalization()(x)

        x = layers.Conv2D(filters, 3, padding="same")(x)
        x = layers.Activation("relu")(x)
        x = layers.BatchNormalization()(x)

        x = layers.MaxPool2D(3, strides=2, padding="same")(x)

    #output
    x = layers.Dropout(rate=0.1)(x)
    x = layers.Flatten()(x)
    x = layers.Dense(128, activation="relu")(x)
    #x = layers.Dense(64, activation="relu")(x)
    #x = layers.Dense(16, activation="sigmoid")(x)

    output = layers.Dense(7, activation=None)(x)

    model = k.Model(inputs=inputs, outputs=output, name="classification")
    return model

classification = classi((256,256,3))
classification.summary()

classification.save_weights("classification.h5")

My weighted loss function (cross entropy loss but with weights)

#weighted binary loss
def get_weights(labels):
    cols = len(labels.columns)-2 #assumes 1 column for image ids
    pos_freqs = []
    neg_freqs = []
    pos_weights = []
    neg_weights = []
    for i in range(cols):
        pos_freqs.append(np.mean(labels[labels.columns[i+1]].tolist())) #get column values and sum
        neg_freqs.append(1-pos_freqs[i]) 
        pos_weights.append(neg_freqs[i]) 
        neg_weights.append(pos_freqs[i])
    
    return pos_weights, neg_weights


def weighted_cross_entropy_loss(y_true, y_pred):
    pos_weights, neg_weights = get_weights(pd.read_csv(cls_train_gt))
    #get frequencies to calculate weights
    loss = 0.0
    #print(k.backend.cast(-(neg_weights[0]*(1-y_true[:, 0])), 'float16'))
    for i in range(len(pos_weights)):
        loss += k.backend.mean(k.backend.cast(-(neg_weights[i]*(1-y_true[:, i])), 'float16')
                                 * k.backend.cast(k.backend.log((1-y_pred[:, i])), 'float16')
                                 + (k.backend.cast(pos_weights[i]*y_true[:, i], 'float16')  
                                    * k.backend.cast(k.backend.log((y_pred[:, i])), 'float16')))
    return loss

This is my code for loading my dataset:

#For loading classification labels and images.

def load_images_and_labels(images_path, labels_path, batch_size, image_shape, verbose=False):
    ds_images = []
    ds_labels = []
    data_indexes = []
    labels = pd.read_csv(labels_path)
    images = os.listdir(images_path)
    if verbose:
        print(f"loading images from {images_path} and labels from {labels_path}")
    for i in range(batch_size):
        random_index = np.random.randint(0, len(images)-2)
        if random_index >= len(images):
            random_index -=1
        img = cv2.imread(os.path.join(images_path, images[random_index]))
        #print(random_index)
        #print(len(labels.columns))
        row = labels.iloc[random_index, 1:]

        if img is not None and row is not None:
            if random_index not in data_indexes:
                data_indexes.append(random_index)
                ds_images.append(np.array(cv2.resize(img, dsize=image_shape)))
                ds_labels.append(row.values)
    return np.array(ds_images).astype(np.int16), np.array(ds_labels).astype(np.int16)

And here is my code for training the model:

datagen = ImageDataGenerator(rescale=1./255,
                             rotation_range=0.1,
                             horizontal_flip=True,
                             vertical_flip=True,
                             )

classification.load_weights('classification.h5') #reset weights
optimizer = tf.keras.optimizers.SGD(learning_rate=0.2)
classification.compile(optimizer=optimizer, loss=weighted_cross_entropy_loss, metrics=["binary_accuracy", 'MeanSquaredError', 'AUC']) 

callback_list = [tf.keras.callbacks.EarlyStopping(patience=1.5)] #can adjust to improve accuracy
batch_size=16
spe = 4 #steps per epoch
epochs = 80 
seed = 123

cls_val = r'validation/ISIC2018_Task3_Validation_Input/'
cls_val_gt = "validation_ground_truth/ISIC2018_Task3_Validation_GroundTruth/ISIC2018_Task3_Validation_GroundTruth.csv"

cls_train = r'train/ISIC2018_Task3_Training_Input/'#r"classi/ISIC2018_Task3_Training_Input/ISIC2018_Task3_Training_Input/" 
cls_train_gt = 'train_ground_truth/ISIC2018_Task3_Training_GroundTruth/ISIC2018_Task3_Training_GroundTruth.csv'#("classi/ISIC2018_Task3_Training_GroundTruth/ISIC2018_Task3_Training_GroundTruth/ISIC2018_Task3_Training_GroundTruth.csv")

#organize_images_to_classes(class_train_gt, class_train)

class_val = r"classi/ISIC2018_Task3_Validation_Input/ISIC2018_Task3_Validation_Input/" 
class_val_gt = pd.read_csv("classi/ISIC2018_Task3_Validation_GroundTruth/ISIC2018_Task3_Validation_GroundTruth/ISIC2018_Task3_Validation_GroundTruth.csv")
organize_images_to_classes(class_val_gt, class_val)
"""


for i in range(epochs):
    train_ds, train_gt = load_images_and_labels(cls_train, cls_train_gt, batch_size, (256,256), True)
    val_ds, val_gt = load_images_and_labels(cls_val, cls_val_gt, batch_size, (256,256), True)
    
    #print(train_ds)
    #print(train_gt)

    print(f"train_ds len: {len(train_ds)}, train labels len: {len(train_gt)}")
    cls_train_gen = datagen.flow(x=train_ds, y=train_gt, seed=seed, batch_size=batch_size)
    val_train_gen = datagen.flow(x=val_ds, y=val_gt, seed=seed, batch_size=batch_size)

    history = classification.fit(x=cls_train_gen.x, y=cls_train_gen.y, steps_per_epoch=spe, callbacks=callback_list, verbose=1)#, validation_data=val_dataset, validation_batch_size=16)
    print(f"--------------- Done epoch {i+1} -----------------")

classification.save_weights("final_class.h5")

Answer 1

First let's fix potential areas for bugs. Overall, prefer using native Tensorflow API implementations because most likely are correctly implemented.

• Switch to using CategoricalCrossentropy loss as you have multiple classes you are predicting for.
• The model.fit supports setting the class_weight to each class, which would achieve a similar weighted loss effect
• Make sure your model output has the softmax activation set, otherwise it returns logits (which you could work with if set from_logits=True in the loss function)
• Make sure the output layer has the same number of classes as in your dataset

I am saying these points because you use weighted binary loss and yet your model has 7 nodes for output. If it is a binary task, I suggest using BinaryCrossentropy loss function, which also supports class weights, in this case your model should have 1 output node.

---

Suggestion for improvements:

• Selecting the metric: Accuracy is not the best metric to use for classification tasks, please read this answer. In addition, use the classification report, ROC-AUC curve, F1 metric to interpret the model performance.
• Handling Imbalance: there are other ways of handling imbalance in data, apart from weighted loss, such as resampling (over/under sampling) and data augmentation (especially for images).
• Regularization: Your model can get quite big, so it can overfit to the data, using techniques like dropout and weight decay might help.
• Transfer Learning: try using a pre-trained model like VGG, ResNet or other as a starting point. Fine-tune the model on your task.