How to get the Confusion matrix, Precision, Recall, F1 score, ROC curve, and AUC graph?

Question

I built and trained the CNN Model but didn't know how to get the Confusion matrix, Precision, Recall, F1 score, ROC curve, and AUC graph.

I'm not splitting the dataset by sklearn. Manually Split dataset into train, test and validation.

import warnings
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
from tensorflow.keras import models
from tensorflow.keras import optimizers
from tensorflow.keras.preprocessing.image import load_img

train_datagen = ImageDataGenerator(rescale = 1./255,
                                  shear_range = 0.2,
                                   zoom_range = 0.2,
                                   horizontal_flip = True,
                                   validation_split=0.2)

training_set = train_datagen.flow_from_directory(
    '/mnt/batch/tasks/shared/LS_root/mounts/clusters/saba19ec117/code/Users/saba19ec117/Project/train', 
    target_size = (250, 250), 
    batch_size = 32, 
    class_mode = 'binary')

validation_generator = train_datagen.flow_from_directory(
    '/mnt/batch/tasks/shared/LS_root/mounts/clusters/saba19ec117/code/Users/saba19ec117/Project/test', 
    target_size = (250, 250), 
    batch_size = 32, 
    class_mode = 'binary')

model = models.Sequential()

model.add(layers.Conv2D(32, (3, 3), padding = 'valid', activation = 'relu', input_shape=(250, 250, 3)))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))

model.add(layers.Conv2D(64, (3, 3), padding = 'valid', activation = 'relu'))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))

model.add(layers.Dense(310, activation = 'relu'))

model.add(layers.Dropout(0.45))

model.add(layers.Dense(270, activation = 'relu'))

model.add(layers.Dense(1, activation = 'sigmoid'))

model.compile(loss = 'binary_crossentropy',
             optimizer = optimizers.Adam(learning_rate = 1e-4),
             metrics = ['accuracy'])
model.fit(training_set, 
      steps_per_epoch = 30,
          epochs = 25,
          validation_data = validation_generator,
          validation_steps = 9)


plt.plot(model.history.history['accuracy'])
plt.plot(model.history.history['val_accuracy'])
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()

    # Plot training & validation loss values
plt.plot(model.history.history['loss'])
plt.plot(model.history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()

Can you tell, me how to get the Confusion matrix, Precision, Recall, F1 score, ROC curve, and AUC graph to my code?

Answer 1

You can add the below snippet in your code at the end. You should have num_of_test_samples and batch_size as defined variables.

# Import libraries
from sklearn.metrics import classification_report, confusion_matrix

predictions = model.predict_generator(validation_generator, num_of_test_samples // batch_size+1, verbose=0)
y_pred = np.argmax(predictions, axis=1)

print('Confusion Matrix')
print(confusion_matrix(validation_generator.classes, y_pred))

print('Classification Report')
target_names=list(train_generator.class_indices.keys())
print(classification_report(validation_generator.classes, y_pred, target_names=target_names))