I am implementing a neural network and I would like to assess its performance with cross validation. Here is my current code:
def recall_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def precision_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
def f1_m(y_true, y_pred):
precision = precision_m(y_true, y_pred)
recall = recall_m(y_true, y_pred)
return 2*((precision*recall)/(precision+recall+K.epsilon()))
def build_model():
hiddenLayers = 1
neurons = 100
#hidden_neurons = int(train_x.shape[0]/(3*(neurons+1)))
hidden_neurons = 500
opt = optimizers.Adam(learning_rate=0.00005, amsgrad=False)
model = Sequential()
model.add(Dense(units=neurons, activation="relu", input_shape=(15,)))
model.add(Dense(units=2*hidden_neurons, activation="relu", input_shape=(18632,)))
model.add(Dense(units=4, activation="softmax"))
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc',f1_m,precision_m, recall_m])
return model
x = df[['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]
y = df[['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]
print(y)
#enforce, this is gone wrong somewhere
y = y.replace(False, 0)
y = y.replace(True, 1)
ep = 1
#fit = model.fit(train_x, train_y, epochs=ep, verbose=1)
#pred = model.predict(test_x)
#loss, accuracy, f1_score, precision, recall = model.evaluate(test_x, test_y, verbose=0)
classifier = KerasClassifier(build_fn=build_model, batch_size=10, epochs=ep)
accuracies = cross_val_score(estimator=classifier, X=x, y=y, cv=10, scoring="f1_macro", verbose=5)
I am using cross_val_score and tried to use a different metric in the function itself than accuracy but I get the error
ValueError: Classification metrics can't handle a mix of multilabel-indicator and binary targets
and read here confusion matrix error "Classification metrics can't handle a mix of multilabel-indicator and multiclass targets" that I need to un-one hot encode the output, before scoring, but I couldn't find any way to do that with this function.
Is there a better way to implement multiple scorings than writing the entire process myself? As you can see I already have the scorings implemented and they work as expected during training, but I can't seem to extract the information because of cross_val_score
Edit:
I've ran just one iteration, with the following code:
train, test = train_test_split(df, test_size=0.1, shuffle=True)
train_x = train[['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]
train_y = train[['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]
test_x = test[['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]
test_y = test[['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]
test_y = test_y.replace(False, 0).replace(True,1)
train_y = train_y.replace(False, 0).replace(True,1)
ep = 500
model = build_model()
print("Train y")
print(train_y)
print("Test y")
print(test_y)
model.fit(train_x, train_y, epochs=1, verbose=1)
pred = model.predict(test_x)
print(pred)
loss, accuracy, f1_score, precision, recall = model.evaluate(test_x, test_y, verbose=0)
This yields the following output:
Train y
wait-categ-none wait-categ-short wait-categ-medium wait-categ-long
4629 1 0 0 0
7643 0 1 0 0
4425 0 1 0 0
10548 1 0 0 0
14180 1 0 0 0
... ... ... ... ...
13661 1 0 0 0
10546 1 0 0 0
1966 1 0 0 0
5506 0 1 0 0
10793 1 0 0 0
[15632 rows x 4 columns]
Test y
wait-categ-none wait-categ-short wait-categ-medium wait-categ-long
10394 0 1 0 0
3804 0 1 0 0
15136 0 1 0 0
7050 1 0 0 0
30 0 1 0 0
... ... ... ... ...
12040 0 1 0 0
4184 0 1 0 0
12345 1 0 0 0
12629 0 1 0 0
664 1 0 0 0
[1737 rows x 4 columns]
Pred
[[2.63620764e-01 5.09552181e-01 1.72765702e-01 5.40613122e-02]
[5.40941073e-07 9.99827385e-01 1.72021420e-04 5.32279255e-11]
[5.91083081e-05 9.97556090e-01 2.38463446e-03 1.01058276e-07]
...
[2.69533932e-01 3.99731129e-01 2.22193986e-01 1.08540975e-01]
[5.87045122e-03 9.67754781e-01 2.62637101e-02 1.11028130e-04]
[2.32783407e-01 4.53738511e-01 2.31750652e-01 8.17274228e-02]]
I've copied the output as it is.
I've been experimenting with @desertnaut 's answer however because I have a multi class problem, I experienced problems not even with the loop itself but the np.argmax() line. After googling I did not find any way to resolve it easily so I ended up (on this user's advice) implementing CV by hand. It was a bit more complicated because I am using a pandas dataframe (and the code can definitely be cleaned up further) but here is the working code:
ep = 120
df_split = np.array_split(df, 10)
test_part = 0
acc = []
f1 = []
prec = []
recalls = []
while test_part < 10:
model = build_model()
train_x = []
train_y = []
test_x = []
test_y = []
print("CV Fold, with test partition i = " , test_part)
for i in range(10):
#on first iter that isnt a test part then set the train set to this
if len(train_x) == 0 and not i == test_part:
train_x = df_split[i][['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]
train_y = df_split[i][['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]
#terminate immediately
continue
#if current is not a test partition then concat with previous version
if not i == test_part:
train_x = pd.concat([train_x, df_split[i][['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]], axis=0)
train_y = pd.concat([train_y, df_split[i][['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]], axis=0)
#set this to test partition
else:
test_x = df_split[i][['start-sin', 'start-cos', 'start-sin-lag', 'start-cos-lag', 'prev-close-sin', 'prev-close-cos', 'prev-length', 'state-lag', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']]
test_y = df_split[i][['wait-categ-none', 'wait-categ-short', 'wait-categ-medium', 'wait-categ-long']]
#enforce
train_y = train_y.replace(False, 0)
train_y = train_y.replace(True, 1)
test_y = test_y.replace(False, 0)
test_y = test_y.replace(True, 1)
#fit
model.fit(train_x, train_y, epochs=ep, verbose=1)
pred = model.predict(test_x)
#score
loss, accuracy, f1_score, precision, recall = model.evaluate(test_x, test_y, verbose=0)
#save
acc.append(accuracy)
f1.append(f1_score)
prec.append(precision)
recalls.append(recall)
test_part += 1
print("CV finished.\n")
print("Mean Accuracy")
print(sum(acc)/len(acc))
print("Mean F1 score")
print(sum(f1)/len(f1))
print("Mean Precision")
print(sum(prec)/len(prec))
print("Mean Recall rate")
print(sum(recalls)/len(recalls))