TypeError: If no scoring is specified, the estimator passed should have a 'score' method. The estimator <keras.Sequential object at 0x04B6A0> does not

Question

model_cnn = Sequential()
model_cnn.add(Convolution1D(filters=32,kernel_size=2, strides=1, activation='relu', input_shape=(x_train.shape[1], 1)))
model_cnn.add(MaxPooling1D())
model_cnn.add(Flatten())
model_cnn.add(Dense(105))
model_cnn.add(Dropout(0.5))
model_cnn.add(Dense(1, activation="relu"))
model_cnn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model_cnn.summary()
x_train = x_train.todense()
x_test = x_test.todense()
x_train = np.expand_dims(x_train, axis=-1)
x_test = np.expand_dims(x_test, axis=-1)
history_cnn = model_cnn.fit(x_train, y_train,validation_data=(x_test, y_test), epochs=3)

Cross-validation score:

print("Mean of Cross validation score(CNN): ", cross_val_score(model_cnn, x_train, y_train, cv=kfold).mean())

I am finding the cross-validation score of CNN model but I got an error:

Error:

TypeError: If no scoring is specified, the estimator passed should have a 'score' method. The estimator <keras.engine.sequential.Sequential object at 0x00000154DF44B6A0> does not.

Answer 1

You could create a custom scikit-learn estimator that implements a score method.

Here's a fairly minimal example:

from sklearn.base import BaseEstimator, ClassifierMixin
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score


class KerasModel(BaseEstimator, ClassifierMixin):
    def fit(self, X, y):
        model = keras.Sequential([
            keras.Input(shape=X.shape[1]),
            layers.Dense(100),
            layers.Dropout(0.1),
            layers.Dense(1, activation="relu")])
        model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
        model.fit(X, y, epochs=25)
        self.model = model
        return self

    def score(self, X, y):
        return accuracy_score(y, (self.model.predict(X) > 0.5).flatten())

Model creation and training occurs inside fit(), and score() returns the accuracy of using the model for prediction. So it implements everything needed for cross_val_score:

if __name__ == "__main__":
    X, y = make_classification()
    clf = KerasModel()
    print(cross_val_score(clf, X, y))

[0.7  0.85 0.95 0.85 0.8 ]