I am performing a hyperparameter tuning optimization (hyperopt) tasks with sklearn on a Keras models. I am trying to optimize KerasClassifiers using the Sklearn cross-validation, Some code follows:
def create_model():
model = Sequential()
model.add(
Dense(output_dim=params['units1'],
input_dim=features_.shape[1],
kernel_initializer="glorot_uniform"))
model.add(Activation(params['activation']))
model.add(Dropout(params['dropout1']))
model.add(BatchNormalization())
...
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
Now what I want to do is to pass the Hyperopt params to KerasClassifier using the following way
def objective(params, n_folds=N_FOLDS):
"""Objective function for Hyperparameter Optimization"""
# Keep track of evals
global ITERATION
ITERATION += 1
clf = KerasClassifier(build_fn=create_model,**params)
start = timer()
# Perform n_folds cross validation
cv_results = cross_val_score(clf,
features_,
labels,
cv=5
).mean()
run_time = timer() - start
# Loss must be minimized
loss = -cv_results
# Dictionary with information for evaluation
return {
'loss': loss,
'params': params,
'iteration': ITERATION,
'train_time': run_time,
'status': STATUS_OK
}
I define the search space as:
space = {'units1': hp.choice('units1', [64, 128, 256, 512]),
'units2': hp.choice('units2', [64, 128, 256, 512]),
'dropout1': hp.choice('dropout1', [0.25, 0.5, 0.75]),
'dropout2': hp.choice('dropout2', [0.25, 0.5, 0.75]),
'batch_size': hp.choice('batch_size', [10, 20, 40, 60, 80, 100]),
'nb_epochs': hp.choice('nb_epochs', [10, 50, 100]),
'optimizer': opt_search_space,
'activation': 'relu' }
Run optimization
best = fmin(fn = objective, space = space, algo = tpe.suggest,
max_evals = MAX_EVALS, trials = bayes_trials, rstate = np.random.RandomState(50))
But it fails giving this error:
ValueError: activation is not a legal parameter
What's the right way to do it?
Make the hyper parameter as the input parameters for create_model function. Then you can feed params dict. Also change the key nb_epochs into epochs in the search space. Read more about the other valid parameter here.
Try the following simplified example of your's.
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from tensorflow.keras import Sequential
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers import Dense, Dropout
import time
def timer():
now = time.localtime(time.time())
return now[5]
X, y = make_classification(n_samples=1000, n_classes=2,
n_informative=4, weights=[0.7, 0.3],
random_state=0)
Defining keras model:
def create_model(units1, activation, dropout):
model = Sequential()
model.add(Dense(units1,
input_dim=X.shape[1],
kernel_initializer="glorot_uniform",
activation=activation))
model.add(Dropout(dropout))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def objective(params, n_folds=2):
"""Objective function for Hyperparameter Optimization"""
# Keep track of evals
global ITERATION
ITERATION += 1
clf = KerasClassifier(build_fn=create_model,**params)
start = timer()
# Perform n_folds cross validation
cv_results = cross_val_score(clf, X, y,
cv=5,
).mean()
run_time = timer() - start
# Loss must be minimized
loss = -cv_results
# Dictionary with information for evaluation
return {
'loss': loss,
'params': params,
'iteration': ITERATION,
'train_time': run_time,
'status': STATUS_OK
}
from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
space = {'units1': hp.choice('units1', [12, 64]),
'dropout': hp.choice('dropout1', [0.25, 0.5]),
'batch_size': hp.choice('batch_size', [10, 20]),
'epochs': hp.choice('nb_epochs', [2, 3]),
'activation': 'relu'
}
global ITERATION
ITERATION = 0
bayes_trials = Trials()
best = fmin(fn = objective, space = space, algo = tpe.suggest,
max_evals = 5, trials = bayes_trials, rstate = np.random.RandomState(50))