I wrote a simple Keras code, in which I use CNN for fashion mnist dataset. Everything works great. I implemented my own class and classification is OK.
However, I wanted to use Optuna, as OptKeras (Optuna wrapper for Keras), you can see an example here: https://medium.com/@Minyus86/optkeras-112bcc34ec73.
However, something is wrong with my code. When I try to use optKeras inside my own class. Here's the code: (ordinary run method works, but optuna_run gives an error: AttributeError: type object 'FrozenTrial' has no attribute '_field_types'.
! pip install optkeras
# -*- coding: utf-8 -*-
#!/usr/bin/env python3
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN
from keras.callbacks import ModelCheckpoint
from keras import backend as K
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import sklearn.metrics
import optuna
from optkeras.optkeras import OptKeras
import sys
import math
import numpy
import scipy.io as sio
import matplotlib.pyplot as plt
class OptunaTest():
def __init__(self):
self.fashion_mnist = keras.datasets.fashion_mnist
(self.train_images, self.train_labels), (self.test_images, self.test_labels) = self.fashion_mnist.load_data()
self.class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
self.train_images = self.train_images / 255.0
self.test_images = self.test_images / 255.0
self.model = None
self.study_name = 'FashionMnist' + '_Simple'
self.ok = OptKeras(study_name=self.study_name)
def run(self):
self.model = keras.Sequential()
self.model.add(keras.layers.Flatten(input_shape=(28, 28)))
self.model.add(keras.layers.Dense(128, activation='relu'))
self.model.add(keras.layers.Dense(10))
self.model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
self.model.fit(self.train_images, self.train_labels, epochs=5)
test_loss, test_acc = self.model.evaluate(self.test_images, self.test_labels, verbose=0)
predictions = self.model.predict(self.test_images)
INDEX = 10
print("\nPREDICTION: " + str(predictions[INDEX]))
print("\nMAX PREDICTION VAL: " + str(numpy.argmax(predictions[INDEX])))
print("\nLABEL: " + str(self.test_labels[INDEX]))
def optuna_run(self, trial):
K.clear_session()
self.model = keras.Sequential()
self.model.add(keras.layers.Flatten(input_shape=(28, 28)))
self.model.add(keras.layers.Dense(units = trial.suggest_categorical('units', [32, 64, 128]), activation = trial.suggest_categorical('activation', ['relu', 'linear'])))
self.model.add(keras.layers.Dense(units = trial.suggest_categorical('units', [32, 64, 128]), activation = trial.suggest_categorical('activation', ['relu', 'linear'])))
self.model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
self.model.fit(self.train_images, self.train_labels, epochs=5, callbacks = self.ok.callbacks(trial), verbose = self.ok.keras_verbose)
test_loss, test_acc = self.model.evaluate(self.test_images, self.test_labels, verbose=0)
predictions = self.model.predict(self.test_images)
print(ok.trial_best_value)
INDEX = 10
print("\nPREDICTION: " + str(predictions[INDEX]))
print("\nMAX PREDICTION VAL: " + str(numpy.argmax(predictions[INDEX])))
print("\nLABEL: " + str(self.test_labels[INDEX]))
if __name__ == "__main__":
ot = OptunaTest()
ot.run()
ot.ok.optimize(ot.optuna_run, timeout = 60)
A code can also be found here: https://colab.research.google.com/drive/1uibWa80BdjatA5Kcw27eMUsS7SmwxaDk?usp=sharing.
The full error message:
[W 2020-06-30 11:09:26,959] Setting status of trial#0 as TrialState.FAIL because of the following error: AttributeError("type object 'FrozenTrial' has no attribute '_field_types'",)
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py", line 230, in synch_with_optuna
self.best_trial = self.study.best_trial
File "/usr/local/lib/python3.6/dist-packages/optuna/study.py", line 97, in best_trial
return copy.deepcopy(self._storage.get_best_trial(self._study_id))
File "/usr/local/lib/python3.6/dist-packages/optuna/storages/in_memory.py", line 293, in get_best_trial
raise ValueError("No trials are completed yet.")
ValueError: No trials are completed yet.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/optuna/study.py", line 734, in _run_trial
result = func(trial)
File "/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py", line 130, in fun_tf
return fun(trial)
File "<ipython-input-11-45495c9f2ae9>", line 65, in optima_run
self.model.fit(self.train_images, self.train_labels, epochs=10, callbacks = self.ok.callbacks(trial), verbose = self.ok.keras_verbose)
File "/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py", line 172, in callbacks
self.synch_with_optuna()
File "/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py", line 232, in synch_with_optuna
self.best_trial = get_trial_default()
File "/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py", line 367, in get_trial_default
num_fields = optuna.structs.FrozenTrial._field_types.__len__()
AttributeError: type object 'FrozenTrial' has no attribute '_field_types'
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py in synch_with_optuna(self)
229 try:
--> 230 self.best_trial = self.study.best_trial
231 except:
12 frames
ValueError: No trials are completed yet.
During handling of the above exception, another exception occurred:
AttributeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/optkeras/optkeras.py in get_trial_default()
365
366 def get_trial_default():
--> 367 num_fields = optuna.structs.FrozenTrial._field_types.__len__()
368 assert num_fields in (10, 11, 12)
369 if num_fields == 12: # possible future version
AttributeError: type object 'FrozenTrial' has no attribute '_field_types'
It seems that optkeras (version I got was 0.0.7) being not quite up-to-date with optuna library is the reason for the issue. I was able to make it work with optuna 1.5.0 by doing the following changes:
First, you'll need to monkey-patch get_default_trial like this before running your code:
import optkeras
optkeras.optkeras.get_trial_default = lambda: optuna.trial.FrozenTrial(
None, None, None, None, None, None, None, None, None, None, None)
After doing so I'm getting an error with Callback saying:
AttributeError: 'OptKeras' object has no attribute '_implements_train_batch_hooks'
To solve this you'll have to manually edit optkeras.py, but not too much - just add tensorflow. to first two lines imports, i.e. make them:
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import Callback, CSVLogger, ModelCheckpoint
instead of:
import keras.backend as K
from keras.callbacks import Callback, CSVLogger, ModelCheckpoint
If you can't change the code after installation it might be a bit of a problem - I would probably just recommend to copy full code of optkeras library (it's just one file optkeras.py) and use fixed version of that in your script or something like that. Unfortunately I don't see a nice way of monkey-patching this import issue. That said I think it can be fairly easy to either change that on-fly even from python (i.e. change optkeras.py lines from within python before importing optkeras) or copying the optkeras.py (also from withing python script) + replacing the strings on fly, then importing from the new location.
After that is done I just had to:
print(ok.trial_best_value) should really be print(self.ok.trial_best_value))validation_split=0.1 to self.model.fit call (or you may use something else for your tuning - just with existing code example callback won't get val_loss value because there is no validation set and optkeras is using val_loss by default - see monitor argument for OptKeras constructor). My guess would be that you probably will either want to create a fixed validation set instead or monitor training loss loss instead of val_loss.return test_loss at the end of optuna_run method.After all of these changes everything seems be working.