So im trying to train a model on colab, and it is going to take me roughly 70-72 hr of continues running. I have a free account, so i get kicked due to over-use or inactivity pretty frequently, which means I cant just dump history in a pickle file.
history = model.fit_generator(custom_generator(train_csv_list,batch_size), steps_per_epoch=len(train_csv_list[:13400])//(batch_size), epochs=1000, verbose=1, callbacks=[stop_training], validation_data=(x_valid,y_valid))
I found the CSVLogger in callback method and added it to my callback as below. But it wont create model_history_log.csv for some reason. I don't get any error or warning. What part am i doing wrong ? My goal is to only save accuracy and loss, throughout the training process
class stop_(Callback):
def on_epoch_end(self, epoch, logs={}):
model.save(Path("/content/drive/MyDrive/.../model" +str(int(epoch))))
CSVLogger("/content/drive/MyDrive/.../model_history_log.csv", append=True)
if(logs.get('accuracy') > ACCURACY_THRESHOLD):
print("\nReached %2.2f%% accuracy, so stopping training!!" %(ACCURACY_THRESHOLD*100))
self.model.stop_training = True
stop_training = stop_()
Also since im saving the model at every epoch, does the model save this information ? so far i havent found anything, and i doubt it saves accuracy, loss, val accuracy,etc
Think you want to write your callback as follows
class STOP(tf.keras.callbacks.Callback):
def __init__ (self, model, csv_path, model_save_dir, epochs, acc_thld): # initialization of the callback
# model is your compiled model
# csv_path is path where csv file will be stored
# model_save_dir is path to directory where model files will be saved
# number of epochs you set in model.fit
self.model=model
self.csv_path=csv_path
self.model_save_dir=model_save_dir
self.epochs=epochs
self.acc_thld=acc_thld
self.acc_list=[] # create empty list to store accuracy
self.loss_list=[] # create empty list to store loss
self.epoch_list=[] # create empty list to store the epoch
def on_epoch_end(self, epoch, logs=None): # method runs on the end of each epoch
savestr='_' + str(epoch+1) + '.h5' # model will be save as an .h5 file with name _epoch.h5
save_path=os.path.join(self.model_save_dir, savestr)
acc= logs.get('accuracy') #get the accuracy for this epoch
loss=logs.get('loss') # get the loss for this epoch
self.model.save (save_path) # save the model
self.acc_list.append(logs.get('accuracy'))
self.loss_list.append(logs.get('loss'))
self.epoch_list.append(epoch + 1)
if acc > self.acc_thld or epoch+1 ==epochs: # see of acc >thld or if this was the last epoch
self.model.stop_training = True # stop training
Eseries=pd.Series(self.epoch_list, name='Epoch')
Accseries =pd.Series(self.acc_list, name='accuracy')
Lseries=pd.Series(self.loss_list, name='loss')
df=pd.concat([Eseries, Lseries, Accseries], axis=1) # create a dataframe with columns epoch loss accuracy
df.to_csv(self.csv_path, index=False) # convert dataframe to a csv file and save it
if acc > self.acc_thld:
print ('\nTraining halted on epoch ', epoch + 1, ' when accuracy exceeded the threshhold')
then before you run model.fit use code
epochs=20 # set number of epoch for model.fit and the callback
sdir=r'C:\Temp\stooges' # set directory where save model files and the csv file will be stored
acc_thld=.98 # set accuracy threshold
csv_path=os.path.join(sdir, 'traindata.csv') # name your csv file to be saved in sdir
callbacks=STOP(model, csv_path, sdir, epochs, acc_thld) # instantiate the callback
Remember in model.fit set callbacks = callbacks. I tested this on a simple dataset. It ran for only 3 epochs before the accuracy exceeded the threshold of .98. So since it ran for 3 epoch it created 3 save model files in the sdir labeled as
_1.h5
_2.h5
_3.h5
It also created the csv file labelled as traindata.csv. The csv file content was
Epoch loss accuracy
1 8.086007 .817778
2 6.911876 .974444
3 6.129871 .987778