Keras handling large dataset which cannot fit into memory

Question

I'm working on facial expression recognition, and I'm using Keras. I've collected many datasets, and then I have applied data augmentation on the images, I've got about 500 000 images saved (as pixels) on a .csv file (same format as fer2013.csv).

This is the code I'm using :

def Zerocenter_ZCA_whitening_Global_Contrast_Normalize(list):
    Intonumpyarray = numpy.asarray(list)
    data = Intonumpyarray.reshape(img_width,img_height)
    data2 = ZeroCenter(data)
    data3 = zca_whitening(flatten_matrix(data2)).reshape(img_width,img_height)
    data4 = global_contrast_normalize(data3)
    data5 = numpy.rot90(data4,3)
    return data5



def load_data():
    train_x = []
    train_y = []
    val_x = []
    val_y = []
    test_x = []
    test_y = []

    f = open('ALL.csv')
    csv_f = csv.reader(f)

    for row in csv_f:
        if str(row[2]) == "Training":
            temp_list_train = []

            for pixel in row[1].split():
                temp_list_train.append(int(pixel))

            data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_train)
            train_y.append(int(row[0]))
            train_x.append(data.reshape(data_resh).tolist())

        elif str(row[2]) == "PublicTest":
            temp_list_validation = []

            for pixel in row[1].split():
                temp_list_validation.append(int(pixel))

            data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_validation)
            val_y.append(int(row[0]))
            val_x.append(data.reshape(data_resh).tolist())

        elif str(row[2]) == "PrivateTest":
            temp_list_test = []

            for pixel in row[1].split():
                temp_list_test.append(int(pixel))

            data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_test)
            test_y.append(int(row[0]))
            test_x.append(data.reshape(data_resh).tolist())

    return train_x, train_y, val_x, val_y, test_x, test_y

And then I load data and feed them to the generator :

Train_x, Train_y, Val_x, Val_y, Test_x, Test_y = load_data()

Train_x = numpy.asarray(Train_x) 
Train_x = Train_x.reshape(Train_x.shape[0],img_rows,img_cols)

Test_x = numpy.asarray(Test_x) 
Test_x = Test_x.reshape(Test_x.shape[0],img_rows,img_cols)

Val_x = numpy.asarray(Val_x)
Val_x = Val_x.reshape(Val_x.shape[0],img_rows,img_cols)

Train_x = Train_x.reshape(Train_x.shape[0], img_rows, img_cols, 1)
Test_x = Test_x.reshape(Test_x.shape[0], img_rows, img_cols, 1)
Val_x = Val_x.reshape(Val_x.shape[0], img_rows, img_cols, 1)

Train_x = Train_x.astype('float32')
Test_x = Test_x.astype('float32')
Val_x = Val_x.astype('float32')

Train_y = np_utils.to_categorical(Train_y, nb_classes)
Test_y = np_utils.to_categorical(Test_y, nb_classes)
Val_y = np_utils.to_categorical(Val_y, nb_classes)


datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    width_shift_range=0.2,
    height_shift_range=0.2,
    horizontal_flip=True,
    shear_range=0.03,
    zoom_range=0.03,
    vertical_flip=False)

datagen.fit(Train_x)

model.fit_generator(datagen.flow(Train_x, Train_y,
    batch_size=batch_size),
    samples_per_epoch=Train_x.shape[0],
    nb_epoch=nb_epoch,
    validation_data=(Val_x, Val_y))

When I run the code, RAM usage gets bigger and bigger until the pc freezes (I've have 16 Gb). It get stuck when loading_data() is called. Any solution for this problem that can fits my code ?

Answer 1

Seems to be a duplicate of this question. Basically, you'll have to use fit_generator() instead of fit() and pass in a function that loads the data into your model one batch at a time instead of all at once.