I followed this tutorial to create the following data generator. However, it takes too much time for training. How can I make it run faster knowing that I have already created all the data files read by the reader object?
ps: The method __data_generation performs 2 disc access per iteration.
import numpy as np
import keras
class DataGenerator(keras.utils.Sequence):
"""
Generates data for Keras
:return: data generator object
"""
def __init__(self, reader, list_IDs, labels, relations_list, batch_size=32, shuffle=True):
# Initialization
self.reader = reader
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.shuffle = shuffle
self.on_epoch_end()
self.relations = relations_list
self.data_num = 0
def __len__(self):
"""
Denotes the number of batches per epoch
:return: int
"""
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
"""
Generate one batch of data
:param index: index of the current training item
:return: tuple
"""
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
return X, y
def on_epoch_end(self):
"""
Updates indexes after each epoch
:return:
"""
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
"""
Generates data containing batch_size samples'
:param list_IDs_temp: the list of IDs of the target batch
:return: tuple
"""
# Initialization
y = []
v_q_words = []
v_d_words = []
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
q_words = self.reader.get_query(self.relations[ID][0]) # corresponds to 1 file read from disc
v_q_words.append(q_words)
d_words = self.reader.get_document(self.relations[ID][1]) # corresponds to another file read from disc
v_d_words.append(d_words)
# Store class
y.append(self.labels[ID])
X = [np.array(v_q_words), np.array(v_d_words)]
return X, np.array(y)
Thanks in advance for your answers.
From your link
Since our code is multicore-friendly, note that you can do more complex operations instead (e.g. computations from source files) without worrying that data generation becomes a bottleneck in the training process.
As suggested by @nabiltos, the most effective way to speed up your training is to use a GPU version of your Keras backend, which implies to have a compatible GPU device installed on your machine.
Once installed, running this code should list your workstation GPUs
>>> from keras import backend as K
>>> K.tensorflow_backend._get_available_gpus()
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.582
pciBusID: 0000:0b:00.0
totalMemory: 10.92GiB freeMemory: 10.32GiB
2018-07-17 14:09:36.190143: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1212] Found device 1 with properties:
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.582
pciBusID: 0000:13:00.0
totalMemory: 10.92GiB freeMemory: 10.54GiB
2018-07-17 14:09:36.395138: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1212] Found device 2 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:1b:00.0
totalMemory: 11.91GiB freeMemory: 11.54GiB
2018-07-17 14:09:36.395451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1312] Adding visible gpu devices: 0, 1, 2
2018-07-17 14:09:37.394013: I tensorflow/core/common_runtime/gpu/gpu_device.cc:993] Creating TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 9990 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:0b:00.0, compute capability: 6.1)
2018-07-17 14:09:37.563166: I tensorflow/core/common_runtime/gpu/gpu_device.cc:993] Creating TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 10203 MB memory) -> physical GPU (device: 1, name: GeForce GTX 1080 Ti, pci bus id: 0000:13:00.0, compute capability: 6.1)
2018-07-17 14:09:37.735253: I tensorflow/core/common_runtime/gpu/gpu_device.cc:993] Creating TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 11170 MB memory) -> physical GPU (device: 2, name: TITAN X (Pascal), pci bus id: 0000:1b:00.0, compute capability: 6.1)
['/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2']
You can see here that I have 3 GPU devices on my machine (2 x GeForce GTX 1080 Ti & 1 x TITAN X (Pascal)). If a TensorFlow operation has both CPU and GPU implementations, the GPU devices will be given priority (read more)