I am trying to implement a classification of audio signals. For that I started compiling the MFCCs of each of the .wav-files I have for training the CNN, split them by label (in some files there's a one sound at the first couple of seconds and another one for the rest). Then I split them in sequences of 2.5 secs and stored each of the MFCCs in a own json file like so: (Since the librosa.features.mfcc returns a nd.array I have to convert it to a list before storing it in json)
for path in wav_paths:
# split stereo .wav-file into channels
filename = os.path.basename(path)
print(filename)
audiosegment = AudioSegment.from_file(path)
arr_mono = audiosegment.get_array_of_samples()
audio_data = (np.asarray(arr_mono)).astype(
np.float32) # audio_data is array.array (int16), ndarray (float32) needed for librosa
sample_rate = audiosegment.frame_rate
# calculate MFCCs for whole audio
mfcc = librosa.feature.mfcc(audio_data, sr=sample_rate, n_mfcc=n_mfcc, n_fft=framesize, hop_length=int(hop_size))
duration = audiosegment.duration_seconds
begin, end, event = create_dataframe.read_json(path_to_json)
list1 = [0, begin, end, duration] # one sound goes from 0secs to begin, the other one from begin to end and then the first one again from end to duration
list2 = list(zip(list1, list1[1:])) # list2=[(0, begin), (begin, end), (end, duration)
lst_mfcc_split_by_label = []
for from_sec, to_sec in list2:
# get label of sequence
label_str = create_dataframe.get_label(begin, end, event, from_sec, to_sec)
label = create_dataframe.label_key(label_str) # label as number between 0 and 3
# split MFCC by label
index_first_frame = librosa.time_to_frames(from_sec, sr=sample_rate, hop_length=hop_size)
index_last_frame = librosa.time_to_frames(to_sec, sr=sample_rate, hop_length=hop_size)
lst_mfcc_split_by_label = np.hsplit(mfcc, [index_first_frame,
index_last_frame + 1]) # returns list of 3 arrays (mfcc-array split at index_first_frame and index_last_frame)
mfcc_split_by_label = lst_mfcc_split_by_label[
1] # returns part between index_first_frame and index_last_frame+1)
# set size of blocks
secs_per_split = 2.5
# nur Blöcke betrachten, die genau secs_per_split entsprechen
n_blocks_in_sequence = int((to_sec - from_sec)/secs_per_split) # abrunden
to_sec_block = n_blocks_in_sequence * secs_per_split # end of last block of sequence
for time in np.arange(0, to_sec_block, secs_per_split):
# get index of frame corresponding to begin and end of block
index_first_frame_block = librosa.time_to_frames(time, sr=sample_rate, hop_length=hop_size)
index_last_frame_block = librosa.time_to_frames(time + 2.5, sr=sample_rate, hop_length=hop_size)
# split
lst_mfcc_split_in_blocks = np.hsplit(mfcc, [index_first_frame_block,
index_last_frame_block + 1]) # returns list of 3 arrays (mfcc-array split at index_first_frame and index_last_frame+1)
mfcc_split_in_blocks = lst_mfcc_split_in_blocks[
1] # returns part between index_first_frame and index_last_frame+1)
# store label and mfcc in dict
data["label"] = label
data["mfcc"] = mfcc_split_in_blocks.tolist()
# save MFCCs to json file
json_filename_data = str(time) + "-" + str(time + secs_per_split) + filename + ".json"
path_to_json_data = os.path.join(dirPath_data, json_filename_data)
with open(path_to_json_data, "w") as fp:
json.dump(data, fp, indent=4)
then, when trying to fit my model(see below), I always get the following error:
ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type list).
I also get this warning:
C:\Users\emmah\OneDrive - rwth-aachen.de\Dokumente\Uni\RWTH\13_Bachelorarbeit\BA Emma Heyen\06 - Repo\ba-emma-heyen-0\src\train_CNN.py:12: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
X = np.array(data["mfcc"])
But when I do specify dtype=object it doesnt change anything.
This is how I load my data and do the Train_test_split:
def load_dataset(data_path):
list_data_X = []
list_data_y = []
files = [f for f in os.listdir(data_path) if os.path.isfile(os.path.join(data_path, f))]
for f in files:
path_to_json = os.path.join(data_path, f)
with open(path_to_json, "r") as fp:
data = json.load(fp)
# extract inputs and targets
X = data["mfcc"]
y = data["label"]
list_data_X.append(X)
list_data_y.append(y)
X_arr = np.array(list_data_X, dtype = object)
y_arr = np.array(list_data_y, dtype = object)
return X_arr, y_arr
def get_data_splits(data_path, test_size=0.1, test_validation=0.1): # train_size=0.9, validation=.9*.9=.09 of all data
# load dataset
X, y = load_dataset_2(data_path)
# create train/validation/test splits
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size=test_validation)
# covert inputs from 2d to 3d arrays bc Im using a CNN
X_train = X_train[..., np.newaxis]
X_validation = X_validation[..., np.newaxis]
X_test = X_test[..., np.newaxis]
return X_train, X_validation, X_test, y_train, y_validation, y_test
Then I build my model like so:
X_train, X_validation, X_test, y_train, y_validation, y_test = train_CNN.get_data_splits(DATA_PATH)
# build CNN model
input_shape = (X_train.shape[0], X_train.shape[1], X_train.shape[2])
model = train_CNN.build_model(input_shape, learning_rate=LEARNING_RATE, num_keywords=NUM_KEYWORDS)
# train model
model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_data=(X_validation, y_validation))
I also tried storing all the mfccs in one json by appending a list with all the mfccs of each segment but I get the same error as soon as I try to train the CNN.
I found a lot of posts regarding the exact same or a similar error where it could be solved by converting the arrays to np.float32 but it didnt help me here.
Does anybody know what might help? Thanks in advance!
So it turned out that in the first json (from 0 to 2.5) for each of the .wav-files the len of the mfcc vectors is by one shorter than the len in all the other jsons.
Still don't know why this happened, but I think this is the reason I get the error mentioned above.