Confusion in the calculation of hidden layer size in CNN

Question

I am trying to understand the convolutional neural network. I am reading the book deep learning by grokking. Here is the code that they have written.

import numpy as np, sys
np.random.seed(1)

from keras.datasets import mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

images, labels = (x_train[0:1000].reshape(1000, 28*28)/255, y_train[0:1000])

one_hot_labels = np.zeros((len(labels), 10))

for i, l in enumerate(labels):
    one_hot_labels[i][l] = 1
labels = one_hot_labels

test_images = x_test.reshape(len(x_test), 28*28) / 255
test_labels = np.zeros((len(y_test), 10))
for i, l in enumerate(y_test):
    test_labels[i][l] = 1

def tanh(x):
    return np.tanh(x)

def tanh2deriv(output):
    return 1 - (output ** 2)

def softmax(x):
    temp = np.exp(x)
    return temp/np.sum(temp, axis=1, keepdims=True)

alpha, iterations = (2, 30)
pixels_per_image, num_labels = (784, 10)
batch_size = 128

input_rows = 28
input_cols = 28

kernel_rows = 3
kernel_cols = 3
num_kernels = 16

hidden_size = ((input_rows - kernel_rows)*(input_cols - kernel_cols))*num_kernels

kernels = 0.02*np.random.random((kernel_rows*kernel_cols, num_kernels))-0.01

weights_1_2 = 0.02*np.random.random((hidden_size, num_labels))-0.1

def get_image_section(layer, row_from, row_to, col_from, col_to):
    section = layer[:, row_from:row_to, col_from:col_to]
    return section.reshape(-1,1, row_to-row_from,col_to-col_from)

for j in range(iterations):
    correct_cnt = 0
    for i in range(int(len(images)/batch_size)):
        batch_start, batch_end = ((i*batch_size), ((i+1)*batch_size))
        layer_0 = images[batch_start:batch_end]
        layer_0 = layer_0.reshape(layer_0.shape[0], 28, 28)
        # print(layer_0.shape)

        sects = list()
        for row_start in range(layer_0.shape[1]-kernel_rows):
            for col_start in range(layer_0.shape[2]-kernel_cols):
                sect = get_image_section(layer_0, row_start, row_start+kernel_rows, col_start, col_start+kernel_cols)
                sects.append(sect)
        
        expanded_input = np.concatenate(sects, axis=1)
        es = expanded_input.shape
        flattened_input = expanded_input.reshape(es[0]*es[1], -1)

        kernel_output = flattened_input.dot(kernels)
        layer_1 = tanh(kernel_output.reshape(es[0], -1))
        dropout_mask = np.random.randint(2, size=layer_1.shape)
        layer_1 *= dropout_mask*2
        layer_2 = softmax(np.dot(layer_1, weights_1_2))

        for k in range(batch_size):
            labelset = labels[batch_start+k:batch_start+k+1]
            _inc = int(np.argmax(layer_2[k:k+1]) == np.argmax(labelset))
            correct_cnt += _inc

        layer_2_delta = (labels[batch_start:batch_end]-layer_2)/(batch_size*layer_2.shape[0])
        layer_1_delta = layer_2_delta.dot(weights_1_2.T)*tanh2deriv(layer_1)
        layer_1_delta *= dropout_mask

        weights_1_2 += alpha*layer_1.T.dot(layer_2_delta)

        l1d_reshape = layer_1_delta.reshape(kernel_output.shape)
        k_update = flattened_input.T.dot(l1d_reshape)
        kernels -= alpha*k_update

    test_correct_cnt = 0

    for i in range(len(test_images)):
        layer_0 = test_images[i:i+1]
        layer_0 = layer_0.reshape(layer_0.shape[0], 28, 28)

        sects = list()

        for row_start in range(layer_0.shape[1]-kernel_rows):
            for col_start in range(layer_0.shape[2]-kernel_cols):
                sect = get_image_section(layer_0, row_start, row_start+kernel_rows, col_start, col_start+kernel_rows)
                sects.append(sect)
        
        expanded_input = np.concatenate(sects, axis=1)
        es = expanded_input.shape
        flattened_input = expanded_input.reshape(es[0]*es[1], -1)

        kernel_output = flattened_input.dot(kernels)
        layer_1 = tanh(kernel_output.reshape(es[0], -1))
        layer_2 = np.dot(layer_1, weights_1_2)

        test_correct_cnt += int(np.argmax(layer_2) == np.argmax(test_labels[i:i+1]))
    if(j%10 == 0):
      print(f"I:{j} Test-Acc:{test_correct_cnt/float(len(test_images))} Train-Acc:{correct_cnt/float(len(images))}")

I am confused about the following line

hidden_size = ((input_rows - kernel_rows)*(input_cols - kernel_cols))*num_kernels

So, if I have a 5x5 image, 3x3 filter, 1 filter, 1 stride and no padding then according to this equation I should have hidden_size as 4. But If I do a convolution operation on paper then I am doing 9 convolution operations. So can anyone please explain what am I doing wrong?

Answer 1

I am not familiar with that book, but the code you presented seems to ignore a column/row at the end. If you add +1 as shown below, you will get 9 convolution operations for a 5x5 image.

# from
hidden_size = ((input_rows - kernel_rows)*(input_cols - kernel_cols))*num_kernels

# to
hidden_size = ((input_rows - kernel_rows + 1)*(input_cols - kernel_cols + 1))*num_kernels

# This changes are required for both training and testing.
# from
        for row_start in range(layer_0.shape[1]-kernel_rows):
            for col_start in range(layer_0.shape[2]-kernel_cols):

# to
        for row_start in range(layer_0.shape[1]-kernel_rows + 1):
            for col_start in range(layer_0.shape[2]-kernel_cols + 1):

In case you didn't get what these changes were about, here are some animations to help you understand. Each cell represents a pixel in the image, and the red box represents the convolution kernel.

before:

after: