kernel logistic regression - wrong prediction

Question

I'm working on a kernel logistic regression functions except they are not returning the correct expected predictions.

It is also throwing exponential overflow warning, which is suppressed for now

    import numpy as np
    import warnings

    warnings.filterwarnings('ignore')

    def monomial_kernel(d):
        def k(x, y, d=d):
            phi_x_y = 0
            prod_xy = np.dot(x.T,y)

            for n in range(d+1):
                phi_x_y += (prod_xy ** n)

            return phi_x_y
            
        return k

    def rbf_kernel(sigma):
        def k(x, y, sigma=sigma):
            numerator = np.linalg.norm(x - y) **2
            denominator = 2 * (sigma ** 2)
            
            return np.exp(-numerator / denominator)

        return k

    def sigmoid(z):
        if type(z) == np.ndarray:
            z = z[0]
        try:
            return 1 / (1 + np.exp(-z)) 
        except:
            print(z)

    def logistic_regression_with_kernel(X, y, k, alpha, iterations):

        n_samples, _ = X.shape
        bias = 0
        kernel_matrix = np.zeros((n_samples, n_samples))
        beta = np.zeros(n_samples)

        #create kernel matrix
        for i in range(n_samples):
            for j in range(n_samples):
                kernel_matrix[i][j] = k(X[i], X[j])
        
        for _ in range(iterations):
            for i in range(n_samples):
                total = 0
                for j in range(n_samples):
                    total += beta[j] * kernel_matrix[i][j]
                total += bias
                sigmoid_value = sigmoid(total)
                t = y[i]

                beta += kernel_matrix[i] * alpha * (t - sigmoid_value)
                        
                bias += (alpha * (t - (sigmoid_value))) 

        def model(x, beta=beta, bias=bias, k=k, ref=X):
            z = sum([k(ref[i], x) * beta[i] for i in range(ref.shape[0])]) + bias 

            sig = sigmoid(z)
            # print(sig)
            return round(sig)
        return model

For some reason it's not correctly learning the following test case:

    def test4():
        
    f = lambda x, y, z, w:  int(x*y*z - y**2*z*w/4 + x**4*w**3/8- y*w/2 >= 0)

    training_examples = [
        ([0.254, 0.782, 0.254, 0.569], 0),
        ([0.237, 0.026, 0.237, 0.638], 0),
        ([0.814, 0.18, 0.814, 0.707], 1),
        ([0.855, 0.117, 0.855, 0.669], 1),
        ([0.776, 0.643, 0.776, 0.628], 1),
        ([0.701, 0.71, 0.701, 0.982], 0),
        ([0.443, 0.039, 0.443, 0.356], 1),
        ([0.278, 0.105, 0.278, 0.158], 0),
        ([0.394, 0.203, 0.394, 0.909], 0),
        ([0.83, 0.197, 0.83, 0.779], 1),
        ([0.277, 0.415, 0.277, 0.357], 0),
        ([0.683, 0.117, 0.683, 0.455], 1),
        ([0.421, 0.631, 0.421, 0.015], 1)
    ]

    X, y = map(np.array, zip(*training_examples))

    h = logistic_regression_with_kernel(X, y, monomial_kernel(10), 0.01, 500)

    test_examples = [
        ([0.157, 0.715, 0.787, 0.644], 0),
        ([0.79, 0.279, 0.761, 0.886], 1),
        ([0.903, 0.544, 0.138, 0.925], 0),
        ([0.129, 0.01, 0.493, 0.658], 0),
        ([0.673, 0.526, 0.672, 0.489], 1),
        ([0.703, 0.716, 0.088, 0.674], 0),
        ([0.276, 0.174, 0.69, 0.358], 1),
        ([0.199, 0.812, 0.825, 0.653], 0),
        ([0.332, 0.721, 0.148, 0.541], 0),
        ([0.51, 0.956, 0.023, 0.249], 0)
    ]
    print(f"{'x' : ^30}{'prediction' : ^11}{'true' : ^6}")
    for x, y in test_examples:
        print(f"{str(x) : ^30}{int(h(x)) : ^11}{y : ^6}")
    #             x               prediction  true
    # [0.157, 0.715, 0.787, 0.644]      0       0
    # [0.79, 0.279, 0.761, 0.886]       1       1
    # [0.903, 0.544, 0.138, 0.925]      0       0
    # [0.129, 0.01, 0.493, 0.658]       0       0
    # [0.673, 0.526, 0.672, 0.489]      1       1
    # [0.703, 0.716, 0.088, 0.674]      0       0
    # [0.276, 0.174, 0.69, 0.358]       1       1
    # [0.199, 0.812, 0.825, 0.653]      0       0
    # [0.332, 0.721, 0.148, 0.541]      0       0
    # [0.51, 0.956, 0.023, 0.249]       0       0

My results are:

I've tried increasing the iteration up to 4000 and it still didn't produce the correct result

Answer 1

Change

# beta += kernel_matrix[i] * alpha * (t - sigmoid_value)
beta[i] +=  alpha * (t - sigmoid_value)

Full Code

import numpy as np
import warnings

warnings.filterwarnings('ignore')

def monomial_kernel(d):
    def k(x, y, d=d):
        phi_x_y = 0
        prod_xy = np.dot(x.T,y)

        for n in range(d+1):
            phi_x_y += (prod_xy ** n)

        return phi_x_y
        
    return k

def rbf_kernel(sigma):
    def k(x, y, sigma=sigma):
        numerator = np.linalg.norm(x - y) **2
        denominator = 2 * (sigma ** 2)
        
        return np.exp(-numerator / denominator)

    return k

def sigmoid(z):
    if type(z) == np.ndarray:
        z = z[0]
    try:
        return 1 / (1 + np.exp(-z)) 
    except:
        print(z)

def logistic_regression_with_kernel(X, y, k, alpha, iterations):

    n_samples, _ = X.shape
    bias = 0
    kernel_matrix = np.zeros((n_samples, n_samples))
    beta = np.zeros(n_samples)

    #create kernel matrix
    for i in range(n_samples):
        for j in range(n_samples):
            kernel_matrix[i][j] = k(X[i], X[j])
    
    for _ in range(iterations):
        for i in range(n_samples):
            total = 0
            for j in range(n_samples):
                total += beta[j] * kernel_matrix[i][j]
            total += bias
            sigmoid_value = sigmoid(total)
            t = y[i]

            # beta += kernel_matrix[i] * alpha * (t - sigmoid_value)
            beta[i] +=  alpha * (t - sigmoid_value)
                    
            bias += (alpha * (t - (sigmoid_value))) 

    def model(x, beta=beta, bias=bias, k=k, ref=X):
        z = sum([k(ref[i], x) * beta[i] for i in range(ref.shape[0])]) + bias 

        sig = sigmoid(z)
        # print(sig)
        return round(sig)
    return model