Learning rate not updating

Question

def make_prediction(x0,t0):
    inputs = torch.vstack([x0,t0])
    layer_1 = torch.matmul(w0,inputs)
    return layer_1

loss1 = nn.MSELoss()
def loss_function():
            u_t=(make_prediction(x,t+inf_s)-make_prediction(x,t))/inf_s
            u_x=(make_prediction(x+inf_s,t)-make_prediction(x,t))/inf_s
            u_xx=(make_prediction(x+inf_s,t)-2*make_prediction(x,t)+make_prediction(x-inf_s,t))/inf_s**2
            return (1/N_i)*(loss1(make_prediction(x0IC,t0IC), u0IC))+(1/N_b)*(loss1(make_prediction(x0BC1,t0BC1), u0BC1))
            +(1/N_b)*(loss1(make_prediction(x0BC2,t0BC2), u0BC2))+(1/N_f)*(np.pi/0.01)*(loss1(u_xx-u_t-make_prediction(x,t)*u_x, 0))

def train_step(w,b, learning_rate):
    trainable_variables = [w,b]
    optimizer = torch.optim.SGD(trainable_variables, lr=learning_rate,momentum=0.9)
    scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.01)
    loss = loss_function()
    loss.backward()
    with torch.no_grad():
        w -= learning_rate * w.grad
        b -= learning_rate * b.grad
        w.grad.zero_()
        b.grad.zero_()
    optimizer.step()
    scheduler.step()
train_step(w,bias,learning_rate)

I run this code (by scheduler.ExponentialLR), but there is no change in the learning rate. Where do you think the problem comes from? I write the full code...thanks from your help

Answer 1

Your are hardcoding the weights and bias calculations. In this case, the optimizer cannot have access to that.

import torch
import torch.nn as nn
import torch.optim as optim


def train_step(w, b, optimizer, scheduler, loss_function):
    optimizer.zero_grad()
    loss = loss_function()
    loss.backward()
    optimizer.step()
    scheduler.step()
    return loss.item()


def loss_function():
    rand_input = torch.randn(64, input_size)
    target = torch.randn(64, output_size)
    output = rand_input.mm(w) + b
    loss = nn.MSELoss()(output, target)
    return loss


input_size, output_size = 100, 3
learning_rate = 0.01
w = torch.randn(input_size, output_size, requires_grad=True)
b = torch.randn(output_size, requires_grad=True)
trainable_variables = [w, b]
optimizer = optim.SGD(trainable_variables, lr=learning_rate, momentum=0.9)
scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99)
steps = 10000
display_step = 100


graph1 = []

for i in range(steps):
    loss = train_step(w, b, optimizer, scheduler, loss_function)
    if i % display_step == 0:
        graph1.append(loss)
        print(f'Epoch {i} \t Training Loss: {loss}')

graph = torch.tensor(graph1)

Prints

Epoch 0      Training Loss: 94.3976058959961
Epoch 100    Training Loss: 1.1436320543289185
Epoch 200    Training Loss: 1.1590440273284912
Epoch 300    Training Loss: 0.9118895530700684
Epoch 400    Training Loss: 0.8449723720550537
Epoch 500    Training Loss: 0.9973302483558655
Epoch 600    Training Loss: 0.9681441783905029
Epoch 700    Training Loss: 1.1692309379577637
Epoch 800    Training Loss: 1.0565043687820435
Epoch 900    Training Loss: 1.0424968004226685
Epoch 1000   Training Loss: 0.9199855923652649
Epoch 1100   Training Loss: 1.1443506479263306
Epoch 1200   Training Loss: 0.9741299748420715
Epoch 1300   Training Loss: 1.1515040397644043
Epoch 1400   Training Loss: 1.2819862365722656
Epoch 1500   Training Loss: 0.9993045926094055
Epoch 1600   Training Loss: 1.066098928451538
Epoch 1700   Training Loss: 1.0772987604141235
...