PyTorch calculate gradient of output with respect to "output"

Question

I was playing around with the backward method of PyTorch tensor to find the gradient of a multidimensional output of the model with respect to intermediate activation layers.

When I try to calculate the gradients of the output with respect to the last activation layer (the output), I get the gradients as 1. Shouldn't the gradients of outputs with respect to itself be 0? Am I wrong here or is there any mistake in my implementation? Below is my code and output for reference.

Code:

import torch.nn as nn
import torch

torch.manual_seed(7986)


class myModel(nn.Module):
    def __init__(self):
        super(myModel, self).__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=2, out_channels=3, kernel_size=1), nn.Sigmoid(),
        )
        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=2, kernel_size=2), nn.Sigmoid(),
        )

        self.gradients = None

    def activations_hook(self, grad):
        self.gradients = grad

    def forward(self, x):
        out1 = self.layer1(x)
        out2 = self.layer2(out1)

        h = out2.register_hook(self.activations_hook)

        return out2

    def get_activations_gradient(self):
        return self.gradients


myNet = myModel()
inputs = torch.randn(1, 2, 2, 2)
outputs = myNet(inputs)

print(f"inputs =\n{inputs}\n")
print(f"outputs =\n{outputs}\n")

outputs.backward(torch.ones_like(outputs))
gradients = myNet.get_activations_gradient()
print(f"gradients =\n{gradients}\n")

Output:

inputs =
tensor([[[[-1.1344, -0.6808],
          [-1.8061, -0.8529]],

         [[ 0.3898, -1.9382],
          [-2.0533, -0.6483]]]])

outputs =
tensor([[[[0.3999]],

         [[0.5760]]]], grad_fn=<SigmoidBackward0>)

gradients =
tensor([[[[1.]],

         [[1.]]]])

Answer 1

Since it is a variable, the derivative of the output with respect to the output itself will be equal to 1.