Search code examples
pytorchtensorrandom-seedequivalence

comparing two tensors in pytorch

I've already tried the solutions described in this question: How to check if two Torch tensors or matrices are equal?

and I'm not having any luck for some reason.

I'm just trying to test reproducibility of my random number seed as an example:

import numpy as np
import os
import random
import torch

def seed_everything(seed):
  random.seed(seed)
  os.environ['PYTHONHASHSEED'] = str(seed)
  np.random.seed(seed)
  torch.manual_seed(seed)
  torch.cuda.manual_seed(seed)
  torch.cuda.manual_seed_all(seed) # for cases of multiple gpus
  torch.backends.cudnn.deterministic = True

seed_everything(4321)
a = torch.rand(1, 3)
print(f"Tensor a: {a}")

a_expect = torch.tensor([[0.1255, 0.5377, 0.6564]])
print(f"Tensor a_expect: {a_expect}")

equal = torch.equal(a, a_expect)
print(f"torch.equal: {equal}")

eq = torch.eq(a, a_expect)
print(f"torch.eq: {eq}")

close = torch.allclose(a, a_expect)
print(f"torch.allclose: {close}")

diff = torch.all(torch.lt(torch.abs(torch.add(a, -a_expect)), 1e-12))
print(f"torch.all(lt(abs(add,1e-12))): {diff}")

output

Tensor a: tensor([[0.1255, 0.5377, 0.6564]])
Tensor a_expect: tensor([[0.1255, 0.5377, 0.6564]])
torch.equal: False
torch.eq: tensor([[False, False, False]])
torch.allclose: False
torch.all(lt(abs(add,1e-12))): False

(I'm using pytorch 1.12.1 and my machine is an apple M1 mac)

(I also tried downgrading to 1.10.2 and it made no difference)

Thank you for any clues

Update: I see the trouble now is the representation of the tensor when printed is truncated. If I turn the random number tensor into a list, the true source of the difference is revealed.

 values = a.tolist()
 print(f"Tensor values: {values}")

output is:

Tensor values: [0.1255376935005188, 0.5376683473587036, 0.6563868522644043]
Solution

  • You have to change atol parameter in allclose method

    The allclose method check using this formula

    enter image description here

    Therefore, since the number of significant figures is up to four decimal places, the value of atol should be 1e-4.

    Below is the example code:

    import numpy as np
import os
import random
import torch

def seed_everything(seed):
  random.seed(seed)
  os.environ['PYTHONHASHSEED'] = str(seed)
  np.random.seed(seed)
  torch.manual_seed(seed)
  torch.cuda.manual_seed(seed)
  torch.cuda.manual_seed_all(seed) # for cases of multiple gpus
  torch.backends.cudnn.deterministic = True

seed_everything(4321)
a = torch.rand(1, 3)

a_expect = torch.tensor([[0.1255, 0.5377, 0.6564]])

close = torch.allclose(a, a_expect, atol=1e-4)
print(f"torch.allclose: {close}")

    result:

    torch.allclose: True