I'm trying to build a Transformer model, but it produces an error of
TypeError: unsupported operand type(s) for +: 'Tensor' and 'NoneType'
It happens in this code:
xe = np.random.randint(0, 20_000, size=(8,512)) # batch size of 8 ;; sequence length 512
xe_t = torch.tensor(xe).to(device)
xd = np.random.randint(0, 10_000, size=(8, 256))
xd_t = torch.tensor(xd).to(device)
maske = np.ones((8, 512))
maske[:, 256:] = 0
maske_t = torch.tensor(maske).to(device)
maskd = np.ones((8, 256))
maskd[:, 128:] = 0
maskd_t = torch.tensor(maskd).to(device)
out = transformer(xe_t, xd_t, maske_t, maskd_t)
out.shape
I'm trying to understand the error. Here is the error message:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[11], line 15
12 maskd[:, 128:] = 0
13 maskd_t = torch.tensor(maskd).to(device)
---> 15 out = transformer(xe_t, xd_t, maske_t, maskd_t)
16 out.shape
File ~\anaconda3\envs\ml\lib\site-packages\torch\nn\modules\module.py:1194, in Module._call_impl(self, *input, **kwargs)
1190 # If we don't have any hooks, we want to skip the rest of the logic in
1191 # this function, and just call forward.
1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1193 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194 return forward_call(*input, **kwargs)
1195 # Do not call functions when jit is used
1196 full_backward_hooks, non_full_backward_hooks = [], []
Cell In[8], line 8, in Transformer.forward(self, enc_input, dec_input, enc_mask, dec_mask)
7 def forward(self, enc_input, dec_input, enc_mask, dec_mask):
----> 8 enc_output = self.encoder(enc_input, enc_mask)
9 dec_output = self.decoder(enc_output, dec_input, enc_mask, dec_mask)
10 return dec_output
File ~\anaconda3\envs\ml\lib\site-packages\torch\nn\modules\module.py:1194, in Module._call_impl(self, *input, **kwargs)
1190 # If we don't have any hooks, we want to skip the rest of the logic in
1191 # this function, and just call forward.
1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1193 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194 return forward_call(*input, **kwargs)
1195 # Do not call functions when jit is used
1196 full_backward_hooks, non_full_backward_hooks = [], []
Cell In[6], line 30, in Encoder.forward(self, x, pad_mask)
28 x = self.pos_encoding(x)
29 for block in self.transformer_blocks:
---> 30 x = block(x, pad_mask)
32 # many-to-one (x has the shape N x T x D)
33 # x = x[:, 0, :]
35 x = self.ln(x)
File ~\anaconda3\envs\ml\lib\site-packages\torch\nn\modules\module.py:1194, in Module._call_impl(self, *input, **kwargs)
1190 # If we don't have any hooks, we want to skip the rest of the logic in
1191 # this function, and just call forward.
1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1193 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194 return forward_call(*input, **kwargs)
1195 # Do not call functions when jit is used
1196 full_backward_hooks, non_full_backward_hooks = [], []
Cell In[3], line 17, in EncoderBlock.forward(self, x, pad_mask)
16 def forward(self, x, pad_mask=None):
---> 17 x = self.ln1(x + self.mha(x, x, x, pad_mask))
18 x = self.ln2(x + self.ann(x))
19 x = self.dropout(x)
TypeError: unsupported operand type(s) for +: 'Tensor' and 'NoneType'`
I can't find anything off in my code, I tried looking for solutions.
bro The error you're encountering indicates that you are trying to add a Tensor object to a NoneType object, which is not supported. This issue is occurring in the line x = self.ln1(x + self.mha(x, x, x, pad_mask)) inside the EncoderBlock class so you have to ensure that the pad_mask parameter passed to the self.mha function is not None you have to midfy the code try this one
class EncoderBlock(nn.Module): def init(self, d_model, num_heads, d_ff, dropout_rate): super(EncoderBlock, self).init() self.mha = MultiheadAttention(d_model, num_heads) self.ln1 = nn.LayerNorm(d_model) self.ann = nn.Sequential( nn.Linear(d_model, d_ff), nn.ReLU(), nn.Linear(d_ff, d_model), ) self.ln2 = nn.LayerNorm(d_model) self.dropout = nn.Dropout(dropout_rate def forward(self, x, pad_mask=None): attn_output = self.mha(x, x, x, pad_mask) x = self.ln1(x + attn_output) x = self.ln2(x + self.ann(x)) x = self.dropout(x) return x
dont forget to implemented the MultiheadAttention class or imported it correctly because it's is not included in the code snippet you provided and if this solution didn't work with you just ge me the implementation of the MultiheadAttention class or module you are using so ican understand it more