PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

added padding_mask entry to decoder and encoder

Browse Source
This commit is contained in:
GassiGiuseppe 2025-10-05 18:46:06 +02:00
parent 9c83d9fa71
commit 0f243eaac2
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Project_Model/Libs/Transformer/Classes/Decoder.py
View File

@ -36,11 +36,11 @@ class Decoder(nn.Module):
def forward(self, x, k_x, v_x, attention_mask) -> torch.Tensor: # k_x = v_x . While x_q = x
def forward(self, x, k_x, v_x, padding_mask = None) -> torch.Tensor: # k_x = v_x . While x_q = x
# 1) Masked Attention
MASKED_ATTENTION = self.__masked_attention(
x, x, x, attention_mask=attention_mask
x, x, x, key_padding_mask=padding_mask
)
# 2) Dropout
@ -57,7 +57,7 @@ class Decoder(nn.Module):
x = self.__layer_norm_1(x)
# 5) Encoder–decoder (cross) attention
CROSS_ATTENTION = self.__cross_attention(x, k_x, v_x)
CROSS_ATTENTION = self.__cross_attention(x, k_x, v_x key_padding_mask=padding_mask)
# 6) Dropout
DROPPED_CROSS_ATTENTION = self.__dropout(CROSS_ATTENTION)

4
Project_Model/Libs/Transformer/Classes/Encoder.py
View File

@ -31,12 +31,12 @@ class Encoder(
self.__dropout = nn.Dropout(0.1) # ...
pass
def forward(self, x):
def forward(self, x, padding_mask = None):
# -> ATTENTION -> dropout -> add and normalize -> FF -> dropout -> add and normalize ->
# Attention with Residual Connection [ input + self-attention]
# 1) Multi Head Attention
ATTENTION = self.__attention(x, x, x)
ATTENTION = self.__attention(x, x, x,key_padding_mask= padding_mask)
# 2) Dropout
DROPPED_ATTENTION = self.__dropout(ATTENTION)