Merge branch 'dev.embedder' of https://repositories.communitynotfound.work/PoliBa-DeepLearning/NanoSocrates into dev.embedder

Project_Model/Libs/Transformer/Classes/Decoder.py

@@ -2,6 +2,9 @@ import torch
 import torch.nn as nn
 from .FeedForwardNetwork import FeedForwardNetwork
 from .TorchMultiHeadAttention import TorchMultiHeadAttention as MultiHeadAttention
+from ..Utils.attention_mask import get_causal_attention_mask
+
+# B, L(T), E_D
 
 
 class Decoder(nn.Module):
@@ -32,11 +35,16 @@ class Decoder(nn.Module):
         )
         self.__layer_norm_3 = nn.LayerNorm(embedding_dimension)
 
-    def forward(self, x, k_x, v_x, attention_mask) -> torch.Tensor:  # k_x = v_x . While x_q = x
+
+
+    def forward(self, x, k_x, v_x, padding_mask = None) -> torch.Tensor:  # k_x = v_x . While x_q = x
+
+        # build of attention mask
+        attention_mask = get_causal_attention_mask(x.size(1))
 
         # 1) Masked Attention
         MASKED_ATTENTION = self.__masked_attention(
-            x, x, x, attention_mask=attention_mask
+            x, x, x, key_padding_mask=padding_mask, attn_mask=attention_mask
         )
 
         # 2) Dropout
@@ -53,7 +61,7 @@ class Decoder(nn.Module):
         x = self.__layer_norm_1(x)
 
         # 5) Encoder–decoder (cross) attention
-        CROSS_ATTENTION = self.__cross_attention(x, k_x, v_x)
+        CROSS_ATTENTION = self.__cross_attention(x, k_x, v_x, key_padding_mask=padding_mask)
 
         # 6) Dropout
         DROPPED_CROSS_ATTENTION = self.__dropout(CROSS_ATTENTION)

Project_Model/Libs/Transformer/Classes/Encoder.py

@@ -31,12 +31,12 @@ class Encoder(
         self.__dropout = nn.Dropout(0.1)  # ...
         pass
 
-    def forward(self, x):
+    def forward(self, x, padding_mask = None):
         # -> ATTENTION -> dropout -> add and normalize -> FF -> dropout -> add and normalize ->
         # Attention with Residual Connection    [ input + self-attention]
 
         # 1) Multi Head Attention
-        ATTENTION = self.__attention(x, x, x)
+        ATTENTION = self.__attention(x, x, x,key_padding_mask= padding_mask)
 
         # 2) Dropout
         DROPPED_ATTENTION = self.__dropout(ATTENTION)

Project_Model/Libs/Transformer/Classes/NanoSocratesCore.py

@@ -0,0 +1,6 @@
+from ..Utils.task_type import TaskType
+
+class NanoSocratesCore():
+
+    def __init__(self) -> None:
+        pass

Project_Model/Libs/Transformer/Classes/TorchMultiHeadAttention.py

@@ -1,6 +1,6 @@
 import torch
 import torch.nn as nn
-
+from typing import Optional
 
 class TorchMultiHeadAttention(nn.Module):
 
@@ -18,21 +18,23 @@ class TorchMultiHeadAttention(nn.Module):
             batch_first=True,
         )
 
+
     def forward(
         self,
         x_q: torch.Tensor,
         x_k: torch.Tensor,
         x_v: torch.Tensor,
-        attention_mask=None,
         key_padding_mask=None,
+        attention_mask: Optional[torch.Tensor] = None
     ) -> torch.Tensor:
 
         # x * Wq -> Q
         # x * Wk -> K
         # x * Wv -> V
-
-        y, _ = self.attention.forward(
-            x_q, x_k, x_v, attn_mask=attention_mask, key_padding_mask=key_padding_mask
+        # REMEMBER: tochAttention uses Batch internally to build the 3 dimension attention mask given the 2 dimension
+        y, _ = self.attention(
+            x_q, x_k, x_v, attn_mask=attention_mask, key_padding_mask=key_padding_mask,
+            need_weights=False
         )
         return y