Added embedder code for "Attention is all you need"

This commit is contained in:
Christian Risi
2025-10-04 19:43:25 +02:00
parent 3b274ad807
commit 9a797a0485
2 changed files with 60 additions and 0 deletions

View File

@@ -0,0 +1,32 @@
import torch
from ..Utils import fixed_positional_encoding
class NanoSocratesEmbedder(torch.nn.Module):
def __init__(
self,
vocabulary_size: int,
embedding_size: int
) -> None:
super().__init__()
self.__embedder = torch.nn.Embedding(
vocabulary_size,
embedding_size
)
def forward(self, tokenized_sentence: list[int]) -> torch.Tensor:
TOKENIZED_TENSOR = torch.tensor(tokenized_sentence)
computed_embeddings: torch.Tensor = self.__embedder(TOKENIZED_TENSOR)
SENTENCE_LENGHT, EMBEDDING_SIZE = computed_embeddings.shape
POSITIONAL_ENCODINGS = fixed_positional_encoding(
SENTENCE_LENGHT,
EMBEDDING_SIZE
)
computed_embeddings = computed_embeddings + POSITIONAL_ENCODINGS
return computed_embeddings