PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'dev.embedder' of https://repositories.communitynotfound.work/PoliBa-DeepLearning/NanoSocrates into dev.embedder

Browse Source
This commit is contained in:
GassiGiuseppe
2025-10-07 23:16:20 +02:00
parent fc44929a7b 0560bc439a
commit 7027414342
13 changed files with 348 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Project_Model/Libs/BPE/Enums/SpecialToken.py
View File

@@ -21,4 +21,7 @@ class SpecialToken(Enum):
# NanoSocrates
START = "<START>"
CORPUS_END = "<END>"
PAD = "<PAD>"
START_OF_SEQUENCE = "<SOS>"
END_OF_SEQUENCE = "<EOS>"
PAD = "<PAD>"

5
Project_Model/Libs/Transformer/Utils/post_tokenization.py
View File

@@ -45,9 +45,8 @@ def normalize_sequence(
pad_token: int,
end_token: int,
) -> tuple[list[int], list[bool]]:
new_sequence = pad_sequence(sequence, max_length, pad_token)
new_sequence = truncate_sequence(new_sequence, max_length, end_token)
new_sequence = truncate_sequence(sequence, max_length, end_token)
new_sequence = pad_sequence(new_sequence, max_length, pad_token)
PADDING_MASK = create_padding_mask(new_sequence, pad_token)
return (new_sequence, PADDING_MASK)