Quick fix to architecture

2025-10-08 12:34:09 +02:00
parent 14c3914571
commit c2e13bc9c6
8 changed files with 89 additions and 126 deletions
--- a/Project_Model/Libs/Transformer/Utils/post_tokenization.py
+++ b/Project_Model/Libs/Transformer/Utils/post_tokenization.py
@@ -2,6 +2,10 @@ def truncate_sequence(
    sequence: list[int], truncate_at: int, end_token: int
 ) -> list[int]:

+    if len(sequence) < truncate_at - 1:
+        sequence.append(end_token)
+        return sequence
+
    if len(sequence) < truncate_at:
        sequence[-1] = end_token
        return sequence