Added EOS token

2025-10-07 22:47:59 +02:00
parent a04f4c7cb7
commit ee12f53f12
3 changed files with 7 additions and 5 deletions
--- a/Scripts/Libs/CleaningPipeline/special_token.py
+++ b/Scripts/Libs/CleaningPipeline/special_token.py
@@ -9,6 +9,7 @@ class SpecialToken(str, Enum):
    RELATIONSHIP = "<PRED>"
    OBJECT = "<OBJ>"
    ABSTRACT = "<ABS>"
+    END_OF_SENTENCE = "<EOS>"
    CORPUS_END = "<END>"

    ## Tasks' Token