updated special token

2025-10-08 11:02:18 +02:00 · 2025-10-08 11:02:18 +02:00 · e353c200d7
commit e353c200d7
parent 159266a603
1 changed files with 1 additions and 0 deletions
--- a/Scripts/Libs/CleaningPipeline/special_token.py
+++ b/Scripts/Libs/CleaningPipeline/special_token.py
@ -10,6 +10,7 @@ class SpecialToken(Enum):
    RELATIONSHIP = "<PRED>"
    OBJECT = "<OBJ>"
    ABSTRACT = "<ABS>"
+    END_OF_SENTENCE = "<EOS>"
    CORPUS_END = "<END>"

    ## Tasks' Token