Added <PAD> and moved <END> Token

2025-10-07 16:36:45 +02:00 · 2025-10-07 16:36:45 +02:00 · fef933da9d
commit fef933da9d
parent c65f5e66fe
1 changed files with 3 additions and 2 deletions
--- a/Project_Model/Libs/BPE/Enums/SpecialToken.py
+++ b/Project_Model/Libs/BPE/Enums/SpecialToken.py
@ -10,7 +10,6 @@ class SpecialToken(Enum):
    RELATIONSHIP = "<PRED>"
    OBJECT = "<OBJ>"
    ABSTRACT = "<ABS>"
-    CORPUS_END = "<END>"

    ## Tasks' Token
    RDF_TO_TEXT = "<RDF2TXT>"
@ -21,3 +20,5 @@ class SpecialToken(Enum):
    # BPE Training:
    # NanoSocrates
    START = "<START>"
+    CORPUS_END = "<END>"
+    PAD = "<PAD>"