PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added <PAD> and moved <END> Token

Browse Source
This commit is contained in:
Christian Risi 2025-10-07 16:36:45 +02:00
parent c65f5e66fe
commit fef933da9d
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Project_Model/Libs/BPE/Enums/SpecialToken.py
View File

@ -10,7 +10,6 @@ class SpecialToken(Enum):
RELATIONSHIP = "<PRED>" RELATIONSHIP = "<PRED>"
OBJECT = "<OBJ>" OBJECT = "<OBJ>"
ABSTRACT = "<ABS>" ABSTRACT = "<ABS>"
CORPUS_END = "<END>"
## Tasks' Token ## Tasks' Token
RDF_TO_TEXT = "<RDF2TXT>" RDF_TO_TEXT = "<RDF2TXT>"
@ -20,4 +19,6 @@ class SpecialToken(Enum):
# BPE Training: # BPE Training:
# NanoSocrates # NanoSocrates
START = "<START>" START = "<START>"
CORPUS_END = "<END>"
PAD = "<PAD>"