PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added new special token

Browse Source
This commit is contained in:
Christian Risi
2025-10-07 20:44:54 +02:00
parent d1ff88da82
commit 533347ee22
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Project_Model/Libs/BPE/Enums/SpecialToken.py
View File

@@ -21,4 +21,7 @@ class SpecialToken(Enum):
# NanoSocrates # NanoSocrates
START = "<START>" START = "<START>"
CORPUS_END = "<END>" CORPUS_END = "<END>"
START_OF_SEQUENCE = "<SOS>"
END_OF_SEQUENCE = "<EOS>"
PAD = "<PAD>" PAD = "<PAD>"