From 533347ee22d793e8c53df0f13e90784ff26565a5 Mon Sep 17 00:00:00 2001 From: Christian Risi <75698846+CnF-Gris@users.noreply.github.com> Date: Tue, 7 Oct 2025 20:44:54 +0200 Subject: [PATCH] Added new special token --- Project_Model/Libs/BPE/Enums/SpecialToken.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Project_Model/Libs/BPE/Enums/SpecialToken.py b/Project_Model/Libs/BPE/Enums/SpecialToken.py index b3c42c1..b14a723 100644 --- a/Project_Model/Libs/BPE/Enums/SpecialToken.py +++ b/Project_Model/Libs/BPE/Enums/SpecialToken.py @@ -21,4 +21,7 @@ class SpecialToken(Enum): # NanoSocrates START = "" CORPUS_END = "" - PAD = "" \ No newline at end of file + START_OF_SEQUENCE = "" + END_OF_SEQUENCE = "" + PAD = "" + \ No newline at end of file