Added BPE

TODO: - complete the fit method
2025-09-28 18:04:44 +02:00
parent b46df4f91a
commit e433941405
3 changed files with 164 additions and 0 deletions
--- a/Project_Model/Libs/BPE/Enums/TokenType.py
+++ b/Project_Model/Libs/BPE/Enums/TokenType.py
@@ -0,0 +1,6 @@
+from enum import Enum, auto
+
+class TokenType(Enum):
+
+    SPECIAL = auto()
+    BPE = auto()