PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Modified imports for BPE

Browse Source
This commit is contained in:
Christian Risi 2025-10-04 19:42:02 +02:00
parent 7188c8678a
commit 03cdca1f00
3 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Project_Model/Libs/BPE/Classes/__init__.py
View File

@ -5,6 +5,8 @@ from .NanoSocraTrainer import NanoSocraTrainer
from .NanoSocraTraineRam import NanoSocraTraineRam
from .NanoSocraTrainerPool import NanoSocraTrainerPool
from .NanoSocratesSpecial import NanoSocratesSpecial
from .TokeNanoCore import TokeNanoCore
from .TokeNano import TokeNano
__all__ = [
"NanoSocratesChunker",
@ -12,5 +14,7 @@ __all__ = [
"NanoSocratesBPE",
"NanoSocraTrainer",
"NanoSocraTraineRam",
"NanoSocraTrainerPool"
"NanoSocraTrainerPool",
"TokeNanoCore",
"TokeNano"
]

5
Project_Model/Libs/BPE/Enums/__init__.py
View File

@ -1 +1,6 @@
from .TokenType import TokenType
from .SpecialToken import SpecialToken
__all__ = [
"SpecialToken"
]

4
Project_Model/Libs/BPE/Utils/__init__.py
View File

@ -3,11 +3,13 @@ from .lag_checker_iterator import iterator_with_checks
from .vocabulary import save_nanos_vocabulary, load_nanos_vocabulary
from .json_utils import save_json, load_json
from .special_regex_maker import special_regex_maker
from .default_special_tokens import default_special_tokens
__all__ = [
"special_regex_maker",
"iterator_with_checks",
"save_nanos_vocabulary",
"load_nanos_vocabulary",
"save_json", "load_json"
"save_json", "load_json",
"default_special_tokens"
]