PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Modified imports for BPE

Browse Source
This commit is contained in:
Christian Risi 2025-10-04 19:42:02 +02:00
parent 7188c8678a
commit 03cdca1f00
3 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Project_Model/Libs/BPE/Classes/__init__.py
View File

@ -5,6 +5,8 @@ from .NanoSocraTrainer import NanoSocraTrainer
from .NanoSocraTraineRam import NanoSocraTraineRam from .NanoSocraTraineRam import NanoSocraTraineRam
from .NanoSocraTrainerPool import NanoSocraTrainerPool from .NanoSocraTrainerPool import NanoSocraTrainerPool
from .NanoSocratesSpecial import NanoSocratesSpecial from .NanoSocratesSpecial import NanoSocratesSpecial
from .TokeNanoCore import TokeNanoCore
from .TokeNano import TokeNano
__all__ = [ __all__ = [
"NanoSocratesChunker", "NanoSocratesChunker",
@ -12,5 +14,7 @@ __all__ = [
"NanoSocratesBPE", "NanoSocratesBPE",
"NanoSocraTrainer", "NanoSocraTrainer",
"NanoSocraTraineRam", "NanoSocraTraineRam",
"NanoSocraTrainerPool" "NanoSocraTrainerPool",
"TokeNanoCore",
"TokeNano"
] ]

5
Project_Model/Libs/BPE/Enums/__init__.py
View File

@ -1 +1,6 @@
from .TokenType import TokenType from .TokenType import TokenType
from .SpecialToken import SpecialToken
__all__ = [
"SpecialToken"
]

4
Project_Model/Libs/BPE/Utils/__init__.py
View File

@ -3,11 +3,13 @@ from .lag_checker_iterator import iterator_with_checks
from .vocabulary import save_nanos_vocabulary, load_nanos_vocabulary from .vocabulary import save_nanos_vocabulary, load_nanos_vocabulary
from .json_utils import save_json, load_json from .json_utils import save_json, load_json
from .special_regex_maker import special_regex_maker from .special_regex_maker import special_regex_maker
from .default_special_tokens import default_special_tokens
__all__ = [ __all__ = [
"special_regex_maker", "special_regex_maker",
"iterator_with_checks", "iterator_with_checks",
"save_nanos_vocabulary", "save_nanos_vocabulary",
"load_nanos_vocabulary", "load_nanos_vocabulary",
"save_json", "load_json" "save_json", "load_json",
"default_special_tokens"
] ]