diff --git a/Project_Model/Libs/BPE/Classes/__init__.py b/Project_Model/Libs/BPE/Classes/__init__.py index bab5bd8..5d89eaf 100644 --- a/Project_Model/Libs/BPE/Classes/__init__.py +++ b/Project_Model/Libs/BPE/Classes/__init__.py @@ -5,6 +5,8 @@ from .NanoSocraTrainer import NanoSocraTrainer from .NanoSocraTraineRam import NanoSocraTraineRam from .NanoSocraTrainerPool import NanoSocraTrainerPool from .NanoSocratesSpecial import NanoSocratesSpecial +from .TokeNanoCore import TokeNanoCore +from .TokeNano import TokeNano __all__ = [ "NanoSocratesChunker", @@ -12,5 +14,7 @@ __all__ = [ "NanoSocratesBPE", "NanoSocraTrainer", "NanoSocraTraineRam", - "NanoSocraTrainerPool" + "NanoSocraTrainerPool", + "TokeNanoCore", + "TokeNano" ] \ No newline at end of file diff --git a/Project_Model/Libs/BPE/Enums/__init__.py b/Project_Model/Libs/BPE/Enums/__init__.py index 8ef388a..c64f9b8 100644 --- a/Project_Model/Libs/BPE/Enums/__init__.py +++ b/Project_Model/Libs/BPE/Enums/__init__.py @@ -1 +1,6 @@ from .TokenType import TokenType +from .SpecialToken import SpecialToken + +__all__ = [ + "SpecialToken" +] \ No newline at end of file diff --git a/Project_Model/Libs/BPE/Utils/__init__.py b/Project_Model/Libs/BPE/Utils/__init__.py index f9213c6..5093d48 100644 --- a/Project_Model/Libs/BPE/Utils/__init__.py +++ b/Project_Model/Libs/BPE/Utils/__init__.py @@ -3,11 +3,13 @@ from .lag_checker_iterator import iterator_with_checks from .vocabulary import save_nanos_vocabulary, load_nanos_vocabulary from .json_utils import save_json, load_json from .special_regex_maker import special_regex_maker +from .default_special_tokens import default_special_tokens __all__ = [ "special_regex_maker", "iterator_with_checks", "save_nanos_vocabulary", "load_nanos_vocabulary", - "save_json", "load_json" + "save_json", "load_json", + "default_special_tokens" ] \ No newline at end of file