diff --git a/Project_Model/Libs/BPE/Utils/default_special_tokens.py b/Project_Model/Libs/BPE/Utils/default_special_tokens.py new file mode 100644 index 0000000..83ef45e --- /dev/null +++ b/Project_Model/Libs/BPE/Utils/default_special_tokens.py @@ -0,0 +1,4 @@ +from ..Enums import SpecialToken + +def default_special_tokens() -> list[str]: + return [token.value for token in SpecialToken]