diff --git a/Project_Model/Libs/BPE/Utils/special_regex_maker.py b/Project_Model/Libs/BPE/Utils/special_regex_maker.py index 414eabf..fd4ac28 100644 --- a/Project_Model/Libs/BPE/Utils/special_regex_maker.py +++ b/Project_Model/Libs/BPE/Utils/special_regex_maker.py @@ -2,6 +2,13 @@ import re def special_regex_maker(special_tokens: list[str]) -> re.Pattern: + """ compile a regex for the special token + Args: + special_tokens (list[str]): the list of special token + + Returns: + re.Pattern: + """ REGEX_STR = "|".join(special_tokens)