diff --git a/Project_Model/Libs/BPE/Utils/special_regex_maker.py b/Project_Model/Libs/BPE/Utils/special_regex_maker.py index fd4ac28..c2d3add 100644 --- a/Project_Model/Libs/BPE/Utils/special_regex_maker.py +++ b/Project_Model/Libs/BPE/Utils/special_regex_maker.py @@ -2,15 +2,14 @@ import re def special_regex_maker(special_tokens: list[str]) -> re.Pattern: - """ compile a regex for the special token + """compile a regex for the special token Args: special_tokens (list[str]): the list of special token Returns: re.Pattern: - """ + """ REGEX_STR = "|".join(special_tokens) return re.compile(REGEX_STR) -