NanoSocrates/Project_Model/Libs/BPE/Utils/special_regex_maker.py

import re


def special_regex_maker(special_tokens: list[str]) -> re.Pattern:
    """ compile a regex for the special token
    Args:
        special_tokens (list[str]): the list of special token

    Returns:
        re.Pattern:
    """    

    REGEX_STR = "|".join(special_tokens)

    return re.compile(REGEX_STR)