16 lines
313 B
Python
16 lines
313 B
Python
import re
|
|
|
|
|
|
def special_regex_maker(special_tokens: list[str]) -> re.Pattern:
|
|
"""compile a regex for the special token
|
|
Args:
|
|
special_tokens (list[str]): the list of special token
|
|
|
|
Returns:
|
|
re.Pattern:
|
|
"""
|
|
|
|
REGEX_STR = "|".join(special_tokens)
|
|
|
|
return re.compile(REGEX_STR)
|