10 lines
155 B
Python
10 lines
155 B
Python
|
import re
|
||
|
|
|
||
|
|
|
||
|
|
def special_regex_maker(special_tokens: list[str]) -> re.Pattern:
|
||
|
|
|
||
|
|
REGEX_STR = "|".join(special_tokens)
|
||
|
|
|
||
|
|
return re.compile(REGEX_STR)
|
||
|
|
|