Added utils to make regexps and iterators that check for last element

2025-09-30 13:33:12 +02:00
parent 2fe1ce9e9a
commit 7020c9e683
2 changed files with 36 additions and 0 deletions
--- a/Project_Model/Libs/BPE/Utils/special_regex_maker.py
+++ b/Project_Model/Libs/BPE/Utils/special_regex_maker.py
@@ -0,0 +1,9 @@
+import re
+
+
+def special_regex_maker(special_tokens: list[str]) -> re.Pattern:
+
+    REGEX_STR = "|".join(special_tokens)
+
+    return re.compile(REGEX_STR)
+