PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added Chunker

Browse Source
This commit is contained in:
Christian Risi 2025-09-28 18:02:06 +02:00
parent ed0255e99b
commit b071145f6e
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Project_Model/Libs/BPE/Classes/NanoSocratesChunker.py
View File

@ -10,6 +10,10 @@ class NanoSocratesChunker:
self.__special_token_regex: re.Pattern = special_token_regex self.__special_token_regex: re.Pattern = special_token_regex
self.__residual: str = "" self.__residual: str = ""
# max theorethical size of chars
# between special tokens:
# - min: size - len(longest_token)
# - MAX: size - len(shortest_token)
def chunk(self, file_path: Path): def chunk(self, file_path: Path):
# read_file # read_file
FILE = open(file_path, "r", encoding="utf-8") FILE = open(file_path, "r", encoding="utf-8")