132 lines
4.2 KiB
Python
132 lines
4.2 KiB
Python
from Project_Model.Libs.BPE.Enums import TokenType
|
|
import Project_Model.Libs.BPE as BPE
|
|
|
|
import re
|
|
|
|
|
|
PATTERN = "<(TOKEN|SOT|SEP|EOT)>"
|
|
SYMBOL_REGEX = re.compile(PATTERN)
|
|
|
|
|
|
class TestSplitter:
|
|
|
|
def test_split(self):
|
|
|
|
TEXT = "<SOT>Lorem <SEP>"
|
|
|
|
SPLITTER = BPE.NanoSocratesSplitter(SYMBOL_REGEX)
|
|
|
|
EXPECTED_CHUNKS = [
|
|
("<SOT>", TokenType.SPECIAL),
|
|
("Lorem ", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
]
|
|
|
|
CHUNKS = list(SPLITTER.split_text(TEXT))
|
|
|
|
assert len(CHUNKS) == len(EXPECTED_CHUNKS)
|
|
|
|
for chunk, expected_chunk in zip(EXPECTED_CHUNKS, CHUNKS):
|
|
print(f"TEST:\n\tCHUNK:\t\t{chunk}\n\tEXPECTED:\t\t{expected_chunk}")
|
|
RECEIVED_TOKEN_STRING, RECEIVED_TOKEN_TYPE = chunk
|
|
EXPECTED_TOKEN_STRING, EXPECTED_TOKEN_TYPE = expected_chunk
|
|
|
|
assert RECEIVED_TOKEN_STRING == EXPECTED_TOKEN_STRING
|
|
assert RECEIVED_TOKEN_TYPE == EXPECTED_TOKEN_TYPE
|
|
|
|
def test_split_trailing_text(self):
|
|
|
|
TEXT = "ipsu<SEP>m d<SEP>olor"
|
|
|
|
SPLITTER = BPE.NanoSocratesSplitter(SYMBOL_REGEX)
|
|
|
|
EXPECTED_CHUNKS = [
|
|
("ipsu", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
("m d", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
]
|
|
|
|
CHUNKS = list(SPLITTER.split_text(TEXT))
|
|
|
|
assert len(CHUNKS) == len(EXPECTED_CHUNKS)
|
|
|
|
for chunk, expected_chunk in zip(EXPECTED_CHUNKS, CHUNKS):
|
|
print(f"TEST:\n\tCHUNK:\t\t{chunk}\n\tEXPECTED:\t\t{expected_chunk}")
|
|
RECEIVED_TOKEN_STRING, RECEIVED_TOKEN_TYPE = chunk
|
|
EXPECTED_TOKEN_STRING, EXPECTED_TOKEN_TYPE = expected_chunk
|
|
|
|
assert RECEIVED_TOKEN_STRING == EXPECTED_TOKEN_STRING
|
|
assert RECEIVED_TOKEN_TYPE == EXPECTED_TOKEN_TYPE
|
|
|
|
def test_split_multi_token(self):
|
|
|
|
TEXT = "ipsu<SEP>m d<SEP><SEP><SEP>dsg<SEP>olor"
|
|
|
|
SPLITTER = BPE.NanoSocratesSplitter(SYMBOL_REGEX)
|
|
|
|
EXPECTED_CHUNKS = [
|
|
("ipsu", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
("m d", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
("dsg", TokenType.BPE),
|
|
("<SEP>", TokenType.SPECIAL),
|
|
]
|
|
|
|
CHUNKS = list(SPLITTER.split_text(TEXT))
|
|
|
|
assert len(CHUNKS) == len(EXPECTED_CHUNKS)
|
|
|
|
for chunk, expected_chunk in zip(EXPECTED_CHUNKS, CHUNKS):
|
|
print(f"TEST:\n\tCHUNK:\t\t{chunk}\n\tEXPECTED:\t\t{expected_chunk}")
|
|
RECEIVED_TOKEN_STRING, RECEIVED_TOKEN_TYPE = chunk
|
|
EXPECTED_TOKEN_STRING, EXPECTED_TOKEN_TYPE = expected_chunk
|
|
|
|
assert RECEIVED_TOKEN_STRING == EXPECTED_TOKEN_STRING
|
|
assert RECEIVED_TOKEN_TYPE == EXPECTED_TOKEN_TYPE
|
|
|
|
def test_split_malformed_1(self):
|
|
|
|
TEXT = "<SEP>lerisque"
|
|
|
|
SPLITTER = BPE.NanoSocratesSplitter(SYMBOL_REGEX)
|
|
|
|
EXPECTED_CHUNKS = [
|
|
("<SEP>", TokenType.SPECIAL),
|
|
]
|
|
|
|
CHUNKS = list(SPLITTER.split_text(TEXT))
|
|
|
|
assert len(CHUNKS) == len(EXPECTED_CHUNKS)
|
|
|
|
for chunk, expected_chunk in zip(EXPECTED_CHUNKS, CHUNKS):
|
|
print(f"TEST:\n\tCHUNK:\t\t{chunk}\n\tEXPECTED:\t\t{expected_chunk}")
|
|
RECEIVED_TOKEN_STRING, RECEIVED_TOKEN_TYPE = chunk
|
|
EXPECTED_TOKEN_STRING, EXPECTED_TOKEN_TYPE = expected_chunk
|
|
|
|
assert RECEIVED_TOKEN_STRING == EXPECTED_TOKEN_STRING
|
|
assert RECEIVED_TOKEN_TYPE == EXPECTED_TOKEN_TYPE
|
|
|
|
def test_split_malformed_2(self):
|
|
|
|
TEXT = "lerisque"
|
|
|
|
SPLITTER = BPE.NanoSocratesSplitter(SYMBOL_REGEX)
|
|
|
|
EXPECTED_CHUNKS = []
|
|
|
|
CHUNKS = list(SPLITTER.split_text(TEXT))
|
|
|
|
assert len(CHUNKS) == len(EXPECTED_CHUNKS)
|
|
|
|
for chunk, expected_chunk in zip(EXPECTED_CHUNKS, CHUNKS):
|
|
print(f"TEST:\n\tCHUNK:\t\t{chunk}\n\tEXPECTED:\t\t{expected_chunk}")
|
|
RECEIVED_TOKEN_STRING, RECEIVED_TOKEN_TYPE = chunk
|
|
EXPECTED_TOKEN_STRING, EXPECTED_TOKEN_TYPE = expected_chunk
|
|
|
|
assert RECEIVED_TOKEN_STRING == EXPECTED_TOKEN_STRING
|
|
assert RECEIVED_TOKEN_TYPE == EXPECTED_TOKEN_TYPE
|