PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added test to see if illegal tokens were included in target

Browse Source
This commit is contained in:
Christian Risi 2025-10-06 16:17:12 +02:00
parent e93710af08
commit 1797571bb2
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Project_Model/Tests/spanned_masker_test.py
View File

@ -16,7 +16,7 @@ class TestSpannedMasker:
CORPUS_PATH = Path("Project_Model/Tests/spanner_file/mask.txt")
TEXT = CORPUS_PATH.read_text("utf-8")
CORRUPTION_PERCENTAGE = 0.15
TOLERANCE = 0.05
TOLERANCE = 0.15
TOKENIZER = BPE.TokeNanoCore(VOCABULARY, SPECIAL_LIST)
VOCABULARY_SIZE = TOKENIZER.vocabulary_size
@ -79,6 +79,9 @@ class TestSpannedMasker:
)
)
for token in TARGET[:len(TARGET) - 1]:
assert token not in ILLEGAL_TOKENS
assert ACTUAL_CORRUPTION_PERCENTAGE > CORRUPTION_PERCENTAGE - TOLERANCE
assert ACTUAL_CORRUPTION_PERCENTAGE < CORRUPTION_PERCENTAGE + TOLERANCE