PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added actual test

Browse Source
This commit is contained in:
Christian Risi 2025-10-06 16:06:17 +02:00
parent b1e7af0607
commit d3bba9b944
1 changed files with 39 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Project_Model/Tests/spanned_masker_test.py
View File

@ -8,30 +8,26 @@ VOCABULARY_PATH = Path("Assets/Model/toy_10/toy_dictionary.json")
VOCABULARY = BPE.load_nanos_vocabulary(VOCABULARY_PATH)
SPECIAL_LIST = BPE.default_special_tokens()
class TestSpannedMasker:
def test_spanned_masking(self):
CORPUS_PATH = Path("Project_Model/Tests/spanner_file/mask.txt")
TEXT = CORPUS_PATH.read_text("utf-8")
CORRUPTION_PERCENTAGE = 0.15
TOLERANCE = 0.05
TOKENIZER = BPE.TokeNanoCore(
VOCABULARY,
SPECIAL_LIST
)
TOKENIZER = BPE.TokeNanoCore(VOCABULARY, SPECIAL_LIST)
VOCABULARY_SIZE = TOKENIZER.vocabulary_size
MASKER = Transformer.SpannedMasker(0.4,average_span=3)
MASKER = Transformer.SpannedMasker(CORRUPTION_PERCENTAGE, 3)
TOKENS = TOKENIZER.encode(TEXT)
LEGAL_TOKENS: set[int] = set(TOKENIZER.encode(
"<SUBJ><OBJ><PRED>"
))
LEGAL_TOKENS: set[int] = set(TOKENIZER.encode("<SUBJ><OBJ><PRED>"))
SPECIAL_TOKENS: set[int] = set(TOKENIZER.encode(
"".join(SPECIAL_LIST)
))
SPECIAL_TOKENS: set[int] = set(TOKENIZER.encode("".join(SPECIAL_LIST)))
ILLEGAL_TOKENS: set[int] = SPECIAL_TOKENS.difference(LEGAL_TOKENS)
@ -40,34 +36,52 @@ class TestSpannedMasker:
OUTPUT, TARGET = MASKER.mask_sequence(TOKENS, VOCABULARY_SIZE, ILLEGAL_TOKENS)
UNCORRUPTED_TOKENS = list(filter(lambda token: token <= VOCABULARY_SIZE, OUTPUT))
UNCORRUPTED_TOKENS = list(
filter(lambda token: token <= VOCABULARY_SIZE, OUTPUT)
)
CORRUPTED_TOKENS = list(filter(lambda token: token <= VOCABULARY_SIZE, TARGET))
TARGET.append(END_FORMATTER)
OUTPUT = list(map(lambda token: SPECIAL_FORMATTER if token > VOCABULARY_SIZE else token, OUTPUT))
TARGET = list(map(lambda token: SPECIAL_FORMATTER if token > VOCABULARY_SIZE else token, TARGET))
OUTPUT = list(
map(
lambda token: SPECIAL_FORMATTER if token > VOCABULARY_SIZE else token,
OUTPUT,
)
)
TARGET = list(
map(
lambda token: SPECIAL_FORMATTER if token > VOCABULARY_SIZE else token,
TARGET,
)
)
OUT_TEXT = TOKENIZER.decode(OUTPUT)
TAR_TEXT = TOKENIZER.decode(TARGET)
ACTUAL_CORRUPTION_PERCENTAGE = len(CORRUPTED_TOKENS) / len(TOKENS)
print(f"Original text:\n\n{TEXT}")
print(f"Inputs:\n\n{OUT_TEXT}")
print(f"Targets:\n\n{TAR_TEXT}")
print(f"Target Tokens:\n\n{OUTPUT}")
print("\n".join([
f"======================",
f"Original length: {len(TOKENS)}",
f"Uncorrupted Chars: {len(UNCORRUPTED_TOKENS)}",
f"Corrupted Chars: {len(CORRUPTED_TOKENS)}",
f"Percentage_corruption: {(len(CORRUPTED_TOKENS)/len(TOKENS))*100}%",
f"======================"
]))
print(
"\n".join(
[
f"======================",
f"Original length: {len(TOKENS)}",
f"Uncorrupted Chars: {len(UNCORRUPTED_TOKENS)}",
f"Corrupted Chars: {len(CORRUPTED_TOKENS)}",
f"Percentage_corruption: {(len(CORRUPTED_TOKENS)/len(TOKENS))*100}%",
f"======================",
]
)
)
assert ACTUAL_CORRUPTION_PERCENTAGE > CORRUPTION_PERCENTAGE - TOLERANCE
assert ACTUAL_CORRUPTION_PERCENTAGE < CORRUPTION_PERCENTAGE + TOLERANCE
if __name__ == "__main__":
TestSpannedMasker().test_spanned_masking()