From e93710af08e1f22283e9f277ddb73a4487e75dde Mon Sep 17 00:00:00 2001
From: Christian Risi <75698846+CnF-Gris@users.noreply.github.com>
Date: Mon, 6 Oct 2025 16:16:47 +0200
Subject: [PATCH] Fixed illegal tokens being added in target output

---
 Project_Model/Libs/Transformer/Classes/SpannedMasker.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Project_Model/Libs/Transformer/Classes/SpannedMasker.py b/Project_Model/Libs/Transformer/Classes/SpannedMasker.py
index 156f512..441a3d8 100644
--- a/Project_Model/Libs/Transformer/Classes/SpannedMasker.py
+++ b/Project_Model/Libs/Transformer/Classes/SpannedMasker.py
@@ -90,6 +90,11 @@ class SpannedMasker:
             SPAN_LENGTH = min(CANDIDATE_SPAN, REMAINING_MASK)
 
             for _ in range(0, SPAN_LENGTH):
+                INNER_TOKEN = sequence[mask_index]
+
+                if self.__is_illegal_token(INNER_TOKEN, forbidden_tokens):
+                    continue
+                
                 MASK[mask_index] = True
                 mask_index += 1