Fixed a masking problem

This commit is contained in:
Christian Risi 2025-10-14 10:34:14 +02:00
parent 80fd7fd600
commit 4968d79403

View File

@ -157,7 +157,7 @@ class Batcher:
X = []
Y = []
for rdf in batch["RDFs"]:
x, y = self._masker.mask_sequence(rdf)
x, y = self._masker.mask_sequence(rdf[:self.__max_length])
X.append(x)
Y.append(y)
return self.__normalization(X, Y)
@ -181,7 +181,7 @@ class Batcher:
def __token_cmpletation_task_special_normalization(self, X: list[list[int]], Y: list[list[int]]
) -> tuple[list[list[int]], list[list[int]], list[list[int]], list[list[int]]]:
def continue_rdf_padding(sequence: list[int], pad_token: int):
for i, x in enumerate(sequence):
if x == pad_token: