Fixed the same bug as before, but this time is correct

This commit is contained in:
Christian Risi 2025-10-03 16:09:53 +02:00
parent 55e0d2ac23
commit 0ee6e48004

View File

@ -163,10 +163,10 @@ class NanoSocratesBPE(Encoder):
# in this case there was a candidate token to substitute the couple of element
NEW_PIECE.append(CANDIDATE_TOKEN)
index += 1
index += 2
if index == PIECE_LENGTH:
NEW_PIECE.append(piece[index])
index += 1
return NEW_PIECE