Fixed the same bug as before, but this time is correct
This commit is contained in:
parent
55e0d2ac23
commit
0ee6e48004
@ -163,10 +163,10 @@ class NanoSocratesBPE(Encoder):
|
||||
# in this case there was a candidate token to substitute the couple of element
|
||||
NEW_PIECE.append(CANDIDATE_TOKEN)
|
||||
|
||||
index += 1
|
||||
index += 2
|
||||
|
||||
if index == PIECE_LENGTH:
|
||||
NEW_PIECE.append(piece[index])
|
||||
index += 1
|
||||
|
||||
return NEW_PIECE
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user