From 845c63dbef1269c4adb7dbdaf0f71a79a9138888 Mon Sep 17 00:00:00 2001 From: GassiGiuseppe Date: Sat, 4 Oct 2025 19:01:21 +0200 Subject: [PATCH] updated tokenano to be more easy to read --- Project_Model/Libs/BPE/Classes/TokeNanoCore.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Project_Model/Libs/BPE/Classes/TokeNanoCore.py b/Project_Model/Libs/BPE/Classes/TokeNanoCore.py index f726a95..3abaeb2 100644 --- a/Project_Model/Libs/BPE/Classes/TokeNanoCore.py +++ b/Project_Model/Libs/BPE/Classes/TokeNanoCore.py @@ -31,11 +31,15 @@ class TokeNanoCore: for piece, token_type in self.__splitter.split_text(corpus): if token_type == TokenType.SPECIAL: - output.extend(self.__special_encoder.encode(piece)) + ENCODED_PIECE = self.__special_encoder.encode(piece) + output.extend(ENCODED_PIECE) + continue # slow but clear if token_type == TokenType.BPE: - output.extend(self.__bpe_encoder.encode(piece)) + ENCODED_PIECE = self.__bpe_encoder.encode(piece) + output.extend(ENCODED_PIECE) + continue return output @@ -46,11 +50,13 @@ class TokeNanoCore: if token_type == TokenType.SPECIAL: output_str += self.__special_encoder.decode( token - ) # it accept an integer + ) + continue # slow but clear if token_type == TokenType.BPE: output_str += self.__bpe_encoder.decode( token - ) # it accept a list of integer + ) + continue return output_str