import torch import Project_Model.Libs.BPE as BPE def decode_batch(batch: torch.Tensor, tokenizer: BPE.TokeNanoCore ,uknonw_token: int) -> list[str]: strings = [] BATCH, _ = batch.shape for i in range(0, BATCH): tokens: list[int] = batch.tolist()[i] tokens = list(map(lambda x: uknonw_token if x > tokenizer.vocabulary_size else x, tokens)) strings.append(tokenizer.decode(tokens)) return strings