diff --git a/Project_Model/Libs/Transformer/Utils/__init__.py b/Project_Model/Libs/Transformer/Utils/__init__.py index d4dfba3..fab00b0 100644 --- a/Project_Model/Libs/Transformer/Utils/__init__.py +++ b/Project_Model/Libs/Transformer/Utils/__init__.py @@ -1,4 +1,14 @@ from .attention_mask import get_causal_attention_mask,get_causal_attention_mask_batched from .task_type import TaskType +from .post_tokenization import truncate_sequence, pad_sequence, normalize_sequence +from .inference_masking import inference_masking -__all__ = ["get_causal_attention_mask", "TaskType", "get_causal_attention_mask_batched"] \ No newline at end of file +__all__ = [ + "TaskType", + "get_causal_attention_mask", + "get_causal_attention_mask_batched", + "truncate_sequence", + "pad_sequence", + "normalize_sequence", + "inference_masking" +] \ No newline at end of file