Added new utils

This commit is contained in:
Christian Risi 2025-10-06 17:00:55 +02:00
parent 1797571bb2
commit ee8e56798c

View File

@ -1,4 +1,14 @@
from .attention_mask import get_causal_attention_mask,get_causal_attention_mask_batched from .attention_mask import get_causal_attention_mask,get_causal_attention_mask_batched
from .task_type import TaskType from .task_type import TaskType
from .post_tokenization import truncate_sequence, pad_sequence, normalize_sequence
from .inference_masking import inference_masking
__all__ = ["get_causal_attention_mask", "TaskType", "get_causal_attention_mask_batched"] __all__ = [
"TaskType",
"get_causal_attention_mask",
"get_causal_attention_mask_batched",
"truncate_sequence",
"pad_sequence",
"normalize_sequence",
"inference_masking"
]