Added new utils
This commit is contained in:
parent
1797571bb2
commit
ee8e56798c
@ -1,4 +1,14 @@
|
|||||||
from .attention_mask import get_causal_attention_mask,get_causal_attention_mask_batched
|
from .attention_mask import get_causal_attention_mask,get_causal_attention_mask_batched
|
||||||
from .task_type import TaskType
|
from .task_type import TaskType
|
||||||
|
from .post_tokenization import truncate_sequence, pad_sequence, normalize_sequence
|
||||||
|
from .inference_masking import inference_masking
|
||||||
|
|
||||||
__all__ = ["get_causal_attention_mask", "TaskType", "get_causal_attention_mask_batched"]
|
__all__ = [
|
||||||
|
"TaskType",
|
||||||
|
"get_causal_attention_mask",
|
||||||
|
"get_causal_attention_mask_batched",
|
||||||
|
"truncate_sequence",
|
||||||
|
"pad_sequence",
|
||||||
|
"normalize_sequence",
|
||||||
|
"inference_masking"
|
||||||
|
]
|
||||||
Loading…
x
Reference in New Issue
Block a user