PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

A set of utils for our pipeline

Browse Source
This commit is contained in:
Christian Risi 2025-10-05 18:37:43 +02:00
parent b303affd18
commit a693cbb77e
1 changed files with 162 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
Project_Model/UML/pipeline-utils.excalidraw.json Normal file
View File

@ -0,0 +1,162 @@
{
"type": "excalidraw",
"version": 2,
"source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor",
"elements": [
{
"id": "MR81S4quQLdw7ZYl2sbpi",
"type": "text",
"x": 309,
"y": 131,
"width": 649,
"height": 200,
"angle": 0,
"strokeColor": "#1e1e1e",
"backgroundColor": "transparent",
"fillStyle": "solid",
"strokeWidth": 2,
"strokeStyle": "solid",
"roughness": 1,
"opacity": 100,
"groupIds": [],
"frameId": null,
"index": "a0",
"roundness": null,
"seed": 194891189,
"version": 243,
"versionNonce": 1614205979,
"isDeleted": false,
"boundElements": null,
"updated": 1759680941870,
"link": null,
"locked": false,
"text": "def truncate(seq: list[int], max_length: int, eos_id: int):\n\n if len(seq) < max_length:\n return seq\n\n seq[max_length - 1] = eos_id\n return seq[:max_length]\n ",
"fontSize": 20,
"fontFamily": 8,
"textAlign": "left",
"verticalAlign": "top",
"containerId": null,
"originalText": "def truncate(seq: list[int], max_length: int, eos_id: int):\n\n if len(seq) < max_length:\n return seq\n\n seq[max_length - 1] = eos_id\n return seq[:max_length]\n ",
"autoResize": true,
"lineHeight": 1.25
},
{
"id": "nzMSk0NIeEpMYeUQ1UvVw",
"type": "text",
"x": 309,
"y": 420.5,
"width": 594,
"height": 350,
"angle": 0,
"strokeColor": "#1e1e1e",
"backgroundColor": "transparent",
"fillStyle": "solid",
"strokeWidth": 2,
"strokeStyle": "solid",
"roughness": 1,
"opacity": 100,
"groupIds": [],
"frameId": null,
"index": "a1",
"roundness": null,
"seed": 807602389,
"version": 533,
"versionNonce": 1455522939,
"isDeleted": false,
"boundElements": null,
"updated": 1759681067736,
"link": null,
"locked": false,
"text": "def pad(seq: list[int], max_length: int, pad_id: int):\n\n if len(seq) > max_length:\n raise Exception()\n\n if len(seq) == max_length:\n return seq\n \n SEQ_LEN = len(seq)\n PAD_LEN = max_length - SEQ_LEN\n PADDING = [pad_id] * PAD_LEN\n seq.extend(PADDING)\n return seq\n ",
"fontSize": 20,
"fontFamily": 8,
"textAlign": "left",
"verticalAlign": "top",
"containerId": null,
"originalText": "def pad(seq: list[int], max_length: int, pad_id: int):\n\n if len(seq) > max_length:\n raise Exception()\n\n if len(seq) == max_length:\n return seq\n \n SEQ_LEN = len(seq)\n PAD_LEN = max_length - SEQ_LEN\n PADDING = [pad_id] * PAD_LEN\n seq.extend(PADDING)\n return seq\n ",
"autoResize": true,
"lineHeight": 1.25
},
{
"id": "8ONOUf0ArJ5DMtxeM_fvG",
"type": "text",
"x": 299,
"y": 805.5,
"width": 473,
"height": 150,
"angle": 0,
"strokeColor": "#1e1e1e",
"backgroundColor": "transparent",
"fillStyle": "solid",
"strokeWidth": 2,
"strokeStyle": "solid",
"roughness": 1,
"opacity": 100,
"groupIds": [],
"frameId": null,
"index": "a2",
"roundness": null,
"seed": 1323527029,
"version": 294,
"versionNonce": 567965627,
"isDeleted": false,
"boundElements": null,
"updated": 1759681345616,
"link": null,
"locked": false,
"text": "def mask(\n seq: list[int], \n max_bpe_voc_id: int\n mask_id: int,\n masking_probability: MaskingProbability\n)",
"fontSize": 20,
"fontFamily": 8,
"textAlign": "left",
"verticalAlign": "top",
"containerId": null,
"originalText": "def mask(\n seq: list[int], \n max_bpe_voc_id: int\n mask_id: int,\n masking_probability: MaskingProbability\n)",
"autoResize": true,
"lineHeight": 1.25
},
{
"id": "cpsIEF7OpTuNkcxiggRpa",
"type": "text",
"x": 997,
"y": 703,
"width": 330,
"height": 150,
"angle": 0,
"strokeColor": "#1e1e1e",
"backgroundColor": "transparent",
"fillStyle": "solid",
"strokeWidth": 2,
"strokeStyle": "solid",
"roughness": 1,
"opacity": 100,
"groupIds": [],
"frameId": null,
"index": "a3",
"roundness": null,
"seed": 516363867,
"version": 192,
"versionNonce": 430327707,
"isDeleted": false,
"boundElements": null,
"updated": 1759681254424,
"link": null,
"locked": false,
"text": "class MaskingProbability:\n\n + change_token_prob: float\n + mask_token_prob: float\n + same_token_prob: float\n + random_token_prob: float",
"fontSize": 20,
"fontFamily": 8,
"textAlign": "left",
"verticalAlign": "top",
"containerId": null,
"originalText": "class MaskingProbability:\n\n + change_token_prob: float\n + mask_token_prob: float\n + same_token_prob: float\n + random_token_prob: float",
"autoResize": true,
"lineHeight": 1.25
}
],
"appState": {
"gridSize": 20,
"gridStep": 5,
"gridModeEnabled": false,
"viewBackgroundColor": "#ffffff"
},
"files": {}
}