Merge branch 'dev.embedder' of https://repositories.communitynotfound.work/PoliBa-DeepLearning/NanoSocrates into dev.embedder

Playgrounds/nanosocrates-sanity-check.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "f5762da9",
    "metadata": {},
    "outputs": [
@@ -127,6 +127,8 @@
     "\n",
     "\n",
     "\n",
+    "\n",
+    "\n",
     "\n"
    ]
   }

Playgrounds/prova.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "4ae47336",
    "metadata": {},
    "outputs": [],
@@ -15,6 +15,77 @@
     "mha = torch.nn.MultiheadAttention(D, num_heads=4, batch_first=True)\n",
     "y, _ = mha(x, x, x, attn_mask=attn_mask, key_padding_mask=pad_mask)  # should work\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e38e3fb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[[0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
+       "         [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],\n",
+       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]],\n",
+       "\n",
+       "        [[0, 0, 1, 0, 0, 0, 0, 0, 0, 0],\n",
+       "         [0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
+       "         [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]]])"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "torch.nn.functional.one_hot(torch.tensor([\n",
+    "    [4, 1, 9],\n",
+    "    [2,4,5]\n",
+    "]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "7119ad53",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "device(type='cpu')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "torch.get_default_device()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "8c95691a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "xpu\n"
+     ]
+    }
+   ],
+   "source": [
+    "from Project_Model.Libs.TorchShims import get_default_device\n",
+    "\n",
+    "print(get_default_device())"
+   ]
   }
  ],
  "metadata": {

Project_Model/Libs/BPE/Enums/SpecialToken.py

@@ -21,4 +21,7 @@ class SpecialToken(Enum):
     # NanoSocrates
     START = "<START>"
     CORPUS_END = "<END>"
-    PAD = "<PAD>"
+    START_OF_SEQUENCE = "<SOS>"
+    END_OF_SEQUENCE = "<EOS>"
+    PAD = "<PAD>"
+    

Project_Model/Libs/Transformer/Utils/post_tokenization.py

@@ -45,9 +45,8 @@ def normalize_sequence(
     pad_token: int,
     end_token: int,
 ) -> tuple[list[int], list[bool]]:
-
-    new_sequence = pad_sequence(sequence, max_length, pad_token)
-    new_sequence = truncate_sequence(new_sequence, max_length, end_token)
+    new_sequence = truncate_sequence(sequence, max_length, end_token)
+    new_sequence = pad_sequence(new_sequence, max_length, pad_token)
     PADDING_MASK = create_padding_mask(new_sequence, pad_token)
 
     return (new_sequence, PADDING_MASK)