2025-10-04 19:43:42 +02:00
|
|
|
{
|
|
|
|
|
"cells": [
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2025-10-07 12:15:03 +02:00
|
|
|
"execution_count": 9,
|
2025-10-04 19:43:42 +02:00
|
|
|
"id": "06229c81",
|
|
|
|
|
"metadata": {
|
|
|
|
|
"slideshow": {
|
|
|
|
|
"slide_type": "slide"
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"outputs": [
|
|
|
|
|
{
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
|
|
|
|
"torch.Size([256, 256])\n"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"data": {
|
|
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA7UAAAKsCAYAAAAz95rSAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXl8XVW5/t+zz5x5Tpo0bdomnaCltNCBeSiUGRQVUAQRULzCVbjqVX9eEb3XWakoCiqzIoOICmgZCmVsKbQUaOncpmmbZp6TM+/z+2OfrOfdZK8mB5K0Ie/38/FzX9fZZ+219z6p95znWc/rSiaTSRIEQRAEQRAEQRCEMYhxqBcgCIIgCIIgCIIgCB8U+VIrCIIgCIIgCIIgjFnkS60gCIIgCIIgCIIwZpEvtYIgCIIgCIIgCMKYRb7UCoIgCIIgCIIgCGMW+VIrCIIgCIIgCIIgjFnkS60gCIIgCIIgCIIwZpEvtYIgCIIgCIIgCMKYRb7UCoIgCIIgCIIgCGMW+VIrCIIgCIIgCIIgjFk+Ml9qb7/9dqqqqqJAIECLFi2itWvXHuolCYIgCIIgCIIgHJa89NJLdP7551N5eTm5XC76+9//Puh7Vq1aRfPnzye/30/V1dV07733DjjmUHwv+0h8qX344YfppptuoptvvpnWr19PRx11FC1btoyampoO9dIEQRAEQRAEQRAOO3p7e+moo46i22+/fUjH7969m84991w69dRTacOGDfTVr36VrrnmGnr66afVMYfqe5krmUwmR/QMo8CiRYvo2GOPpd/85jdERGSaJlVWVtINN9xA3/zmNw/x6gRBEARBEARBEA5fXC4XPf7443TRRRdpj/nv//5veuqpp2jjxo1q7NJLL6WOjg5asWIFER2672WeEZt5lIhGo7Ru3Tr61re+pcYMw6ClS5fS6tWrHd8TiUQoEomo/26aJrW1tVFhYSG5XK4RX7MgCIIgCIIgCHqSySR1d3dTeXk5GcbYMpeGw2GKRqOH5NzJZHLA9xm/309+v/9Dz7169WpaunSpbWzZsmX01a9+lYg+2Pey4WLMf6ltaWmhRCJBpaWltvHS0lLasmWL43t+9KMf0S233DIayxMEQRAEQRAE4QOyd+9emjhx4qFexpAJh8M0ZXIWNTQlDsn5s7KyqKenxzZ288030/e+970PPXdDQ4Pjd66uri4KhULU3t6e9vey4WLMf6n9IHzrW9+im266Sf33zs5OmjRpElXe8h0yAgHa8LG7iYho3uOfV8f0j43m+Hg5Jx+Xey73/KN6Tj4u91zu+Uf1nHxc7rnc84/qOfm43PORO2dXj0mT59dSdnY2jSWi0Sg1NCVoz7oqysk2RvXcXd0mTV5QS3v37qWcnBw1Phwq7eHOmP9SW1RURG63mxobG23jjY2NVFZW5vgenQR/38X3UVa2Qd9uOYGIiO7+1P3qtZ91HaXqX33iUVX/Pjxd1T/42BOqftjEL0rfuPA5Vf/LVaTq/zjvFVW/7M5U9WfPfpOIiN7w+NTYx858V9Xv+bAN+qzT8avHLn9M1SeevFvVewOwWh97wj5VNwdDqj7quANERNQR7FNjMxY1q7o7A+NVC9pVHc7AHBVHd6k6lhlWdcncXlUnMrGWgiPxXsq0LBq5s/E+IxPXkzkDtTcrrupgTdxx3F+NX8f8WaaqfVNNx3FPlXVPg1m4t+5JsG7wcWOi87irAv9wZWUz28cEt/N4mdthzON8bAnGc7IxHxV7HceTRZrxwoHjyQKf87GaccrXjOdpxnP9Bx23jeVojtWNZ6c5nuVwzqyDHztgPDPwocddGc7HDtt48ODj6Rw7XOOH4px8fLjmNgIfbHw45hgL5+Tjh+KcfHy8nJOPyz2Xe/5ROycRjdmtgVnZLvv/LzcKmGSdLycnx/aldrgoKytz/M6Vk5NDwWCQ3G532t/LhovR/flgBPD5fLRgwQJauXKlGjNNk1auXElLliw5hCsTBEEQBEEQBEH4aLBkyRLbdy4iomeffVZ95zqU38vGvFJLRHTTTTfRlVdeSccccwwtXLiQli9fTr29vXTVVVcd6qUJgiAIgiAIgiAcdvT09NCOHTvUf9+9ezdt2LCBCgoKaNKkSfStb32L9u/fT/ffb7lXr7vuOvrNb35D3/jGN+jzn/88Pf/88/TII4/QU089peY4VN/LPhJfai+55BJqbm6m7373u9TQ0EDz5s2jFStWDNikPBil7ghluw1at3w+ERH9+McvqdduvOtEVd/yrXdU/c17z1T1pv/4napr/nSdqrdffoeqp/7zWlXvuOBOVU9/HnsJ3j3VGl+4Fg9/1bF/UPVFmy5X9Z9nwSJ9w+5PqPqHVY+r+ucHluE6KtBH6jfNJ6v6PyqeJyKiezsWqrFrJr6s6r91H6HqKyrXqPpfvVNV/anKdVhvCBaDCytxv16P5Kn6zImwTr8dDRIR0SkV+MPaHIP9ZElFrap3xGAbnl8OO/WeOOzHR5Y1qHpfApbnmhJYqhsTsDpPKWklIqJWE2MVxbBZt7Px0iLYrLuSmLuooFvVPSbG8wp6Hcdz8i1Ld5+JdLyMHFiyI0lYrgM5Ecdxn248G3PGkrgvnqzYgHF35sAxIiJ3BsZNwj13BeOO4xRMpDceSGjHBuA3HYeTunGfbtyhg5lX09VMO+48N3k0xzuMJzXHasfdIzee1Ph10h0nncPKaTydYw8zkpo1pjM+HHOM9DkFQRCED0ciaVJilBunJpKa/x9Fw5tvvkmnnnqq+u/9mUNXXnkl3XvvvXTgwAGqq6tTr0+ZMoWeeuopuvHGG+lXv/oVTZw4kf74xz/SsmX4rjFc38vS5SPxpZaI6Prrr6frr7/+UC9DEARBEARBEAThsOeUU06hZFL/zfvee+91fM9bb7110HkPxfeyj8yX2uHgrBe/REYwQDV/fp2IiM78zGfUa2V3bVD1dZ89QdVVdyOQ6VeXVam65v42Va+4GAE60x6E+rX5bKh/5Y/hmO6TrWOCT2CDd9ZCvN71NFTQSXOQCLdtFVTTI67NUPXLa2ar+p5PQX399IajVf3Ts6xr/vIbx6qx9SfgA3vSlnNwPfPuUvVlWy9T9e+n/0XV36y7SNXfnYgArdubTlP1Z4teVfUTndZaluZuUmPP985S9Wl5m1X9aqga68rbpup14UpVL8nfqep3I/hl6NiCPareGstX9VH59UREtDOG+3Zk/gFV703g/s/Mx+b3hgRkjql5eOaNCSiOU9h4mwmVsyK3k4iIOpnCWpoDtbfTxHhRDqLZe5JQYQuyEODVx+bJyYLiG2LjWSzAK5JSZTNZeFecWMBWMMaOZSpwBld2cbwv6DzuCWCc/4Lo9icGjBl+vI+rxi6fs9praMZdTE21jacUXJs67HM+VqfIatVU7fEDx3VzkEZhTX/cedgxRUE3h5HeeDKNce2xOjVxOFTjdNXhca5gijosCILw4TApSSaNrlQ72uc7nBjzQVGCIAiCIAiCIAjC+EWUWsb0X/WQxx2jxJK5RETkvw3tT4wCKF/r7pms6pJ2qJm//fvZqq7a+Jqq/+Nl7IGteRnHX/veZ1Wd+zRa9nzngLVPt3hFrRq7479qVF3xTKuqV3wZCuLElVjjps9BwSt/Eb/atH4C+zuLXsXjN89KtbRZi9ZCWSchXj20vkDVJQuyVL1nY7mqp83G+LqtVaqeOQX38flduI6fl7+g6hv2zSQiopvmQb39Ze0Zqh4OFZiI6MTsrapeH8Ia52fWEhHRuxGovUdl7lX1lgjU8SOy6lW9PVqs6lnZUHZr43mqnpbVour9cdzfKVnWc2xIoHVMVTZU3VYTvzlVZGIfbxvboDEhC+OdJlTLkkwou91M8SxgrZl6UuO5rC1THzs2NwOqbpgpr1lBvo+XtVcK8H28CTYOpZYrwb5UCyo+5vXz/bpJx3E+t5sptTYVmKmvNiXYO1Addjkoqe8ft6m92vHkkMd159TuhR2mvbaOqqxOkdUquM7DaY3rlDrdWlzpjTuqw8O0X3VYFF9RKh0Z6b3GgiAIo4lp93+N2jnHK6LUCoIgCIIgCIIgCGM
|
|
|
|
|
"text/plain": [
|
|
|
|
|
"<Figure size 1200x800 with 2 Axes>"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"output_type": "display_data"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"source": [
|
2025-10-07 12:15:03 +02:00
|
|
|
"import random\n",
|
2025-10-04 19:43:42 +02:00
|
|
|
"import torch\n",
|
|
|
|
|
"import matplotlib.pyplot as plt\n",
|
|
|
|
|
"import Project_Model.Libs.Embedder as Embedder\n",
|
|
|
|
|
"\n",
|
2025-10-07 12:15:03 +02:00
|
|
|
"# set a fixed seed\n",
|
|
|
|
|
"torch.manual_seed(0)\n",
|
|
|
|
|
"random.seed(0)\n",
|
|
|
|
|
"\n",
|
2025-10-04 19:43:42 +02:00
|
|
|
"TOKENS = 256\n",
|
|
|
|
|
"DIMENSIONS = 256\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"# Custom code made by Christian Risi and Giuseppe Gassi\n",
|
|
|
|
|
"TENSOR = Embedder.fixed_positional_encoding(TOKENS, DIMENSIONS)\n",
|
|
|
|
|
"# print(TENSOR)\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"# Code taken from\n",
|
|
|
|
|
"# https://github.com/jalammar/jalammar.github.io/blob/master/notebookes/transformer/transformer_positional_encoding_graph.ipynb\n",
|
|
|
|
|
"# to test for correctness of custom code\n",
|
|
|
|
|
"print (TENSOR.shape)\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"plt.figure(figsize=(12,8))\n",
|
|
|
|
|
"plt.pcolormesh(TENSOR, cmap='viridis')\n",
|
|
|
|
|
"plt.xlabel('Embedding Dimensions')\n",
|
|
|
|
|
"plt.xlim((0, DIMENSIONS))\n",
|
|
|
|
|
"plt.ylim((TOKENS,0))\n",
|
|
|
|
|
"plt.ylabel('Token Position')\n",
|
|
|
|
|
"plt.colorbar()\n",
|
|
|
|
|
"plt.show()"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2025-10-07 12:15:03 +02:00
|
|
|
"execution_count": 10,
|
2025-10-04 19:43:42 +02:00
|
|
|
"id": "c7ad6593",
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [
|
|
|
|
|
{
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
2025-10-07 12:15:03 +02:00
|
|
|
"[[7706, 290, 756, 4270, 7357, 115, 351, 1507, 1213, 410, 3382, 317, 497, 4740, 2784, 7700], [7706, 290, 756, 4270, 7357, 115, 351, 1507, 1213, 410, 3382, 317, 497, 4740, 2784, 7700]]\n",
|
|
|
|
|
"2\n",
|
|
|
|
|
"torch.Size([2, 16, 256])\n",
|
|
|
|
|
"tensor([[[-0.6981, 0.0804, -2.1672, ..., 0.3919, 0.3341, 1.0794],\n",
|
|
|
|
|
" [ 2.5818, -0.2308, 0.6001, ..., -0.0500, -0.0408, -0.9852],\n",
|
|
|
|
|
" [-0.6967, 0.8109, 1.3108, ..., 2.1693, 1.4143, -0.1236],\n",
|
|
|
|
|
" ...,\n",
|
|
|
|
|
" [ 2.1226, 2.5695, -1.6178, ..., -0.0652, -0.0802, 0.1103],\n",
|
|
|
|
|
" [ 0.8770, -2.4782, 0.8536, ..., 2.0471, -1.5702, 0.7387],\n",
|
|
|
|
|
" [-0.0495, -1.8601, 0.0405, ..., 2.3944, -0.4297, 1.1141]],\n",
|
|
|
|
|
"\n",
|
|
|
|
|
" [[-0.6981, 0.0804, -2.1672, ..., 0.3919, 0.3341, 1.0794],\n",
|
|
|
|
|
" [ 2.5818, -0.2308, 0.6001, ..., -0.0500, -0.0408, -0.9852],\n",
|
|
|
|
|
" [-0.6967, 0.8109, 1.3108, ..., 2.1693, 1.4143, -0.1236],\n",
|
|
|
|
|
" ...,\n",
|
|
|
|
|
" [ 2.1226, 2.5695, -1.6178, ..., -0.0652, -0.0802, 0.1103],\n",
|
|
|
|
|
" [ 0.8770, -2.4782, 0.8536, ..., 2.0471, -1.5702, 0.7387],\n",
|
|
|
|
|
" [-0.0495, -1.8601, 0.0405, ..., 2.3944, -0.4297, 1.1141]]],\n",
|
2025-10-04 19:43:42 +02:00
|
|
|
" grad_fn=<AddBackward0>)\n"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"data": {
|
2025-10-07 12:15:03 +02:00
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA5YAAAKsCAYAAACXnSIBAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAl8ZJREFUeJzs3Xl8VPW9+P/3zCQz2ROyJ5BAWGQHEQQBRVBkEXGpba21LS4/bXtxqXit0l5Fetsv1fZ6bdVr7Sbaaq37WhdEARf2RfZ9C4EkQPZtJpk5vz8GPu+TBpRhwmSA1/PxmAfvnJz5bOdzPmc+fE7OOCzLsgQAAAAAgJPk7OgCAAAAAABOb0wsAQAAAABhYWIJAAAAAAgLE0sAAAAAQFiYWAIAAAAAwsLEEgAAAAAQFiaWAAAAAICwMLEEAAAAAISFiSUAAAAAICxMLAEAAAAAYTktJpZPPvmkdOvWTeLi4mTEiBGybNmyji4SAAAAAOCIqJ9Y/vOf/5QZM2bIrFmzZNWqVTJ48GCZOHGilJeXd3TRAAAAAAAi4rAsy+roQnyVESNGyPnnny9PPPGEiIgEAgEpKCiQO+64Q+6///4OLh0AAAAAIKajC/BVfD6frFy5UmbOnGm2OZ1OGT9+vCxevPiY7/F6veL1es3PgUBAKioqJCMjQxwOxykvMwAAAIDjsyxLamtrJT8/X5zOqL+Bso2mpibx+XwRz9ftdktcXFzE8z1RUT2xPHTokPj9fsnJyWm1PScnRzZv3nzM98yZM0dmz54dieIBAAAAOEnFxcXSpUuXji5GSJqamqSoa5KUlvsjnndubq7s2rUraieXUT2xPBkzZ86UGTNmmJ+rq6ulsLBQet/yoLjccdKUrfsWfNhg4t1Xxps4abeubNb0axERkbzCw2bbwaokE7c0uDVBh95VnLxet9f0azaxuyzWxM2p2iGdLfq/Ncndq7T8e1NERCSmzmXLR0N/XMDElkvzHzRwj4kv6LTTxM+8PV7zT9H3du19wMR79meJiEjBK5pR6Qgtd1yF5t+izSbebM0/kKb/izO61w4TL1vYT/NP0/rH1Gr9hl0Y/E+DL9/to2lnatoOLbak9tHCZN2jK9U7Z6eauPuDlSbefmtnE2f0O2Ti8uJOJo6tDJ4WnTZpnocHaZ5On7bLZeNXmvjdNYNNHLdfTy1vltYztkbradn+g87VvTa4b3mC2dZpve5b08NWf7/m35yufctVr3l6CmpNnPxmsokrJmufb6nUQcmKCaafvFXTqOuqDd3lnDITJ8Xqsd3/VlcTB2yjScDWXb2DNc/4VVq/2j5a9hF9gn20+PFeZltDjjZQ9Xl6bJ3Vtr5Yqvt4qjTPyoG2c8ur+8Qf0LbzH6n+Hde9Zbb9duUEEyenNmp9Fmr/SLj0oInLSnS7/Zinbde2a4nXPCsG6HG0jpy7aev0fbnf0PN282bbhdaj6bkqdH93dz3OfbP0787X7c8zcdKiRBP3uH6biTe9HWxrX5pmY9mHGdt/xMbrqSIpl+tYsW+bbUBN1DYvekHLu+ubWt7Yo2XvXme2tZRpn7Cc2j6eg1qYtBFat7K96ZpepyYTN1foYJRVoOd87dJMEzfl+4+UQ9P2J2qecaV6rPLGF5t4R4nW85r+a0z83lsjNJ1+9Vqnci3LOQM0neJ5wfPlmm9/arY9v2SUifv03mfiresKTBywjRWpW/WHBj3Mra4LMefU6A+rgtcQb1/tz8nLtHzVvfRYxVZr2jF9NY0Wv25vKdXj5a7Q7YE+Wv+EL7TP1RYF2zd9vRbp6js/MfFb/zPOxBUDdB/3Ya1QU76W0dmkeQ4YqdeWtauLtB6Vus/R8dfVyTaG2PpZc4Xtw5ltbO3VT4/FrvIMLddarVuzhhKjXdFcI5u0q0pjvp4fnnLtf81pWpbYSs0/ab9urxmvbZv2nmbadHW1ifN/ridsn2eD48j8F84327wX6DkXs1Y/w9ivrU6v5t+Spel59ujnmaYuOm7H1Oi57eys4/zRP7BqqbK1ra3NUzbq+2wfm6S+i218tPXnlrQWE8cX6/jvy7CNizXBY567TMvXlKn5lI/SNDJWaBr263BdocaXX7bcxAue03a0H1P7Z5GE4cHPiJUVenwsWyViSj1an1QtS0K2tts5mXpt2fBpD00nVtvleOdF4Mg13H1Y+5bP9tnD/vkw1nYNsW/PXqHxoWu1XDFrtL+4tRuJ60ifb9LLoBRO1GuYfQyTgO2AZgXfGGj0SvEdj0hysn5GOV34fD4pLffLnpXdJCU5cqutNbUB6Tp0t/h8PiaWJyMzM1NcLpeUlZW12l5WVia5ubnHfI/H4xGPx9Nmu8sdJy5PnDhtxyEmxnaxsh0gl0dPAGd8cACISdQ0nT7d12kde2Lp8uh2Z7ye6K44HdD88bYPv83aMV0Jtrzig3k5W449sbTijz2xjE3U/OOS9DC7bPX0297bqn5H8oyJddjep+V22ZrXssXOONsVIl7r406ytYUtf2er+mv9jpbd5bHtG3fsiaW9rWJs57YzIc623VY3W/7HqrOIiLMx2F4ut+2Ca78+OrVdPEnaLvY0XB7bBddeT99xJpYJzW3TcOu+rervt/dP2z5+23FO0A8FLretbgn2D2htJ5aty33s/hGrVW51jBy20cRh6672PFsd0/i2xzwm1l5/p21fW5299r5oO29sp2KrNnfY9rGd23KkSvG288Peb1y2cjs8X99v7G0XE2s7L93249V2Yuly2yZeifZxw9bpPPax6tjHudV77fWwHX/7PkePhb1v2yeW9juT7G17vPqLrc1bja3xtvY9UnZHgn6wCtjSsE8sXR4tzHHbXOc44m+0bbft37rPHZlkxNnOwzh7nrYPgsfJ037O29O2EvzH3D/mGGU53rjR6hjaPzTYjkur88L+ucLWtV0JOomSo8c5wVZP+5hgO89dXvt1SNOw/FoA+/Fyxdk6ia3+rdI/0r72PmS/JtnPeXt9Wl2Hbf+B6rQ9c/B47WUfF44ec2eCLT1bP3Pa+o19Ytnq+NvPJ9sxD9jLa7v8Ha2r/VppH5Psfdt/nP5nv/64jtO2rgSdzca42l6X7GV12c65411bnQ77WGUfN4/9ecbps4+dtjHvSJJO77Enlvax0n49t5fFPrE8+jks+F7buWPrFy5fsLwxsbbPW2779cyWhvvYE0t7/2t1ntvb3HZMj/VZxN6f7BNLZ5x9PLGVxXZsW43Ptv4ciD12H7HXX47sYx/b7H3O/vnQfg2xb4+x53Oc67bL9h+OrsDR39uKcbwxzD6xtI3bInJa/5laUrJDkpIjV/6ARH9bRfVNzW63W4YOHSrz58832wKBgMyfP19GjhzZgSUDAAAAABwV1SuWIiIzZsyQadOmybBhw2T48OHy2GOPSX19vdx0000dXTQAAAAAgJwGE8vrrrtODh48KA8++KCUlpbKueeeK++//36bB/oAAAAAQCT4rYD4I/iljX4r8PU7dbCon1iKiNx+++1y++23d3QxAAAAAADHcFpMLAEAAAAgWgTEkoBEbskyknmdrKh+eA8AAAAAIPqxYgkAAAAAIQhIQCL5V4+Rze3ksGIJAAAAAAgLE0sAAAAAQFi4FRYAAAAAQuC3LPFbkXugTiTzOlmsWAIAAAAAwsKKJQAAAACEgK8baYsVSwAAAABAWJhYAgAAAADCwq2wAAAAABCCgFji51bYVlixBAAAAACEhYklAAAAAITg6MN7IvkKx69//WtxOBzyk5/8pH0a4BiYWAIAAADAGWr58uXy9NNPy6BBg05pPkwsAQAAACAEfsuK+EtEpKamptXL6/V+ZTnr6urkhhtukD/96U/SqVOnU9omTCwBAAAA4DRQUFAgqamp5jVnzpyv3H/69OkyZcoUGT9+/CkvG0+FBQAAAIDTQHFxsaSkpJifPR7Pcfd98cUXZdWqVbJ8+fJIFI2JJQAAAACEInDkFcn8RERSUlJaTSyPp7i4WO666y6ZN2+exMXFndrCHcHEEgAAAADOICtXrpTy8nI577zzzDa/3y+LFi2SJ554Qrxer7hcrnbNk4klAAAAAIT
|
|
|
|
|
"text/plain": [
|
|
|
|
|
"<Figure size 1200x800 with 2 Axes>"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"output_type": "display_data"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"data": {
|
|
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA5YAAAKsCAYAAACXnSIBAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAl8ZJREFUeJzs3Xl8VPW9+P/3zCQz2ROyJ5BAWGQHEQQBRVBkEXGpba21LS4/bXtxqXit0l5Fetsv1fZ6bdVr7Sbaaq37WhdEARf2RfZ9C4EkQPZtJpk5vz8GPu+TBpRhwmSA1/PxmAfvnJz5bOdzPmc+fE7OOCzLsgQAAAAAgJPk7OgCAAAAAABOb0wsAQAAAABhYWIJAAAAAAgLE0sAAAAAQFiYWAIAAAAAwsLEEgAAAAAQFiaWAAAAAICwMLEEAAAAAISFiSUAAAAAICxMLAEAAAAAYTktJpZPPvmkdOvWTeLi4mTEiBGybNmyji4SAAAAAOCIqJ9Y/vOf/5QZM2bIrFmzZNWqVTJ48GCZOHGilJeXd3TRAAAAAAAi4rAsy+roQnyVESNGyPnnny9PPPGEiIgEAgEpKCiQO+64Q+6///4OLh0AAAAAIKajC/BVfD6frFy5UmbOnGm2OZ1OGT9+vCxevPiY7/F6veL1es3PgUBAKioqJCMjQxwOxykvMwAAAIDjsyxLamtrJT8/X5zOqL+Bso2mpibx+XwRz9ftdktcXFzE8z1RUT2xPHTokPj9fsnJyWm1PScnRzZv3nzM98yZM0dmz54dieIBAAAAOEnFxcXSpUuXji5GSJqamqSoa5KUlvsjnndubq7s2rUraieXUT2xPBkzZ86UGTNmmJ+rq6ulsLBQet/yoLjccdKUrfsWfNhg4t1Xxps4abeubNb0axERkbzCw2bbwaokE7c0uDVBh95VnLxet9f0azaxuyzWxM2p2iGdLfq/Ncndq7T8e1NERCSmzmXLR0N/XMDElkvzHzRwj4kv6LTTxM+8PV7zT9H3du19wMR79meJiEjBK5pR6Qgtd1yF5t+izSbebM0/kKb/izO61w4TL1vYT/NP0/rH1Gr9hl0Y/E+DL9/to2lnatoOLbak9tHCZN2jK9U7Z6eauPuDlSbefmtnE2f0O2Ti8uJOJo6tDJ4WnTZpnocHaZ5On7bLZeNXmvjdNYNNHLdfTy1vltYztkbradn+g87VvTa4b3mC2dZpve5b08NWf7/m35yufctVr3l6CmpNnPxmsokrJmufb6nUQcmKCaafvFXTqOuqDd3lnDITJ8Xqsd3/VlcTB2yjScDWXb2DNc/4VVq/2j5a9hF9gn20+PFeZltDjjZQ9Xl6bJ3Vtr5Yqvt4qjTPyoG2c8ur+8Qf0LbzH6n+Hde9Zbb9duUEEyenNmp9Fmr/SLj0oInLSnS7/Zinbde2a4nXPCsG6HG0jpy7aev0fbnf0PN282bbhdaj6bkqdH93dz3OfbP0787X7c8zcdKiRBP3uH6biTe9HWxrX5pmY9mHGdt/xMbrqSIpl+tYsW+bbUBN1DYvekHLu+ubWt7Yo2XvXme2tZRpn7Cc2j6eg1qYtBFat7K96ZpepyYTN1foYJRVoOd87dJMEzfl+4+UQ9P2J2qecaV6rPLGF5t4R4nW85r+a0z83lsjNJ1+9Vqnci3LOQM0neJ5wfPlmm9/arY9v2SUifv03mfiresKTBywjRWpW/WHBj3Mra4LMefU6A+rgtcQb1/tz8nLtHzVvfRYxVZr2jF9NY0Wv25vKdXj5a7Q7YE+Wv+EL7TP1RYF2zd9vRbp6js/MfFb/zPOxBUDdB/3Ya1QU76W0dmkeQ4YqdeWtauLtB6Vus/R8dfVyTaG2PpZc4Xtw5ltbO3VT4/FrvIMLddarVuzhhKjXdFcI5u0q0pjvp4fnnLtf81pWpbYSs0/ab9urxmvbZv2nmbadHW1ifN/ridsn2eD48j8F84327wX6DkXs1Y/w9ivrU6v5t+Spel59ujnmaYuOm7H1Oi57eys4/zRP7BqqbK1ra3NUzbq+2wfm6S+i218tPXnlrQWE8cX6/jvy7CNizXBY567TMvXlKn5lI/SNDJWaBr263BdocaXX7bcxAue03a0H1P7Z5GE4cHPiJUVenwsWyViSj1an1QtS0K2tts5mXpt2fBpD00nVtvleOdF4Mg13H1Y+5bP9tnD/vkw1nYNsW/PXqHxoWu1XDFrtL+4tRuJ60ifb9LLoBRO1GuYfQyTgO2AZgXfGGj0SvEdj0hysn5GOV34fD4pLffLnpXdJCU5cqutNbUB6Tp0t/h8PiaWJyMzM1NcLpeUlZW12l5WVia5ubnHfI/H4xGPx9Nmu8sdJy5PnDhtxyEmxnaxsh0gl0dPAGd8cACISdQ0nT7d12kde2Lp8uh2Z7ye6K44HdD88bYPv83aMV0Jtrzig3k5W449sbTijz2xjE3U/OOS9DC7bPX0297bqn5H8oyJddjep+V22ZrXssXOONsVIl7r406ytYUtf2er+mv9jpbd5bHtG3fsiaW9rWJs57YzIc623VY3W/7HqrOIiLMx2F4ut+2Ca78+OrVdPEnaLvY0XB7bBddeT99xJpYJzW3TcOu+rervt/dP2z5+23FO0A8FLretbgn2D2htJ5aty33s/hGrVW51jBy20cRh6672PFsd0/i2xzwm1l5/p21fW5299r5oO29sp2KrNnfY9rGd23KkSvG288Peb1y2cjs8X99v7G0XE2s7L93249V2Yuly2yZeifZxw9bpPPax6tjHudV77fWwHX/7PkePhb1v2yeW9juT7G17vPqLrc1bja3xtvY9UnZHgn6wCtjSsE8sXR4tzHHbXOc44m+0bbft37rPHZlkxNnOwzh7nrYPgsfJ037O29O2EvzH3D/mGGU53rjR6hjaPzTYjkur88L+ucLWtV0JOomSo8c5wVZP+5hgO89dXvt1SNOw/FoA+/Fyxdk6ia3+rdI/0r72PmS/JtnPeXt9Wl2Hbf+B6rQ9c/B47WUfF44ec2eCLT1bP3Pa+o19Ytnq+NvPJ9sxD9jLa7v8Ha2r/VppH5Psfdt/nP5nv/64jtO2rgSdzca42l6X7GV12c65411bnQ77WGUfN4/9ecbps4+dtjHvSJJO77Enlvax0n49t5fFPrE8+jks+F7buWPrFy5fsLwxsbbPW2779cyWhvvYE0t7/2t1ntvb3HZMj/VZxN6f7BNLZ5x9PLGVxXZsW43Ptv4ciD12H7HXX47sYx/b7H3O/vnQfg2xb4+x53Oc67bL9h+OrsDR39uKcbwxzD6xtI3bInJa/5laUrJDkpIjV/6ARH9bRfVNzW63W4YOHSrz58832wKBgMyfP19GjhzZgSUDAAAAABwV1SuWIiIzZsyQadOmybBhw2T48OHy2GOPSX19vdx0000dXTQAAAAAgJwGE8vrrrtODh48KA8++KCUlpbKueeeK++//36bB/oAAAAAQCT4rYD4I/iljX4r8PU7dbCon1iKiNx+++1y++23d3QxAAAAAADHcFpMLAEAAAAgWgTEkoBEbskyknmdrKh+eA8AAAAAIPqxYgkAAAAAIQhIQCL5V4+Rze3ksGIJAAAAAAgLE0sAAAAAQFi4FRYAAAAAQuC3LPFbkXugTiTzOlmsWAIAAAAAwsKKJQAAAACEgK8baYsVSwAAAABAWJhYAgAAAADCwq2wAAAAABCCgFji51bYVlixBAAAAACEhYklAAAAAITg6MN7IvkKx69//WtxOBzyk5/8pH0a4BiYWAIAAADAGWr58uXy9NNPy6BBg05pPkwsAQAAACAEfsuK+EtEpKamptXL6/V+ZTnr6urkhhtukD/96U/SqVOnU9omTCwBAAAA4DRQUFAgqamp5jVnzpyv3H/69OkyZcoUGT9+/CkvG0+FBQAAAIDTQHFxsaSkpJifPR7Pcfd98cUXZdWqVbJ8+fJIFI2JJQAAAACEInDkFcn8RERSUlJaTSyPp7i4WO666y6ZN2+exMXFndrCHcHEEgAAAADOICtXrpTy8nI577zzzDa/3y+LFi2SJ554Qrxer7hcrnbNk4klAAAAAIT
|
2025-10-04 19:43:42 +02:00
|
|
|
"text/plain": [
|
|
|
|
|
"<Figure size 1200x800 with 2 Axes>"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"output_type": "display_data"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"source": [
|
|
|
|
|
"from pathlib import Path\n",
|
|
|
|
|
"import Project_Model.Libs.BPE as BPE\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"TEXT = \"<ABS>The Dark Knight is a 2008 superhero film directed by Christopher Nolan,<SOTL>\"\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"VOCABULARY_PATH = Path(\"Assets/Model/toy_10/toy_dictionary.json\")\n",
|
|
|
|
|
"SPECIAL_VOC = BPE.default_special_tokens()\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"VOCABULARY = BPE.load_nanos_vocabulary(VOCABULARY_PATH)\n",
|
|
|
|
|
"TOKENANO = BPE.TokeNanoCore(\n",
|
|
|
|
|
" VOCABULARY,\n",
|
|
|
|
|
" SPECIAL_VOC\n",
|
|
|
|
|
")\n",
|
|
|
|
|
"\n",
|
2025-10-07 12:15:03 +02:00
|
|
|
"TOKENIZATION = [TOKENANO.encode(TEXT), TOKENANO.encode(TEXT)]\n",
|
2025-10-04 19:43:42 +02:00
|
|
|
"print(TOKENIZATION)\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"TOKEN_SPACE_SIZE = TOKENANO.vocabulary_size\n",
|
|
|
|
|
"EMBEDDED_SIZE = 256\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"EMBEDDER = Embedder.NanoSocratesEmbedder(TOKEN_SPACE_SIZE, EMBEDDED_SIZE)\n",
|
|
|
|
|
"TENSOR: torch.Tensor = EMBEDDER(TOKENIZATION)\n",
|
|
|
|
|
"print(len(TOKENIZATION))\n",
|
|
|
|
|
"print(TENSOR.shape)\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"print(TENSOR)\n",
|
|
|
|
|
"\n",
|
2025-10-07 12:15:03 +02:00
|
|
|
"BATCH_SIZE, TOKENS, DIMENSIONS = TENSOR.shape\n",
|
2025-10-04 19:43:42 +02:00
|
|
|
"\n",
|
2025-10-07 12:15:03 +02:00
|
|
|
"for i in range(0, BATCH_SIZE):\n",
|
|
|
|
|
"\n",
|
|
|
|
|
" PLOT_TENSOR = TENSOR[i,:,:].squeeze()\n",
|
|
|
|
|
"\n",
|
|
|
|
|
" plt.figure(figsize=(12,8))\n",
|
|
|
|
|
" plt.pcolormesh(PLOT_TENSOR.detach().numpy(), cmap='viridis')\n",
|
|
|
|
|
" plt.xlabel('Embedding Dimensions')\n",
|
|
|
|
|
" plt.xlim((0, DIMENSIONS))\n",
|
|
|
|
|
" plt.ylim((TOKENS,0))\n",
|
|
|
|
|
" plt.ylabel('Token Position')\n",
|
|
|
|
|
" plt.colorbar()\n",
|
|
|
|
|
" plt.show()\n"
|
2025-10-04 19:43:42 +02:00
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"metadata": {
|
|
|
|
|
"kernelspec": {
|
|
|
|
|
"display_name": "deep_learning",
|
|
|
|
|
"language": "python",
|
|
|
|
|
"name": "python3"
|
|
|
|
|
},
|
|
|
|
|
"language_info": {
|
|
|
|
|
"codemirror_mode": {
|
|
|
|
|
"name": "ipython",
|
|
|
|
|
"version": 3
|
|
|
|
|
},
|
|
|
|
|
"file_extension": ".py",
|
|
|
|
|
"mimetype": "text/x-python",
|
|
|
|
|
"name": "python",
|
|
|
|
|
"nbconvert_exporter": "python",
|
|
|
|
|
"pygments_lexer": "ipython3",
|
|
|
|
|
"version": "3.13.7"
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"nbformat": 4,
|
|
|
|
|
"nbformat_minor": 5
|
|
|
|
|
}
|
