NanoSocrates/Playgrounds/embedder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "06229c81",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([256, 256])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7UAAAKsCAYAAAAz95rSAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXl8XVW5/t+zz5x5Tpo0bdomnaCltNCBeSiUGRQVUAQRULzCVbjqVX9eEb3XWakoCiqzIoOICmgZCmVsKbQUaOncpmmbZp6TM+/z+2OfrOfdZK8mB5K0Ie/38/FzX9fZZ+219z6p95znWc/rSiaTSRIEQRAEQRAEQRCEMYhxqBcgCIIgCIIgCIIgCB8U+VIrCIIgCIIgCIIgjFnkS60gCIIgCIIgCIIwZpEvtYIgCIIgCIIgCMKYRb7UCoIgCIIgCIIgCGMW+VIrCIIgCIIgCIIgjFnkS60gCIIgCIIgCIIwZpEvtYIgCIIgCIIgCMKYRb7UCoIgCIIgCIIgCGMW+VIrCIIgCIIgCIIgjFk+Ml9qb7/9dqqqqqJAIECLFi2itWvXHuolCYIgCIIgCIIgHJa89NJLdP7551N5eTm5XC76+9//Puh7Vq1aRfPnzye/30/V1dV07733DjjmUHwv+0h8qX344YfppptuoptvvpnWr19PRx11FC1btoyampoO9dIEQRAEQRAEQRAOO3p7e+moo46i22+/fUjH7969m84991w69dRTacOGDfTVr36VrrnmGnr66afVMYfqe5krmUwmR/QMo8CiRYvo2GOPpd/85jdERGSaJlVWVtINN9xA3/zmNw/x6gRBEARBEARBEA5fXC4XPf7443TRRRdpj/nv//5veuqpp2jjxo1q7NJLL6WOjg5asWIFER2672WeEZt5lIhGo7Ru3Tr61re+pcYMw6ClS5fS6tWrHd8TiUQoEomo/26aJrW1tVFhYSG5XK4RX7MgCIIgCIIgCHqSySR1d3dTeXk5GcbYMpeGw2GKRqOH5NzJZHLA9xm/309+v/9Dz7169WpaunSpbWzZsmX01a9+lYg+2Pey4WLMf6ltaWmhRCJBpaWltvHS0lLasmWL43t+9KMf0S233DIayxMEQRAEQRAE4QOyd+9emjhx4qFexpAJh8M0ZXIWNTQlDsn5s7KyqKenxzZ288030/e+970PPXdDQ4Pjd66uri4KhULU3t6e9vey4WLMf6n9IHzrW9+im266Sf33zs5OmjRpElXe8h0yAgHa8LG7iYho3uOfV8f0j43m+Hg5Jx+Xey73/KN6Tj4u91zu+Uf1nHxc7rnc84/qOfm43PORO2dXj0mT59dSdnY2jSWi0Sg1NCVoz7oqysk2RvXcXd0mTV5QS3v37qWcnBw1Phwq7eHOmP9SW1RURG63mxobG23jjY2NVFZW5vgenQR/38X3UVa2Qd9uOYGIiO7+1P3qtZ91HaXqX33iUVX/Pjxd1T/42BOqftjEL0rfuPA5Vf/LVaTq/zjvFVW/7M5U9WfPfpOIiN7w+NTYx858V9Xv+bAN+qzT8avHLn9M1SeevFvVewOwWh97wj5VNwdDqj7quANERNQR7FNjMxY1q7o7A+NVC9pVHc7AHBVHd6k6lhlWdcncXlUnMrGWgiPxXsq0LBq5s/E+IxPXkzkDtTcrrupgTdxx3F+NX8f8WaaqfVNNx3FPlXVPg1m4t+5JsG7wcWOi87irAv9wZWUz28cEt/N4mdthzON8bAnGc7IxHxV7HceTRZrxwoHjyQKf87GaccrXjOdpxnP9Bx23jeVojtWNZ6c5nuVwzqyDHztgPDPwocddGc7HDtt48ODj6Rw7XOOH4px8fLjmNgIfbHw45hgL5+Tjh+KcfHy8nJOPyz2Xe/5ROycRjdmtgVnZLvv/LzcKmGSdLycnx/aldrgoKytz/M6Vk5NDwWCQ3G532t/LhovR/flgBPD5fLRgwQJauXKlGjNNk1auXElLliw5hCsTBEEQBEEQBEH4aLBkyRLbdy4iomeffVZ95zqU38vGvFJLRHTTTTfRlVdeSccccwwtXLiQli9fTr29vXTVVVcd6qUJgiAIgiAIgiAcdvT09NCOHTvUf9+9ezdt2LCBCgoKaNKkSfStb32L9u/fT/ffb7lXr7vuOvrNb35D3/jGN+jzn/88Pf/88/TII4/QU089peY4VN/LPhJfai+55BJqbm6m7373u9TQ0EDz5s2jFStWDNikPBil7ghluw1at3w+ERH9+McvqdduvOtEVd/yrXdU/c17z1T1pv/4napr/nSdqrdffoeqp/7zWlXvuOBOVU9/HnsJ3j3VGl+4Fg9/1bF/UPVFmy5X9Z9nwSJ9w+5PqPqHVY+r+ucHluE6KtBH6jfNJ6v6PyqeJyKiezsWqrFrJr6s6r91H6HqKyrXqPpfvVNV/anKdVhvCBaDCytxv16P5Kn6zImwTr8dDRIR0SkV+MPaHIP9ZElFrap3xGAbnl8OO/WeOOzHR5Y1qHpfApbnmhJYqhsTsDpPKWklIqJWE2MVxbBZt7Px0iLYrLuSmLuooFvVPSbG8wp6Hcdz8i1Ld5+JdLyMHFiyI0lYrgM5Ecdxn248G3PGkrgvnqzYgHF35sAxIiJ3BsZNwj13BeOO4xRMpDceSGjHBuA3HYeTunGfbtyhg5lX09VMO+48N3k0xzuMJzXHasfdIzee1Ph10h0nncPKaTydYw8zkpo1pjM+HHOM9DkFQRCED0ciaVJilBunJpKa/x9Fw5tvvkmnnnqq+u/9mUNXXnkl3XvvvXTgwAGqq6tTr0+ZMoWeeuopuvHGG+lXv/oVTZw4kf74xz/SsmX4rjFc38vS5SPxpZaI6Prrr6frr7/+UC9DEARBEARBEAThsOeUU06hZFL/zfvee+91fM9bb7110HkPxfeyj8yX2uHgrBe/REYwQDV/fp2IiM78zGfUa2V3bVD1dZ89QdVVdyOQ6VeXVam65v42Va+4GAE60x6E+rX5bKh/5Y/hmO6TrWOCT2CDd9ZCvN71NFTQSXOQCLdtFVTTI67NUPXLa2ar+p5PQX399IajVf3Ts6xr/vIbx6qx9SfgA3vSlnNwPfPuUvVlWy9T9e+n/0XV36y7SNXfnYgArdubTlP1Z4teVfUTndZaluZuUmPP985S9Wl5m1X9aqga68rbpup14UpVL8nfqep3I/hl6NiCPareGstX9VH59UREtDOG+3Zk/gFV703g/s/Mx+b3hgRkjql5eOaNCSiOU9h4mwmVsyK3k4iIOpnCWpoDtbfTxHhRDqLZe5JQYQuyEODVx+bJyYLiG2LjWSzAK5JSZTNZeFecWMBWMMaOZSpwBld2cbwv6DzuCWCc/4Lo9icGjBl+vI+rxi6fs9praMZdTE21jacUXJs67HM+VqfIatVU7fEDx3VzkEZhTX/cedgxRUE3h5HeeDKNce2xOjVxOFTjdNXhca5gijosCILw4TApSSaNrlQ72uc7nBjzQVGCIAiCIAiCIAjC+EWUWsb0X/WQxx2jxJK5RETkvw3tT4wCKF/r7pms6pJ2qJm//fvZqq7a+Jqq/+Nl7IGteRnHX/veZ1Wd+zRa9nzngLVPt3hFrRq7479qVF3xTKuqV3wZCuLElVjjps9BwSt/Eb/atH4C+zuLXsXjN89KtbRZi9ZCWSchXj20vkDVJQuyVL1nY7mqp83G+LqtVaqeOQX38flduI6fl7+g6hv2zSQiopvmQb39Ze0Zqh4OFZiI6MTsrapeH8Ia52fWEhHRuxGovUdl7lX1lgjU8SOy6lW9PVqs6lnZUHZr43mqnpbVour9cdzfKVnWc2xIoHVMVTZU3VYTvzlVZGIfbxvboDEhC+OdJlTLkkwou91M8SxgrZl6UuO5rC1THzs2NwOqbpgpr1lBvo+XtVcK8H28CTYOpZYrwb5UCyo+5vXz/bpJx3E+t5sptTYVmKmvNiXYO1Addjkoqe8ft6m92vHkkMd159TuhR2mvbaOqqxOkdUquM7DaY3rlDrdWlzpjTuqw8O0X3VYFF9RKh0Z6b3GgiAIo4lp93+N2jnHK6LUCoIgCIIgCIIgCGM
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import Project_Model.Libs.Embedder as Embedder\n",
    "\n",
    "TOKENS = 256\n",
    "DIMENSIONS = 256\n",
    "\n",
    "# Custom code made by Christian Risi and Giuseppe Gassi\n",
    "TENSOR = Embedder.fixed_positional_encoding(TOKENS, DIMENSIONS)\n",
    "# print(TENSOR)\n",
    "\n",
    "\n",
    "# Code taken from\n",
    "#   https://github.com/jalammar/jalammar.github.io/blob/master/notebookes/transformer/transformer_positional_encoding_graph.ipynb\n",
    "#   to test for correctness of custom code\n",
    "print (TENSOR.shape)\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.pcolormesh(TENSOR, cmap='viridis')\n",
    "plt.xlabel('Embedding Dimensions')\n",
    "plt.xlim((0, DIMENSIONS))\n",
    "plt.ylim((TOKENS,0))\n",
    "plt.ylabel('Token Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c7ad6593",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7706, 290, 756, 4270, 7357, 115, 351, 1507, 1213, 410, 3382, 317, 497, 4740, 2784, 7700]\n",
      "16\n",
      "torch.Size([16, 256])\n",
      "tensor([[-0.4328,  0.2221,  0.0101,  ...,  0.8186, -0.3063,  0.1974],\n",
      "        [ 0.1821, -0.0650,  2.6001,  ...,  1.2106, -0.0669,  1.9826],\n",
      "        [ 2.0355, -0.6010,  1.0185,  ...,  0.2779, -0.1062,  0.8631],\n",
      "        ...,\n",
      "        [ 1.3615,  0.0708,  0.2655,  ..., -0.3767, -0.3460,  1.4408],\n",
      "        [-0.4671, -0.8220,  2.1557,  ...,  1.6037, -1.0323,  0.8597],\n",
      "        [ 0.4647, -0.7747,  2.5598,  ...,  0.6946,  0.4063,  2.0707]],\n",
      "       grad_fn=<AddBackward0>)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA5YAAAKsCAYAAACXnSIBAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAmFtJREFUeJzs3Xl8VNXd+PHvzCSZ7AlJyEYW9n2VTRYRBEVE1NaqtdTi8mj7VK0V60KfRxFbS7WbfVp/2vZ5KrZ1q/u+IAqIsu8IhC1ACIQA2bdJMnN/fwyc740BZRiYTODzfr3mxZeTO/ece865587JubnjsCzLEgAAAAAATpGzrQsAAAAAAGjfmFgCAAAAAILCxBIAAAAAEBQmlgAAAACAoDCxBAAAAAAEhYklAAAAACAoTCwBAAAAAEFhYgkAAAAACAoTSwAAAABAUJhYAgAAAACC0i4mlk8++aR07txZoqOjZeTIkbJixYq2LhIAAAAA4Kiwn1i+9NJLMnPmTJk9e7asWbNGBg0aJJMnT5bS0tK2LhoAAAAAQEQclmVZbV2IrzNy5EgZPny4/PnPfxYREZ/PJ7m5uXLnnXfKAw880MalAwAAAABEtHUBvk5jY6OsXr1aZs2aZdKcTqdMmjRJli5detz3eDwe8Xg85v8+n0/KysokNTVVHA7HGS8zAAAAgBOzLEuqq6slOztbnM6wv4GylYaGBmlsbAx5vlFRURIdHR3yfE9WWE8sDx8+LF6vVzIyMlqkZ2RkyNatW4/7nrlz58qcOXNCUTwAAAAAp6ioqEhycnLauhgBaWhokC758VJS6g153pmZmVJYWBi2k8uwnlieilmzZsnMmTPN/ysrKyUvL0+63vWQuNzR4kn1mZ85m3QF82dT3jLx71+/0sRWlP9O4dT+h0xa4zvpuv9eur+4/CqNX0o0cen5Wj53To2Je6QdNnHRS11N3Gu6TprXLOgtIiLNSbZyN+hvdpxdq03cXBSv6c2aZ3OsvjcxV8son3bQ4+irb0je5O8WDSm6qWVb7I0aWGnihoIkzdNj28aWTeT4MhOX7042cVoXTT+8V8six27OjrMdhE1e1hET793X0cTRSfUmdqzV+o+o0/fW5umd3+4jelDNMbqNN86/TXyh/txy6c+rB+mBxm51m7gpQbfxuTWftD7ad6oXad+xbGefq8H/b4MejjSlNpnY6dbBy6rTN6bnlps4J0HbJTlSD/qTgl4mfv2Cp00840936fZT9vvzfy7LpB0+T8uSuF3roqKflsXp0b7oqtNtOmzT4y/vqelNqdqmA/vsNfGGTfkiIhK/Sys6ulz3cXiY9uFevfdpWRq04Q4W2Tqs7dwW+40KcVp2R6W/HuP26DHUZ2ueMfv1jU7btaPftVtMvHRDTxN367HfxHuX5Zr4gokbTLyosLuJfYf8F4WEXZr/vT96ycQvlw7XbW0nYNGrXfRwppaY+PDyTBMnD9M+NzC12MQLlg80sbuTfyxqbIg0adYR7c9XjFltYqdo/RfVaz2v2qjjlhWhdeew9YtLRujxryzNExGR8v16fua+re8rGaVliR2gfbt6R7KJfZG2NirRfBrStYzzpv7FxLc+958mbkz1N2Rkuo4VGUk6hh5eov1/8pUrTfzGusEmnthP23/TUwNMfESrVpJs50vX72838Y4Xe4iISGVPLav7iO0YOmq6Q0M5f1iBiVfuzTNxbprW0b7l+sGs37idJt75ejcREen1bS3H9pd7mLhurF6Tmqq1/Z2VOs74OtjGogg9GXyVUSaOytAxx7XONhie578YePZomsuj9RNZoZumXWQ7h7brL5QzuuqYX3rYNrbv0Q9WTTm6cjC4W5GJ123qLCIiue9rhR4arP2sPk+PLWmTprts17PKXtrncvoeMHHF+9kmbtRiiae7f0CPKLZ98LO1Z6R2OanrpeXukqPPjti3QtvT0VPf4GvW/jI8T8fQVcU65iTF+vOv3JCm5UvX40zcrMeZcImOISWH9XoeuV3HVoftD6aihmqfq9+g1+2hF+nnlrXz/Z9bLpqy1qSV2y6yy3boGNbjKa1o56903wWb9fgjq/S64NDDEEsPQ5pS/NcWd4n2W/vnkB5X7dCyeI7/gXzv+k4mjsi3nRce3ae9XhpTbI16VPxubZ+6YTrOeMv1XOmQr9dqeV/H0/Lh2hc6v6CVfuBWTW/cH2fingO0n5v6itdrbESp5tllqG5b81et28Pf0jImLog1cfO0Cs1zubZzx3F6jhYf9qdbh3XcyO+rP99zMFXLskvrreno51lfQ4Psm/1LSUiwjRftRGNjo5SUemXP6s6SmBC61daqap/kD90tjY2NTCxPRVpamrhcLjl48GCL9IMHD0pmZuZx3+N2u8XtdrdKd7mjxeWOFmeMbYLm0otbTLxWhcvWWL6jE8uION2nN0p/7ozW/blidYCMiLRvYytHrI6KkXF60rts+2yRfrQsPls+Ttszl+z789nKbZ9Y2o/ZXkZx27aP0Te4oiKO5q2b2ieWrqMXLRERpy1Pl30bj31724eVmOhvTDcTy5jjTyztbdFyf7YPtrZjc9kmBc5o3cbltk0c7dkf3cYVZfu57Uxxxmi6y9bXvPZz3DaxtJfXZStXi4mldax89nz0YuqMtk0sbW+079veb6KitO7sdRRvGwDtZTm2H1fUCfqtrS6cMbaJpcO2P5+tXqJs9Ryt6V5bm9rLe6yMLrces30f9j5sP+YI1wn6UMQJJpa2sjsaj/Zzt+0XNSfoH/aJZdRxyv3VctnHkKh42/axtjIeO+YozT82QY8/slbfZ59Y2tvoRHna01vk3+J88beF02mbWNbp+9zxmm6fWEY6j7+/FhNL221NUbb9uGrcrd4XEWnvK7Zt7eOD/QJq71v2trP1kRb93D4uHm1/+1gREacf2uznRIvjjzl+e55wnLedL8cb51uMySc4BvvEskWfi/3m9m+R59FjOtH15lg/EBHxNtvq3PZhWuxjUaTtZGi07TPWfky2yjh6zbG3odNhH0N10xON7S3S62xlt+3TG6P1eLyxJSLSXj5727qOm+6yTabs48KJxnP7cThjjr1Pf25vT1ejfVst94na02G7zottYmnvF65Ye5tarfI/0XGesG5tx2afWNrPyxP2uaPp9nM/sukE46btl7bOE7S/s9HW/2zb+2wTy2OfYVxu22c5W5vYyxcR0fozoshXPs/YzwuXfZ/2Om09sWxxPtvGGavB3lb6GUrs11x7X7CNp65Y+xhxgvPiWLrtGuuMjjruti3GLVsZ7eOCFXv8fn68/mJFu4//89gTnast6609/5lafIJD4hNCV36fhH9dhfVNzVFRUTJ06FBZsGCBSfP5fLJgwQIZNWpUG5YMAAAAAHBMWK9YiojMnDlTZsyYIcOGDZMRI0bIE088IbW1tXLTTTe1ddEAAAAAANIOJpbXXXedHDp0SB566CEpKSmRwYMHywcffNDqgT4AAAAAEApeyyfeEH5po9dqfft1uAn7iaWIyB133CF33HFHWxcDAAAAAHAc7WJiCQAAAADhwieW+CR0S5ahzOtUhfXDewAAAAAA4Y8VSwAAAAAIgE98Esq/egxtbqeGFUsAAAAAQFCYWAIAAAAAgsKtsAAAAAAQAK9lidcK3QN1QpnXqWLFEgAAAAAQFFYsAQAAACAAfN1Ia6xYAgAAAACCwsQSAAAAABAUboUFAAAAgAD4xBIvt8K2wIolAAAAACAorFgCAAAAQAB4eE9rrFgCAAAAAILCiiUAAAAABMBrWeK1QreKGMq8ThUrlgAAAACAoDCxBAAAAAAEhVthAQAAACAAvqOvUOYX7lixBAAAAAAEhRVLAAAAAAiAVyzxhvArQEKZ16lixRIAAAAAEBQmlgAAAACAoHArLAAAAAAEwGv5X6HML9yxYgkAAAAACAorlgAAAAAQAL5upDVWLAEAAAAAQWHFEgAAAAAC4BOHeMUR0vzCHSuWAAAAAICgMLEEAAAAAASFW2EBAAAAIAA+y/8KZX7hjhVLAAAAAEBQzpkVS0+qT5wxPnHV61z69mnvm/iX73/bxDFV+j5
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import Project_Model.Libs.BPE as BPE\n",
    "\n",
    "TEXT = \"<ABS>The Dark Knight is a 2008 superhero film directed by Christopher Nolan,<SOTL>\"\n",
    "\n",
    "VOCABULARY_PATH = Path(\"Assets/Model/toy_10/toy_dictionary.json\")\n",
    "SPECIAL_VOC = BPE.default_special_tokens()\n",
    "\n",
    "VOCABULARY = BPE.load_nanos_vocabulary(VOCABULARY_PATH)\n",
    "TOKENANO = BPE.TokeNanoCore(\n",
    "    VOCABULARY,\n",
    "    SPECIAL_VOC\n",
    ")\n",
    "\n",
    "TOKENIZATION = TOKENANO.encode(TEXT)\n",
    "print(TOKENIZATION)\n",
    "\n",
    "TOKEN_SPACE_SIZE = TOKENANO.vocabulary_size\n",
    "EMBEDDED_SIZE = 256\n",
    "\n",
    "EMBEDDER = Embedder.NanoSocratesEmbedder(TOKEN_SPACE_SIZE, EMBEDDED_SIZE)\n",
    "TENSOR: torch.Tensor = EMBEDDER(TOKENIZATION)\n",
    "print(len(TOKENIZATION))\n",
    "print(TENSOR.shape)\n",
    "\n",
    "print(TENSOR)\n",
    "\n",
    "TOKENS, DIMENSIONS = TENSOR.shape\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.pcolormesh(TENSOR.detach().numpy(), cmap='viridis')\n",
    "plt.xlabel('Embedding Dimensions')\n",
    "plt.xlim((0, DIMENSIONS))\n",
    "plt.ylim((TOKENS,0))\n",
    "plt.ylabel('Token Position')\n",
    "plt.colorbar()\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}