{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "06229c81",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([256, 256])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 201416 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import Project_Model.Libs.Embedder as Embedder\n",
    "\n",
    "# set a fixed seed\n",
    "torch.manual_seed(0)\n",
    "random.seed(0)\n",
    "\n",
    "TOKENS = 256\n",
    "DIMENSIONS = 256\n",
    "\n",
    "# Custom code made by Christian Risi and Giuseppe Gassi\n",
    "TENSOR = Embedder.fixed_positional_encoding(TOKENS, DIMENSIONS)\n",
    "# print(TENSOR)\n",
    "\n",
    "\n",
    "# Code taken from\n",
    "#   https://github.com/jalammar/jalammar.github.io/blob/master/notebookes/transformer/transformer_positional_encoding_graph.ipynb\n",
    "#   to test for correctness of custom code\n",
    "print (TENSOR.shape)\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.pcolormesh(TENSOR, cmap='viridis')\n",
    "plt.xlabel('Embedding Dimensions')\n",
    "plt.xlim((0, DIMENSIONS))\n",
    "plt.ylim((TOKENS,0))\n",
    "plt.ylabel('Token Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c7ad6593",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7706, 290, 756, 4270, 7357, 115, 351, 1507, 1213, 410, 3382, 317, 497, 4740, 2784, 7700], [7706, 290, 756, 4270, 7357, 115, 351, 1507, 1213, 410, 3382, 317, 497, 4740, 2784, 7700]]\n",
      "2\n",
      "torch.Size([2, 16, 256])\n",
      "tensor([[[-0.6981,  0.0804, -2.1672,  ...,  0.3919,  0.3341,  1.0794],\n",
      "         [ 2.5818, -0.2308,  0.6001,  ..., -0.0500, -0.0408, -0.9852],\n",
      "         [-0.6967,  0.8109,  1.3108,  ...,  2.1693,  1.4143, -0.1236],\n",
      "         ...,\n",
      "         [ 2.1226,  2.5695, -1.6178,  ..., -0.0652, -0.0802,  0.1103],\n",
      "         [ 0.8770, -2.4782,  0.8536,  ...,  2.0471, -1.5702,  0.7387],\n",
      "         [-0.0495, -1.8601,  0.0405,  ...,  2.3944, -0.4297,  1.1141]],\n",
      "\n",
      "        [[-0.6981,  0.0804, -2.1672,  ...,  0.3919,  0.3341,  1.0794],\n",
      "         [ 2.5818, -0.2308,  0.6001,  ..., -0.0500, -0.0408, -0.9852],\n",
      "         [-0.6967,  0.8109,  1.3108,  ...,  2.1693,  1.4143, -0.1236],\n",
      "         ...,\n",
      "         [ 2.1226,  2.5695, -1.6178,  ..., -0.0652, -0.0802,  0.1103],\n",
      "         [ 0.8770, -2.4782,  0.8536,  ...,  2.0471, -1.5702,  0.7387],\n",
      "         [-0.0495, -1.8601,  0.0405,  ...,  2.3944, -0.4297,  1.1141]]],\n",
      "       grad_fn=<AddBackward0>)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 52004 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52004 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import Project_Model.Libs.BPE as BPE\n",
    "\n",
    "TEXT = \"<ABS>The Dark Knight is a 2008 superhero film directed by Christopher Nolan,<SOTL>\"\n",
    "\n",
    "VOCABULARY_PATH = Path(\"Assets/Model/toy_10/toy_dictionary.json\")\n",
    "SPECIAL_VOC = BPE.default_special_tokens()\n",
    "\n",
    "VOCABULARY = BPE.load_nanos_vocabulary(VOCABULARY_PATH)\n",
    "TOKENANO = BPE.TokeNanoCore(\n",
    "    VOCABULARY,\n",
    "    SPECIAL_VOC\n",
    ")\n",
    "\n",
    "TOKENIZATION = [TOKENANO.encode(TEXT), TOKENANO.encode(TEXT)]\n",
    "print(TOKENIZATION)\n",
    "\n",
    "TOKEN_SPACE_SIZE = TOKENANO.vocabulary_size\n",
    "EMBEDDED_SIZE = 256\n",
    "\n",
    "EMBEDDER = Embedder.NanoSocratesEmbedder(TOKEN_SPACE_SIZE, EMBEDDED_SIZE)\n",
    "TENSOR: torch.Tensor = EMBEDDER(TOKENIZATION)\n",
    "print(len(TOKENIZATION))\n",
    "print(TENSOR.shape)\n",
    "\n",
    "print(TENSOR)\n",
    "\n",
    "BATCH_SIZE, TOKENS, DIMENSIONS = TENSOR.shape\n",
    "\n",
    "for i in range(0, BATCH_SIZE):\n",
    "\n",
    "    PLOT_TENSOR = TENSOR[i,:,:].squeeze()\n",
    "\n",
    "    plt.figure(figsize=(12,8))\n",
    "    plt.pcolormesh(PLOT_TENSOR.detach().numpy(), cmap='viridis')\n",
    "    plt.xlabel('Embedding Dimensions')\n",
    "    plt.xlim((0, DIMENSIONS))\n",
    "    plt.ylim((TOKENS,0))\n",
    "    plt.ylabel('Token Position')\n",
    "    plt.colorbar()\n",
    "    plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
