NanoSocrates/Playgrounds/nanosocrates-train-toy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adbd9598",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH 1\n",
      "\tLoss: 9.161508560180664\n",
      "ACTUAL:\n",
      "\tdbp-dbr:Category:Afpurl:linguistics/gold/hypernydbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award=dbp-dbr:Category:Af city beginSt girdbp-dbr:Category:Films_set_in_a_movie_thea_C Colin performance. performance. In Kong,dbp-dbr:Category:Films_shot_in_Tokyo folkloredbp-dbr:Jamiedbp-dbr:Ingenious_Film_PRogRog Agat=ahlberdbp-dbr:I'm_Sdbp-dbr:Category:Films_about_paralledbp-dbr:Matt_Ddbp-dbr:Category:Films_about begin official secon slu\u0013 slu slujangodbp-dbr:Roy_Buchadbp-dbr:Matt_D official charact slu begin .. officialdbp-dbr:Category:American_films_about_9720.0dbp-dbr:Megamind_dbp-dbr:Roy_Buchauzanne_Pleshettedbp-dbr:Category:American_films_about_ Endgamso2.9dbp-dbr:Category:Films_about_American_s138.0so back gro ..ialldbp-dbr:Sweet_Dreams_(dbp-dbr:Nobody_but_Me_(The_Isley_Bdbp-dbr:Category:2000_inddbp-dbr:Jamie novel,am_Bergmaneldbp-dbr:Category:Films_featuring 21st Nai folklore direct-to cityam_Bergman_Actor_Golden_Globe_winning_performance-noun-1_crime_itdbp-dbo:sound folkloredbp-dbr:Category:American_films_about_dbp-dbr:Category:Films_set_in_acinematographer)dbp-dbr:Ingenious_Film_P dialogue official neighborhood. pdbp-dbr:Category:Best_Drama_Picture_cinematographer) reunrounddbp-dbr:Category:Sdbp-dbr:David_S._G manufacturers Winstdbp-dbr:Matt_D selectedeinstince ban_R see 21stdbp-dbr:Category:Films_based_on_works_by_Bret_dbp-dbr:List_of_accolades_receiveddbp-dbr:Category:Films_about_telepresence citych causedbp-dbr:Category:Films_about_cannibalman, creative p Martelldbp-dbr:Category:American_films_about_ictur styli city replacitrover Review roledbp-dbr:Category:Yō bedbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award sets8.3am_Bergman tridbp-dbr:Category:Films_about_paraplegics_or_quadriplegdbp-dbr:Category:2000s_ official twodbp-dbr:Category:Films_featuring Blason Theroux,dbp-dbr:Category:Necrophilia_in_filmdbp-dbr:Category:Films_using_motion_c official operated8.3dbp-dbr:James_Remar save Nai official time.S ..dbp-dbr:Matt_D pdbp-dbr:Matt_D official Bla<START>dbp-dbr:Category:Films_featuring city two creativew3:2002/07/owl#round begin Begins. folkloreman, inspirhttp://www.w3.org/2006/03/wn/wn20/instances/synset-movie threedbp-dbr:MRC_(dbp-dbr:Reginald_Hudlindbp-dbr:Category:Yōs.dbp-dbr:Category:Cultdbp-dbr:Megamind_am_Bergman Yorksongdbp-dbr:Chadbp-dbr:Category:Films_set_in_adbp-dbr:Category:Cultarsgenre matchdbp-dbr:Category:Cult Batman novelhttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_re .. back acclaidbp-dbr:Charles_ officialdbp-dbr:Category:Cult Wyoming, city ten-yeardbp-dbr:Category:Films_set_in_adbp-dbr:Category:Films_featuring Theroux, sciencecontroversdbp-dbr:Category:Films_shot_in_Louisiana Cindydbp-dbr:Category:Yōdbp-dbr:Jamie friendtodbp-dbr:Category:2000s_chdbp-dbr:Category:Edgar_Award featur two1.006E9 Redbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Awarddbp-dbr:Category:Yō play,dbp-dbr:I'm_S8.3lan_Silvest Redbp-dbr:Philip_Proctorman,900000dbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Awarddbp-dbr:Category:Yōdbp-dbr:Category:Films_shot_in_Bedfordshiredbp-dbr:Category:Yō IsbinStephendbp-dbr:Category:Films_sho bot toget informatdbp-dbr:Category:Yōdbp-dbr:Category:The_Weinstein_Company_filmsling dialoguedbp-dbr:Category:Yōtra noveldbp-dbr:Category:American_films_about_ mastermindbp-dbr:Category:Cultmerican_1.006E9dbp-dbr:Category:Japanese_ghost_films novel dialogue1.006E9 Japanestrover<END>\n",
      "EXPECTED:\n",
      "\tThe Dark Knight is a 2008 superhero film directed by Christopher Nolan, from a screenplay co-written with his brother Jonathan. Based on the DC Comics superhero Batman, it is the sequel to Batman Begins (2005), and the second installment in The Dark Knight trilogy. The plot follows the vigilante Batman, police lieutenant James Gordon, and district attorney Harvey Dent, who form an alliance to dismantle organized crime in Gotham City. Their efforts are derailed by the Joker, an anarchistic mastermind who seeks to test how far Batman will go to save the city from chaos. The ensemble cast includes Christian Bale, Michael Caine, Heath Ledger, Gary Oldman, Aaron Eckhart, Maggie Gyllenhaal, and Morgan Freeman.Warner Bros. Pictures prioritized a sequel following the successful reinvention of the Batman film series with Batman Begins. Christopher and Batman Begins co-writer David S. Goyer developed the story elements, making Dent the central protagonist caught up in the battle between Batman and the Joker. In writing the screenplay, the Nolans were influenced by 1980s Batman comics and crime drama films, and sought to continue Batman Begins' heightened sense of realism. From April to November 2007, filming took place with a $185 million budget in Chicago and Hong Kong, and on sets<END><END>\n",
      "EPOCH 2\n",
      "\tLoss: 9.131484031677246\n",
      "ACTUAL:\n",
      "\tdbp-dbr:Category:Afpurl:linguistics/gold/hyperny2.9dbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_AwardGraham_ writdbp-dbr:20th_Century_St city beginSt girdbp-dbr:Category:Films_set_in_a_movie_thea_Co performance. performance.dbp-dbr:Category:Afscience_fictiondbp-dbr:Category:Films_shot_in_Tokyo folklore begin official BlaRog this= positivedbp-dbr:I'm_Sdbp-dbr:Category:Films_about_paralledbp-dbr:Matt_D Christ begin official.og match\u0013 slu slujangodbp-dbr:Roy_Buchadbp-dbr:Matt_D official charact back begincinematographer) officialdbp-dbr:Category:American_films_about_9720.0dbp-dbr:Megamind_uzanne_Pleshettedbp-dbr:Category:American_films_about_dbp-dbr:Category:Films_set_in_a folklore2.9comedy_dbp-dbr:One_Way execut back characters'dbp-dbr:Category:Best_Animated_Feature_Broadcast_ialldbp-dbr:Sweet_Dreams_(dbp-dbr:Nobody_but_Me_(The_Isley_Bdbp-dbr:Category:2000_inddbp-dbr:Jamie novel,3.0eldbp-dbr:Category:Films_featuring 21st Nai_F direct-to city BAFTA,_Actor_Golden_Globe_winning_performance-noun-1 Naiitdbp-dbo:sound inspirdbp-dbr:Category:American_films_about_ pcinematographer)dbp-dbr:Ingenious_Film_P dialogue ban p frecinematographer) reun_Actor_Golden_Globe_winning_performancedbp-dbr:Roy_Bsing manufacturers Winstdbp-dbr:Matt_D selectedeinstincears_R see 21stdbp-dbr:Category:Afdbp-dbp:namedbp-dbr:Category:Films_about_telepresence citych causedbp-dbr:Category:Films_about_cannibalman, creative p Martelldbp-dbr:Category:American_films_about_ emphasized styli Re replacitrovereghttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_redbp-dbr:Category:Yō targetsdbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award put8.3am_Bergman tridbp-dbr:Category:Yōdbp-dbr:Category:2000s_ official twodbp-dbr:Category:Films_featuring high-reso8.3 Theroux,dbp-dbr:Category:Necrophilia_in_filmdbp-dbr:Category:Films_using_motion_c official operated8.3drama_films save Nai official time.S ..dbp-dbr:Matt_Dent_dbp-dbr:Matt_D official Bla<START>dbp-dbr:Category:Films_featuringo twodbp-dbr:Category:Films_featuringcontroversies_ advertis begin Begins. folkloreman, inspirhttp://www.w3.org/2006/03/wn/wn20/instances/synset-movie .. noveldbp-dbr:Reginald_Hudlindbp-dbr:Category:Yōs.dbp-dbr:Category:Cultdbp-dbr:James_Remaram_Bergman Yorksongdbp-dbr:Chadbp-dbr:Category:Films_set_in_a (Evanars match Begins.genre matchdbp-dbr:Category:Cult Batman novelhttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_refter back acclai folklore officialdbp-dbr:The_Dark_Knight__Sound__1 noveldbp-dbr:Category:Films_featuring_a ten-yeardbp-dbr:Category:Films_set_in_adbp-dbr:Category:Films_featuring Theroux,dbp-dbr:Category:2010s_American_filmssuitle Cindydbp-dbr:Category:Yōdbp-dbr:Jamiemerican_todbp-dbr:Category:2000s_chdbp-dbr:Category:Edgar_Award featur two1.006E9 Redbp-dbr:Roy_Buchadbp-dbr:Category:Yōdbp-dbr:Category:Yōdbp-dbr:I'm_S8.3lan_Silvest (particularlydbp-dbr:Philip_Proctorman,900000 Naidbp-dbr:Category:Yō Goggindbp-dbr:Category:Yō IsbinStephendbp-dbr:Category:Films_sho botEnglish-languag informatdbp-dbr:Category:Yōdbp-dbr:Category:The_Weinstein_Company_films Japanesdbp-dbr:Category:Af BAFTA,tra noveldbp-dbr:Category:American_films_about_ mastermindbp-dbr:Category:Cultmerican_1.006E9dbp-dbr:Category:Japanese_ghost_films novel dialogue1.006E9 Japanes Behind<END>\n",
      "EXPECTED:\n",
      "\tThe Dark Knight is a 2008 superhero film directed by Christopher Nolan, from a screenplay co-written with his brother Jonathan. Based on the DC Comics superhero Batman, it is the sequel to Batman Begins (2005), and the second installment in The Dark Knight trilogy. The plot follows the vigilante Batman, police lieutenant James Gordon, and district attorney Harvey Dent, who form an alliance to dismantle organized crime in Gotham City. Their efforts are derailed by the Joker, an anarchistic mastermind who seeks to test how far Batman will go to save the city from chaos. The ensemble cast includes Christian Bale, Michael Caine, Heath Ledger, Gary Oldman, Aaron Eckhart, Maggie Gyllenhaal, and Morgan Freeman.Warner Bros. Pictures prioritized a sequel following the successful reinvention of the Batman film series with Batman Begins. Christopher and Batman Begins co-writer David S. Goyer developed the story elements, making Dent the central protagonist caught up in the battle between Batman and the Joker. In writing the screenplay, the Nolans were influenced by 1980s Batman comics and crime drama films, and sought to continue Batman Begins' heightened sense of realism. From April to November 2007, filming took place with a $185 million budget in Chicago and Hong Kong, and on sets<END><END>\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import torch\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import Project_Model.Libs.Embedder as Embedder\n",
    "import Project_Model.Libs.BPE as BPE\n",
    "import Project_Model.Libs.Transformer as Transformer\n",
    "import Project_Model.Libs.TorchShims as torch_shims\n",
    "\n",
    "# set a fixed seed\n",
    "torch.manual_seed(0)\n",
    "random.seed(0)\n",
    "DEVICE = torch_shims.get_default_device()\n",
    "torch.set_default_device(DEVICE)\n",
    "\n",
    "# set a default device\n",
    "\n",
    "# BPE Init\n",
    "VOCABULARY_PATH = Path(\"Assets/Model/toy_10/toy_dictionary.json\")\n",
    "SPECIAL_VOC = BPE.default_special_tokens()\n",
    "\n",
    "VOCABULARY = BPE.load_nanos_vocabulary(VOCABULARY_PATH)\n",
    "TOKENANO = BPE.TokeNanoCore(VOCABULARY, SPECIAL_VOC)\n",
    "\n",
    "\n",
    "# Constants\n",
    "TOKEN_SPACE_SIZE = TOKENANO.vocabulary_size + 1\n",
    "EMBEDDED_SIZE = 256\n",
    "FEED_FORWARD_MULTIPLIER =  4\n",
    "ATTENTION_HEADS = 4\n",
    "SENTENCE_LENGTH = 256\n",
    "NUMBER_OF_BLOCKS = 2\n",
    "MAX_EPOCHS = int(1e3)\n",
    "\n",
    "\n",
    "PAD_TOKEN = TOKENANO.encode(\"<PAD>\")[0]\n",
    "END_TOKEN = TOKENANO.encode(\"<END>\")[0]\n",
    "\n",
    "\n",
    "# Load CSV\n",
    "TOY_DATASET_PATH = Path(\"Assets/Dataset/1-hop/toy/rdf_text.csv\")\n",
    "\n",
    "TOY_DATASET = pd.read_csv(TOY_DATASET_PATH)\n",
    "\n",
    "TOY_BATCH_INPUT_LIST: list[list[int]] = []\n",
    "TOY_BATCH_PADDING_LIST: list[list[bool]] = []\n",
    "TOY_BATCH_TARGET_LIST: list[list[int]] = []\n",
    "TOY_BATCH_DECODER_DEFAULT: list[list[int]]= []\n",
    "\n",
    "\n",
    "for index, row in TOY_DATASET.iterrows():\n",
    "\n",
    "    RDFs: str = row[\"RDFs\"]\n",
    "    Abstract: str = row[\"Abstract\"]\n",
    "\n",
    "    input_tokens = TOKENANO.encode(RDFs)\n",
    "    output_tokens = TOKENANO.encode(Abstract)[1:]\n",
    "    decoder_default_tokens = TOKENANO.encode(\"<SOS>\")\n",
    "\n",
    "    input_tokens, padding = Transformer.normalize_sequence(\n",
    "        input_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN\n",
    "    )\n",
    "    output_tokens, _ = Transformer.normalize_sequence(\n",
    "        output_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN\n",
    "    )\n",
    "    decoder_default_tokens, _ = Transformer.normalize_sequence(\n",
    "        decoder_default_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN, False\n",
    "    )\n",
    "\n",
    "    TOY_BATCH_INPUT_LIST.append(input_tokens)\n",
    "    TOY_BATCH_PADDING_LIST.append(padding)\n",
    "    TOY_BATCH_TARGET_LIST.append(output_tokens)\n",
    "    TOY_BATCH_DECODER_DEFAULT.append(decoder_default_tokens)\n",
    "\n",
    "# Training loop\n",
    "LOSS_HISTORY = []\n",
    "NANOSOCRATES = Transformer.TrainingModel(\n",
    "    TOKEN_SPACE_SIZE,\n",
    "    EMBEDDED_SIZE,\n",
    "    FEED_FORWARD_MULTIPLIER,\n",
    "    ATTENTION_HEADS,\n",
    "    NUMBER_OF_BLOCKS\n",
    ")\n",
    "cross_entropy = torch.nn.CrossEntropyLoss(ignore_index=PAD_TOKEN)\n",
    "optimizer = torch.optim.AdamW(NANOSOCRATES.parameters())\n",
    "scheduler = Transformer.WarmupLR(optimizer, 4000, EMBEDDED_SIZE)\n",
    "last_loss = 0\n",
    "current_epoch = 0\n",
    "\n",
    "while current_epoch < MAX_EPOCHS:\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    encoder_list = torch.tensor([TOY_BATCH_INPUT_LIST[0]])\n",
    "    decoder_list = torch.tensor([TOY_BATCH_DECODER_DEFAULT[0]])\n",
    "    src_padding = torch.tensor([TOY_BATCH_PADDING_LIST[0]], dtype=torch.bool)\n",
    "\n",
    "    # Transform target into logits\n",
    "    target_logits = torch.tensor([TOY_BATCH_TARGET_LIST[0]])\n",
    "\n",
    "    last_loss = 0\n",
    "    last_prediction: torch.Tensor\n",
    "\n",
    "    for i in range(0, SENTENCE_LENGTH):\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        tgt_padding = decoder_list.eq(PAD_TOKEN)\n",
    "\n",
    "        logits: torch.Tensor = NANOSOCRATES((encoder_list, src_padding, decoder_list, tgt_padding))\n",
    "        prob = torch.softmax(logits, 2)\n",
    "\n",
    "        most_probable_tokens = torch.argmax(prob, 2)\n",
    "        last_prediction = most_probable_tokens\n",
    "\n",
    "        logits = logits[:,:i,:]\n",
    "        logits = logits.permute(0, 2, 1)\n",
    "\n",
    "        loss : torch.Tensor = cross_entropy(logits, target_logits[:, 0:i])\n",
    "        # loss : torch.Tensor = cross_entropy(logits, target_logits)\n",
    "\n",
    "        last_loss = loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        if i < SENTENCE_LENGTH - 1:\n",
    "            decoder_list[:,i+1] = target_logits[:,i]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    current_epoch += 1\n",
    "\n",
    "    if current_epoch % 1 == 0:\n",
    "        print(f\"EPOCH {current_epoch}\\n\\tLoss: {last_loss}\")\n",
    "\n",
    "        for encoded_sentence, expected_sentence in zip(\n",
    "            Transformer.tensor2token(last_prediction[:,:], END_TOKEN), # type: ignore\n",
    "            Transformer.tensor2token(target_logits[:,:], END_TOKEN)\n",
    "        ):\n",
    "            decoded_sentence = TOKENANO.decode(encoded_sentence)\n",
    "            decoded_target = TOKENANO.decode(expected_sentence)\n",
    "            print(f\"ACTUAL:\\n\\t{decoded_sentence}\\nEXPECTED:\\n\\t{decoded_target}\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}