diff --git a/Playgrounds/nanosocrates-train-toy.ipynb b/Playgrounds/nanosocrates-train-toy.ipynb index f69a836..41e1d51 100644 --- a/Playgrounds/nanosocrates-train-toy.ipynb +++ b/Playgrounds/nanosocrates-train-toy.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "adbd9598", "metadata": {}, "outputs": [ @@ -11,52 +11,30 @@ "output_type": "stream", "text": [ "c:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\utils\\_device.py:103: UserWarning: Aten Op fallback from XPU to CPU happends. This may have performance implications. If need debug the fallback ops please set environment variable `PYTORCH_DEBUG_XPU_FALLBACK=1` (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\build\\xpu\\ATen\\RegisterXPU_0.cpp:54528.)\n", - " return func(*args, **kwargs)\n" + " return func(*args, **kwargs)\n", + "252.87s - name 'tensor' is not defined\n", + "Traceback (most recent call last):\n", + " File \"c:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\debugpy\\_vendored\\pydevd\\_pydevd_bundle\\pydevd_vars.py\", line 636, in change_attr_expression\n", + " value = eval(expression, frame.f_globals, frame.f_locals)\n", + " File \"\", line 1, in \n", + "NameError: name 'tensor' is not defined\n" ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "EPOCH 0\n", - "\tCurrent Loss: 8.951058387756348\n", - "EPOCH 10\n", - "\tCurrent Loss: 8.913984298706055\n", - "EPOCH 20\n", - "\tCurrent Loss: 8.911956787109375\n", - "EPOCH 30\n", - "\tCurrent Loss: 8.911856651306152\n", - "EPOCH 40\n", - "\tCurrent Loss: 8.911840438842773\n", - "EPOCH 50\n", - "\tCurrent Loss: 8.911835670471191\n", - "EPOCH 60\n", - "\tCurrent Loss: 8.911831855773926\n", - "EPOCH 70\n", - "\tCurrent Loss: 8.91179084777832\n", - "EPOCH 80\n", - "\tCurrent Loss: 8.899038314819336\n", - "EPOCH 90\n", - "\tCurrent Loss: 8.898558616638184\n" - ] - }, - { - "ename": "KeyboardInterrupt", + "ename": "", "evalue": "", "output_type": "error", "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 133\u001b[39m\n\u001b[32m 130\u001b[39m LOSS_HISTORY.append(loss)\n\u001b[32m 132\u001b[39m loss.backward()\n\u001b[32m--> \u001b[39m\u001b[32m133\u001b[39m \u001b[43moptimizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 134\u001b[39m \u001b[38;5;66;03m# scheduler.step()\u001b[39;00m\n\u001b[32m 136\u001b[39m most_probable_tokens = torch.argmax(logits, \u001b[32m2\u001b[39m)\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\optimizer.py:516\u001b[39m, in \u001b[36mOptimizer.profile_hook_step..wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 511\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 512\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 513\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m must return None or a tuple of (new_args, new_kwargs), but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresult\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 514\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m516\u001b[39m out = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 517\u001b[39m \u001b[38;5;28mself\u001b[39m._optimizer_step_code()\n\u001b[32m 519\u001b[39m \u001b[38;5;66;03m# call optimizer step post hooks\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\optimizer.py:81\u001b[39m, in \u001b[36m_use_grad_for_differentiable.._use_grad\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 79\u001b[39m torch.set_grad_enabled(\u001b[38;5;28mself\u001b[39m.defaults[\u001b[33m\"\u001b[39m\u001b[33mdifferentiable\u001b[39m\u001b[33m\"\u001b[39m])\n\u001b[32m 80\u001b[39m torch._dynamo.graph_break()\n\u001b[32m---> \u001b[39m\u001b[32m81\u001b[39m ret = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 82\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 83\u001b[39m torch._dynamo.graph_break()\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\adam.py:247\u001b[39m, in \u001b[36mAdam.step\u001b[39m\u001b[34m(self, closure)\u001b[39m\n\u001b[32m 235\u001b[39m beta1, beta2 = group[\u001b[33m\"\u001b[39m\u001b[33mbetas\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m 237\u001b[39m has_complex = \u001b[38;5;28mself\u001b[39m._init_group(\n\u001b[32m 238\u001b[39m group,\n\u001b[32m 239\u001b[39m params_with_grad,\n\u001b[32m (...)\u001b[39m\u001b[32m 244\u001b[39m state_steps,\n\u001b[32m 245\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m247\u001b[39m \u001b[43madam\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 248\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams_with_grad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 249\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 250\u001b[39m \u001b[43m \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 251\u001b[39m \u001b[43m \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 252\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 253\u001b[39m \u001b[43m \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 254\u001b[39m \u001b[43m \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mamsgrad\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 255\u001b[39m \u001b[43m \u001b[49m\u001b[43mhas_complex\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhas_complex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 256\u001b[39m \u001b[43m \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 257\u001b[39m \u001b[43m \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 258\u001b[39m \u001b[43m \u001b[49m\u001b[43mlr\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlr\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 259\u001b[39m \u001b[43m \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mweight_decay\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 260\u001b[39m \u001b[43m \u001b[49m\u001b[43meps\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43meps\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 261\u001b[39m \u001b[43m \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmaximize\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 262\u001b[39m \u001b[43m \u001b[49m\u001b[43mforeach\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mforeach\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 263\u001b[39m \u001b[43m \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcapturable\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 264\u001b[39m \u001b[43m \u001b[49m\u001b[43mdifferentiable\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mdifferentiable\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 265\u001b[39m \u001b[43m \u001b[49m\u001b[43mfused\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mfused\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 266\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mgrad_scale\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 267\u001b[39m \u001b[43m \u001b[49m\u001b[43mfound_inf\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mfound_inf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 268\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecoupled_weight_decay\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mdecoupled_weight_decay\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 269\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 271\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m loss\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\optimizer.py:149\u001b[39m, in \u001b[36m_disable_dynamo_if_unsupported..wrapper..maybe_fallback\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 147\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m disabled_func(*args, **kwargs)\n\u001b[32m 148\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m149\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\adam.py:949\u001b[39m, in \u001b[36madam\u001b[39m\u001b[34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, differentiable, fused, grad_scale, found_inf, has_complex, decoupled_weight_decay, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[39m\n\u001b[32m 946\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 947\u001b[39m func = _single_tensor_adam\n\u001b[32m--> \u001b[39m\u001b[32m949\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 950\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 951\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 952\u001b[39m \u001b[43m \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 953\u001b[39m \u001b[43m \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 954\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 955\u001b[39m \u001b[43m \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 956\u001b[39m \u001b[43m \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[43m=\u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 957\u001b[39m \u001b[43m \u001b[49m\u001b[43mhas_complex\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhas_complex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 958\u001b[39m \u001b[43m \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 959\u001b[39m \u001b[43m \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 960\u001b[39m \u001b[43m \u001b[49m\u001b[43mlr\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlr\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 961\u001b[39m \u001b[43m \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[43m=\u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 962\u001b[39m \u001b[43m \u001b[49m\u001b[43meps\u001b[49m\u001b[43m=\u001b[49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 963\u001b[39m \u001b[43m \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmaximize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 964\u001b[39m \u001b[43m \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcapturable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 965\u001b[39m \u001b[43m \u001b[49m\u001b[43mdifferentiable\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdifferentiable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 966\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 967\u001b[39m \u001b[43m \u001b[49m\u001b[43mfound_inf\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfound_inf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 968\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecoupled_weight_decay\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecoupled_weight_decay\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 969\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\adam.py:755\u001b[39m, in \u001b[36m_multi_tensor_adam\u001b[39m\u001b[34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, grad_scale, found_inf, amsgrad, has_complex, beta1, beta2, lr, weight_decay, eps, maximize, capturable, differentiable, decoupled_weight_decay)\u001b[39m\n\u001b[32m 752\u001b[39m torch._foreach_addcdiv_(device_params, device_exp_avgs, exp_avg_sq_sqrt)\n\u001b[32m 753\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 754\u001b[39m bias_correction1 = [\n\u001b[32m--> \u001b[39m\u001b[32m755\u001b[39m \u001b[32m1\u001b[39m - beta1 ** \u001b[43m_get_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstep\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m step \u001b[38;5;129;01min\u001b[39;00m device_state_steps\n\u001b[32m 756\u001b[39m ]\n\u001b[32m 757\u001b[39m bias_correction2 = [\n\u001b[32m 758\u001b[39m \u001b[32m1\u001b[39m - beta2 ** _get_value(step) \u001b[38;5;28;01mfor\u001b[39;00m step \u001b[38;5;129;01min\u001b[39;00m device_state_steps\n\u001b[32m 759\u001b[39m ]\n\u001b[32m 761\u001b[39m step_size = _stack_if_compiling([(lr / bc) * -\u001b[32m1\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m bc \u001b[38;5;129;01min\u001b[39;00m bias_correction1])\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\optim\\optimizer.py:96\u001b[39m, in \u001b[36m_get_value\u001b[39m\u001b[34m(x)\u001b[39m\n\u001b[32m 94\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m x\n\u001b[32m 95\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m96\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mx\u001b[49m\u001b[43m.\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(x, torch.Tensor) \u001b[38;5;28;01melse\u001b[39;00m x\n", - "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\utils\\_device.py:103\u001b[39m, in \u001b[36mDeviceContext.__torch_function__\u001b[39m\u001b[34m(self, func, types, args, kwargs)\u001b[39m\n\u001b[32m 101\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m func \u001b[38;5;129;01min\u001b[39;00m _device_constructors() \u001b[38;5;129;01mand\u001b[39;00m kwargs.get(\u001b[33m\"\u001b[39m\u001b[33mdevice\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 102\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mdevice\u001b[39m\u001b[33m\"\u001b[39m] = \u001b[38;5;28mself\u001b[39m.device\n\u001b[32m--> \u001b[39m\u001b[32m103\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[31mKeyboardInterrupt\u001b[39m: " + "\u001b[1;31mCannot execute code, session has been disposed. Please try restarting the Kernel." + ] + }, + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mCannot execute code, session has been disposed. Please try restarting the Kernel. \n", + "\u001b[1;31mView Jupyter log for further details." ] } ], @@ -89,29 +67,13 @@ "# Constants\n", "TOKEN_SPACE_SIZE = TOKENANO.vocabulary_size + 1\n", "EMBEDDED_SIZE = 256\n", - "FEED_FORWARD_DIM = EMBEDDED_SIZE * 4\n", + "FEED_FORWARD_MULTIPLIER = 4\n", "ATTENTION_HEADS = 4\n", "SENTENCE_LENGTH = 256\n", "NUMBER_OF_BLOCKS = 2\n", "MAX_EPOCHS = int(1e3)\n", "\n", "\n", - "# Model Init\n", - "ENCODER_EMBEDDER = Embedder.NanoSocratesEmbedder(TOKEN_SPACE_SIZE, EMBEDDED_SIZE)\n", - "DECODER_EMBEDDER = Embedder.NanoSocratesEmbedder(TOKEN_SPACE_SIZE, EMBEDDED_SIZE)\n", - "\n", - "ENCODERS = [\n", - " Transformer.Encoder(EMBEDDED_SIZE, FEED_FORWARD_DIM, ATTENTION_HEADS)\n", - "] * NUMBER_OF_BLOCKS\n", - "ENCODER = torch.nn.Sequential(*ENCODERS)\n", - "\n", - "DECODERS = [\n", - " Transformer.Decoder(EMBEDDED_SIZE, FEED_FORWARD_DIM, ATTENTION_HEADS)\n", - "] * NUMBER_OF_BLOCKS\n", - "DECODER = torch.nn.Sequential(*DECODERS)\n", - "\n", - "DETOKENER = Transformer.DeToken(EMBEDDED_SIZE, TOKEN_SPACE_SIZE)\n", - "\n", "PAD_TOKEN = TOKENANO.encode(\"\")[0]\n", "END_TOKEN = TOKENANO.encode(\"\")[0]\n", "\n", @@ -133,7 +95,7 @@ " Abstract: str = row[\"Abstract\"]\n", "\n", " input_tokens = TOKENANO.encode(RDFs)\n", - " output_tokens = TOKENANO.encode(Abstract)\n", + " output_tokens = TOKENANO.encode(Abstract)[1:]\n", " decoder_default_tokens = TOKENANO.encode(\"\")\n", "\n", " input_tokens, padding = Transformer.normalize_sequence(\n", @@ -153,16 +115,16 @@ "\n", "# Training loop\n", "LOSS_HISTORY = []\n", - "NANOSOCRATES = torch.nn.ModuleList([\n", - " ENCODER_EMBEDDER,\n", - " ENCODER,\n", - " DECODER_EMBEDDER,\n", - " DECODER,\n", - " DETOKENER\n", - "])\n", - "cross_entropy = torch.nn.CrossEntropyLoss()\n", + "NANOSOCRATES = Transformer.TrainingModel(\n", + " TOKEN_SPACE_SIZE,\n", + " EMBEDDED_SIZE,\n", + " FEED_FORWARD_MULTIPLIER,\n", + " ATTENTION_HEADS,\n", + " NUMBER_OF_BLOCKS\n", + ")\n", + "cross_entropy = torch.nn.CrossEntropyLoss(ignore_index=PAD_TOKEN)\n", "optimizer = torch.optim.AdamW(NANOSOCRATES.parameters())\n", - "# scheduler = torch.optim.lr_scheduler.LRScheduler(optimizer)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 4)\n", "last_loss = 0\n", "current_epoch = 0\n", "\n", @@ -170,47 +132,39 @@ "\n", " optimizer.zero_grad()\n", "\n", - " INPUT_LIST = TOY_BATCH_INPUT_LIST[:]\n", - " TARGET_LIST = TOY_BATCH_TARGET_LIST[:]\n", - " # Transform target into logits\n", - " target_logits = torch.tensor(TOY_BATCH_TARGET_LIST[:])\n", - " DECODER_DEFAULT_LIST = TOY_BATCH_DECODER_DEFAULT[:]\n", - " PADDINGS = torch.tensor(TOY_BATCH_PADDING_LIST, dtype=torch.bool)\n", - " ENCODER_INPUTS = ENCODER_EMBEDDER(INPUT_LIST)\n", - " DECODER_INPUTS = DECODER_EMBEDDER(DECODER_DEFAULT_LIST)\n", + " encoder_list = torch.tensor([TOY_BATCH_INPUT_LIST[0]])\n", + " decoder_list = torch.tensor([TOY_BATCH_DECODER_DEFAULT[0]])\n", + " padding_list = torch.tensor([TOY_BATCH_PADDING_LIST[0]], dtype=torch.bool)\n", "\n", - " for _ in range(0, SENTENCE_LENGTH):\n", + " # Transform target into logits\n", + " target_logits = torch.tensor([TOY_BATCH_TARGET_LIST[0]])\n", + "\n", + " last_loss = 0\n", + "\n", + " for i in range(0, SENTENCE_LENGTH):\n", "\n", " optimizer.zero_grad()\n", "\n", - " \n", - "\n", - " encoder_output, _ = ENCODER((ENCODER_INPUTS, PADDINGS))\n", - "\n", - " decoder_output, _, _, _ = DECODER(\n", - " (DECODER_INPUTS, encoder_output, encoder_output, None)\n", - " )\n", - "\n", - " logits: torch.Tensor = DETOKENER(decoder_output)\n", - " logits = logits.permute(0, 2, 1)\n", - "\n", - "\n", - " loss: torch.Tensor= cross_entropy(logits, target_logits)\n", - " last_loss = loss\n", - " LOSS_HISTORY.append(loss)\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - " # scheduler.step()\n", + " logits: torch.Tensor = NANOSOCRATES((encoder_list, padding_list, decoder_list))\n", "\n", " most_probable_tokens = torch.argmax(logits, 2)\n", "\n", - " if current_epoch % 10 == 0:\n", + " logits = logits[:,i,:]\n", + "\n", + " loss = cross_entropy(logits, target_logits[:,i])\n", + " last_loss = loss\n", + " optimizer.step()\n", + " scheduler.step()\n", + "\n", + " if i < SENTENCE_LENGTH - 1:\n", + " decoder_list[:,i+1] = most_probable_tokens[:,i]\n", "\n", - " print(f\"EPOCH {current_epoch}\\n\\tCurrent Loss: {last_loss}\")\n", "\n", " current_epoch += 1\n", "\n", + " if current_epoch % 1 == 0:\n", + " print(f\"EPOCH {current_epoch}\\n\\tLoss: {last_loss}\")\n", + "\n", "\n", "\n", "\n", diff --git a/Project_Model/Libs/Embedder/Classes/NanoSocratesEmbedder.py b/Project_Model/Libs/Embedder/Classes/NanoSocratesEmbedder.py index b5eaca9..dd11fb1 100644 --- a/Project_Model/Libs/Embedder/Classes/NanoSocratesEmbedder.py +++ b/Project_Model/Libs/Embedder/Classes/NanoSocratesEmbedder.py @@ -9,11 +9,9 @@ class NanoSocratesEmbedder(torch.nn.Module): super().__init__() self.__embedder = torch.nn.Embedding(vocabulary_size, embedding_size) - def forward(self, tokenized_sentence: list[list[int]]) -> torch.Tensor: + def forward(self, tokenized_sentence: torch.Tensor) -> torch.Tensor: - TOKENIZED_TENSOR = torch.tensor(tokenized_sentence) - - computed_embeddings: torch.Tensor = self.__embedder(TOKENIZED_TENSOR) + computed_embeddings: torch.Tensor = self.__embedder(tokenized_sentence) _, SENTENCE_LENGHT, EMBEDDING_SIZE = computed_embeddings.shape # for batching diff --git a/Project_Model/Libs/Transformer/Classes/Decoder.py b/Project_Model/Libs/Transformer/Classes/Decoder.py index a9c7907..8074da1 100644 --- a/Project_Model/Libs/Transformer/Classes/Decoder.py +++ b/Project_Model/Libs/Transformer/Classes/Decoder.py @@ -56,12 +56,12 @@ class Decoder(nn.Module): ) # 2) Dropout - DROPPED_MASKED_ATTENTION = self.__dropout(MASKED_ATTENTION) - del MASKED_ATTENTION + # DROPPED_MASKED_ATTENTION = self.__dropout(MASKED_ATTENTION) + # del MASKED_ATTENTION # 3) Residual Connection - x = x + DROPPED_MASKED_ATTENTION - del DROPPED_MASKED_ATTENTION + x = x + MASKED_ATTENTION + del MASKED_ATTENTION # 4) Layer Normalization x = self.__layer_norm_1(x) @@ -72,12 +72,12 @@ class Decoder(nn.Module): ) # 6) Dropout - DROPPED_CROSS_ATTENTION = self.__dropout(CROSS_ATTENTION) - del CROSS_ATTENTION + # DROPPED_CROSS_ATTENTION = self.__dropout(CROSS_ATTENTION) + # del CROSS_ATTENTION # 7) Residual Connection - x = x + DROPPED_CROSS_ATTENTION - del DROPPED_CROSS_ATTENTION + x = x + CROSS_ATTENTION + del CROSS_ATTENTION # 8) Layer Normalization x = self.__layer_norm_2(x) @@ -86,12 +86,12 @@ class Decoder(nn.Module): FEED_FORWARD = self.__feed_forward_network(x) # 10) Dropout - DROPPED_FEED_FORWARD = self.__dropout(FEED_FORWARD) - del FEED_FORWARD + # DROPPED_FEED_FORWARD = self.__dropout(FEED_FORWARD) + # del FEED_FORWARD # 11) Residual Connection - x = x + DROPPED_FEED_FORWARD - del DROPPED_FEED_FORWARD + x = x + FEED_FORWARD + del FEED_FORWARD # 12) Layer Normalization x = self.__layer_norm_3(x) diff --git a/Project_Model/Libs/Transformer/Classes/Encoder.py b/Project_Model/Libs/Transformer/Classes/Encoder.py index e232a18..78fca5e 100644 --- a/Project_Model/Libs/Transformer/Classes/Encoder.py +++ b/Project_Model/Libs/Transformer/Classes/Encoder.py @@ -43,11 +43,12 @@ class Encoder( ATTENTION = self.__attention(x, x, x, key_padding_mask=padding_mask) # 2) Dropout - DROPPED_ATTENTION = self.__dropout(ATTENTION) - del ATTENTION + # DROPPED_ATTENTION = self.__dropout(ATTENTION) + # del ATTENTION # 3) Residual Connection - x = x + DROPPED_ATTENTION + x = x + ATTENTION + del ATTENTION # 4) Layer Normalization x = self.__layer_norm_1(x) @@ -56,12 +57,12 @@ class Encoder( FEED_FORWARD = self.__feed_forward(x) # 6) Dropout - DROPPED_FEED_FORWARD = self.__dropout(FEED_FORWARD) - del FEED_FORWARD + # DROPPED_FEED_FORWARD = self.__dropout(FEED_FORWARD) + # del FEED_FORWARD # 7) Residual Connection - x = x + DROPPED_FEED_FORWARD - del DROPPED_FEED_FORWARD + x = x + FEED_FORWARD + del FEED_FORWARD # 8) Layer Normalization x = self.__layer_norm_2(x) diff --git a/Project_Model/Libs/Transformer/Models/TrainingModel.py b/Project_Model/Libs/Transformer/Models/TrainingModel.py index ebd8010..2a72717 100644 --- a/Project_Model/Libs/Transformer/Models/TrainingModel.py +++ b/Project_Model/Libs/Transformer/Models/TrainingModel.py @@ -37,12 +37,11 @@ class TrainingModel(torch.nn.Module): self.__detokener = DeToken(latent_space, vocabulary_size) - def forward(self, args: tuple[list[list[int]], list[list[bool]], list[list[int]]]): - - encoder_embedder_input, padding_input, decoder_embedder_input = args + def forward(self, args: tuple[torch.Tensor, torch.Tensor, torch.Tensor]): + + encoder_embedder_input, padding_tensor, decoder_embedder_input = args encoder_tensor = self.__encoder_embedder(encoder_embedder_input) - padding_tensor = torch.tensor(padding_input, dtype=torch.bool) decoder_tensor = self.__decoder_embedder(decoder_embedder_input) encoder_output, _ = self.__encoder((encoder_tensor, padding_tensor)) diff --git a/Project_Model/Libs/Transformer/Models/__init__.py b/Project_Model/Libs/Transformer/Models/__init__.py new file mode 100644 index 0000000..8afa378 --- /dev/null +++ b/Project_Model/Libs/Transformer/Models/__init__.py @@ -0,0 +1,5 @@ +from .TrainingModel import TrainingModel + +__all__ = [ + "TrainingModel" +] \ No newline at end of file diff --git a/Project_Model/Libs/Transformer/Utils/post_tokenization.py b/Project_Model/Libs/Transformer/Utils/post_tokenization.py index bac6b6d..5a7c988 100644 --- a/Project_Model/Libs/Transformer/Utils/post_tokenization.py +++ b/Project_Model/Libs/Transformer/Utils/post_tokenization.py @@ -2,6 +2,10 @@ def truncate_sequence( sequence: list[int], truncate_at: int, end_token: int ) -> list[int]: + if len(sequence) < truncate_at - 1: + sequence.append(end_token) + return sequence + if len(sequence) < truncate_at: sequence[-1] = end_token return sequence diff --git a/Project_Model/Libs/Transformer/__init__.py b/Project_Model/Libs/Transformer/__init__.py index d906699..5f9e637 100644 --- a/Project_Model/Libs/Transformer/__init__.py +++ b/Project_Model/Libs/Transformer/__init__.py @@ -1,5 +1,7 @@ from .Classes import * from .Utils import * +from .Models import * from . import Classes -from . import Utils \ No newline at end of file +from . import Utils +from . import Models \ No newline at end of file