Deep-Learning/Chapters/15-Appendix-A/python-experiments/pca.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8c14ea22",
   "metadata": {},
   "source": [
    "# Computing PCA\n",
    "\n",
    "Here I'll be taking data from [Geeks4Geeks](https://www.geeksforgeeks.org/machine-learning/mathematical-approach-to-pca/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b32eb5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.8        1.87777778]\n",
      "[[ 0.7         0.52222222]\n",
      " [-1.3        -1.17777778]\n",
      " [ 0.4         1.02222222]\n",
      " [ 1.3         1.12222222]\n",
      " [ 0.5         0.82222222]\n",
      " [ 0.2        -0.27777778]\n",
      " [-0.8        -0.77777778]\n",
      " [-0.3        -0.27777778]\n",
      " [-0.7        -0.97777778]]\n",
      "[[0.6925     0.68875   ]\n",
      " [0.68875    0.79444444]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "X : np.ndarray = np.array([\n",
    "    [2.5, 2.4],\n",
    "    [0.5, 0.7],\n",
    "    [2.2, 2.9],\n",
    "    [3.1, 3.0],\n",
    "    [2.3, 2.7],\n",
    "    [2.0, 1.6],\n",
    "    [1.0, 1.1],\n",
    "    [1.5, 1.6],\n",
    "    [1.1, 0.9]\n",
    "])\n",
    "\n",
    "# Compute mean values for features\n",
    "mu_X = np.mean(X, 0)\n",
    "\n",
    "print(mu_X)\n",
    "# \"Normalize\" Features\n",
    "X = X - mu_X\n",
    "print(X)\n",
    "\n",
    "# Compute covariance matrix applying\n",
    "#   Bessel's correction (n-1) instead of n\n",
    "Cov = (X.T @ X) / (X.shape[0] - 1)\n",
    "\n",
    "print(Cov)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78e9429f",
   "metadata": {},
   "source": [
    "As you can notice, we did $X^T \\times X$ instead of $X \\times X^T$. This is because our \n",
    "dataset had datapoints over rows instead of features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "f93b7a92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.05283865 1.43410579]\n",
      "[[-0.73273632 -0.68051267]\n",
      " [ 0.68051267 -0.73273632]]\n"
     ]
    }
   ],
   "source": [
    "# Computing eigenvalues\n",
    "eigen = np.linalg.eig(Cov)\n",
    "eigen_values = eigen.eigenvalues\n",
    "eigen_vectors = eigen.eigenvectors\n",
    "\n",
    "print(eigen_values)\n",
    "print(eigen_vectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfbdd9c3",
   "metadata": {},
   "source": [
    "Now we'll generate the new X matrix by only using the first eigen vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "7ce6c540",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9, 1)\n",
      "Compressed\n",
      "[[-0.85901005]\n",
      " [ 1.74766702]\n",
      " [-1.02122441]\n",
      " [-1.70695945]\n",
      " [-0.94272842]\n",
      " [ 0.06743533]\n",
      " [ 1.11431616]\n",
      " [ 0.40769167]\n",
      " [ 1.19281215]]\n",
      "Reconstruction\n",
      "[[ 0.58456722  0.62942786]\n",
      " [-1.18930955 -1.28057909]\n",
      " [ 0.69495615  0.74828821]\n",
      " [ 1.16160753  1.25075117]\n",
      " [ 0.64153863  0.69077135]\n",
      " [-0.0458906  -0.04941232]\n",
      " [-0.75830626 -0.81649992]\n",
      " [-0.27743934 -0.29873049]\n",
      " [-0.81172378 -0.87401678]]\n",
      "Difference\n",
      "[[0.11543278 0.10720564]\n",
      " [0.11069045 0.10280131]\n",
      " [0.29495615 0.27393401]\n",
      " [0.13839247 0.12852895]\n",
      " [0.14153863 0.13145088]\n",
      " [0.2458906  0.22836546]\n",
      " [0.04169374 0.03872214]\n",
      " [0.02256066 0.02095271]\n",
      " [0.11172378 0.10376099]]\n"
     ]
    }
   ],
   "source": [
    "# Computing X coming from only 1st eigen vector\n",
    "Z_pca = X @ eigen_vectors[:,1]\n",
    "Z_pca = Z_pca.reshape([Z_pca.shape[0], 1])\n",
    "\n",
    "print(Z_pca.shape)\n",
    "\n",
    "\n",
    "# X reconstructed\n",
    "eigen_v = (eigen_vectors[:, 1].reshape([eigen_vectors[:, 1].shape[0], 1]))\n",
    "X_rec = Z_pca @ eigen_v.T\n",
    "\n",
    "print(\"Compressed\")\n",
    "print(Z_pca)\n",
    "\n",
    "print(\"Reconstruction\")\n",
    "print(X_rec)\n",
    "\n",
    "print(\"Difference\")\n",
    "print(abs(X - X_rec))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}