preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic = np.loadtxt(\"datasets/Lending-company-Numeric.csv\", delimiter=',')\n",
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.isnan(lending_data_numreic).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[False False False False False False]\n",
      " [False False False False False False]\n",
      " [False False False False False False]\n",
      " ...\n",
      " [ True False False False False False]\n",
      " [False False False False False False]\n",
      " [False False False False False False]]\n",
      "260\n"
     ]
    }
   ],
   "source": [
    "lending_data_numreic_nan = np.genfromtxt(\"datasets/Lending-company-Numeric-NAN.csv\", delimiter=';')\n",
    "# lending_data_numreic_nan\n",
    "print(np.isnan(lending_data_numreic_nan))\n",
    "print(np.isnan(lending_data_numreic_nan).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "lending_data_numreic_nan_imputed = np.genfromtxt(\"datasets/Lending-company-Numeric-NAN.csv\", delimiter=';', filling_values=0)\n",
    "print(np.isnan(lending_data_numreic_nan_imputed).sum())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Substituting the missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64002.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adding one to the value keeps the  null value out of range\n",
    "temp_fill = np.nanmax(lending_data_numreic_nan).round(2) +1\n",
    "temp_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now ill values \n",
    "lending_data_numreic_nan_imputed = np.genfromtxt(\"datasets/Lending-company-Numeric-NAN.csv\", delimiter=';', filling_values=temp_fill)\n",
    "np.isnan(lending_data_numreic_nan_imputed).sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if a value is equal to maximum value of the array,  it is actually a missing values \n",
    "# changing the data imputation to mean imputation will help much, as it does not change the interpretation data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [   nan,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2250.25,    46.11,   365.  ,  3895.99,  5160.75, 16571.44])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_mean = np.nanmean(lending_data_numreic_nan, axis=0).round(2)\n",
    "temp_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is actual mean from data where the missing values are present. Now, lets see the avergae of imputed data. The actual mean was 2250.25, and imputed data has 4263.25 mean that is misleading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4263.25"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(np.isnan(lending_data_numreic_nan_imputed).sum())\n",
    "np.mean(lending_data_numreic_nan_imputed, axis=0).round(2)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2250.25"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic_nan_imputed[:,0] = np.where(lending_data_numreic_nan_imputed[:,0] == temp_fill, temp_mean[0], lending_data_numreic_nan_imputed[:,0])\n",
    "np.mean(lending_data_numreic_nan_imputed, axis=0).round(2)[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "imputing the entire dataset with mean valeus. We could have done fillna_mean at the begininig, but we explored the difference. we will impute each column of the dataset, where we have temp fill values i.e. max +1.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.  ,    40.  ,   365.  ,  3121.  ,  4241.  , 13621.  ],\n",
       "       [ 2000.  ,    40.  ,   365.  ,  3061.  ,  4171.  , 15041.  ],\n",
       "       [ 1000.  ,    40.  ,   365.  ,  2160.  ,  3280.  , 15340.  ],\n",
       "       ...,\n",
       "       [ 2250.25,    40.  ,   365.  ,  4201.  ,  5001.  , 16600.  ],\n",
       "       [ 1000.  ,    40.  ,   365.  ,  2080.  ,  3320.  , 15600.  ],\n",
       "       [ 2000.  ,    40.  ,   365.  ,  4601.  ,  4601.  , 16600.  ]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for col in range(lending_data_numreic_nan_imputed.shape[1]):\n",
    "    lending_data_numreic_nan_imputed[:, col] = np.where(lending_data_numreic_nan_imputed[:,col] == temp_fill, temp_mean[col], lending_data_numreic_nan_imputed[:,col])\n",
    "\n",
    "lending_data_numreic_nan_imputed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Reshape (Efficiently)\n",
    "\n",
    "Why it is useful: certain conditions about shapes and sizes to be met."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1043, 6)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365., ...,   365.,  1581.,  3041.],\n",
       "       [12277.,  2000.,    40., ...,    50.,   365.,  5350.],\n",
       "       [ 6850., 15150.,  1000., ...,  2000.,    40.,   365.],\n",
       "       [ 3101.,  4351., 16600., ..., 16600.,  2000.,    40.],\n",
       "       [  365.,  3441.,  4661., ...,  8450., 22250.,  2000.],\n",
       "       [   40.,   365.,  3701., ...,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.reshape(lending_data_numreic, (6,1043))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,  2000.,  1000., ...,  2000.,  1000.,  2000.],\n",
       "       [   40.,    40.,    40., ...,    40.,    40.,    40.],\n",
       "       [  365.,   365.,   365., ...,   365.,   365.,   365.],\n",
       "       [ 3121.,  3061.,  2160., ...,  4201.,  2080.,  4601.],\n",
       "       [ 4241.,  4171.,  3280., ...,  5001.,  3320.,  4601.],\n",
       "       [13621., 15041., 15340., ..., 16600., 15600., 16600.]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove valeus using delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   40.,   365.,  3121., ...,  4601.,  4601., 16600.])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.delete(lending_data_numreic, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6257,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.delete(lending_data_numreic, 0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   40.,  3121., 13621.],\n",
       "       [   40.,  3061., 15041.],\n",
       "       [   40.,  2160., 15340.],\n",
       "       ...,\n",
       "       [   40.,  4201., 16600.],\n",
       "       [   40.,  2080., 15600.],\n",
       "       [   40.,  4601., 16600.]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.delete(lending_data_numreic, [0,2, 4], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sorting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2870., -2550., -2450., ..., 16751., 17650., 19001.])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic[:,3].sort()\n",
    "lending_data_numreic[:, 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.8700e+03, -3.5000e+02,  5.0000e+01,  3.6500e+02,  1.0000e+03,\n",
       "         1.1600e+03],\n",
       "       [-2.8700e+03,  4.0000e+01,  1.0000e+02,  3.6500e+02,  1.0000e+03,\n",
       "         1.5000e+03],\n",
       "       [-2.5500e+03,  4.0000e+01,  1.5000e+02,  3.6500e+02,  1.0000e+03,\n",
       "         2.2000e+03],\n",
       "       ...,\n",
       "       [ 1.2500e+02,  3.6500e+02,  7.2510e+03,  9.2010e+03,  1.8751e+04,\n",
       "         5.4625e+04],\n",
       "       [ 1.2500e+02,  3.6500e+02,  9.0000e+03,  1.2126e+04,  2.0001e+04,\n",
       "         5.4625e+04],\n",
       "       [ 1.6500e+02,  3.6500e+02,  9.0000e+03,  1.3900e+04,  2.2001e+04,\n",
       "         6.4001e+04]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic.sort(axis=0)\n",
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;31mSignature:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDocstring:\u001b[0m\n",
      "Returns the indices that would sort an array.\n",
      "\n",
      "Perform an indirect sort along the given axis using the algorithm specified\n",
      "by the `kind` keyword. It returns an array of indices of the same shape as\n",
      "`a` that index data along the given axis in sorted order.\n",
      "\n",
      "Parameters\n",
      "----------\n",
      "a : array_like\n",
      "    Array to sort.\n",
      "axis : int or None, optional\n",
      "    Axis along which to sort.  The default is -1 (the last axis). If None,\n",
      "    the flattened array is used.\n",
      "kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional\n",
      "    Sorting algorithm. The default is 'quicksort'. Note that both 'stable'\n",
      "    and 'mergesort' use timsort under the covers and, in general, the\n",
      "    actual implementation will vary with data type. The 'mergesort' option\n",
      "    is retained for backwards compatibility.\n",
      "\n",
      "    .. versionchanged:: 1.15.0.\n",
      "       The 'stable' option was added.\n",
      "order : str or list of str, optional\n",
      "    When `a` is an array with fields defined, this argument specifies\n",
      "    which fields to compare first, second, etc.  A single field can\n",
      "    be specified as a string, and not all fields need be specified,\n",
      "    but unspecified fields will still be used, in the order in which\n",
      "    they come up in the dtype, to break ties.\n",
      "\n",
      "Returns\n",
      "-------\n",
      "index_array : ndarray, int\n",
      "    Array of indices that sort `a` along the specified `axis`.\n",
      "    If `a` is one-dimensional, ``a[index_array]`` yields a sorted `a`.\n",
      "    More generally, ``np.take_along_axis(a, index_array, axis=axis)``\n",
      "    always yields the sorted `a`, irrespective of dimensionality.\n",
      "\n",
      "See Also\n",
      "--------\n",
      "sort : Describes sorting algorithms used.\n",
      "lexsort : Indirect stable sort with multiple keys.\n",
      "ndarray.sort : Inplace sort.\n",
      "argpartition : Indirect partial sort.\n",
      "take_along_axis : Apply ``index_array`` from argsort\n",
      "                  to an array as if by calling sort.\n",
      "\n",
      "Notes\n",
      "-----\n",
      "See `sort` for notes on the different sorting algorithms.\n",
      "\n",
      "As of NumPy 1.4.0 `argsort` works with real/complex arrays containing\n",
      "nan values. The enhanced sort order is documented in `sort`.\n",
      "\n",
      "Examples\n",
      "--------\n",
      "One dimensional array:\n",
      "\n",
      ">>> x = np.array([3, 1, 2])\n",
      ">>> np.argsort(x)\n",
      "array([1, 2, 0])\n",
      "\n",
      "Two-dimensional array:\n",
      "\n",
      ">>> x = np.array([[0, 3], [2, 2]])\n",
      ">>> x\n",
      "array([[0, 3],\n",
      "       [2, 2]])\n",
      "\n",
      ">>> ind = np.argsort(x, axis=0)  # sorts along first axis (down)\n",
      ">>> ind\n",
      "array([[0, 1],\n",
      "       [1, 0]])\n",
      ">>> np.take_along_axis(x, ind, axis=0)  # same as np.sort(x, axis=0)\n",
      "array([[0, 2],\n",
      "       [2, 3]])\n",
      "\n",
      ">>> ind = np.argsort(x, axis=1)  # sorts along last axis (across)\n",
      ">>> ind\n",
      "array([[0, 1],\n",
      "       [0, 1]])\n",
      ">>> np.take_along_axis(x, ind, axis=1)  # same as np.sort(x, axis=1)\n",
      "array([[0, 3],\n",
      "       [2, 2]])\n",
      "\n",
      "Indices of the sorted elements of a N-dimensional array:\n",
      "\n",
      ">>> ind = np.unravel_index(np.argsort(x, axis=None), x.shape)\n",
      ">>> ind\n",
      "(array([0, 1, 1, 0]), array([0, 0, 1, 1]))\n",
      ">>> x[ind]  # same as np.sort(x, axis=None)\n",
      "array([0, 2, 2, 3])\n",
      "\n",
      "Sorting with keys:\n",
      "\n",
      ">>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])\n",
      ">>> x\n",
      "array([(1, 0), (0, 1)],\n",
      "      dtype=[('x', '<i4'), ('y', '<i4')])\n",
      "\n",
      ">>> np.argsort(x, order=('x','y'))\n",
      "array([1, 0])\n",
      "\n",
      ">>> np.argsort(x, order=('y','x'))\n",
      "array([0, 1])\n",
      "\u001b[0;31mFile:\u001b[0m      /opt/miniconda3/lib/python3.9/site-packages/numpy/core/fromnumeric.py\n",
      "\u001b[0;31mType:\u001b[0m      function\n"
     ]
    }
   ],
   "source": [
    "?np.argsort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2, 3, 4, 5],\n",
       "       [0, 1, 2, 3, 4, 5],\n",
       "       [0, 1, 2, 3, 4, 5],\n",
       "       ...,\n",
       "       [0, 1, 2, 3, 4, 5],\n",
       "       [0, 1, 2, 3, 4, 5],\n",
       "       [0, 1, 2, 3, 4, 5]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# argsort\n",
    "np.argsort(lending_data_numreic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic = np.loadtxt(\"datasets/Lending-company-Numeric.csv\", delimiter=',')\n",
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,    0],\n",
       "       [   0,    1],\n",
       "       [   0,    2],\n",
       "       ...,\n",
       "       [1042,    3],\n",
       "       [1042,    4],\n",
       "       [1042,    5]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argwhere(lending_data_numreic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,    0],\n",
       "       [   0,    3],\n",
       "       [   0,    4],\n",
       "       ...,\n",
       "       [1042,    3],\n",
       "       [1042,    4],\n",
       "       [1042,    5]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argwhere(lending_data_numreic > 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  11,    3],\n",
       "       [  15,    3],\n",
       "       [  27,    3],\n",
       "       [  58,    3],\n",
       "       [  60,    4],\n",
       "       [  85,    4],\n",
       "       [ 117,    5],\n",
       "       [ 152,    1],\n",
       "       [ 152,    2],\n",
       "       [ 152,    4],\n",
       "       [ 172,    1],\n",
       "       [ 175,    1],\n",
       "       [ 175,    2],\n",
       "       [ 176,    3],\n",
       "       [ 177,    4],\n",
       "       [ 178,    5],\n",
       "       [ 211,    3],\n",
       "       [ 229,    0],\n",
       "       [ 230,    1],\n",
       "       [ 237,    1],\n",
       "       [ 247,    3],\n",
       "       [ 251,    5],\n",
       "       [ 252,    4],\n",
       "       [ 258,    1],\n",
       "       [ 260,    3],\n",
       "       [ 262,    4],\n",
       "       [ 271,    5],\n",
       "       [ 272,    4],\n",
       "       [ 284,    2],\n",
       "       [ 284,    3],\n",
       "       [ 297,    1],\n",
       "       [ 297,    2],\n",
       "       [ 300,    3],\n",
       "       [ 315,    3],\n",
       "       [ 315,    5],\n",
       "       [ 327,    4],\n",
       "       [ 336,    4],\n",
       "       [ 343,    0],\n",
       "       [ 344,    2],\n",
       "       [ 346,    2],\n",
       "       [ 363,    3],\n",
       "       [ 375,    3],\n",
       "       [ 377,    2],\n",
       "       [ 398,    5],\n",
       "       [ 416,    4],\n",
       "       [ 428,    0],\n",
       "       [ 432,    1],\n",
       "       [ 433,    3],\n",
       "       [ 434,    2],\n",
       "       [ 440,    3],\n",
       "       [ 441,    1],\n",
       "       [ 446,    2],\n",
       "       [ 455,    1],\n",
       "       [ 456,    2],\n",
       "       [ 458,    2],\n",
       "       [ 461,    3],\n",
       "       [ 467,    4],\n",
       "       [ 468,    0],\n",
       "       [ 469,    3],\n",
       "       [ 470,    0],\n",
       "       [ 481,    1],\n",
       "       [ 482,    3],\n",
       "       [ 484,    4],\n",
       "       [ 489,    0],\n",
       "       [ 493,    5],\n",
       "       [ 499,    0],\n",
       "       [ 504,    3],\n",
       "       [ 506,    1],\n",
       "       [ 517,    4],\n",
       "       [ 518,    0],\n",
       "       [ 521,    2],\n",
       "       [ 530,    5],\n",
       "       [ 532,    3],\n",
       "       [ 534,    5],\n",
       "       [ 547,    3],\n",
       "       [ 549,    5],\n",
       "       [ 553,    5],\n",
       "       [ 554,    5],\n",
       "       [ 559,    3],\n",
       "       [ 566,    0],\n",
       "       [ 566,    1],\n",
       "       [ 566,    2],\n",
       "       [ 567,    0],\n",
       "       [ 567,    1],\n",
       "       [ 567,    2],\n",
       "       [ 581,    1],\n",
       "       [ 581,    2],\n",
       "       [ 582,    1],\n",
       "       [ 582,    2],\n",
       "       [ 583,    1],\n",
       "       [ 583,    2],\n",
       "       [ 593,    3],\n",
       "       [ 594,    3],\n",
       "       [ 595,    3],\n",
       "       [ 596,    3],\n",
       "       [ 609,    4],\n",
       "       [ 609,    5],\n",
       "       [ 610,    4],\n",
       "       [ 610,    5],\n",
       "       [ 611,    4],\n",
       "       [ 611,    5],\n",
       "       [ 612,    4],\n",
       "       [ 612,    5],\n",
       "       [ 623,    2],\n",
       "       [ 623,    3],\n",
       "       [ 624,    2],\n",
       "       [ 624,    3],\n",
       "       [ 625,    2],\n",
       "       [ 625,    3],\n",
       "       [ 626,    2],\n",
       "       [ 626,    3],\n",
       "       [ 627,    2],\n",
       "       [ 627,    3],\n",
       "       [ 636,    0],\n",
       "       [ 637,    0],\n",
       "       [ 637,    4],\n",
       "       [ 637,    5],\n",
       "       [ 638,    0],\n",
       "       [ 638,    4],\n",
       "       [ 638,    5],\n",
       "       [ 639,    4],\n",
       "       [ 639,    5],\n",
       "       [ 640,    4],\n",
       "       [ 640,    5],\n",
       "       [ 641,    4],\n",
       "       [ 641,    5],\n",
       "       [ 653,    1],\n",
       "       [ 653,    2],\n",
       "       [ 653,    3],\n",
       "       [ 653,    4],\n",
       "       [ 654,    1],\n",
       "       [ 654,    2],\n",
       "       [ 654,    3],\n",
       "       [ 654,    4],\n",
       "       [ 655,    1],\n",
       "       [ 655,    2],\n",
       "       [ 655,    3],\n",
       "       [ 655,    4],\n",
       "       [ 685,    2],\n",
       "       [ 686,    0],\n",
       "       [ 701,    4],\n",
       "       [ 704,    2],\n",
       "       [ 724,    1],\n",
       "       [ 724,    2],\n",
       "       [ 730,    1],\n",
       "       [ 730,    2],\n",
       "       [ 730,    3],\n",
       "       [ 731,    1],\n",
       "       [ 731,    2],\n",
       "       [ 731,    3],\n",
       "       [ 732,    1],\n",
       "       [ 732,    2],\n",
       "       [ 732,    3],\n",
       "       [ 733,    1],\n",
       "       [ 733,    2],\n",
       "       [ 733,    3],\n",
       "       [ 742,    4],\n",
       "       [ 747,    3],\n",
       "       [ 755,    1],\n",
       "       [ 758,    1],\n",
       "       [ 760,    1],\n",
       "       [ 764,    0],\n",
       "       [ 764,    1],\n",
       "       [ 764,    2],\n",
       "       [ 764,    3],\n",
       "       [ 764,    4],\n",
       "       [ 764,    5],\n",
       "       [ 765,    3],\n",
       "       [ 775,    1],\n",
       "       [ 775,    2],\n",
       "       [ 775,    3],\n",
       "       [ 776,    1],\n",
       "       [ 776,    2],\n",
       "       [ 776,    3],\n",
       "       [ 777,    1],\n",
       "       [ 777,    2],\n",
       "       [ 777,    3],\n",
       "       [ 778,    1],\n",
       "       [ 778,    2],\n",
       "       [ 778,    3],\n",
       "       [ 787,    4],\n",
       "       [ 788,    1],\n",
       "       [ 790,    2],\n",
       "       [ 794,    3],\n",
       "       [ 795,    5],\n",
       "       [ 796,    1],\n",
       "       [ 796,    4],\n",
       "       [ 800,    0],\n",
       "       [ 801,    1],\n",
       "       [ 802,    1],\n",
       "       [ 803,    2],\n",
       "       [ 805,    0],\n",
       "       [ 809,    0],\n",
       "       [ 810,    0],\n",
       "       [ 811,    0],\n",
       "       [ 812,    1],\n",
       "       [ 820,    0],\n",
       "       [ 820,    3],\n",
       "       [ 820,    4],\n",
       "       [ 822,    2],\n",
       "       [ 823,    2],\n",
       "       [ 825,    0],\n",
       "       [ 830,    1],\n",
       "       [ 830,    2],\n",
       "       [ 836,    3],\n",
       "       [ 837,    3],\n",
       "       [ 838,    3],\n",
       "       [ 839,    4],\n",
       "       [ 843,    2],\n",
       "       [ 856,    0],\n",
       "       [ 856,    1],\n",
       "       [ 856,    3],\n",
       "       [ 856,    4],\n",
       "       [ 856,    5],\n",
       "       [ 857,    0],\n",
       "       [ 858,    1],\n",
       "       [ 870,    0],\n",
       "       [ 870,    2],\n",
       "       [ 870,    3],\n",
       "       [ 872,    0],\n",
       "       [ 873,    0],\n",
       "       [ 874,    1],\n",
       "       [ 890,    1],\n",
       "       [ 890,    2],\n",
       "       [ 890,    3],\n",
       "       [ 909,    1],\n",
       "       [ 910,    0],\n",
       "       [ 912,    0],\n",
       "       [ 912,    2],\n",
       "       [ 912,    3],\n",
       "       [ 912,    4],\n",
       "       [ 928,    1],\n",
       "       [ 928,    2],\n",
       "       [ 928,    3],\n",
       "       [ 929,    1],\n",
       "       [ 929,    2],\n",
       "       [ 929,    3],\n",
       "       [ 930,    3],\n",
       "       [ 958,    2],\n",
       "       [ 959,    3],\n",
       "       [ 960,    3],\n",
       "       [1024,    0],\n",
       "       [1024,    2],\n",
       "       [1026,    2],\n",
       "       [1026,    4],\n",
       "       [1028,    0],\n",
       "       [1028,    3],\n",
       "       [1028,    4],\n",
       "       [1029,    0],\n",
       "       [1030,    1],\n",
       "       [1030,    5],\n",
       "       [1032,    0],\n",
       "       [1032,    5],\n",
       "       [1034,    3],\n",
       "       [1035,    4],\n",
       "       [1036,    2],\n",
       "       [1037,    1],\n",
       "       [1038,    4],\n",
       "       [1039,    3],\n",
       "       [1040,    0]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the co-ordiates of missing values \n",
    "np.argwhere(np.isnan(lending_data_numreic_nan))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### casting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000,    40,   365,  3121,  4241, 13621],\n",
       "       [ 2000,    40,   365,  3061,  4171, 15041],\n",
       "       [ 1000,    40,   365,  2160,  3280, 15340],\n",
       "       ...,\n",
       "       [ 2000,    40,   365,  4201,  5001, 16600],\n",
       "       [ 1000,    40,   365,  2080,  3320, 15600],\n",
       "       [ 2000,    40,   365,  4601,  4601, 16600]], dtype=int32)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic.astype(dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/1r/3db931yx4ys8zd46l5tbk6x80000gn/T/ipykernel_1137/2269807719.py:1: DeprecationWarning: `np.str` is a deprecated alias for the builtin `str`. To silence this warning, use `str` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.str_` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  data_string = lending_data_numreic.astype(dtype=np.str)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([['2000.0', '40.0', '365.0', '3121.0', '4241.0', '13621.0'],\n",
       "       ['2000.0', '40.0', '365.0', '3061.0', '4171.0', '15041.0'],\n",
       "       ['1000.0', '40.0', '365.0', '2160.0', '3280.0', '15340.0'],\n",
       "       ...,\n",
       "       ['2000.0', '40.0', '365.0', '4201.0', '5001.0', '16600.0'],\n",
       "       ['1000.0', '40.0', '365.0', '2080.0', '3320.0', '15600.0'],\n",
       "       ['2000.0', '40.0', '365.0', '4601.0', '4601.0', '16600.0']],\n",
       "      dtype='<U32')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_string = lending_data_numreic.astype(dtype=np.str)\n",
    "data_string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stacking "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.,    40.,   365.,  3121.,  4241., 13621.],\n",
       "       [ 2000.,    40.,   365.,  3061.,  4171., 15041.],\n",
       "       [ 1000.,    40.,   365.,  2160.,  3280., 15340.],\n",
       "       ...,\n",
       "       [ 2000.,    40.,   365.,  4201.,  5001., 16600.],\n",
       "       [ 1000.,    40.,   365.,  2080.,  3320., 15600.],\n",
       "       [ 2000.,    40.,   365.,  4601.,  4601., 16600.]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2000.  ,    40.  ,   365.  ,  3121.  ,  4241.  , 13621.  ],\n",
       "       [ 2000.  ,    40.  ,   365.  ,  3061.  ,  4171.  , 15041.  ],\n",
       "       [ 1000.  ,    40.  ,   365.  ,  2160.  ,  3280.  , 15340.  ],\n",
       "       ...,\n",
       "       [ 2250.25,    40.  ,   365.  ,  4201.  ,  5001.  , 16600.  ],\n",
       "       [ 1000.  ,    40.  ,   365.  ,  2080.  ,  3320.  , 15600.  ],\n",
       "       [ 2000.  ,    40.  ,   365.  ,  4601.  ,  4601.  , 16600.  ]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lending_data_numreic_nan_imputed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  40.,   40.,   40., ...,   40.,   40.,   40.],\n",
       "       [2000., 2000., 1000., ..., 2000., 1000., 2000.]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example\n",
    "np.stack((lending_data_numreic[:,1], lending_data_numreic[:,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  40., 2000.],\n",
       "       [  40., 2000.],\n",
       "       [  40., 1000.],\n",
       "       ...,\n",
       "       [  40., 2000.],\n",
       "       [  40., 1000.],\n",
       "       [  40., 2000.]])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.stack((lending_data_numreic[:,1], lending_data_numreic[:,0]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1043, 6) (1043, 6)\n"
     ]
    }
   ],
   "source": [
    "# conforming the shape\n",
    "print(lending_data_numreic.shape, lending_data_numreic_nan.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2086, 6)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((lending_data_numreic, lending_data_numreic_nan)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1043, 12)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.hstack((lending_data_numreic, lending_data_numreic_nan)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 2000.,  2000.],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 3121.,  3121.],\n",
       "        [ 4241.,  4241.],\n",
       "        [13621., 13621.]],\n",
       "\n",
       "       [[ 2000.,  2000.],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 3061.,  3061.],\n",
       "        [ 4171.,  4171.],\n",
       "        [15041., 15041.]],\n",
       "\n",
       "       [[ 1000.,  1000.],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 2160.,  2160.],\n",
       "        [ 3280.,  3280.],\n",
       "        [15340., 15340.]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[ 2000.,    nan],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 4201.,  4201.],\n",
       "        [ 5001.,  5001.],\n",
       "        [16600., 16600.]],\n",
       "\n",
       "       [[ 1000.,  1000.],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 2080.,  2080.],\n",
       "        [ 3320.,  3320.],\n",
       "        [15600., 15600.]],\n",
       "\n",
       "       [[ 2000.,  2000.],\n",
       "        [   40.,    40.],\n",
       "        [  365.,   365.],\n",
       "        [ 4601.,  4601.],\n",
       "        [ 4601.,  4601.],\n",
       "        [16600., 16600.]]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dstack((lending_data_numreic, lending_data_numreic_nan))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1043, 6, 2)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# it adds the depth [row * cols * original array]\n",
    "np.dstack((lending_data_numreic, lending_data_numreic_nan)).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenation\n",
    "Linking together objects in chain. Creating a new (larger) array by meriging existing smaller arrays along a given axis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2000.,    40.,   365.,  3121.,  4241., 13621.,  2000.,    40.,\n",
       "         365.,  3061.,  4171., 15041.])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.concatenate((lending_data_numreic[0,:], lending_data_numreic[1,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2086, 6)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# acts as hstack, Another array is concatenated at the bottom of one array\n",
    "np.concatenate((lending_data_numreic, lending_data_numreic_nan)).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 35.,  40.,  50., 125., 165.]),\n",
       " array([327,   0,   4,  19,  27]),\n",
       " array([  4, 567, 451,  19,   2]))"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(lending_data_numreic[:,1], return_counts=True, return_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "3d597f4c481aa0f25dceb95d2a0067e73c0966dcbd003d741d821a7208527ecf"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}