{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working With Image Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "digits_data = load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'target_names', 'images', 'DESCR'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits_data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.Series(digits_data['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>54</th>\n",
       "      <th>55</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2     3    4    5    6    7    8    9   ...   54   55   56   57  \\\n",
       "0  0.0  0.0  5.0  13.0  9.0  1.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0  0.0   \n",
       "\n",
       "    58    59    60   61   62   63  \n",
       "0  6.0  13.0  10.0  0.0  0.0  0.0  \n",
       "\n",
       "[1 rows x 64 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(digits_data['data'])\n",
    "data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7ffb8fa43250>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAKt0lEQVR4nO3dXYhc9RnH8d+vq9L6EoxNKJINXRckIIWauAQkIDR2S6yivaiSgEKl4E0VpQWjveud3oi9KIJErWCqZKOCiNUKKq3QWneS2BpXSxJTMlWbhEZ8KTREn17sBKJd3TNnzts+/X5gcV+G/T/D5uuZmT17/o4IAcjjK20PAKBaRA0kQ9RAMkQNJEPUQDKn1fFNV6xYERMTE3V861YdO3as0fX6/X5jay1btqyxtcbHxxtba2xsrLG1mnTw4EEdPXrUC32tlqgnJiY0Oztbx7du1czMTKPrbd26tbG1pqenG1vrrrvuamyt5cuXN7ZWk6ampr7wazz8BpIhaiAZogaSIWogGaIGkiFqIBmiBpIhaiAZogaSKRS17U2237K9z/YddQ8FoLxFo7Y9JulXkq6QdJGkLbYvqnswAOUUOVKvl7QvIg5ExHFJj0m6pt6xAJRVJOpVkg6d8nF/8LnPsH2T7Vnbs0eOHKlqPgBDKhL1Qn/e9T9XK4yI+yNiKiKmVq5cOfpkAEopEnVf0upTPh6X9E494wAYVZGoX5V0oe0LbJ8habOkp+odC0BZi14kISJO2L5Z0nOSxiQ9GBF7a58MQCmFrnwSEc9IeqbmWQBUgDPKgGSIGkiGqIFkiBpIhqiBZIgaSIaogWRq2aEjqyZ3zJCkt99+u7G1mtxS6LzzzmtsrR07djS2liRde+21ja63EI7UQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kU2SHjgdtH7b9ehMDARhNkSP1ryVtqnkOABVZNOqI+L2kfzUwC4AKVPacmm13gG6oLGq23QG6gVe/gWSIGkimyK+0HpX0R0lrbPdt/7j+sQCUVWQvrS1NDAKgGjz8BpIhaiAZogaSIWogGaIGkiFqIBmiBpJZ8tvu9Hq9xtZqchscSdq/f39ja01OTja21vT0dGNrNfnvQ2LbHQA1IGogGaIGkiFqIBmiBpIhaiAZogaSIWogGaIGkiFqIJki1yhbbftF23O299q+tYnBAJRT5NzvE5J+FhG7bJ8jqWf7+Yh4o+bZAJRQZNuddyNi1+D9DyXNSVpV92AAyhnqObXtCUlrJb2ywNfYdgfogMJR2z5b0uOSbouIDz7/dbbdAbqhUNS2T9d80Nsj4ol6RwIwiiKvflvSA5LmIuKe+kcCMIoiR+oNkm6QtNH2nsHb92ueC0BJRbbdeVmSG5gFQAU4owxIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZJb8XlrHjh1rbK1169Y1tpbU7P5WTbrkkkvaHiE1jtRAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJFLjz4Vdt/tv3aYNudXzQxGIByipwm+h9JGyPio8Glgl+2/duI+FPNswEoociFB0PSR4MPTx+8RZ1DASiv6MX8x2zvkXRY0vMRwbY7QEcVijoiPomIiyWNS1pv+1sL3IZtd4AOGOrV74h4X9JLkjbVMg2AkRV59Xul7XMH739N0nclvVn3YADKKfLq9/mSHrY9pvn/CeyIiKfrHQtAWUVe/f6L5vekBrAEcEYZkAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8mw7c4QpqenG1srsyZ/ZsuXL29sra7gSA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDKFox5c0H+3bS46CHTYMEfqWyXN1TUIgGoU3XZnXNKVkrbVOw6AURU9Ut8r6XZJn37RDdhLC+iGIjt0XCXpcET0vux27KUFdEORI/UGSVfbPijpMUkbbT9S61QASls06oi4MyLGI2JC0mZJL0TE9bVPBqAUfk8NJDPU5Ywi4iXNb2ULoKM4UgPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJLPltd5rcVqXX+9LT35e0JrfCmZ2dbWyt6667rrG1uoIjNZAMUQPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRQ6TXRwJdEPJX0i6URETNU5FIDyhjn3+zsRcbS2SQBUgoffQDJFow5Jv7Pds33TQjdg2x2gG4pGvSEi1km6QtJPbF/2+Ruw7Q7QDYWijoh3Bv89LOlJSevrHApAeUU2yDvL9jkn35f0PUmv1z0YgHKKvPr9DUlP2j55+99ExLO1TgWgtEWjjogDkr7dwCwAKsCvtIBkiBpIhqiBZIgaSIaogWSIGkiGqIFklvy2O5OTk42t1eR2MZI0MzOTcq0mbd26te0RGseRGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZApFbftc2zttv2l7zvaldQ8GoJyi537/UtKzEfFD22dIOrPGmQCMYNGobS+TdJmkH0lSRByXdLzesQCUVeTh96SkI5Iesr3b9rbB9b8/g213gG4oEvVpktZJui8i1kr6WNIdn78R2+4A3VAk6r6kfkS8Mvh4p+YjB9BBi0YdEe9JOmR7zeBTl0t6o9apAJRW9NXvWyRtH7zyfUDSjfWNBGAUhaKOiD2SpmqeBUAFOKMMSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWTYS2sId999d2NrSc3uAzU11dy5Rb1er7G1/h9xpAaSIWogGaIGkiFqIBmiBpIhaiAZogaSIWogGaIGklk0attrbO855e0D27c1MRyA4S16mmhEvCXpYkmyPSbpH5KerHkuACUN+/D7ckn7I+LvdQwDYHTDRr1Z0qMLfYFtd4BuKBz14JrfV0uaWejrbLsDdMMwR+orJO2KiH/WNQyA0Q0T9RZ9wUNvAN1RKGrbZ0qalvREveMAGFXRbXf+LenrNc8CoAKcUQYkQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMo6I6r+pfUTSsH+euULS0cqH6Yas94371Z5vRsSCfzlVS9Rl2J6NiOY2dGpQ1vvG/eomHn4DyRA1kEyXor6/7QFqlPW+cb86qDPPqQFUo0tHagAVIGogmU5EbXuT7bds77N9R9vzVMH2atsv2p6zvdf2rW3PVCXbY7Z323667VmqZPtc2zttvzn42V3a9kzDav059WCDgL9p/nJJfUmvStoSEW+0OtiIbJ8v6fyI2GX7HEk9ST9Y6vfrJNs/lTQlaVlEXNX2PFWx/bCkP0TEtsEVdM+MiPfbnmsYXThSr5e0LyIORMRxSY9JuqblmUYWEe9GxK7B+x9KmpO0qt2pqmF7XNKVkra1PUuVbC+TdJmkByQpIo4vtaClbkS9StKhUz7uK8k//pNsT0haK+mVdiepzL2Sbpf0aduDVGxS0hFJDw2eWmyzfVbbQw2rC1F7gc+l+T2b7bMlPS7ptoj4oO15RmX7KkmHI6LX9iw1OE3SOkn3RcRaSR9LWnKv8XQh6r6k1ad8PC7pnZZmqZTt0zUf9PaIyHJ55Q2SrrZ9UPNPlTbafqTdkSrTl9SPiJOPqHZqPvIlpQtRvyrpQtsXDF6Y2CzpqZZnGplta/652VxE3NP2PFWJiDsjYjwiJjT/s3ohIq5veaxKRMR7kg7ZXjP41OWSltwLm4Wu+12niDhh+2ZJz0kak/RgROxteawqbJB0g6S/2t4z+NzPI+KZFmfC4m6RtH1wgDkg6caW5xla67/SAlCtLjz8BlAhogaSIWogGaIGkiFqIBmiBpIhaiCZ/wLr8rHX1UUh+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "first_image = data.iloc[0]\n",
    "np_image = first_image.values\n",
    "np_image = np_image.reshape(8,8)\n",
    "\n",
    "plt.imshow(np_image, cmap='gray_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7ffb8ea4d690>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5892 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 8 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, axarr = plt.subplots(2, 4)\n",
    "\n",
    "axarr[0, 0].imshow(data.iloc[0].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[0, 1].imshow(data.iloc[99].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[0, 2].imshow(data.iloc[199].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[0, 3].imshow(data.iloc[299].values.reshape(8,8), cmap='gray_r')\n",
    "\n",
    "axarr[1, 0].imshow(data.iloc[999].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[1, 1].imshow(data.iloc[1099].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[1, 2].imshow(data.iloc[1199].values.reshape(8,8), cmap='gray_r')\n",
    "axarr[1, 3].imshow(data.iloc[1299].values.reshape(8,8), cmap='gray_r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbors Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9677233358079684"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# 50% Train / test validation\n",
    "def train_knn(nneighbors, train_features, train_labels):\n",
    "    knn = KNeighborsClassifier(n_neighbors = nneighbors)\n",
    "    knn.fit(train_features, train_labels)\n",
    "    return knn\n",
    "\n",
    "def test(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    train_test_df = pd.DataFrame()\n",
    "    train_test_df['correct_label'] = test_labels\n",
    "    train_test_df['predicted_label'] = predictions\n",
    "    overall_accuracy = sum(train_test_df[\"predicted_label\"] == train_test_df[\"correct_label\"])/len(train_test_df)    \n",
    "    return overall_accuracy\n",
    "\n",
    "def cross_validate(k):\n",
    "    fold_accuracies = []\n",
    "    kf = KFold(n_splits = 4, random_state=2)\n",
    "    for train_index, test_index in kf.split(data):\n",
    "        train_features, test_features = data.loc[train_index], data.loc[test_index]\n",
    "        train_labels, test_labels = labels.loc[train_index], labels.loc[test_index]\n",
    "        model = train_knn(k, train_features, train_labels)\n",
    "        overall_accuracy = test(model, test_features, test_labels)\n",
    "        fold_accuracies.append(overall_accuracy)\n",
    "    return fold_accuracies\n",
    "        \n",
    "knn_one_accuracies = cross_validate(1)\n",
    "np.mean(knn_one_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ffb8eb2b250>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19380 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "k_values = list(range(1,10))\n",
    "k_overall_accuracies = []\n",
    "\n",
    "for k in k_values:\n",
    "    k_accuracies = cross_validate(k)\n",
    "    k_mean_accuracy = np.mean(k_accuracies)\n",
    "    k_overall_accuracies.append(k_mean_accuracy)\n",
    "    \n",
    "plt.figure(figsize=(8,4))\n",
    "plt.title(\"Mean Accuracy vs. k\")\n",
    "plt.plot(k_values, k_overall_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network With One Hidden Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "# 50% Train / test validation\n",
    "def train_nn(neuron_arch, train_features, train_labels):\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=neuron_arch)\n",
    "    mlp.fit(train_features, train_labels)\n",
    "    return mlp\n",
    "\n",
    "def test(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    train_test_df = pd.DataFrame()\n",
    "    train_test_df['correct_label'] = test_labels\n",
    "    train_test_df['predicted_label'] = predictions\n",
    "    overall_accuracy = sum(train_test_df[\"predicted_label\"] == train_test_df[\"correct_label\"])/len(train_test_df)    \n",
    "    return overall_accuracy\n",
    "\n",
    "def cross_validate(neuron_arch):\n",
    "    fold_accuracies = []\n",
    "    kf = KFold(n_splits = 4, random_state=2)\n",
    "    for train_index, test_index in kf.split(data):\n",
    "        train_features, test_features = data.loc[train_index], data.loc[test_index]\n",
    "        train_labels, test_labels = labels.loc[train_index], labels.loc[test_index]\n",
    "       \n",
    "        model = train_nn(neuron_arch, train_features, train_labels)\n",
    "        overall_accuracy = test(model, test_features, test_labels)\n",
    "        fold_accuracies.append(overall_accuracy)\n",
    "    return fold_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ffb8c136190>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 15564 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn_one_neurons = [\n",
    "    (8,),\n",
    "    (16,),\n",
    "    (32,),\n",
    "    (64,),\n",
    "    (128,),\n",
    "    (256,)\n",
    "]\n",
    "nn_one_accuracies = []\n",
    "\n",
    "for n in nn_one_neurons:\n",
    "    nn_accuracies = cross_validate(n)\n",
    "    nn_mean_accuracy = np.mean(nn_accuracies)\n",
    "    nn_one_accuracies.append(nn_mean_accuracy)\n",
    "\n",
    "plt.figure(figsize=(8,4))\n",
    "plt.title(\"Mean Accuracy vs. Neurons In Single Hidden Layer\")\n",
    "\n",
    "x = [i[0] for i in nn_one_neurons]\n",
    "plt.plot(x, nn_one_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary:\n",
    "\n",
    "It looks like adding more neurons to the single hidden layer helped massively improved simple accuracy from approximately `86%` to approximately `94%`. Simple accuracy computes the number of correct classifications the model made, but doesn't tell us anything about false or true positives or false or true negatives.\n",
    "\n",
    "Given that k-nearest neighbors achieved approximately `96%` accuracy, there doesn't seem to be any advantages to using a single hidden layer neural network for this problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network With Two Hidden Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ffb8deee050>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 21248 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn_two_neurons = [\n",
    "    (64,64),\n",
    "    (128, 128),\n",
    "    (256, 256)\n",
    "]\n",
    "nn_two_accuracies = []\n",
    "\n",
    "for n in nn_two_neurons:\n",
    "    nn_accuracies = cross_validate(n)\n",
    "    nn_mean_accuracy = np.mean(nn_accuracies)\n",
    "    nn_two_accuracies.append(nn_mean_accuracy)\n",
    "\n",
    "plt.figure(figsize=(8,4))\n",
    "plt.title(\"Mean Accuracy vs. Neurons In Two Hidden Layers\")\n",
    "\n",
    "x = [i[0] for i in nn_two_neurons]\n",
    "plt.plot(x, nn_two_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9398861667903984, 0.9571492204899779, 0.9515837663944569]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_two_accuracies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary\n",
    "\n",
    "Using 2 hidden layers improved our simple accuracy to `95%`. While I'd traditionally be worried about overfitting, using 4-fold cross validation also gives me a bit more assurance that the model is generalizing to achieve the extra `1%` in simple accuracy over the single hidden layer networks we tried earlier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network With Three Hidden Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# 50% Train / test validation\n",
    "def train_nn(neuron_arch, train_features, train_labels):\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=neuron_arch)\n",
    "    mlp.fit(train_features, train_labels)\n",
    "    return mlp\n",
    "\n",
    "def test(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    train_test_df = pd.DataFrame()\n",
    "    train_test_df['correct_label'] = test_labels\n",
    "    train_test_df['predicted_label'] = predictions\n",
    "    overall_accuracy = sum(train_test_df[\"predicted_label\"] == train_test_df[\"correct_label\"])/len(train_test_df)    \n",
    "    return overall_accuracy\n",
    "\n",
    "def cross_validate_six(neuron_arch):\n",
    "    fold_accuracies = []\n",
    "    kf = KFold(n_splits = 6, random_state=2)\n",
    "    for train_index, test_index in kf.split(data):\n",
    "        train_features, test_features = data.loc[train_index], data.loc[test_index]\n",
    "        train_labels, test_labels = labels.loc[train_index], labels.loc[test_index]\n",
    "       \n",
    "        model = train_nn(neuron_arch, train_features, train_labels)\n",
    "        overall_accuracy = test(model, test_features, test_labels)\n",
    "        fold_accuracies.append(overall_accuracy)\n",
    "    return fold_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/home/yunoac/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ffb8c4f7550>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 16720 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn_three_neurons = [\n",
    "    (10, 10, 10),\n",
    "    (64, 64, 64),\n",
    "    (128, 128, 128)\n",
    "]\n",
    "\n",
    "nn_three_accuracies = []\n",
    "\n",
    "for n in nn_three_neurons:\n",
    "    nn_accuracies = cross_validate_six(n)\n",
    "    nn_mean_accuracy = np.mean(nn_accuracies)\n",
    "    nn_three_accuracies.append(nn_mean_accuracy)\n",
    "\n",
    "plt.figure(figsize=(8,4))\n",
    "plt.title(\"Mean Accuracy vs. Neurons In Three Hidden Layers\")\n",
    "\n",
    "x = [i[0] for i in nn_three_neurons]\n",
    "plt.plot(x, nn_three_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9154124860646601, 0.950479375696767, 0.9582552954292085]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_three_accuracies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary\n",
    "\n",
    "Using 3 hidden layers improved our simple accuracy to `96%`, even with 6-fold cross validation. This seems to be in line with the research literature out there about deep neural networks for computer vision. Having more layers and more neurons tends to improve the network's performance."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
