{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introducing Data Science Workflows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "train = pd.read_csv(\"train.csv\")\n",
    "holdout = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load functions.py\n",
    "def process_missing(df):\n",
    "    \"\"\"Handle various missing values from the data set\n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    holdout = process_missing(holdout)\n",
    "    \"\"\"\n",
    "    df[\"Fare\"] = df[\"Fare\"].fillna(train[\"Fare\"].mean())\n",
    "    df[\"Embarked\"] = df[\"Embarked\"].fillna(\"S\")\n",
    "    return df\n",
    "\n",
    "def process_age(df):\n",
    "    \"\"\"Process the Age column into pre-defined 'bins' \n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    train = process_age(train)\n",
    "    \"\"\"\n",
    "    df[\"Age\"] = df[\"Age\"].fillna(-0.5)\n",
    "    cut_points = [-1,0,5,12,18,35,60,100]\n",
    "    label_names = [\"Missing\",\"Infant\",\"Child\",\"Teenager\",\"Young Adult\",\"Adult\",\"Senior\"]\n",
    "    df[\"Age_categories\"] = pd.cut(df[\"Age\"],cut_points,labels=label_names)\n",
    "    return df\n",
    "\n",
    "def process_fare(df):\n",
    "    \"\"\"Process the Fare column into pre-defined 'bins' \n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    train = process_fare(train)\n",
    "    \"\"\"\n",
    "    cut_points = [-1,12,50,100,1000]\n",
    "    label_names = [\"0-12\",\"12-50\",\"50-100\",\"100+\"]\n",
    "    df[\"Fare_categories\"] = pd.cut(df[\"Fare\"],cut_points,labels=label_names)\n",
    "    return df\n",
    "\n",
    "def process_cabin(df):\n",
    "    \"\"\"Process the Cabin column into pre-defined 'bins' \n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    train process_cabin(train)\n",
    "    \"\"\"\n",
    "    df[\"Cabin_type\"] = df[\"Cabin\"].str[0]\n",
    "    df[\"Cabin_type\"] = df[\"Cabin_type\"].fillna(\"Unknown\")\n",
    "    df = df.drop('Cabin',axis=1)\n",
    "    return df\n",
    "\n",
    "def process_titles(df):\n",
    "    \"\"\"Extract and categorize the title from the name column \n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    train = process_titles(train)\n",
    "    \"\"\"\n",
    "    titles = {\n",
    "        \"Mr\" :         \"Mr\",\n",
    "        \"Mme\":         \"Mrs\",\n",
    "        \"Ms\":          \"Mrs\",\n",
    "        \"Mrs\" :        \"Mrs\",\n",
    "        \"Master\" :     \"Master\",\n",
    "        \"Mlle\":        \"Miss\",\n",
    "        \"Miss\" :       \"Miss\",\n",
    "        \"Capt\":        \"Officer\",\n",
    "        \"Col\":         \"Officer\",\n",
    "        \"Major\":       \"Officer\",\n",
    "        \"Dr\":          \"Officer\",\n",
    "        \"Rev\":         \"Officer\",\n",
    "        \"Jonkheer\":    \"Royalty\",\n",
    "        \"Don\":         \"Royalty\",\n",
    "        \"Sir\" :        \"Royalty\",\n",
    "        \"Countess\":    \"Royalty\",\n",
    "        \"Dona\":        \"Royalty\",\n",
    "        \"Lady\" :       \"Royalty\"\n",
    "    }\n",
    "    extracted_titles = df[\"Name\"].str.extract(' ([A-Za-z]+)\\.',expand=False)\n",
    "    df[\"Title\"] = extracted_titles.map(titles)\n",
    "    return df\n",
    "\n",
    "def create_dummies(df,column_name):\n",
    "    \"\"\"Create Dummy Columns (One Hot Encoding) from a single Column\n",
    "\n",
    "    Usage\n",
    "    ------\n",
    "\n",
    "    train = create_dummies(train,\"Age\")\n",
    "    \"\"\"\n",
    "    dummies = pd.get_dummies(df[column_name],prefix=column_name)\n",
    "    df = pd.concat([df,dummies],axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_process(df):\n",
    "    df = process_missing(df)\n",
    "    df = process_age(df)\n",
    "    df = process_fare(df)\n",
    "    df = process_titles(df)\n",
    "    df = process_cabin(df)\n",
    "\n",
    "    for col in [\"Age_categories\",\"Fare_categories\",\n",
    "                \"Title\",\"Cabin_type\",\"Sex\"]:\n",
    "        df = create_dummies(df,col)\n",
    "    \n",
    "    return df\n",
    "\n",
    "train = pre_process(train)\n",
    "holdout = pre_process(holdout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 3 columns):\n",
      "SibSp       891 non-null int64\n",
      "Parch       891 non-null int64\n",
      "Survived    891 non-null int64\n",
      "dtypes: int64(3)\n",
      "memory usage: 21.0 KB\n"
     ]
    }
   ],
   "source": [
    "explore_cols = [\"SibSp\",\"Parch\",\"Survived\"]\n",
    "explore = train[explore_cols].copy()\n",
    "explore.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10056 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11425cda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "explore.drop(\"Survived\",axis=1).plot.hist(alpha=0.5,bins=8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12140 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114488630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "explore[\"familysize\"] = explore[[\"SibSp\",\"Parch\"]].sum(axis=1)\n",
    "explore.drop(\"Survived\",axis=1).plot.hist(alpha=0.5,bins=10)\n",
    "plt.xticks(range(11))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7936 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11450b710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 7840 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1145c0f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 8708 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1145f9b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "for col in explore.columns.drop(\"Survived\"):\n",
    "    pivot = explore.pivot_table(index=col,values=\"Survived\")\n",
    "    pivot.plot.bar(ylim=(0,1),yticks=np.arange(0,1,.1))\n",
    "    plt.axhspan(.3, .6, alpha=0.2, color='red')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `SibSp` column shows the number of siblings and/or spouses each passenger had on board, while the `Parch` columns shows the number of parents or children each passenger had onboard. Neither column has any missing values.\n",
    "\n",
    "The distribution of values in both columns is skewed right, with the majority of values being zero.\n",
    "\n",
    "You can sum these two columns to explore the total number of family members each passenger had onboard.  The shape of the distribution of values in this case is similar, however there are less values at zero, and the quantity tapers off less rapidly as the values increase.\n",
    "\n",
    "Looking at the survival rates of the the combined family members, you can see that few of the over 500 passengers with no family members survived, while greater numbers of passengers with family members survived."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Engineering New Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_isalone(df):\n",
    "    df[\"familysize\"] = df[[\"SibSp\",\"Parch\"]].sum(axis=1)\n",
    "    df[\"isalone\"] = 0\n",
    "    df.loc[(df[\"familysize\"] == 0),\"isalone\"] = 1\n",
    "    df = df.drop(\"familysize\",axis=1)\n",
    "    return df\n",
    "\n",
    "train = process_isalone(train)\n",
    "holdout = process_isalone(holdout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selecting the Best-Performing Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Columns \n",
      "------------\n",
      "['Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Age_categories_Missing', 'Age_categories_Infant', 'Age_categories_Young Adult', 'Fare_categories_12-50', 'Title_Miss', 'Title_Mr', 'Title_Mrs', 'Cabin_type_D', 'Cabin_type_E', 'Cabin_type_Unknown', 'Sex_female', 'Sex_male', 'isalone']\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joshuadevlin/.virtualenvs/dscontent/lib/python3.4/site-packages/sklearn/utils/__init__.py:54: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(mask.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_selection import RFECV\n",
    "\n",
    "def select_features(df):\n",
    "    # Remove non-numeric columns, columns that have null values\n",
    "    df = df.select_dtypes([np.number]).dropna(axis=1)\n",
    "    all_X = df.drop([\"Survived\",\"PassengerId\"],axis=1)\n",
    "    all_y = df[\"Survived\"]\n",
    "    \n",
    "    clf = RandomForestClassifier(random_state=1)\n",
    "    selector = RFECV(clf,cv=10)\n",
    "    selector.fit(all_X,all_y)\n",
    "    \n",
    "    best_columns = list(all_X.columns[selector.support_])\n",
    "    print(\"Best Columns \\n\"+\"-\"*12+\"\\n{}\\n\".format(best_columns))\n",
    "    \n",
    "    return best_columns\n",
    "\n",
    "cols = select_features(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selecting and Tuning Different Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "------------------\n",
      "Best Score: 0.8204264870931538\n",
      "Best Parameters: {'solver': 'liblinear'}\n",
      "\n",
      "KNeighborsClassifier\n",
      "--------------------\n",
      "Best Score: 0.7755331088664422\n",
      "Best Parameters: {'weights': 'uniform', 'p': 1, 'n_neighbors': 5, 'algorithm': 'brute'}\n",
      "\n",
      "RandomForestClassifier\n",
      "----------------------\n",
      "Best Score: 0.8294051627384961\n",
      "Best Parameters: {'max_features': 'log2', 'criterion': 'entropy', 'n_estimators': 4, 'min_samples_leaf': 1, 'min_samples_split': 5, 'max_depth': 5}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "def select_model(df,features):\n",
    "    \n",
    "    all_X = df[features]\n",
    "    all_y = df[\"Survived\"]\n",
    "\n",
    "    # List of dictionaries, each containing a model name,\n",
    "    # it's estimator and a dict of hyperparameters\n",
    "    models = [\n",
    "        {\n",
    "            \"name\": \"LogisticRegression\",\n",
    "            \"estimator\": LogisticRegression(),\n",
    "            \"hyperparameters\":\n",
    "                {\n",
    "                    \"solver\": [\"newton-cg\", \"lbfgs\", \"liblinear\"]\n",
    "                }\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"KNeighborsClassifier\",\n",
    "            \"estimator\": KNeighborsClassifier(),\n",
    "            \"hyperparameters\":\n",
    "                {\n",
    "                    \"n_neighbors\": range(1,20,2),\n",
    "                    \"weights\": [\"distance\", \"uniform\"],\n",
    "                    \"algorithm\": [\"ball_tree\", \"kd_tree\", \"brute\"],\n",
    "                    \"p\": [1,2]\n",
    "                }\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"RandomForestClassifier\",\n",
    "            \"estimator\": RandomForestClassifier(random_state=1),\n",
    "            \"hyperparameters\":\n",
    "                {\n",
    "                    \"n_estimators\": [4, 6, 9],\n",
    "                    \"criterion\": [\"entropy\", \"gini\"],\n",
    "                    \"max_depth\": [2, 5, 10],\n",
    "                    \"max_features\": [\"log2\", \"sqrt\"],\n",
    "                    \"min_samples_leaf\": [1, 5, 8],\n",
    "                    \"min_samples_split\": [2, 3, 5]\n",
    "\n",
    "                }\n",
    "        }\n",
    "    ]\n",
    "\n",
    "    for model in models:\n",
    "        print(model['name'])\n",
    "        print('-'*len(model['name']))\n",
    "\n",
    "        grid = GridSearchCV(model[\"estimator\"],\n",
    "                            param_grid=model[\"hyperparameters\"],\n",
    "                            cv=10)\n",
    "        grid.fit(all_X,all_y)\n",
    "        model[\"best_params\"] = grid.best_params_\n",
    "        model[\"best_score\"] = grid.best_score_\n",
    "        model[\"best_model\"] = grid.best_estimator_\n",
    "\n",
    "        print(\"Best Score: {}\".format(model[\"best_score\"]))\n",
    "        print(\"Best Parameters: {}\\n\".format(model[\"best_params\"]))\n",
    "\n",
    "    return models\n",
    "\n",
    "result = select_model(train,cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making a Submission to Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_submission_file(model,cols,filename=\"submission.csv\"):\n",
    "    holdout_data = holdout[cols]\n",
    "    predictions = model.predict(holdout_data)\n",
    "    \n",
    "    holdout_ids = holdout[\"PassengerId\"]\n",
    "    submission_df = {\"PassengerId\": holdout_ids,\n",
    "                 \"Survived\": predictions}\n",
    "    submission = pd.DataFrame(submission_df)\n",
    "\n",
    "    submission.to_csv(filename,index=False)\n",
    "\n",
    "best_rf_model = result[2][\"best_model\"]\n",
    "save_submission_file(best_rf_model,cols)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dscontent",
   "language": "python",
   "name": "dscontent"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
