{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy\n",
    "import re\n",
    "\n",
    "data_files = [\n",
    "    \"ap_2010.csv\",\n",
    "    \"class_size.csv\",\n",
    "    \"demographics.csv\",\n",
    "    \"graduation.csv\",\n",
    "    \"hs_directory.csv\",\n",
    "    \"sat_results.csv\"\n",
    "]\n",
    "\n",
    "data = {}\n",
    "\n",
    "for f in data_files:\n",
    "    d = pd.read_csv(\"schools/{0}\".format(f))\n",
    "    data[f.replace(\".csv\", \"\")] = d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in the Surveys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_survey = pd.read_csv(\"schools/survey_all.txt\", delimiter=\"\\t\", encoding='windows-1252')\n",
    "d75_survey = pd.read_csv(\"schools/survey_d75.txt\", delimiter=\"\\t\", encoding='windows-1252')\n",
    "survey = pd.concat([all_survey, d75_survey], axis=0)\n",
    "\n",
    "survey[\"DBN\"] = survey[\"dbn\"]\n",
    "\n",
    "survey_fields = [\n",
    "    \"DBN\", \n",
    "    \"rr_s\", \n",
    "    \"rr_t\", \n",
    "    \"rr_p\", \n",
    "    \"N_s\", \n",
    "    \"N_t\", \n",
    "    \"N_p\", \n",
    "    \"saf_p_11\", \n",
    "    \"com_p_11\", \n",
    "    \"eng_p_11\", \n",
    "    \"aca_p_11\", \n",
    "    \"saf_t_11\", \n",
    "    \"com_t_11\", \n",
    "    \"eng_t_11\", \n",
    "    \"aca_t_11\", \n",
    "    \"saf_s_11\", \n",
    "    \"com_s_11\", \n",
    "    \"eng_s_11\", \n",
    "    \"aca_s_11\", \n",
    "    \"saf_tot_11\", \n",
    "    \"com_tot_11\", \n",
    "    \"eng_tot_11\", \n",
    "    \"aca_tot_11\",\n",
    "]\n",
    "survey = survey[survey_fields]\n",
    "data[\"survey\"] = survey"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add DBN Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"hs_directory\"][\"DBN\"] = data[\"hs_directory\"][\"dbn\"]\n",
    "\n",
    "def pad_csd(num):\n",
    "    string_representation = str(num)\n",
    "    if len(string_representation) > 1:\n",
    "        return string_representation\n",
    "    else:\n",
    "        return \"0\" + string_representation\n",
    "    \n",
    "data[\"class_size\"][\"padded_csd\"] = data[\"class_size\"][\"CSD\"].apply(pad_csd)\n",
    "data[\"class_size\"][\"DBN\"] = data[\"class_size\"][\"padded_csd\"] + data[\"class_size\"][\"SCHOOL CODE\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert Columns to Numeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['SAT Math Avg. Score', 'SAT Critical Reading Avg. Score', 'SAT Writing Avg. Score']\n",
    "for c in cols:\n",
    "    data[\"sat_results\"][c] = pd.to_numeric(data[\"sat_results\"][c], errors=\"coerce\")\n",
    "\n",
    "data['sat_results']['sat_score'] = data['sat_results'][cols[0]] + data['sat_results'][cols[1]] + data['sat_results'][cols[2]]\n",
    "\n",
    "def find_lat(loc):\n",
    "    coords = re.findall(\"\\(.+, .+\\)\", loc)\n",
    "    lat = coords[0].split(\",\")[0].replace(\"(\", \"\")\n",
    "    return lat\n",
    "\n",
    "def find_lon(loc):\n",
    "    coords = re.findall(\"\\(.+, .+\\)\", loc)\n",
    "    lon = coords[0].split(\",\")[1].replace(\")\", \"\").strip()\n",
    "    return lon\n",
    "\n",
    "data[\"hs_directory\"][\"lat\"] = data[\"hs_directory\"][\"Location 1\"].apply(find_lat)\n",
    "data[\"hs_directory\"][\"lon\"] = data[\"hs_directory\"][\"Location 1\"].apply(find_lon)\n",
    "\n",
    "data[\"hs_directory\"][\"lat\"] = pd.to_numeric(data[\"hs_directory\"][\"lat\"], errors=\"coerce\")\n",
    "data[\"hs_directory\"][\"lon\"] = pd.to_numeric(data[\"hs_directory\"][\"lon\"], errors=\"coerce\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Condense Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_size = data[\"class_size\"]\n",
    "class_size = class_size[class_size[\"GRADE \"] == \"09-12\"]\n",
    "class_size = class_size[class_size[\"PROGRAM TYPE\"] == \"GEN ED\"]\n",
    "\n",
    "class_size = class_size.groupby(\"DBN\").agg(numpy.mean)\n",
    "class_size.reset_index(inplace=True)\n",
    "data[\"class_size\"] = class_size\n",
    "\n",
    "data[\"demographics\"] = data[\"demographics\"][data[\"demographics\"][\"schoolyear\"] == 20112012]\n",
    "\n",
    "data[\"graduation\"] = data[\"graduation\"][data[\"graduation\"][\"Cohort\"] == \"2006\"]\n",
    "data[\"graduation\"] = data[\"graduation\"][data[\"graduation\"][\"Demographic\"] == \"Total Cohort\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert AP Scores to Numeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['AP Test Takers ', 'Total Exams Taken', 'Number of Exams with scores 3 4 or 5']\n",
    "\n",
    "for col in cols:\n",
    "    data[\"ap_2010\"][col] = pd.to_numeric(data[\"ap_2010\"][col], errors=\"coerce\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine the Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = data[\"sat_results\"]\n",
    "\n",
    "combined = combined.merge(data[\"ap_2010\"], on=\"DBN\", how=\"left\")\n",
    "combined = combined.merge(data[\"graduation\"], on=\"DBN\", how=\"left\")\n",
    "\n",
    "to_merge = [\"class_size\", \"demographics\", \"survey\", \"hs_directory\"]\n",
    "\n",
    "for m in to_merge:\n",
    "    combined = combined.merge(data[m], on=\"DBN\", how=\"inner\")\n",
    "\n",
    "combined = combined.fillna(combined.mean())\n",
    "combined = combined.fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add a School District Column for Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_first_two_chars(dbn):\n",
    "    return dbn[0:2]\n",
    "\n",
    "combined[\"school_dist\"] = combined[\"DBN\"].apply(get_first_two_chars)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SAT Critical Reading Avg. Score    0.986820\n",
      "SAT Math Avg. Score                0.972643\n",
      "SAT Writing Avg. Score             0.987771\n",
      "sat_score                          1.000000\n",
      "AP Test Takers                     0.523140\n",
      "                                     ...   \n",
      "priority08                              NaN\n",
      "priority09                              NaN\n",
      "priority10                              NaN\n",
      "lat                               -0.121029\n",
      "lon                               -0.132222\n",
      "Name: sat_score, Length: 67, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "correlations = combined.corr()\n",
    "correlations = correlations[\"sat_score\"]\n",
    "print(correlations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting Survey Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Remove DBN since it's a unique identifier, not a useful numerical value for correlation.\n",
    "survey_fields.remove(\"DBN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf6c66760>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 23996 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "combined.corr()[\"sat_score\"][survey_fields].plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are high correlations between `N_s`, `N_t`, `N_p`, and `sat_score`. Since these columns are correlated with `total_enrollment`, it makes sense that they would be high.  \n",
    "\n",
    "It is more interesting that `rr_s`, the student response rate, or the percentage of students that completed the survey, correlates with `sat_score`. This might make sense because students who are more likely to fill out surveys may be more likely to also be doing well academically.\n",
    "\n",
    "How students and teachers percieved safety (`saf_t_11` and `saf_s_11`) correlate with `sat_score`. This make sense — it's difficult to teach or learn in an unsafe environment.\n",
    "\n",
    "The last interesting correlation is the `aca_s_11`, which indicates how the student perceives academic standards, correlates with `sat_score`, but this is not true for `aca_t_11`, how teachers perceive academic standards, or `aca_p_11`, how parents perceive academic standards."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring Safety"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf5e63ac0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 29244 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "combined.plot.scatter(\"saf_s_11\", \"sat_score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There appears to be a correlation between SAT scores and safety, although it isn't very strong. It looks like there are a few schools with extremely high SAT scores and high safety scores. There are a few schools with low safety scores and low SAT scores. No school with a safety score lower than `6.5` has an average SAT score higher than 1500 or so."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Borough Safety"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "boro\n",
      "Bronx            6.606577\n",
      "Brooklyn         6.370755\n",
      "Manhattan        6.831370\n",
      "Queens           6.721875\n",
      "Staten Island    6.530000\n",
      "Name: saf_s_11, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "boros = combined.groupby(\"boro\").agg(numpy.mean)[\"saf_s_11\"]\n",
    "print(boros)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like Manhattan and Queens tend to have higher safety scores, whereas Brooklyn has low safety scores."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Racial Differences in SAT Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf741a9d0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 14880 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "race_fields = [\"white_per\", \"asian_per\", \"black_per\", \"hispanic_per\"]\n",
    "combined.corr()[\"sat_score\"][race_fields].plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like a higher percentage of white or Asian students at a school correlates positively with SAT scores, whereas a higher percentage of black or Hispanic students correlates negatively with SAT score. This may be due to a lack of funding for schools in certain areas, which are more likely to have a higher percentage of black or Hispanic students."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf6565eb0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 34776 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "combined.plot.scatter(\"hispanic_per\", \"sat_score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44                         MANHATTAN BRIDGES HIGH SCHOOL\n",
      "82      WASHINGTON HEIGHTS EXPEDITIONARY LEARNING SCHOOL\n",
      "89     GREGORIO LUPERON HIGH SCHOOL FOR SCIENCE AND M...\n",
      "125                  ACADEMY FOR LANGUAGE AND TECHNOLOGY\n",
      "141                INTERNATIONAL SCHOOL FOR LIBERAL ARTS\n",
      "176     PAN AMERICAN INTERNATIONAL HIGH SCHOOL AT MONROE\n",
      "253                            MULTICULTURAL HIGH SCHOOL\n",
      "286               PAN AMERICAN INTERNATIONAL HIGH SCHOOL\n",
      "Name: SCHOOL NAME, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(combined[combined[\"hispanic_per\"] > 95][\"SCHOOL NAME\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The schools listed above appear to primarily serve recent immigrants to the U.S. These schools have many students who are learning English, which would explain the lower SAT scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37                                STUYVESANT HIGH SCHOOL\n",
      "151                         BRONX HIGH SCHOOL OF SCIENCE\n",
      "187                       BROOKLYN TECHNICAL HIGH SCHOOL\n",
      "327    QUEENS HIGH SCHOOL FOR THE SCIENCES AT YORK CO...\n",
      "356                  STATEN ISLAND TECHNICAL HIGH SCHOOL\n",
      "Name: SCHOOL NAME, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(combined[(combined[\"hispanic_per\"] < 10) & (combined[\"sat_score\"] > 1800)][\"SCHOOL NAME\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many of the schools above appear to be specialized science and technology schools that receive extra funding and only admit students who pass an entrance exam. This doesn't explain the low `hispanic_per`, but it does explain why their students tend to do better on the SAT — they are students from all over New York City who did well on a standardized test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gender Differences in SAT Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf649a610>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 11740 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gender_fields = [\"male_per\", \"female_per\"]\n",
    "combined.corr()[\"sat_score\"][gender_fields].plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the plot above, we can see that a high percentage of females at a school positively correlates with SAT scores, whereas a high percentage of males at a school negatively correlates with SAT scores. Neither correlation is extremely strong."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf649a3a0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 30312 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "combined.plot.scatter(\"female_per\", \"sat_score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the scatter plot, there doesn't seem to be any real correlation between `sat_score` and `female_per`.  However, there is a cluster of schools with a high percentage of females (`60` to `80`) and high SAT scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5                         BARD HIGH SCHOOL EARLY COLLEGE\n",
      "26                         ELEANOR ROOSEVELT HIGH SCHOOL\n",
      "60                                    BEACON HIGH SCHOOL\n",
      "61     FIORELLO H. LAGUARDIA HIGH SCHOOL OF MUSIC & A...\n",
      "302                          TOWNSEND HARRIS HIGH SCHOOL\n",
      "Name: SCHOOL NAME, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(combined[(combined[\"female_per\"] > 60) & (combined[\"sat_score\"] > 1700)][\"SCHOOL NAME\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These schools appear to be very selective liberal arts schools that have high academic standards."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AP Exam Scores vs. SAT Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fbdf5fc2160>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 30848 characters]\n",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "combined[\"ap_per\"] = combined[\"AP Test Takers \"] / combined[\"total_enrollment\"]\n",
    "\n",
    "combined.plot.scatter(x='ap_per', y='sat_score')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like there is a relationship between the percentage of students in a school who take the AP exam and their average SAT scores. It's not a very strong correlation, however."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
