2.DataClean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Postive Covid-19 Test Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Alabama': 'AL',\n",
       " 'Alaska': 'AK',\n",
       " 'Arizona': 'AZ',\n",
       " 'Arkansas': 'AR',\n",
       " 'California': 'CA',\n",
       " 'Colorado': 'CO',\n",
       " 'Connecticut': 'CT',\n",
       " 'Delaware': 'DE',\n",
       " 'Florida': 'FL',\n",
       " 'Georgia': 'GA',\n",
       " 'Hawaii': 'HI',\n",
       " 'Idaho': 'ID',\n",
       " 'Illinois': 'IL',\n",
       " 'Indiana': 'IN',\n",
       " 'Iowa': 'IA',\n",
       " 'Kansas': 'KS',\n",
       " 'Kentucky': 'KY',\n",
       " 'Louisiana': 'LA',\n",
       " 'Maine': 'ME',\n",
       " 'Maryland': 'MD',\n",
       " 'Massachusetts': 'MA',\n",
       " 'Michigan': 'MI',\n",
       " 'Minnesota': 'MN',\n",
       " 'Mississippi': 'MS',\n",
       " 'Missouri': 'MO',\n",
       " 'Montana': 'MT',\n",
       " 'Nebraska': 'NE',\n",
       " 'Nevada': 'NV',\n",
       " 'New Hampshire': 'NH',\n",
       " 'New Jersey': 'NJ',\n",
       " 'New Mexico': 'NM',\n",
       " 'New York': 'NY',\n",
       " 'North Carolina': 'NC',\n",
       " 'North Dakota': 'ND',\n",
       " 'Ohio': 'OH',\n",
       " 'Oklahoma': 'OK',\n",
       " 'Oregon': 'OR',\n",
       " 'Pennsylvania': 'PA',\n",
       " 'Rhode Island': 'RI',\n",
       " 'South Carolina': 'SC',\n",
       " 'South Dakota': 'SD',\n",
       " 'Tennessee': 'TN',\n",
       " 'Texas': 'TX',\n",
       " 'Utah': 'UT',\n",
       " 'Vermont': 'VT',\n",
       " 'Virginia': 'VA',\n",
       " 'Washington': 'WA',\n",
       " 'West Virginia': 'WV',\n",
       " 'Wisconsin': 'WI',\n",
       " 'Wyoming': 'WY',\n",
       " 'American Samoa': 'AS',\n",
       " 'Guam': 'GU',\n",
       " 'Northern Mariana Islands': 'MP',\n",
       " 'Puerto Rico': 'PR',\n",
       " 'Virgin Islands': 'VI',\n",
       " 'WashtingtonDC': 'DC',\n",
       " 'Palau': 'PW'}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_fips(county_id):\n",
    "    State, county = county_id.split('-')\n",
    "    state_id = State_to_FIPS[State]\n",
    "    return state_id+ county\n",
    "\n",
    "\n",
    "# BroadBoardSpeedMobile#[cols]\n",
    "def modifyGEOID(v):\n",
    "    if type(v) == int:\n",
    "        v = str(v)\n",
    "    if len(v) == 4:\n",
    "        v = '0' + v\n",
    "    return v\n",
    "\n",
    "\n",
    "string  = '''Alabama\tAL\t01\n",
    "Alaska\tAK\t02\n",
    "Arizona\tAZ\t04\n",
    "Arkansas\tAR\t05\n",
    "California\tCA\t06\n",
    "Colorado\tCO\t08\n",
    "Connecticut\tCT\t09\n",
    "Delaware\tDE\t10\n",
    "Florida\tFL\t12\n",
    "Georgia\tGA\t13\n",
    "Hawaii\tHI\t15\n",
    "Idaho\tID\t16\n",
    "Illinois\tIL\t17\n",
    "Indiana\tIN\t18\n",
    "Iowa\tIA\t19\n",
    "Kansas\tKS\t20\n",
    "Kentucky\tKY\t21\n",
    "Louisiana\tLA\t22\n",
    "Maine\tME\t23\n",
    "Maryland\tMD\t24\n",
    "Massachusetts\tMA\t25\n",
    "Michigan\tMI\t26\n",
    "Minnesota\tMN\t27\n",
    "Mississippi\tMS\t28\n",
    "Missouri\tMO\t29\n",
    "Montana\tMT\t30\n",
    "Nebraska\tNE\t31\n",
    "Nevada\tNV\t32\n",
    "New Hampshire\tNH\t33\n",
    "New Jersey\tNJ\t34\n",
    "New Mexico\tNM\t35\n",
    "New York\tNY\t36\n",
    "North Carolina\tNC\t37\n",
    "North Dakota\tND\t38\n",
    "Ohio\tOH\t39\n",
    "Oklahoma\tOK\t40\n",
    "Oregon\tOR\t41\n",
    "Pennsylvania\tPA\t42\n",
    "Rhode Island\tRI\t44\n",
    "South Carolina\tSC\t45\n",
    "South Dakota\tSD\t46\n",
    "Tennessee\tTN\t47\n",
    "Texas\tTX\t48\n",
    "Utah\tUT\t49\n",
    "Vermont\tVT\t50\n",
    "Virginia\tVA\t51\n",
    "Washington\tWA\t53\n",
    "West Virginia\tWV\t54\n",
    "Wisconsin\tWI\t55\n",
    "Wyoming\tWY\t56\n",
    "American Samoa\tAS\t60\n",
    "Guam\tGU\t66\n",
    "Northern Mariana Islands\tMP\t69\n",
    "Puerto Rico\tPR\t72\n",
    "Virgin Islands\tVI\t78\n",
    "WashtingtonDC\tDC\t11\n",
    "Palau\tPW\t00'''\n",
    "\n",
    "# State_to_FIPS = {k:v for k, v in [i.split('\\t')[1:] for i in string.split('\\n')]}\n",
    "# State_to_FIPS\n",
    "State_to_ABBR = {k:v for k, v in [i.split('\\t')[:2] for i in string.split('\\n')]}\n",
    "State_to_ABBR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'California': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_CA_external',\n",
       " 'Illinois': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_IL_external',\n",
       " 'Indiana': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_IN_external',\n",
       " 'Maine': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_ME_external',\n",
       " 'New Jersey': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_NJ_external',\n",
       " 'New York': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_NY_external',\n",
       " 'North Carolina': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_NC_external',\n",
       " 'Ohio': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_OH_external',\n",
       " 'Oregon': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_OR_external',\n",
       " 'Pennsylvania': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_PA_external',\n",
       " 'South Carolina': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_SC_external',\n",
       " 'Tennessee': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_TN_external',\n",
       " 'Texas': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_TX_external',\n",
       " 'Virginia': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_VA_external',\n",
       " 'West Virginia': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_WV_external',\n",
       " 'Wisconsin': 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_WI_external'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# url = 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_latest_external_data'\n",
    "Available_States = {\n",
    " 'California': 'CA',\n",
    " 'Illinois': 'IL',\n",
    " 'Indiana': 'IN',\n",
    " 'Maine': 'ME',\n",
    " 'New Jersey': 'NJ',\n",
    " 'New York': 'NY',\n",
    " 'North Carolina': 'NC',\n",
    " 'Ohio': 'OH',\n",
    " 'Oregon': 'OR',\n",
    " 'Pennsylvania': 'PA',\n",
    " 'South Carolina': 'SC',\n",
    " 'Tennessee': 'TN',\n",
    " 'Texas': 'TX',\n",
    " 'Virginia': 'VA',\n",
    " 'West Virginia': 'WV',\n",
    " 'Wisconsin': 'WI',\n",
    " }\n",
    "\n",
    "# len(Available_States)\n",
    "\n",
    "Available_States_Links = {}\n",
    "\n",
    "for state, state_abbr in Available_States.items():\n",
    "    url = 'https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_state_' \n",
    "    Available_States_Links[state] = url + state_abbr + '_external'\n",
    "\n",
    "    \n",
    "Available_States_Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fips_code</th>\n",
       "      <th>state</th>\n",
       "      <th>state_name</th>\n",
       "      <th>cbsa_code</th>\n",
       "      <th>county</th>\n",
       "      <th>new_cases_week_over_week_percent_change</th>\n",
       "      <th>new_cases_7_day_rolling_average</th>\n",
       "      <th>new_cases_per_100k_7_day_rolling_average</th>\n",
       "      <th>new_deaths_7_day_rolling_average</th>\n",
       "      <th>new_deaths_week_over_week_percent_change</th>\n",
       "      <th>...</th>\n",
       "      <th>percent_adult_icu_beds_used_confirmed_covid_week_over_week_absolute_change</th>\n",
       "      <th>hospitals_included_in_percent_adult_icu_beds_used_confirmed_covid</th>\n",
       "      <th>cbsa_daily_cli_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_cli_percentage_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_ili_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_ili_percentage_7_day_rolling_average</th>\n",
       "      <th>date</th>\n",
       "      <th>report_date_window_start</th>\n",
       "      <th>report_date_window_end</th>\n",
       "      <th>community_transmission_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>-50.00</td>\n",
       "      <td>31.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-16</td>\n",
       "      <td>2020-05-10</td>\n",
       "      <td>2020-05-16</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>-42.99</td>\n",
       "      <td>35.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-17</td>\n",
       "      <td>2020-05-11</td>\n",
       "      <td>2020-05-17</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>-37.96</td>\n",
       "      <td>36.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-15</td>\n",
       "      <td>2020-05-09</td>\n",
       "      <td>2020-05-15</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>-31.16</td>\n",
       "      <td>39.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-18</td>\n",
       "      <td>2020-05-12</td>\n",
       "      <td>2020-05-18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>-29.11</td>\n",
       "      <td>35.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-21</td>\n",
       "      <td>2020-05-15</td>\n",
       "      <td>2020-05-21</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10327</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10900</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>200.00</td>\n",
       "      <td>4.29</td>\n",
       "      <td>4.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>11.14</td>\n",
       "      <td>1.18</td>\n",
       "      <td>4.43</td>\n",
       "      <td>0.47</td>\n",
       "      <td>2020-07-24</td>\n",
       "      <td>2020-07-18</td>\n",
       "      <td>2020-07-24</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10328</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10900</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>227.27</td>\n",
       "      <td>5.14</td>\n",
       "      <td>4.89</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.86</td>\n",
       "      <td>1.13</td>\n",
       "      <td>4.71</td>\n",
       "      <td>0.49</td>\n",
       "      <td>2020-07-26</td>\n",
       "      <td>2020-07-20</td>\n",
       "      <td>2020-07-26</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10329</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10900</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>250.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>4.75</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>11.00</td>\n",
       "      <td>1.16</td>\n",
       "      <td>4.43</td>\n",
       "      <td>0.46</td>\n",
       "      <td>2020-07-25</td>\n",
       "      <td>2020-07-19</td>\n",
       "      <td>2020-07-25</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10330</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10900</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>300.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>3.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.57</td>\n",
       "      <td>1.12</td>\n",
       "      <td>3.86</td>\n",
       "      <td>0.40</td>\n",
       "      <td>2020-07-23</td>\n",
       "      <td>2020-07-17</td>\n",
       "      <td>2020-07-23</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10331</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10900</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>366.67</td>\n",
       "      <td>4.00</td>\n",
       "      <td>3.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.43</td>\n",
       "      <td>1.10</td>\n",
       "      <td>4.14</td>\n",
       "      <td>0.43</td>\n",
       "      <td>2020-07-22</td>\n",
       "      <td>2020-07-16</td>\n",
       "      <td>2020-07-22</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10332 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       fips_code state  state_name cbsa_code           county  \\\n",
       "0          34001    NJ        None      None  Atlantic County   \n",
       "1          34001    NJ        None      None  Atlantic County   \n",
       "2          34001    NJ        None      None  Atlantic County   \n",
       "3          34001    NJ        None      None  Atlantic County   \n",
       "4          34001    NJ        None      None  Atlantic County   \n",
       "...          ...   ...         ...       ...              ...   \n",
       "10327      34041    NJ  New Jersey     10900    Warren County   \n",
       "10328      34041    NJ  New Jersey     10900    Warren County   \n",
       "10329      34041    NJ  New Jersey     10900    Warren County   \n",
       "10330      34041    NJ  New Jersey     10900    Warren County   \n",
       "10331      34041    NJ  New Jersey     10900    Warren County   \n",
       "\n",
       "       new_cases_week_over_week_percent_change  \\\n",
       "0                                       -50.00   \n",
       "1                                       -42.99   \n",
       "2                                       -37.96   \n",
       "3                                       -31.16   \n",
       "4                                       -29.11   \n",
       "...                                        ...   \n",
       "10327                                   200.00   \n",
       "10328                                   227.27   \n",
       "10329                                   250.00   \n",
       "10330                                   300.00   \n",
       "10331                                   366.67   \n",
       "\n",
       "       new_cases_7_day_rolling_average  \\\n",
       "0                                31.71   \n",
       "1                                35.43   \n",
       "2                                36.43   \n",
       "3                                39.14   \n",
       "4                                35.14   \n",
       "...                                ...   \n",
       "10327                             4.29   \n",
       "10328                             5.14   \n",
       "10329                             5.00   \n",
       "10330                             4.00   \n",
       "10331                             4.00   \n",
       "\n",
       "       new_cases_per_100k_7_day_rolling_average  \\\n",
       "0                                           NaN   \n",
       "1                                           NaN   \n",
       "2                                           NaN   \n",
       "3                                           NaN   \n",
       "4                                           NaN   \n",
       "...                                         ...   \n",
       "10327                                      4.07   \n",
       "10328                                      4.89   \n",
       "10329                                      4.75   \n",
       "10330                                      3.80   \n",
       "10331                                      3.80   \n",
       "\n",
       "       new_deaths_7_day_rolling_average  \\\n",
       "0                                  5.43   \n",
       "1                                  5.71   \n",
       "2                                  5.57   \n",
       "3                                  5.00   \n",
       "4                                  4.14   \n",
       "...                                 ...   \n",
       "10327                              0.00   \n",
       "10328                              0.00   \n",
       "10329                              0.00   \n",
       "10330                              0.00   \n",
       "10331                              0.00   \n",
       "\n",
       "       new_deaths_week_over_week_percent_change  ...  \\\n",
       "0                                           NaN  ...   \n",
       "1                                           NaN  ...   \n",
       "2                                           NaN  ...   \n",
       "3                                           NaN  ...   \n",
       "4                                           NaN  ...   \n",
       "...                                         ...  ...   \n",
       "10327                                    -100.0  ...   \n",
       "10328                                    -100.0  ...   \n",
       "10329                                    -100.0  ...   \n",
       "10330                                    -100.0  ...   \n",
       "10331                                    -100.0  ...   \n",
       "\n",
       "       percent_adult_icu_beds_used_confirmed_covid_week_over_week_absolute_change  \\\n",
       "0                                                    NaN                            \n",
       "1                                                    NaN                            \n",
       "2                                                    NaN                            \n",
       "3                                                    NaN                            \n",
       "4                                                    NaN                            \n",
       "...                                                  ...                            \n",
       "10327                                                0.0                            \n",
       "10328                                                0.0                            \n",
       "10329                                                0.0                            \n",
       "10330                                                0.0                            \n",
       "10331                                                0.0                            \n",
       "\n",
       "       hospitals_included_in_percent_adult_icu_beds_used_confirmed_covid  \\\n",
       "0                                                    NaN                   \n",
       "1                                                    NaN                   \n",
       "2                                                    NaN                   \n",
       "3                                                    NaN                   \n",
       "4                                                    NaN                   \n",
       "...                                                  ...                   \n",
       "10327                                                2.0                   \n",
       "10328                                                2.0                   \n",
       "10329                                                2.0                   \n",
       "10330                                                2.0                   \n",
       "10331                                                2.0                   \n",
       "\n",
       "       cbsa_daily_cli_7_day_rolling_average  \\\n",
       "0                                       NaN   \n",
       "1                                       NaN   \n",
       "2                                       NaN   \n",
       "3                                       NaN   \n",
       "4                                       NaN   \n",
       "...                                     ...   \n",
       "10327                                 11.14   \n",
       "10328                                 10.86   \n",
       "10329                                 11.00   \n",
       "10330                                 10.57   \n",
       "10331                                 10.43   \n",
       "\n",
       "       cbsa_daily_cli_percentage_7_day_rolling_average  \\\n",
       "0                                                  NaN   \n",
       "1                                                  NaN   \n",
       "2                                                  NaN   \n",
       "3                                                  NaN   \n",
       "4                                                  NaN   \n",
       "...                                                ...   \n",
       "10327                                             1.18   \n",
       "10328                                             1.13   \n",
       "10329                                             1.16   \n",
       "10330                                             1.12   \n",
       "10331                                             1.10   \n",
       "\n",
       "       cbsa_daily_ili_7_day_rolling_average  \\\n",
       "0                                       NaN   \n",
       "1                                       NaN   \n",
       "2                                       NaN   \n",
       "3                                       NaN   \n",
       "4                                       NaN   \n",
       "...                                     ...   \n",
       "10327                                  4.43   \n",
       "10328                                  4.71   \n",
       "10329                                  4.43   \n",
       "10330                                  3.86   \n",
       "10331                                  4.14   \n",
       "\n",
       "       cbsa_daily_ili_percentage_7_day_rolling_average        date  \\\n",
       "0                                                  NaN  2020-05-16   \n",
       "1                                                  NaN  2020-05-17   \n",
       "2                                                  NaN  2020-05-15   \n",
       "3                                                  NaN  2020-05-18   \n",
       "4                                                  NaN  2020-05-21   \n",
       "...                                                ...         ...   \n",
       "10327                                             0.47  2020-07-24   \n",
       "10328                                             0.49  2020-07-26   \n",
       "10329                                             0.46  2020-07-25   \n",
       "10330                                             0.40  2020-07-23   \n",
       "10331                                             0.43  2020-07-22   \n",
       "\n",
       "       report_date_window_start  report_date_window_end  \\\n",
       "0                    2020-05-10              2020-05-16   \n",
       "1                    2020-05-11              2020-05-17   \n",
       "2                    2020-05-09              2020-05-15   \n",
       "3                    2020-05-12              2020-05-18   \n",
       "4                    2020-05-15              2020-05-21   \n",
       "...                         ...                     ...   \n",
       "10327                2020-07-18              2020-07-24   \n",
       "10328                2020-07-20              2020-07-26   \n",
       "10329                2020-07-19              2020-07-25   \n",
       "10330                2020-07-17              2020-07-23   \n",
       "10331                2020-07-16              2020-07-22   \n",
       "\n",
       "       community_transmission_level  \n",
       "0                              None  \n",
       "1                              None  \n",
       "2                              None  \n",
       "3                              None  \n",
       "4                              None  \n",
       "...                             ...  \n",
       "10327                          None  \n",
       "10328                          None  \n",
       "10329                          None  \n",
       "10330                          None  \n",
       "10331                          None  \n",
       "\n",
       "[10332 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from datetime import datetime\n",
    "\n",
    "\n",
    "def get_data_from_url(url):\n",
    "    \n",
    "    Data = pd.read_json(url)\n",
    "    L = []\n",
    "    for idx, row in Data.iterrows():\n",
    "        d = row.to_dict()\n",
    "        data = d['integrated_county_timeseries_external_data']\n",
    "        L.append(data)\n",
    "\n",
    "    Data = pd.DataFrame(L)\n",
    "    return Data\n",
    "\n",
    "\n",
    "url = Available_States_Links['New Jersey']\n",
    "Data = get_data_from_url(url)\n",
    "Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max(Data['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fips_code</th>\n",
       "      <th>state</th>\n",
       "      <th>state_name</th>\n",
       "      <th>cbsa_code</th>\n",
       "      <th>county</th>\n",
       "      <th>new_cases_week_over_week_percent_change</th>\n",
       "      <th>new_cases_7_day_rolling_average</th>\n",
       "      <th>new_cases_per_100k_7_day_rolling_average</th>\n",
       "      <th>new_deaths_7_day_rolling_average</th>\n",
       "      <th>new_deaths_week_over_week_percent_change</th>\n",
       "      <th>...</th>\n",
       "      <th>percent_adult_icu_beds_used_confirmed_covid_week_over_week_absolute_change</th>\n",
       "      <th>hospitals_included_in_percent_adult_icu_beds_used_confirmed_covid</th>\n",
       "      <th>cbsa_daily_cli_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_cli_percentage_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_ili_7_day_rolling_average</th>\n",
       "      <th>cbsa_daily_ili_percentage_7_day_rolling_average</th>\n",
       "      <th>date</th>\n",
       "      <th>report_date_window_start</th>\n",
       "      <th>report_date_window_end</th>\n",
       "      <th>community_transmission_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>34001</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Atlantic County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>130.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.86</td>\n",
       "      <td>50.00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>34003</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Bergen County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>480.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.00</td>\n",
       "      <td>27.27</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.60</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1207</th>\n",
       "      <td>34005</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Burlington County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>162.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-36.36</td>\n",
       "      <td>...</td>\n",
       "      <td>1.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1685</th>\n",
       "      <td>34007</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Camden County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>165.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-30.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2193</th>\n",
       "      <td>34009</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Cape May County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37.29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.43</td>\n",
       "      <td>200.00</td>\n",
       "      <td>...</td>\n",
       "      <td>-11.61</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2646</th>\n",
       "      <td>34011</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Cumberland County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.71</td>\n",
       "      <td>-16.67</td>\n",
       "      <td>...</td>\n",
       "      <td>1.28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3190</th>\n",
       "      <td>34013</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Essex County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>422.86</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.57</td>\n",
       "      <td>60.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3661</th>\n",
       "      <td>34015</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Gloucester County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>101.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>16.67</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4177</th>\n",
       "      <td>34017</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Hudson County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>382.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.43</td>\n",
       "      <td>9.09</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4673</th>\n",
       "      <td>34019</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Hunterdon County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>54.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.29</td>\n",
       "      <td>100.00</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.79</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5143</th>\n",
       "      <td>34021</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Mercer County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>109.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.43</td>\n",
       "      <td>42.86</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5649</th>\n",
       "      <td>34023</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Middlesex County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>390.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.86</td>\n",
       "      <td>-48.00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.71</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6143</th>\n",
       "      <td>34025</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Monmouth County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>380.57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.14</td>\n",
       "      <td>-42.31</td>\n",
       "      <td>...</td>\n",
       "      <td>4.21</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6618</th>\n",
       "      <td>34027</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Morris County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>231.57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.43</td>\n",
       "      <td>-23.08</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7122</th>\n",
       "      <td>34029</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Ocean County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>343.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.00</td>\n",
       "      <td>34.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7598</th>\n",
       "      <td>34031</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Passaic County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>249.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.14</td>\n",
       "      <td>-44.44</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.63</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8104</th>\n",
       "      <td>34033</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Salem County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.29</td>\n",
       "      <td>100.00</td>\n",
       "      <td>...</td>\n",
       "      <td>6.77</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8606</th>\n",
       "      <td>34035</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Somerset County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>137.86</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.43</td>\n",
       "      <td>50.00</td>\n",
       "      <td>...</td>\n",
       "      <td>4.66</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9105</th>\n",
       "      <td>34037</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Sussex County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>104.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.57</td>\n",
       "      <td>100.00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.80</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9572</th>\n",
       "      <td>34039</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Union County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>243.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.43</td>\n",
       "      <td>30.77</td>\n",
       "      <td>...</td>\n",
       "      <td>3.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10055</th>\n",
       "      <td>34041</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>None</td>\n",
       "      <td>Warren County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>45.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.29</td>\n",
       "      <td>100.00</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.54</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       fips_code state  state_name cbsa_code             county  \\\n",
       "232        34001    NJ  New Jersey      None    Atlantic County   \n",
       "737        34003    NJ  New Jersey      None      Bergen County   \n",
       "1207       34005    NJ  New Jersey      None  Burlington County   \n",
       "1685       34007    NJ  New Jersey      None      Camden County   \n",
       "2193       34009    NJ  New Jersey      None    Cape May County   \n",
       "2646       34011    NJ  New Jersey      None  Cumberland County   \n",
       "3190       34013    NJ  New Jersey      None       Essex County   \n",
       "3661       34015    NJ  New Jersey      None  Gloucester County   \n",
       "4177       34017    NJ  New Jersey      None      Hudson County   \n",
       "4673       34019    NJ  New Jersey      None   Hunterdon County   \n",
       "5143       34021    NJ  New Jersey      None      Mercer County   \n",
       "5649       34023    NJ  New Jersey      None   Middlesex County   \n",
       "6143       34025    NJ  New Jersey      None    Monmouth County   \n",
       "6618       34027    NJ  New Jersey      None      Morris County   \n",
       "7122       34029    NJ  New Jersey      None       Ocean County   \n",
       "7598       34031    NJ  New Jersey      None     Passaic County   \n",
       "8104       34033    NJ  New Jersey      None       Salem County   \n",
       "8606       34035    NJ  New Jersey      None    Somerset County   \n",
       "9105       34037    NJ  New Jersey      None      Sussex County   \n",
       "9572       34039    NJ  New Jersey      None       Union County   \n",
       "10055      34041    NJ  New Jersey      None      Warren County   \n",
       "\n",
       "       new_cases_week_over_week_percent_change  \\\n",
       "232                                        NaN   \n",
       "737                                        NaN   \n",
       "1207                                       NaN   \n",
       "1685                                       NaN   \n",
       "2193                                       NaN   \n",
       "2646                                       NaN   \n",
       "3190                                       NaN   \n",
       "3661                                       NaN   \n",
       "4177                                       NaN   \n",
       "4673                                       NaN   \n",
       "5143                                       NaN   \n",
       "5649                                       NaN   \n",
       "6143                                       NaN   \n",
       "6618                                       NaN   \n",
       "7122                                       NaN   \n",
       "7598                                       NaN   \n",
       "8104                                       NaN   \n",
       "8606                                       NaN   \n",
       "9105                                       NaN   \n",
       "9572                                       NaN   \n",
       "10055                                      NaN   \n",
       "\n",
       "       new_cases_7_day_rolling_average  \\\n",
       "232                             130.43   \n",
       "737                             480.71   \n",
       "1207                            162.71   \n",
       "1685                            165.71   \n",
       "2193                             37.29   \n",
       "2646                             37.14   \n",
       "3190                            422.86   \n",
       "3661                            101.00   \n",
       "4177                            382.00   \n",
       "4673                             54.14   \n",
       "5143                            109.00   \n",
       "5649                            390.14   \n",
       "6143                            380.57   \n",
       "6618                            231.57   \n",
       "7122                            343.43   \n",
       "7598                            249.00   \n",
       "8104                             23.43   \n",
       "8606                            137.86   \n",
       "9105                            104.00   \n",
       "9572                            243.71   \n",
       "10055                            45.00   \n",
       "\n",
       "       new_cases_per_100k_7_day_rolling_average  \\\n",
       "232                                         NaN   \n",
       "737                                         NaN   \n",
       "1207                                        NaN   \n",
       "1685                                        NaN   \n",
       "2193                                        NaN   \n",
       "2646                                        NaN   \n",
       "3190                                        NaN   \n",
       "3661                                        NaN   \n",
       "4177                                        NaN   \n",
       "4673                                        NaN   \n",
       "5143                                        NaN   \n",
       "5649                                        NaN   \n",
       "6143                                        NaN   \n",
       "6618                                        NaN   \n",
       "7122                                        NaN   \n",
       "7598                                        NaN   \n",
       "8104                                        NaN   \n",
       "8606                                        NaN   \n",
       "9105                                        NaN   \n",
       "9572                                        NaN   \n",
       "10055                                       NaN   \n",
       "\n",
       "       new_deaths_7_day_rolling_average  \\\n",
       "232                                0.86   \n",
       "737                                4.00   \n",
       "1207                               1.00   \n",
       "1685                               1.00   \n",
       "2193                               0.43   \n",
       "2646                               0.71   \n",
       "3190                               4.57   \n",
       "3661                               1.00   \n",
       "4177                               3.43   \n",
       "4673                               0.29   \n",
       "5143                               1.43   \n",
       "5649                               1.86   \n",
       "6143                               2.14   \n",
       "6618                               1.43   \n",
       "7122                               5.00   \n",
       "7598                               2.14   \n",
       "8104                               0.29   \n",
       "8606                               0.43   \n",
       "9105                               0.57   \n",
       "9572                               2.43   \n",
       "10055                              0.29   \n",
       "\n",
       "       new_deaths_week_over_week_percent_change  ...  \\\n",
       "232                                       50.00  ...   \n",
       "737                                       27.27  ...   \n",
       "1207                                     -36.36  ...   \n",
       "1685                                     -30.00  ...   \n",
       "2193                                     200.00  ...   \n",
       "2646                                     -16.67  ...   \n",
       "3190                                      60.00  ...   \n",
       "3661                                      16.67  ...   \n",
       "4177                                       9.09  ...   \n",
       "4673                                     100.00  ...   \n",
       "5143                                      42.86  ...   \n",
       "5649                                     -48.00  ...   \n",
       "6143                                     -42.31  ...   \n",
       "6618                                     -23.08  ...   \n",
       "7122                                      34.62  ...   \n",
       "7598                                     -44.44  ...   \n",
       "8104                                     100.00  ...   \n",
       "8606                                      50.00  ...   \n",
       "9105                                     100.00  ...   \n",
       "9572                                      30.77  ...   \n",
       "10055                                    100.00  ...   \n",
       "\n",
       "       percent_adult_icu_beds_used_confirmed_covid_week_over_week_absolute_change  \\\n",
       "232                                                 1.04                            \n",
       "737                                                -0.60                            \n",
       "1207                                                1.20                            \n",
       "1685                                                0.68                            \n",
       "2193                                              -11.61                            \n",
       "2646                                                1.28                            \n",
       "3190                                                0.00                            \n",
       "3661                                                0.00                            \n",
       "4177                                               -0.86                            \n",
       "4673                                               -1.79                            \n",
       "5143                                               -1.04                            \n",
       "5649                                                1.71                            \n",
       "6143                                                4.21                            \n",
       "6618                                               -1.28                            \n",
       "7122                                                0.74                            \n",
       "7598                                               -1.63                            \n",
       "8104                                                6.77                            \n",
       "8606                                                4.66                            \n",
       "9105                                                1.80                            \n",
       "9572                                                3.69                            \n",
       "10055                                              -1.54                            \n",
       "\n",
       "       hospitals_included_in_percent_adult_icu_beds_used_confirmed_covid  \\\n",
       "232                                                  0.0                   \n",
       "737                                                  0.0                   \n",
       "1207                                                 0.0                   \n",
       "1685                                                 0.0                   \n",
       "2193                                                 0.0                   \n",
       "2646                                                 0.0                   \n",
       "3190                                                 0.0                   \n",
       "3661                                                 0.0                   \n",
       "4177                                                 0.0                   \n",
       "4673                                                 0.0                   \n",
       "5143                                                 0.0                   \n",
       "5649                                                 0.0                   \n",
       "6143                                                 0.0                   \n",
       "6618                                                 0.0                   \n",
       "7122                                                 0.0                   \n",
       "7598                                                 0.0                   \n",
       "8104                                                 0.0                   \n",
       "8606                                                 0.0                   \n",
       "9105                                                 0.0                   \n",
       "9572                                                 0.0                   \n",
       "10055                                                0.0                   \n",
       "\n",
       "       cbsa_daily_cli_7_day_rolling_average  \\\n",
       "232                                     0.0   \n",
       "737                                     0.0   \n",
       "1207                                    0.0   \n",
       "1685                                    0.0   \n",
       "2193                                    0.0   \n",
       "2646                                    0.0   \n",
       "3190                                    0.0   \n",
       "3661                                    0.0   \n",
       "4177                                    0.0   \n",
       "4673                                    0.0   \n",
       "5143                                    0.0   \n",
       "5649                                    0.0   \n",
       "6143                                    0.0   \n",
       "6618                                    0.0   \n",
       "7122                                    0.0   \n",
       "7598                                    0.0   \n",
       "8104                                    0.0   \n",
       "8606                                    0.0   \n",
       "9105                                    0.0   \n",
       "9572                                    0.0   \n",
       "10055                                   0.0   \n",
       "\n",
       "       cbsa_daily_cli_percentage_7_day_rolling_average  \\\n",
       "232                                                0.0   \n",
       "737                                                0.0   \n",
       "1207                                               0.0   \n",
       "1685                                               0.0   \n",
       "2193                                               0.0   \n",
       "2646                                               0.0   \n",
       "3190                                               0.0   \n",
       "3661                                               0.0   \n",
       "4177                                               0.0   \n",
       "4673                                               0.0   \n",
       "5143                                               0.0   \n",
       "5649                                               0.0   \n",
       "6143                                               0.0   \n",
       "6618                                               0.0   \n",
       "7122                                               0.0   \n",
       "7598                                               0.0   \n",
       "8104                                               0.0   \n",
       "8606                                               0.0   \n",
       "9105                                               0.0   \n",
       "9572                                               0.0   \n",
       "10055                                              0.0   \n",
       "\n",
       "       cbsa_daily_ili_7_day_rolling_average  \\\n",
       "232                                     0.0   \n",
       "737                                     0.0   \n",
       "1207                                    0.0   \n",
       "1685                                    0.0   \n",
       "2193                                    0.0   \n",
       "2646                                    0.0   \n",
       "3190                                    0.0   \n",
       "3661                                    0.0   \n",
       "4177                                    0.0   \n",
       "4673                                    0.0   \n",
       "5143                                    0.0   \n",
       "5649                                    0.0   \n",
       "6143                                    0.0   \n",
       "6618                                    0.0   \n",
       "7122                                    0.0   \n",
       "7598                                    0.0   \n",
       "8104                                    0.0   \n",
       "8606                                    0.0   \n",
       "9105                                    0.0   \n",
       "9572                                    0.0   \n",
       "10055                                   0.0   \n",
       "\n",
       "       cbsa_daily_ili_percentage_7_day_rolling_average        date  \\\n",
       "232                                                0.0  2021-03-27   \n",
       "737                                                0.0  2021-03-27   \n",
       "1207                                               0.0  2021-03-27   \n",
       "1685                                               0.0  2021-03-27   \n",
       "2193                                               0.0  2021-03-27   \n",
       "2646                                               0.0  2021-03-27   \n",
       "3190                                               0.0  2021-03-27   \n",
       "3661                                               0.0  2021-03-27   \n",
       "4177                                               0.0  2021-03-27   \n",
       "4673                                               0.0  2021-03-27   \n",
       "5143                                               0.0  2021-03-27   \n",
       "5649                                               0.0  2021-03-27   \n",
       "6143                                               0.0  2021-03-27   \n",
       "6618                                               0.0  2021-03-27   \n",
       "7122                                               0.0  2021-03-27   \n",
       "7598                                               0.0  2021-03-27   \n",
       "8104                                               0.0  2021-03-27   \n",
       "8606                                               0.0  2021-03-27   \n",
       "9105                                               0.0  2021-03-27   \n",
       "9572                                               0.0  2021-03-27   \n",
       "10055                                              0.0  2021-03-27   \n",
       "\n",
       "       report_date_window_start  report_date_window_end  \\\n",
       "232                  2021-03-21              2021-03-27   \n",
       "737                  2021-03-21              2021-03-27   \n",
       "1207                 2021-03-21              2021-03-27   \n",
       "1685                 2021-03-21              2021-03-27   \n",
       "2193                 2021-03-21              2021-03-27   \n",
       "2646                 2021-03-21              2021-03-27   \n",
       "3190                 2021-03-21              2021-03-27   \n",
       "3661                 2021-03-21              2021-03-27   \n",
       "4177                 2021-03-21              2021-03-27   \n",
       "4673                 2021-03-21              2021-03-27   \n",
       "5143                 2021-03-21              2021-03-27   \n",
       "5649                 2021-03-21              2021-03-27   \n",
       "6143                 2021-03-21              2021-03-27   \n",
       "6618                 2021-03-21              2021-03-27   \n",
       "7122                 2021-03-21              2021-03-27   \n",
       "7598                 2021-03-21              2021-03-27   \n",
       "8104                 2021-03-21              2021-03-27   \n",
       "8606                 2021-03-21              2021-03-27   \n",
       "9105                 2021-03-21              2021-03-27   \n",
       "9572                 2021-03-21              2021-03-27   \n",
       "10055                2021-03-21              2021-03-27   \n",
       "\n",
       "       community_transmission_level  \n",
       "232                            high  \n",
       "737                            high  \n",
       "1207                           high  \n",
       "1685                           high  \n",
       "2193                           high  \n",
       "2646                           high  \n",
       "3190                           high  \n",
       "3661                           high  \n",
       "4177                           high  \n",
       "4673                           high  \n",
       "5143                           high  \n",
       "5649                           high  \n",
       "6143                           high  \n",
       "6618                           high  \n",
       "7122                           high  \n",
       "7598                           high  \n",
       "8104                           high  \n",
       "8606                           high  \n",
       "9105                           high  \n",
       "9572                           high  \n",
       "10055                          high  \n",
       "\n",
       "[21 rows x 40 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Data[Data['date'] == '2021-03-27']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-05-29 21:15:38.851604 \t California\n",
      "2021-05-29 21:15:58.519354 \t (58, 5)\n",
      "2021-05-29 21:15:58.520656 \t Illinois\n",
      "2021-05-29 21:16:28.244083 \t (102, 5)\n",
      "2021-05-29 21:16:28.244282 \t Indiana\n",
      "2021-05-29 21:16:55.443747 \t (92, 5)\n",
      "2021-05-29 21:16:55.443952 \t Maine\n",
      "2021-05-29 21:17:00.558030 \t (16, 5)\n",
      "2021-05-29 21:17:00.558262 \t New Jersey\n",
      "2021-05-29 21:17:08.075454 \t (21, 5)\n",
      "2021-05-29 21:17:08.075703 \t New York\n",
      "2021-05-29 21:17:20.794621 \t (62, 5)\n",
      "2021-05-29 21:17:20.794851 \t North Carolina\n",
      "2021-05-29 21:17:49.276505 \t (100, 5)\n",
      "2021-05-29 21:17:49.276690 \t Ohio\n",
      "2021-05-29 21:18:07.950788 \t (88, 5)\n",
      "2021-05-29 21:18:07.950990 \t Oregon\n",
      "2021-05-29 21:18:19.777560 \t (36, 5)\n",
      "2021-05-29 21:18:19.777821 \t Pennsylvania\n",
      "2021-05-29 21:18:48.358214 \t (67, 5)\n",
      "2021-05-29 21:18:48.358446 \t South Carolina\n",
      "2021-05-29 21:19:03.578944 \t (46, 5)\n",
      "2021-05-29 21:19:03.579159 \t Tennessee\n",
      "2021-05-29 21:19:30.689766 \t (95, 5)\n",
      "2021-05-29 21:19:30.689982 \t Texas\n",
      "2021-05-29 21:20:49.730623 \t (254, 5)\n",
      "2021-05-29 21:20:49.731098 \t Virginia\n",
      "2021-05-29 21:21:15.778664 \t (133, 5)\n",
      "2021-05-29 21:21:15.778904 \t West Virginia\n",
      "2021-05-29 21:21:32.481103 \t (55, 5)\n",
      "2021-05-29 21:21:32.481315 \t Wisconsin\n",
      "2021-05-29 21:21:55.489783 \t (72, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fips_code</th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>county</th>\n",
       "      <th>percent_new_test_results_reported_positive_7_day_rolling_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6001</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alameda County</td>\n",
       "      <td>1.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6003</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alpine County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6005</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Amador County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6007</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Butte County</td>\n",
       "      <td>2.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6009</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Calaveras County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>55133</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waukesha County</td>\n",
       "      <td>5.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>55135</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waupaca County</td>\n",
       "      <td>2.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>55137</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waushara County</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>55139</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Winnebago County</td>\n",
       "      <td>3.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>55141</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Wood County</td>\n",
       "      <td>3.56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1297 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    fips_code        date state            county  \\\n",
       "0        6001  2021-03-27    CA    Alameda County   \n",
       "1        6003  2021-03-27    CA     Alpine County   \n",
       "2        6005  2021-03-27    CA     Amador County   \n",
       "3        6007  2021-03-27    CA      Butte County   \n",
       "4        6009  2021-03-27    CA  Calaveras County   \n",
       "..        ...         ...   ...               ...   \n",
       "67      55133  2021-03-27    WI   Waukesha County   \n",
       "68      55135  2021-03-27    WI    Waupaca County   \n",
       "69      55137  2021-03-27    WI   Waushara County   \n",
       "70      55139  2021-03-27    WI  Winnebago County   \n",
       "71      55141  2021-03-27    WI       Wood County   \n",
       "\n",
       "    percent_new_test_results_reported_positive_7_day_rolling_average  \n",
       "0                                                1.38                 \n",
       "1                                                0.00                 \n",
       "2                                                0.00                 \n",
       "3                                                2.72                 \n",
       "4                                                0.00                 \n",
       "..                                                ...                 \n",
       "67                                               5.42                 \n",
       "68                                               2.11                 \n",
       "69                                               0.29                 \n",
       "70                                               3.62                 \n",
       "71                                               3.56                 \n",
       "\n",
       "[1297 rows x 5 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "date = '2021-03-27'\n",
    "# url = Available_States_Links['New Jersey']\n",
    "L = []\n",
    "for state, url in Available_States_Links.items():\n",
    "    print(datetime.now(), '\\t', state,)\n",
    "    NewReportRate = get_data_from_url(url)\n",
    "    NewReportRate = NewReportRate[NewReportRate['date'] == date]\n",
    "    NewReportRate = NewReportRate[['fips_code', 'date', 'state', 'county', 'percent_new_test_results_reported_positive_7_day_rolling_average']]\n",
    "    NewReportRate = NewReportRate.reset_index(drop = True)\n",
    "    print(datetime.now(), '\\t', NewReportRate.shape)\n",
    "    \n",
    "    L.append(NewReportRate)\n",
    "    \n",
    "\n",
    "NewCaseRate = pd.concat(L)\n",
    "NewCaseRate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fips_code</th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>county</th>\n",
       "      <th>percent_new_test_results_reported_positive_7_day_rolling_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6001</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alameda County</td>\n",
       "      <td>1.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6003</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alpine County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6005</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Amador County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6007</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Butte County</td>\n",
       "      <td>2.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6009</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Calaveras County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>55133</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waukesha County</td>\n",
       "      <td>5.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>55135</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waupaca County</td>\n",
       "      <td>2.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>55137</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waushara County</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>55139</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Winnebago County</td>\n",
       "      <td>3.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>55141</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Wood County</td>\n",
       "      <td>3.56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1297 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    fips_code        date state            county  \\\n",
       "0        6001  2021-03-27    CA    Alameda County   \n",
       "1        6003  2021-03-27    CA     Alpine County   \n",
       "2        6005  2021-03-27    CA     Amador County   \n",
       "3        6007  2021-03-27    CA      Butte County   \n",
       "4        6009  2021-03-27    CA  Calaveras County   \n",
       "..        ...         ...   ...               ...   \n",
       "67      55133  2021-03-27    WI   Waukesha County   \n",
       "68      55135  2021-03-27    WI    Waupaca County   \n",
       "69      55137  2021-03-27    WI   Waushara County   \n",
       "70      55139  2021-03-27    WI  Winnebago County   \n",
       "71      55141  2021-03-27    WI       Wood County   \n",
       "\n",
       "    percent_new_test_results_reported_positive_7_day_rolling_average  \n",
       "0                                                1.38                 \n",
       "1                                                0.00                 \n",
       "2                                                0.00                 \n",
       "3                                                2.72                 \n",
       "4                                                0.00                 \n",
       "..                                                ...                 \n",
       "67                                               5.42                 \n",
       "68                                               2.11                 \n",
       "69                                               0.29                 \n",
       "70                                               3.62                 \n",
       "71                                               3.56                 \n",
       "\n",
       "[1297 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NewCaseRate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NewCaseRate.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>county</th>\n",
       "      <th>percent_new_test_results_reported_positive_7_day_rolling_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6001</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alameda County</td>\n",
       "      <td>1.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6003</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Alpine County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6005</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Amador County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6007</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Butte County</td>\n",
       "      <td>2.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6009</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>CA</td>\n",
       "      <td>Calaveras County</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>55133</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waukesha County</td>\n",
       "      <td>5.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>55135</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waupaca County</td>\n",
       "      <td>2.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>55137</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Waushara County</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>55139</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Winnebago County</td>\n",
       "      <td>3.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>55141</td>\n",
       "      <td>2021-03-27</td>\n",
       "      <td>WI</td>\n",
       "      <td>Wood County</td>\n",
       "      <td>3.56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1297 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    GEOID        date state            county  \\\n",
       "0    6001  2021-03-27    CA    Alameda County   \n",
       "1    6003  2021-03-27    CA     Alpine County   \n",
       "2    6005  2021-03-27    CA     Amador County   \n",
       "3    6007  2021-03-27    CA      Butte County   \n",
       "4    6009  2021-03-27    CA  Calaveras County   \n",
       "..    ...         ...   ...               ...   \n",
       "67  55133  2021-03-27    WI   Waukesha County   \n",
       "68  55135  2021-03-27    WI    Waupaca County   \n",
       "69  55137  2021-03-27    WI   Waushara County   \n",
       "70  55139  2021-03-27    WI  Winnebago County   \n",
       "71  55141  2021-03-27    WI       Wood County   \n",
       "\n",
       "    percent_new_test_results_reported_positive_7_day_rolling_average  \n",
       "0                                                1.38                 \n",
       "1                                                0.00                 \n",
       "2                                                0.00                 \n",
       "3                                                2.72                 \n",
       "4                                                0.00                 \n",
       "..                                                ...                 \n",
       "67                                               5.42                 \n",
       "68                                               2.11                 \n",
       "69                                               0.29                 \n",
       "70                                               3.62                 \n",
       "71                                               3.56                 \n",
       "\n",
       "[1297 rows x 5 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NewCaseRate = NewCaseRate.rename(columns = {'fips_code': 'GEOID'}) # ['GEOID'] = NewCaseRate['fips_code'].apply(get_fips).apply(modifyGEOID)\n",
    "NewCaseRate.to_csv('DataMerge/1-NewCaseRate-2021-03-27.csv', index = False)\n",
    "NewCaseRate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A. Racial Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>racial_bias</th>\n",
       "      <th>racial_explicit_bias</th>\n",
       "      <th>racial_weighted_bias</th>\n",
       "      <th>racial_weighted_explicit_bias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00030</td>\n",
       "      <td>0.461000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01001</td>\n",
       "      <td>0.462460</td>\n",
       "      <td>1.160714</td>\n",
       "      <td>0.432357</td>\n",
       "      <td>1.070972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01003</td>\n",
       "      <td>0.440297</td>\n",
       "      <td>1.260521</td>\n",
       "      <td>0.432839</td>\n",
       "      <td>1.130820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01005</td>\n",
       "      <td>0.472971</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.429265</td>\n",
       "      <td>1.018059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01007</td>\n",
       "      <td>0.465729</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.425924</td>\n",
       "      <td>0.997510</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  racial_bias  racial_explicit_bias  racial_weighted_bias  \\\n",
       "0  00030     0.461000                   NaN                   NaN   \n",
       "1  01001     0.462460              1.160714              0.432357   \n",
       "2  01003     0.440297              1.260521              0.432839   \n",
       "3  01005     0.472971              0.976190              0.429265   \n",
       "4  01007     0.465729              0.791667              0.425924   \n",
       "\n",
       "   racial_weighted_explicit_bias  \n",
       "0                            NaN  \n",
       "1                       1.070972  \n",
       "2                       1.130820  \n",
       "3                       1.018059  \n",
       "4                       0.997510  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'DataMerge/Data/AB-Pysch/county_means_2016.csv'\n",
    "\n",
    "RaceBias = pd.read_csv(path)\n",
    "RaceBias['GEOID'] = RaceBias['county_id'].apply(get_fips).apply(modifyGEOID)\n",
    "RaceBias = RaceBias.sort_values(by = 'GEOID').reset_index(drop = True)\n",
    "\n",
    "cols = ['GEOID',  'bias', 'explicit_diff', 'weighted_bias', 'weighted_explicit_diff', ]\n",
    "RaceBias = RaceBias[cols]\n",
    "RaceBias.columns = ['GEOID', 'racial_bias', 'racial_explicit_bias', 'racial_weighted_bias', 'racial_weighted_explicit_bias', ]\n",
    "RaceBias.to_csv('DataMerge/A-RaceBias.csv', index = False)\n",
    "RaceBias.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# B. Sex Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>sex_bias</th>\n",
       "      <th>sex_explicit_bias</th>\n",
       "      <th>sex_weighted_bias</th>\n",
       "      <th>sex_weighted_explicit_bias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00030</td>\n",
       "      <td>-0.268537</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01001</td>\n",
       "      <td>0.347608</td>\n",
       "      <td>1.278571</td>\n",
       "      <td>0.397887</td>\n",
       "      <td>1.461195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01003</td>\n",
       "      <td>0.424211</td>\n",
       "      <td>2.267974</td>\n",
       "      <td>0.443720</td>\n",
       "      <td>2.185791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01005</td>\n",
       "      <td>0.474008</td>\n",
       "      <td>1.162162</td>\n",
       "      <td>0.445093</td>\n",
       "      <td>1.509029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01007</td>\n",
       "      <td>0.381588</td>\n",
       "      <td>2.055556</td>\n",
       "      <td>0.421544</td>\n",
       "      <td>2.029959</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  sex_bias  sex_explicit_bias  sex_weighted_bias  \\\n",
       "0  00030 -0.268537           0.000000                NaN   \n",
       "1  01001  0.347608           1.278571           0.397887   \n",
       "2  01003  0.424211           2.267974           0.443720   \n",
       "3  01005  0.474008           1.162162           0.445093   \n",
       "4  01007  0.381588           2.055556           0.421544   \n",
       "\n",
       "   sex_weighted_explicit_bias  \n",
       "0                         NaN  \n",
       "1                    1.461195  \n",
       "2                    2.185791  \n",
       "3                    1.509029  \n",
       "4                    2.029959  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "path = 'DataMerge/Data/AB-Pysch/county_means_sexuality_2016.csv'\n",
    "SexBias = pd.read_csv(path)\n",
    "\n",
    "SexBias['GEOID'] = SexBias['county_id'].apply(get_fips).apply(modifyGEOID)\n",
    "SexBias = SexBias.sort_values(by = 'GEOID').reset_index(drop = True)\n",
    "\n",
    "\n",
    "cols = ['GEOID',  'bias', 'explicit', 'weighted_bias', 'weighted_explicit', ]\n",
    "SexBias = SexBias[cols]\n",
    "\n",
    "SexBias.columns = ['GEOID',  'sex_bias', 'sex_explicit_bias', 'sex_weighted_bias', 'sex_weighted_explicit_bias', ]\n",
    "SexBias.to_csv('DataMerge/B-SexBias.csv', index = False)\n",
    "\n",
    "SexBias.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# C. Age (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>TOT_POP</th>\n",
       "      <th>WA</th>\n",
       "      <th>BA</th>\n",
       "      <th>NHWA</th>\n",
       "      <th>NHBA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>55869</td>\n",
       "      <td>42607</td>\n",
       "      <td>11237</td>\n",
       "      <td>41215</td>\n",
       "      <td>11098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>223234</td>\n",
       "      <td>195198</td>\n",
       "      <td>19593</td>\n",
       "      <td>185747</td>\n",
       "      <td>19215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>24686</td>\n",
       "      <td>12134</td>\n",
       "      <td>11906</td>\n",
       "      <td>11235</td>\n",
       "      <td>11807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>22394</td>\n",
       "      <td>17191</td>\n",
       "      <td>4763</td>\n",
       "      <td>16663</td>\n",
       "      <td>4719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>57826</td>\n",
       "      <td>55412</td>\n",
       "      <td>978</td>\n",
       "      <td>50176</td>\n",
       "      <td>872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3137</th>\n",
       "      <td>56037</td>\n",
       "      <td>42343</td>\n",
       "      <td>39698</td>\n",
       "      <td>598</td>\n",
       "      <td>33561</td>\n",
       "      <td>502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3138</th>\n",
       "      <td>56039</td>\n",
       "      <td>23464</td>\n",
       "      <td>22285</td>\n",
       "      <td>172</td>\n",
       "      <td>19000</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3139</th>\n",
       "      <td>56041</td>\n",
       "      <td>20226</td>\n",
       "      <td>19277</td>\n",
       "      <td>152</td>\n",
       "      <td>17657</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3140</th>\n",
       "      <td>56043</td>\n",
       "      <td>7805</td>\n",
       "      <td>7377</td>\n",
       "      <td>44</td>\n",
       "      <td>6417</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3141</th>\n",
       "      <td>56045</td>\n",
       "      <td>6927</td>\n",
       "      <td>6454</td>\n",
       "      <td>48</td>\n",
       "      <td>6236</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3142 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  TOT_POP      WA     BA    NHWA   NHBA\n",
       "0     01001    55869   42607  11237   41215  11098\n",
       "1     01003   223234  195198  19593  185747  19215\n",
       "2     01005    24686   12134  11906   11235  11807\n",
       "3     01007    22394   17191   4763   16663   4719\n",
       "4     01009    57826   55412    978   50176    872\n",
       "...     ...      ...     ...    ...     ...    ...\n",
       "3137  56037    42343   39698    598   33561    502\n",
       "3138  56039    23464   22285    172   19000    145\n",
       "3139  56041    20226   19277    152   17657    126\n",
       "3140  56043     7805    7377     44    6417     38\n",
       "3141  56045     6927    6454     48    6236     45\n",
       "\n",
       "[3142 rows x 6 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def get_county_id(v):\n",
    "    v = str(v)\n",
    "    if len(v) == 1:\n",
    "        return '00' + str(v)\n",
    "    elif len(v) == 2:\n",
    "        return '0' + str(v)\n",
    "    elif len(v) == 3:\n",
    "        return v\n",
    "    else:\n",
    "        raise ValueError('Error')\n",
    "    \n",
    "def get_state_id(v):\n",
    "    v = str(v)\n",
    "    if len(v) == 1:\n",
    "        return '0' + str(v)\n",
    "    elif len(v) == 2:\n",
    "        return v\n",
    "    else:\n",
    "        raise ValueError('Error')\n",
    "    \n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "def getGEOID(idx):\n",
    "    return idx.split('US')[-1]\n",
    "\n",
    "\n",
    "def change_col_name(name):\n",
    "    d = {}\n",
    "    \n",
    "    d['TOT_POP'] = 'WholeNum'\n",
    "    d['WA'] = 'WhiteNum'\n",
    "    d['BA'] = 'BlackNum'\n",
    "    d['NHWA'] = 'NHWhiteNum'\n",
    "    d['NHBA'] = 'NHBlackNum'\n",
    "    \n",
    "    for k, v in d.items():\n",
    "        if k in name:\n",
    "            name = name.replace(k, v)\n",
    "    \n",
    "    return name\n",
    "\n",
    "def get_Age_Race_County(path_cc, ageidx = 0):\n",
    "    \n",
    "\n",
    "    CCALLDATA = pd.read_csv(path_cc)\n",
    "\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['YEAR'] == 12]  # 12: 2019-7-1\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['AGEGRP'] == ageidx] #  0: Age Group Total\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['SUMLEV'] == 50] #  50: County and/or Statistical Equivalent\n",
    "    CCALLDATA = CCALLDATA.reset_index(drop = True)\n",
    "\n",
    "\n",
    "\n",
    "    CCALLDATA['GEOID'] = CCALLDATA['STATE'].apply(get_state_id) + CCALLDATA['COUNTY'].apply(get_county_id)\n",
    "    # POPULATION\n",
    "\n",
    "    PrefixCols = ['GEOID', 'TOT_POP'] \n",
    "    RaceCols = ['WA', 'BA', 'NHWA', 'NHBA']\n",
    "\n",
    "    for i in RaceCols:\n",
    "\n",
    "        CCALLDATA[i] = CCALLDATA[i + '_MALE'] + CCALLDATA[i +'_FEMALE']\n",
    "\n",
    "\n",
    "\n",
    "    CCALLDATA = CCALLDATA[PrefixCols + RaceCols]\n",
    "    \n",
    "    # return US_COUNTY_POPULATION\n",
    "    return CCALLDATA\n",
    "\n",
    "\n",
    "\n",
    "path_cc = 'DataMerge/Data/C-AnnualCountyPopulation/cc-est2019-alldata.csv'\n",
    "\n",
    "df0 = get_Age_Race_County(path_cc, ageidx = 0)\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 (3142, 6)\n",
      "1 (3142, 6)\n",
      "2 (3142, 6)\n",
      "3 (3142, 6)\n",
      "4 (3142, 6)\n",
      "5 (3142, 6)\n",
      "6 (3142, 6)\n",
      "7 (3142, 6)\n",
      "8 (3142, 6)\n",
      "9 (3142, 6)\n",
      "10 (3142, 6)\n",
      "11 (3142, 6)\n",
      "12 (3142, 6)\n",
      "13 (3142, 6)\n",
      "14 (3142, 6)\n",
      "15 (3142, 6)\n",
      "16 (3142, 6)\n",
      "17 (3142, 6)\n",
      "18 (3142, 6)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['GEOID', 'TOT_POP', 'WA', 'BA', 'NHWA', 'NHBA', 'TOT_POP_1', 'WA_1',\n",
       "       'BA_1', 'NHWA_1', 'NHBA_1', 'TOT_POP_2', 'WA_2', 'BA_2', 'NHWA_2',\n",
       "       'NHBA_2', 'TOT_POP_3', 'WA_3', 'BA_3', 'NHWA_3', 'NHBA_3', 'TOT_POP_4',\n",
       "       'WA_4', 'BA_4', 'NHWA_4', 'NHBA_4', 'TOT_POP_5', 'WA_5', 'BA_5',\n",
       "       'NHWA_5', 'NHBA_5', 'TOT_POP_6', 'WA_6', 'BA_6', 'NHWA_6', 'NHBA_6',\n",
       "       'TOT_POP_7', 'WA_7', 'BA_7', 'NHWA_7', 'NHBA_7', 'TOT_POP_8', 'WA_8',\n",
       "       'BA_8', 'NHWA_8', 'NHBA_8', 'TOT_POP_9', 'WA_9', 'BA_9', 'NHWA_9',\n",
       "       'NHBA_9', 'TOT_POP_10', 'WA_10', 'BA_10', 'NHWA_10', 'NHBA_10',\n",
       "       'TOT_POP_11', 'WA_11', 'BA_11', 'NHWA_11', 'NHBA_11', 'TOT_POP_12',\n",
       "       'WA_12', 'BA_12', 'NHWA_12', 'NHBA_12', 'TOT_POP_13', 'WA_13', 'BA_13',\n",
       "       'NHWA_13', 'NHBA_13', 'TOT_POP_14', 'WA_14', 'BA_14', 'NHWA_14',\n",
       "       'NHBA_14', 'TOT_POP_15', 'WA_15', 'BA_15', 'NHWA_15', 'NHBA_15',\n",
       "       'TOT_POP_16', 'WA_16', 'BA_16', 'NHWA_16', 'NHBA_16', 'TOT_POP_17',\n",
       "       'WA_17', 'BA_17', 'NHWA_17', 'NHBA_17', 'TOT_POP_18', 'WA_18', 'BA_18',\n",
       "       'NHWA_18', 'NHBA_18'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "print(0, df0.shape)\n",
    "for ageidx in list(range(1,19)):\n",
    "    \n",
    "    df = get_Age_Race_County(path_cc, ageidx = ageidx)\n",
    "    new_names = dict(zip(['TOT_POP', 'WA', 'BA', 'NHWA', 'NHBA'], [i+'_' + str(ageidx) for i in ['TOT_POP', 'WA', 'BA',  'NHWA', 'NHBA']]))\n",
    "    df = df.rename(columns = new_names)\n",
    "    print(ageidx, df.shape)\n",
    "    df0 = pd.merge(df0, df, on ='GEOID')\n",
    "    \n",
    "    \n",
    "df0.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(range(1, 19))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Above75_WholeNum</th>\n",
       "      <th>Above75_WhiteNum</th>\n",
       "      <th>Above75_NHWhiteNum</th>\n",
       "      <th>Above75_BlackNum</th>\n",
       "      <th>Above75_NHBlackNum</th>\n",
       "      <th>Above75_WholeRate</th>\n",
       "      <th>Above75_WhiteRate</th>\n",
       "      <th>Above75_NHWhiteRate</th>\n",
       "      <th>Above75_BlackRate</th>\n",
       "      <th>...</th>\n",
       "      <th>A15T74_WholeNum</th>\n",
       "      <th>A15T74_WhiteNum</th>\n",
       "      <th>A15T74_NHWhiteNum</th>\n",
       "      <th>A15T74_BlackNum</th>\n",
       "      <th>A15T74_NHBlackNum</th>\n",
       "      <th>A15T74_WholeRate</th>\n",
       "      <th>A15T74_WhiteRate</th>\n",
       "      <th>A15T74_NHWhiteRate</th>\n",
       "      <th>A15T74_BlackRate</th>\n",
       "      <th>A15T74_NHBlackRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>3911</td>\n",
       "      <td>3358</td>\n",
       "      <td>3303</td>\n",
       "      <td>471</td>\n",
       "      <td>468</td>\n",
       "      <td>7.000304</td>\n",
       "      <td>7.881334</td>\n",
       "      <td>8.014073</td>\n",
       "      <td>4.191510</td>\n",
       "      <td>...</td>\n",
       "      <td>41365</td>\n",
       "      <td>31614</td>\n",
       "      <td>30669</td>\n",
       "      <td>8467</td>\n",
       "      <td>8365</td>\n",
       "      <td>74.039270</td>\n",
       "      <td>74.199075</td>\n",
       "      <td>74.412229</td>\n",
       "      <td>75.349293</td>\n",
       "      <td>75.373941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>18786</td>\n",
       "      <td>17643</td>\n",
       "      <td>17453</td>\n",
       "      <td>840</td>\n",
       "      <td>832</td>\n",
       "      <td>8.415385</td>\n",
       "      <td>9.038515</td>\n",
       "      <td>9.396114</td>\n",
       "      <td>4.287245</td>\n",
       "      <td>...</td>\n",
       "      <td>165264</td>\n",
       "      <td>144920</td>\n",
       "      <td>138638</td>\n",
       "      <td>14716</td>\n",
       "      <td>14437</td>\n",
       "      <td>74.031734</td>\n",
       "      <td>74.242564</td>\n",
       "      <td>74.638083</td>\n",
       "      <td>75.108457</td>\n",
       "      <td>75.134010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>1995</td>\n",
       "      <td>1327</td>\n",
       "      <td>1313</td>\n",
       "      <td>635</td>\n",
       "      <td>631</td>\n",
       "      <td>8.081504</td>\n",
       "      <td>10.936212</td>\n",
       "      <td>11.686693</td>\n",
       "      <td>5.333445</td>\n",
       "      <td>...</td>\n",
       "      <td>18498</td>\n",
       "      <td>9035</td>\n",
       "      <td>8494</td>\n",
       "      <td>9031</td>\n",
       "      <td>8967</td>\n",
       "      <td>74.933160</td>\n",
       "      <td>74.460194</td>\n",
       "      <td>75.603026</td>\n",
       "      <td>75.852511</td>\n",
       "      <td>75.946472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>1547</td>\n",
       "      <td>1344</td>\n",
       "      <td>1330</td>\n",
       "      <td>178</td>\n",
       "      <td>176</td>\n",
       "      <td>6.908100</td>\n",
       "      <td>7.818044</td>\n",
       "      <td>7.981756</td>\n",
       "      <td>3.737140</td>\n",
       "      <td>...</td>\n",
       "      <td>17134</td>\n",
       "      <td>12988</td>\n",
       "      <td>12581</td>\n",
       "      <td>3885</td>\n",
       "      <td>3859</td>\n",
       "      <td>76.511566</td>\n",
       "      <td>75.551160</td>\n",
       "      <td>75.502611</td>\n",
       "      <td>81.566240</td>\n",
       "      <td>81.775800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>4465</td>\n",
       "      <td>4334</td>\n",
       "      <td>4231</td>\n",
       "      <td>48</td>\n",
       "      <td>47</td>\n",
       "      <td>7.721440</td>\n",
       "      <td>7.821411</td>\n",
       "      <td>8.432318</td>\n",
       "      <td>4.907975</td>\n",
       "      <td>...</td>\n",
       "      <td>42426</td>\n",
       "      <td>40804</td>\n",
       "      <td>37339</td>\n",
       "      <td>713</td>\n",
       "      <td>637</td>\n",
       "      <td>73.368381</td>\n",
       "      <td>73.637479</td>\n",
       "      <td>74.416055</td>\n",
       "      <td>72.903885</td>\n",
       "      <td>73.050459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3137</th>\n",
       "      <td>56037</td>\n",
       "      <td>1875</td>\n",
       "      <td>1804</td>\n",
       "      <td>1656</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>4.428123</td>\n",
       "      <td>4.544310</td>\n",
       "      <td>4.934299</td>\n",
       "      <td>3.511706</td>\n",
       "      <td>...</td>\n",
       "      <td>31322</td>\n",
       "      <td>29486</td>\n",
       "      <td>25257</td>\n",
       "      <td>443</td>\n",
       "      <td>384</td>\n",
       "      <td>73.972085</td>\n",
       "      <td>74.275782</td>\n",
       "      <td>75.256995</td>\n",
       "      <td>74.080268</td>\n",
       "      <td>76.494024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3138</th>\n",
       "      <td>56039</td>\n",
       "      <td>1356</td>\n",
       "      <td>1338</td>\n",
       "      <td>1306</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.779066</td>\n",
       "      <td>6.004039</td>\n",
       "      <td>6.873684</td>\n",
       "      <td>0.581395</td>\n",
       "      <td>...</td>\n",
       "      <td>18578</td>\n",
       "      <td>17627</td>\n",
       "      <td>15344</td>\n",
       "      <td>158</td>\n",
       "      <td>138</td>\n",
       "      <td>79.176611</td>\n",
       "      <td>79.098048</td>\n",
       "      <td>80.757895</td>\n",
       "      <td>91.860465</td>\n",
       "      <td>95.172414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3139</th>\n",
       "      <td>56041</td>\n",
       "      <td>1023</td>\n",
       "      <td>998</td>\n",
       "      <td>964</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5.057846</td>\n",
       "      <td>5.177154</td>\n",
       "      <td>5.459591</td>\n",
       "      <td>1.315789</td>\n",
       "      <td>...</td>\n",
       "      <td>14464</td>\n",
       "      <td>13848</td>\n",
       "      <td>12780</td>\n",
       "      <td>100</td>\n",
       "      <td>87</td>\n",
       "      <td>71.511915</td>\n",
       "      <td>71.836904</td>\n",
       "      <td>72.379226</td>\n",
       "      <td>65.789474</td>\n",
       "      <td>69.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3140</th>\n",
       "      <td>56043</td>\n",
       "      <td>782</td>\n",
       "      <td>763</td>\n",
       "      <td>712</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>10.019218</td>\n",
       "      <td>10.342958</td>\n",
       "      <td>11.095528</td>\n",
       "      <td>4.545455</td>\n",
       "      <td>...</td>\n",
       "      <td>5625</td>\n",
       "      <td>5323</td>\n",
       "      <td>4666</td>\n",
       "      <td>33</td>\n",
       "      <td>32</td>\n",
       "      <td>72.069186</td>\n",
       "      <td>72.156703</td>\n",
       "      <td>72.713106</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>84.210526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3141</th>\n",
       "      <td>56045</td>\n",
       "      <td>627</td>\n",
       "      <td>615</td>\n",
       "      <td>608</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.051537</td>\n",
       "      <td>9.528974</td>\n",
       "      <td>9.749840</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5142</td>\n",
       "      <td>4783</td>\n",
       "      <td>4614</td>\n",
       "      <td>46</td>\n",
       "      <td>43</td>\n",
       "      <td>74.231269</td>\n",
       "      <td>74.109080</td>\n",
       "      <td>73.989737</td>\n",
       "      <td>95.833333</td>\n",
       "      <td>95.555556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3142 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  Above75_WholeNum  Above75_WhiteNum  Above75_NHWhiteNum  \\\n",
       "0     01001              3911              3358                3303   \n",
       "1     01003             18786             17643               17453   \n",
       "2     01005              1995              1327                1313   \n",
       "3     01007              1547              1344                1330   \n",
       "4     01009              4465              4334                4231   \n",
       "...     ...               ...               ...                 ...   \n",
       "3137  56037              1875              1804                1656   \n",
       "3138  56039              1356              1338                1306   \n",
       "3139  56041              1023               998                 964   \n",
       "3140  56043               782               763                 712   \n",
       "3141  56045               627               615                 608   \n",
       "\n",
       "      Above75_BlackNum  Above75_NHBlackNum  Above75_WholeRate  \\\n",
       "0                  471                 468           7.000304   \n",
       "1                  840                 832           8.415385   \n",
       "2                  635                 631           8.081504   \n",
       "3                  178                 176           6.908100   \n",
       "4                   48                  47           7.721440   \n",
       "...                ...                 ...                ...   \n",
       "3137                21                  21           4.428123   \n",
       "3138                 1                   1           5.779066   \n",
       "3139                 2                   2           5.057846   \n",
       "3140                 2                   2          10.019218   \n",
       "3141                 0                   0           9.051537   \n",
       "\n",
       "      Above75_WhiteRate  Above75_NHWhiteRate  Above75_BlackRate  ...  \\\n",
       "0              7.881334             8.014073           4.191510  ...   \n",
       "1              9.038515             9.396114           4.287245  ...   \n",
       "2             10.936212            11.686693           5.333445  ...   \n",
       "3              7.818044             7.981756           3.737140  ...   \n",
       "4              7.821411             8.432318           4.907975  ...   \n",
       "...                 ...                  ...                ...  ...   \n",
       "3137           4.544310             4.934299           3.511706  ...   \n",
       "3138           6.004039             6.873684           0.581395  ...   \n",
       "3139           5.177154             5.459591           1.315789  ...   \n",
       "3140          10.342958            11.095528           4.545455  ...   \n",
       "3141           9.528974             9.749840           0.000000  ...   \n",
       "\n",
       "      A15T74_WholeNum  A15T74_WhiteNum  A15T74_NHWhiteNum  A15T74_BlackNum  \\\n",
       "0               41365            31614              30669             8467   \n",
       "1              165264           144920             138638            14716   \n",
       "2               18498             9035               8494             9031   \n",
       "3               17134            12988              12581             3885   \n",
       "4               42426            40804              37339              713   \n",
       "...               ...              ...                ...              ...   \n",
       "3137            31322            29486              25257              443   \n",
       "3138            18578            17627              15344              158   \n",
       "3139            14464            13848              12780              100   \n",
       "3140             5625             5323               4666               33   \n",
       "3141             5142             4783               4614               46   \n",
       "\n",
       "      A15T74_NHBlackNum  A15T74_WholeRate  A15T74_WhiteRate  \\\n",
       "0                  8365         74.039270         74.199075   \n",
       "1                 14437         74.031734         74.242564   \n",
       "2                  8967         74.933160         74.460194   \n",
       "3                  3859         76.511566         75.551160   \n",
       "4                   637         73.368381         73.637479   \n",
       "...                 ...               ...               ...   \n",
       "3137                384         73.972085         74.275782   \n",
       "3138                138         79.176611         79.098048   \n",
       "3139                 87         71.511915         71.836904   \n",
       "3140                 32         72.069186         72.156703   \n",
       "3141                 43         74.231269         74.109080   \n",
       "\n",
       "      A15T74_NHWhiteRate  A15T74_BlackRate  A15T74_NHBlackRate  \n",
       "0              74.412229         75.349293           75.373941  \n",
       "1              74.638083         75.108457           75.134010  \n",
       "2              75.603026         75.852511           75.946472  \n",
       "3              75.502611         81.566240           81.775800  \n",
       "4              74.416055         72.903885           73.050459  \n",
       "...                  ...               ...                 ...  \n",
       "3137           75.256995         74.080268           76.494024  \n",
       "3138           80.757895         91.860465           95.172414  \n",
       "3139           72.379226         65.789474           69.047619  \n",
       "3140           72.713106         75.000000           84.210526  \n",
       "3141           73.989737         95.833333           95.555556  \n",
       "\n",
       "[3142 rows x 61 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "age_groups = {\n",
    "    'Above75': [16, 17, 18],\n",
    "    'Above65': [14, 15, 16, 17, 18],\n",
    "    # 'Below14': [1, 2, 3],\n",
    "    # 'Below19': [1, 2, 3, 4],\n",
    "    # 'Below24': [1, 2, 3, 4, 5],\n",
    "    # 'Below29': [1, 2, 3, 4, 5, 6],\n",
    "    # 'Below35': [1, 2, 3, 4, 5, 6, 7],\n",
    "    'Above15': [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], \n",
    "    'Above20': [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18],\n",
    "    'A20T74': [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ],\n",
    "    'A15T74': [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ], \n",
    "}\n",
    "\n",
    "\n",
    "# def get_age_list_rate(age_idx_list):\n",
    "\n",
    "for name, age_idx_list in age_groups.items():\n",
    "    range_dict = {}\n",
    "    range_dict['WA'] = ['WA_' + str(ageidx) for ageidx in age_idx_list] # Total White People in the corresponding age period\n",
    "    range_dict['BA'] = ['BA_' + str(ageidx) for ageidx in age_idx_list] # Total Black People in the corresponding age period\n",
    "    range_dict['NHWA'] = ['NHWA_' + str(ageidx) for ageidx in age_idx_list] # Total White People in the corresponding age period\n",
    "    range_dict['NHBA'] = ['NHBA_' + str(ageidx) for ageidx in age_idx_list] # Total Black People in the corresponding age period\n",
    "    \n",
    "    range_dict['TOT_POP'] = ['TOT_POP_' + str(ageidx) for ageidx in age_idx_list] # # Total Popluation in the corresponding age period\n",
    "    \n",
    "    for item, item_list in range_dict.items():\n",
    "        df0[name + '_' + item] = df0[item_list].sum(axis = 1)\n",
    "        # print(name + '_' + item)\n",
    "    \n",
    "    \n",
    "    # df0[name + '_WhiteRate'] = df0[name + '_WA'] / df0['WA']  \n",
    "    # df0[name + '_BlackRate'] = df0[name + '_BA'] / df0['BA']\n",
    "    # df0[name + '_WholeRate'] = df0[name + '_TOT_POP'] / df0['TOT_POP']\n",
    "\n",
    "    # df0[name + '_Diff']      = df0[name + '_WhiteRate'] - df0[name + '_BlackRate']\n",
    "    # df0[name + '_Avg']       = (df0[name + '_WA']   + df0[name + '_BA']) / (df0['WA'] + df0['BA'])\n",
    "    # df0[name + '_Disparity'] = df0[name + '_Diff'] / df0[name + '_Avg']\n",
    "    \n",
    "    \n",
    "    # Which WhiteNumToUse\n",
    "    df0[name + '_WholeNum'] = df0[name + '_TOT_POP']\n",
    "    \n",
    "    df0[name + '_WhiteNum'] = df0[name + '_WA']\n",
    "    df0[name + '_NHWhiteNum'] = df0[name + '_NHWA']\n",
    "    \n",
    "    df0[name + '_BlackNum'] = df0[name + '_BA']\n",
    "    df0[name + '_NHBlackNum'] = df0[name + '_NHBA']\n",
    "    \n",
    "    \n",
    "    df0[name + '_WholeRate'] = df0[name + '_TOT_POP'] / df0['TOT_POP'] * 100\n",
    "    df0[name + '_WhiteRate'] = df0[name + '_WA'] / df0['WA'] * 100\n",
    "    df0[name + '_NHWhiteRate'] = df0[name + '_NHWA'] / df0['NHWA']  * 100\n",
    "    df0[name + '_BlackRate'] = df0[name + '_BA'] / df0['BA'] * 100\n",
    "    df0[name + '_NHBlackRate'] = df0[name + '_NHBA'] / df0['NHBA'] * 100\n",
    "    \n",
    "\n",
    "cols = ['GEOID'] +  [i for i in df0.columns if 'Rate' in i or 'Num' in i]\n",
    "AgeDisparity = df0[cols]\n",
    "AgeDisparity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['GEOID', 'Above75_WholeNum', 'Above75_WhiteNum', 'Above75_NHWhiteNum',\n",
       "       'Above75_BlackNum', 'Above75_NHBlackNum', 'Above75_WholeRate',\n",
       "       'Above75_WhiteRate', 'Above75_NHWhiteRate', 'Above75_BlackRate',\n",
       "       'Above75_NHBlackRate', 'Above65_WholeNum', 'Above65_WhiteNum',\n",
       "       'Above65_NHWhiteNum', 'Above65_BlackNum', 'Above65_NHBlackNum',\n",
       "       'Above65_WholeRate', 'Above65_WhiteRate', 'Above65_NHWhiteRate',\n",
       "       'Above65_BlackRate', 'Above65_NHBlackRate', 'Above15_WholeNum',\n",
       "       'Above15_WhiteNum', 'Above15_NHWhiteNum', 'Above15_BlackNum',\n",
       "       'Above15_NHBlackNum', 'Above15_WholeRate', 'Above15_WhiteRate',\n",
       "       'Above15_NHWhiteRate', 'Above15_BlackRate', 'Above15_NHBlackRate',\n",
       "       'Above20_WholeNum', 'Above20_WhiteNum', 'Above20_NHWhiteNum',\n",
       "       'Above20_BlackNum', 'Above20_NHBlackNum', 'Above20_WholeRate',\n",
       "       'Above20_WhiteRate', 'Above20_NHWhiteRate', 'Above20_BlackRate',\n",
       "       'Above20_NHBlackRate', 'A20T74_WholeNum', 'A20T74_WhiteNum',\n",
       "       'A20T74_NHWhiteNum', 'A20T74_BlackNum', 'A20T74_NHBlackNum',\n",
       "       'A20T74_WholeRate', 'A20T74_WhiteRate', 'A20T74_NHWhiteRate',\n",
       "       'A20T74_BlackRate', 'A20T74_NHBlackRate', 'A15T74_WholeNum',\n",
       "       'A15T74_WhiteNum', 'A15T74_NHWhiteNum', 'A15T74_BlackNum',\n",
       "       'A15T74_NHBlackNum', 'A15T74_WholeRate', 'A15T74_WhiteRate',\n",
       "       'A15T74_NHWhiteRate', 'A15T74_BlackRate', 'A15T74_NHBlackRate'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AgeDisparity.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "AgeDisparity.to_csv('DataMerge/C-AgeDisparity.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# D. Mean Income (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Per Capita Average Income\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "path = 'DataMerge/Data/DE-INCOME/MeanIncome/ACSST5Y2019.S1902_2021-03-28T123828/ACSST5Y2019.S1902_data_with_overlays_2021-03-28T123801.csv'\n",
    "DATA = pd.read_csv(path, skiprows=[0], low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3220, 86)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "cols = list(DATA.columns)\n",
    "cols = [i for i in cols if 'Margin of Error!' not in i]\n",
    "DATA = DATA[cols]\n",
    "\n",
    "print(DATA.shape)\n",
    "new_names = []\n",
    "\n",
    "for t in [i.split('!!') for i in list(DATA.columns)]:\n",
    "    if len(t) <= 2:\n",
    "        name = t[0]\n",
    "    else:\n",
    "        name = t[1] +'_' + t[-1]\n",
    "    new_names.append(name)\n",
    "    \n",
    "\n",
    "new_names_dict = dict(zip(list(DATA.columns), new_names))\n",
    "# [v for k, v in new_names_dict.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean income (dollars)_Total population\n",
      "Mean income (dollars)_White\n",
      "Mean income (dollars)_White alone, not Hispanic or Latino\n",
      "Mean income (dollars)_Black or African American\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>MeanInc_Whole</th>\n",
       "      <th>MeanInc_White</th>\n",
       "      <th>MeanInc_NHWhite</th>\n",
       "      <th>MeanInc_Black</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>29819.0</td>\n",
       "      <td>33021.0</td>\n",
       "      <td>33009.0</td>\n",
       "      <td>19610.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>32626.0</td>\n",
       "      <td>34639.0</td>\n",
       "      <td>35159.0</td>\n",
       "      <td>18565.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>18473.0</td>\n",
       "      <td>25581.0</td>\n",
       "      <td>25916.0</td>\n",
       "      <td>12369.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>20778.0</td>\n",
       "      <td>23509.0</td>\n",
       "      <td>23610.0</td>\n",
       "      <td>10938.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>24747.0</td>\n",
       "      <td>25176.0</td>\n",
       "      <td>26575.0</td>\n",
       "      <td>15911.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  MeanInc_Whole  MeanInc_White  MeanInc_NHWhite  MeanInc_Black\n",
       "0  01001        29819.0        33021.0          33009.0        19610.0\n",
       "1  01003        32626.0        34639.0          35159.0        18565.0\n",
       "2  01005        18473.0        25581.0          25916.0        12369.0\n",
       "3  01007        20778.0        23509.0          23610.0        10938.0\n",
       "4  01009        24747.0        25176.0          26575.0        15911.0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "\n",
    "DATA = DATA.rename(columns = new_names_dict)\n",
    "\n",
    "DATA['Geographic Area Name'] = DATA['Geographic Area Name'].apply(remove_string)\n",
    "DATA['County'] = DATA['Geographic Area Name'].apply(get_county)\n",
    "DATA['State']  = DATA['Geographic Area Name'].apply(get_state)\n",
    "\n",
    "def getGEOID(idx):\n",
    "    return idx.split('US')[-1]\n",
    "\n",
    "cols = ['GEOID', 'Mean income (dollars)_Total population',\n",
    " 'Mean income (dollars)_White',\n",
    " 'Mean income (dollars)_White alone, not Hispanic or Latino',\n",
    " 'Mean income (dollars)_Black or African American',]\n",
    "\n",
    "DATA['GEOID'] = DATA['id'].apply(getGEOID)\n",
    "\n",
    "DATA = DATA[cols]\n",
    "DATA = DATA.replace(['-', 'N'], np.nan)\n",
    "for col in cols[1:]:\n",
    "    print(col)\n",
    "    DATA[col] = DATA[col].astype(float)\n",
    "\n",
    "MeanIncome = DATA\n",
    "MeanIncome.columns = ['GEOID', 'MeanInc_Whole', 'MeanInc_White', 'MeanInc_NHWhite',  'MeanInc_Black']\n",
    "\n",
    "# MeanIncome['MeanInc_Disparity'] = MeanIncome['MeanInc_White'] - MeanIncome['MeanInc_Black']\n",
    "MeanIncome.to_csv('DataMerge/D-MeanIncome.csv', index = False)\n",
    "MeanIncome.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# E. Median Income (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3220, 122)\n"
     ]
    }
   ],
   "source": [
    "# Househod Median Income\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "path = 'DataMerge/Data/DE-INCOME/MedianIncome/ACSST5Y2019.S1903_2021-03-28T121313/ACSST5Y2019.S1903_data_with_overlays_2021-03-28T121240.csv'\n",
    "DATA = pd.read_csv(path, skiprows=[0], low_memory=False)\n",
    "# How to understand total population in household?\n",
    "\n",
    "cols = list(DATA.columns)\n",
    "cols = [i for i in cols if 'Margin of Error!' not in i]\n",
    "DATA = DATA[cols]\n",
    "# [i.split('!!')[-1] for i in list(INTERNET_SUBSCRIPTIONS.columns)]\n",
    "\n",
    "print(DATA.shape)\n",
    "\n",
    "new_names = []\n",
    "\n",
    "for t in [i.split('!!') for i in list(DATA.columns)]:\n",
    "    # print(t)\n",
    "    \n",
    "    if len(t) <= 2:\n",
    "        name = t[0]\n",
    "    else:\n",
    "        name = t[1] +'_' + t[-1]\n",
    "    \n",
    "    # print(name)\n",
    "    \n",
    "    new_names.append(name)\n",
    "    \n",
    "new_names_dict = dict(zip(list(DATA.columns), new_names))\n",
    "\n",
    "# [v for k,v in new_names_dict.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median income (dollars)_Households\n",
      "Median income (dollars)_White\n",
      "Median income (dollars)_White alone, not Hispanic or Latino\n",
      "Median income (dollars)_Black or African American\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>MedianInc_Whole</th>\n",
       "      <th>MedianInc_White</th>\n",
       "      <th>MedianInc_NHWhite</th>\n",
       "      <th>MedianInc_Black</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>58731.0</td>\n",
       "      <td>66540.0</td>\n",
       "      <td>65992.0</td>\n",
       "      <td>28808.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>58320.0</td>\n",
       "      <td>61505.0</td>\n",
       "      <td>61872.0</td>\n",
       "      <td>36616.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>32525.0</td>\n",
       "      <td>46815.0</td>\n",
       "      <td>47175.0</td>\n",
       "      <td>22357.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>47542.0</td>\n",
       "      <td>52388.0</td>\n",
       "      <td>52543.0</td>\n",
       "      <td>21964.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>49358.0</td>\n",
       "      <td>49289.0</td>\n",
       "      <td>49529.0</td>\n",
       "      <td>77518.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  MedianInc_Whole  MedianInc_White  MedianInc_NHWhite  MedianInc_Black\n",
       "0  01001          58731.0          66540.0            65992.0          28808.0\n",
       "1  01003          58320.0          61505.0            61872.0          36616.0\n",
       "2  01005          32525.0          46815.0            47175.0          22357.0\n",
       "3  01007          47542.0          52388.0            52543.0          21964.0\n",
       "4  01009          49358.0          49289.0            49529.0          77518.0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# new_names_dict\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "\n",
    "\n",
    "DATA = DATA.rename(columns = new_names_dict)\n",
    "\n",
    "DATA['Geographic Area Name'] = DATA['Geographic Area Name'].apply(remove_string)\n",
    "DATA['County'] = DATA['Geographic Area Name'].apply(get_county)\n",
    "DATA['State']  = DATA['Geographic Area Name'].apply(get_state)\n",
    "\n",
    "def getGEOID(idx):\n",
    "    return idx.split('US')[-1]\n",
    "\n",
    "\n",
    "cols = ['GEOID', 'Median income (dollars)_Households',\n",
    " 'Median income (dollars)_White',\n",
    " 'Median income (dollars)_White alone, not Hispanic or Latino',\n",
    " 'Median income (dollars)_Black or African American',]\n",
    "\n",
    "\n",
    "DATA['GEOID'] = DATA['id'].apply(getGEOID)\n",
    "\n",
    "\n",
    "DATA = DATA[cols]\n",
    "DATA = DATA.replace(['-', 'N'], np.nan)\n",
    "DATA = DATA.replace(['2,500-'], 2500)\n",
    "DATA = DATA.replace(['250,000+'], 250000)\n",
    "\n",
    "for col in cols[1:]:\n",
    "    print(col)\n",
    "    DATA[col] = DATA[col].astype(float)\n",
    "    \n",
    "MedianIncome = DATA\n",
    "MedianIncome.columns = ['GEOID', 'MedianInc_Whole','MedianInc_White', 'MedianInc_NHWhite', 'MedianInc_Black']\n",
    "# MedianIncome['MedianInc_Disparity'] = MedianIncome['MedianInc_White'] - MedianIncome['MedianInc_Black']\n",
    "\n",
    "MedianIncome.to_csv('DataMerge/E-MedianIncome.csv', index = False)\n",
    "\n",
    "MedianIncome.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# F. Political Ideology"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>winner</th>\n",
       "      <th>votes_R_2020</th>\n",
       "      <th>votes_D_2020</th>\n",
       "      <th>votes_third_party_2020</th>\n",
       "      <th>votes_total_2020</th>\n",
       "      <th>republican</th>\n",
       "      <th>republican_Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>R</td>\n",
       "      <td>19838</td>\n",
       "      <td>7503</td>\n",
       "      <td>350</td>\n",
       "      <td>27691</td>\n",
       "      <td>1</td>\n",
       "      <td>0.716406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>R</td>\n",
       "      <td>83544</td>\n",
       "      <td>24578</td>\n",
       "      <td>1229</td>\n",
       "      <td>109351</td>\n",
       "      <td>1</td>\n",
       "      <td>0.763999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>R</td>\n",
       "      <td>5622</td>\n",
       "      <td>4816</td>\n",
       "      <td>68</td>\n",
       "      <td>10506</td>\n",
       "      <td>1</td>\n",
       "      <td>0.535123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>R</td>\n",
       "      <td>7525</td>\n",
       "      <td>1986</td>\n",
       "      <td>74</td>\n",
       "      <td>9585</td>\n",
       "      <td>1</td>\n",
       "      <td>0.785081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>R</td>\n",
       "      <td>24711</td>\n",
       "      <td>2640</td>\n",
       "      <td>209</td>\n",
       "      <td>27560</td>\n",
       "      <td>1</td>\n",
       "      <td>0.896626</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID winner  votes_R_2020  votes_D_2020  votes_third_party_2020  \\\n",
       "0  01001      R         19838          7503                     350   \n",
       "1  01003      R         83544         24578                    1229   \n",
       "2  01005      R          5622          4816                      68   \n",
       "3  01007      R          7525          1986                      74   \n",
       "4  01009      R         24711          2640                     209   \n",
       "\n",
       "   votes_total_2020  republican  republican_Rate  \n",
       "0             27691           1         0.716406  \n",
       "1            109351           1         0.763999  \n",
       "2             10506           1         0.535123  \n",
       "3              9585           1         0.785081  \n",
       "4             27560           1         0.896626  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path = 'DataMerge/Data/F-PoliticalIdeology/political_ideology.xlsx'\n",
    "\n",
    "def convertFIPS(v):\n",
    "    v = str(int(v))\n",
    "    if len(v) == 4:\n",
    "        v = '0' + v\n",
    "    else:\n",
    "        v = v\n",
    "    return v\n",
    "\n",
    "\n",
    "DATA = pd.read_excel(path)\n",
    "\n",
    "\n",
    "DATA['GEOID'] = DATA['FIPS'].apply(convertFIPS)\n",
    "\n",
    "cols = 'GEOID\twinner\tvotes_R_2020\tvotes_D_2020\tvotes_third_party_2020\tvotes_total_2020'.split('\\t')\n",
    "\n",
    "\n",
    "DATA = DATA[cols].sort_values(by ='GEOID').reset_index(drop = True)\n",
    "PoliticalIdeology = DATA\n",
    "PoliticalIdeology['republican'] = (PoliticalIdeology['winner'] == 'R').astype(int)\n",
    "PoliticalIdeology['republican_Rate'] = PoliticalIdeology['votes_R_2020'] / PoliticalIdeology['votes_total_2020']\n",
    "\n",
    "PoliticalIdeology.to_csv('DataMerge/F-PoliticalIdeology.csv', index = False)\n",
    "PoliticalIdeology.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>votes_R_2020</th>\n",
       "      <th>votes_D_2020</th>\n",
       "      <th>votes_third_party_2020</th>\n",
       "      <th>votes_total_2020</th>\n",
       "      <th>republican</th>\n",
       "      <th>republican_Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.112000e+03</td>\n",
       "      <td>3.112000e+03</td>\n",
       "      <td>3112.000000</td>\n",
       "      <td>3.112000e+03</td>\n",
       "      <td>3112.000000</td>\n",
       "      <td>3112.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.378962e+04</td>\n",
       "      <td>2.606944e+04</td>\n",
       "      <td>812.925450</td>\n",
       "      <td>5.067199e+04</td>\n",
       "      <td>0.631427</td>\n",
       "      <td>0.651396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.434651e+04</td>\n",
       "      <td>9.752098e+04</td>\n",
       "      <td>2668.739323</td>\n",
       "      <td>1.500756e+05</td>\n",
       "      <td>0.482495</td>\n",
       "      <td>0.161409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.600000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.054606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.680000e+03</td>\n",
       "      <td>1.299500e+03</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>5.342750e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.559612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.295500e+03</td>\n",
       "      <td>3.698000e+03</td>\n",
       "      <td>165.000000</td>\n",
       "      <td>1.251100e+04</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.684850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.079925e+04</td>\n",
       "      <td>1.217925e+04</td>\n",
       "      <td>547.500000</td>\n",
       "      <td>3.355700e+04</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.776368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.145530e+06</td>\n",
       "      <td>3.028885e+06</td>\n",
       "      <td>89028.000000</td>\n",
       "      <td>4.263443e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.961818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       votes_R_2020  votes_D_2020  votes_third_party_2020  votes_total_2020  \\\n",
       "count  3.112000e+03  3.112000e+03             3112.000000      3.112000e+03   \n",
       "mean   2.378962e+04  2.606944e+04              812.925450      5.067199e+04   \n",
       "std    5.434651e+04  9.752098e+04             2668.739323      1.500756e+05   \n",
       "min    6.000000e+01  4.000000e+00                0.000000      6.600000e+01   \n",
       "25%    3.680000e+03  1.299500e+03               61.000000      5.342750e+03   \n",
       "50%    8.295500e+03  3.698000e+03              165.000000      1.251100e+04   \n",
       "75%    2.079925e+04  1.217925e+04              547.500000      3.355700e+04   \n",
       "max    1.145530e+06  3.028885e+06            89028.000000      4.263443e+06   \n",
       "\n",
       "        republican  republican_Rate  \n",
       "count  3112.000000      3112.000000  \n",
       "mean      0.631427         0.651396  \n",
       "std       0.482495         0.161409  \n",
       "min       0.000000         0.054606  \n",
       "25%       0.000000         0.559612  \n",
       "50%       1.000000         0.684850  \n",
       "75%       1.000000         0.776368  \n",
       "max       1.000000         0.961818  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PoliticalIdeology.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# G. VaxHesitancy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3142, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>hesitancy</th>\n",
       "      <th>strong_hesitancy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  hesitancy  strong_hesitancy\n",
       "0  01001       0.22              0.10\n",
       "1  01003       0.20              0.10\n",
       "2  01005       0.23              0.11\n",
       "3  01007       0.24              0.12\n",
       "4  01009       0.23              0.11"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "path = 'DataMerge/Data/G-VACCINE HESITANCY/county_week26_data_fixed.xlsx'\n",
    "\n",
    "VaxHesitancy = pd.read_excel(path)\n",
    "\n",
    "def convertFIPS(v):\n",
    "    v = str(int(v))\n",
    "    if len(v) == 4:\n",
    "        v = '0' + v\n",
    "    else:\n",
    "        v = v\n",
    "    return v\n",
    "\n",
    "\n",
    "VaxHesitancy['GEOID'] = VaxHesitancy['FIPS Code'].apply(convertFIPS)\n",
    "VaxHesitancy = VaxHesitancy[['GEOID', 'Estimated hesitant', 'Estimated strongly hesitant']]\n",
    "VaxHesitancy.columns = ['GEOID', 'hesitancy', 'strong_hesitancy']\n",
    "VaxHesitancy = VaxHesitancy.sort_values(by = 'GEOID').reset_index(drop = True)\n",
    "\n",
    "print(VaxHesitancy.shape)\n",
    "VaxHesitancy.to_csv('DataMerge/G-VaxHesitancy.csv', index = False)\n",
    "VaxHesitancy.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# H. Covid-19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Confirmed</th>\n",
       "      <th>Deaths</th>\n",
       "      <th>Incident_Rate</th>\n",
       "      <th>Case_Fatality_Ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>6675</td>\n",
       "      <td>102</td>\n",
       "      <td>11947.591688</td>\n",
       "      <td>1.528090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>20573</td>\n",
       "      <td>301</td>\n",
       "      <td>9215.890053</td>\n",
       "      <td>1.463083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>2239</td>\n",
       "      <td>55</td>\n",
       "      <td>9069.918172</td>\n",
       "      <td>2.456454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>2557</td>\n",
       "      <td>58</td>\n",
       "      <td>11418.237028</td>\n",
       "      <td>2.268283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>6469</td>\n",
       "      <td>133</td>\n",
       "      <td>11187.009304</td>\n",
       "      <td>2.055959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>90053</td>\n",
       "      <td>1521</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.262985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>90054</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3263</th>\n",
       "      <td>90055</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3264</th>\n",
       "      <td>90056</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3265</th>\n",
       "      <td>99999</td>\n",
       "      <td>103</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.912621</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3266 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  Confirmed  Deaths  Incident_Rate  Case_Fatality_Ratio\n",
       "0     01001       6675     102   11947.591688             1.528090\n",
       "1     01003      20573     301    9215.890053             1.463083\n",
       "2     01005       2239      55    9069.918172             2.456454\n",
       "3     01007       2557      58   11418.237028             2.268283\n",
       "4     01009       6469     133   11187.009304             2.055959\n",
       "...     ...        ...     ...            ...                  ...\n",
       "3261  90053       1521       4            NaN             0.262985\n",
       "3262  90054          0       0            NaN                  NaN\n",
       "3263  90055          0       0            NaN                  NaN\n",
       "3264  90056          0       0            NaN                  NaN\n",
       "3265  99999        103       3            NaN             2.912621\n",
       "\n",
       "[3266 rows x 5 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#########\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-2021-04-07.csv'\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-05-20-2021.csv'\n",
    "path = 'DataMerge/Data/H-JHU-covid/jhu-04-07-2021.csv'\n",
    "#########\n",
    "\n",
    "\n",
    "COVID = pd.read_csv(path)\n",
    "\n",
    "COVID = COVID[- COVID['FIPS'].isna()]\n",
    "\n",
    "COVID['GEOID'] = COVID['FIPS'].apply(convertFIPS)\n",
    "COVID = COVID[['GEOID', 'Confirmed', 'Deaths', 'Incident_Rate', 'Case_Fatality_Ratio']].sort_values(by ='GEOID').reset_index(drop = True)\n",
    "# COVID.rename(columns = {'mortality': 'mortality_rate'})\n",
    "COVID.to_csv('DataMerge/H-COVID-04-07.csv', index = False)\n",
    "\n",
    "COVID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Confirmed</th>\n",
       "      <th>Deaths</th>\n",
       "      <th>Incident_Rate</th>\n",
       "      <th>Case_Fatality_Ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>6543</td>\n",
       "      <td>99</td>\n",
       "      <td>11711.324710</td>\n",
       "      <td>1.513067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>20423</td>\n",
       "      <td>301</td>\n",
       "      <td>9148.695987</td>\n",
       "      <td>1.473829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>2221</td>\n",
       "      <td>54</td>\n",
       "      <td>8997.002350</td>\n",
       "      <td>2.431337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>2535</td>\n",
       "      <td>58</td>\n",
       "      <td>11319.996430</td>\n",
       "      <td>2.287968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>6415</td>\n",
       "      <td>131</td>\n",
       "      <td>11093.625700</td>\n",
       "      <td>2.042089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>90053</td>\n",
       "      <td>1546</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.323415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>90054</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>90055</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>90056</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3263</th>\n",
       "      <td>99999</td>\n",
       "      <td>103</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.912621</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3264 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  Confirmed  Deaths  Incident_Rate  Case_Fatality_Ratio\n",
       "0     01001       6543      99   11711.324710             1.513067\n",
       "1     01003      20423     301    9148.695987             1.473829\n",
       "2     01005       2221      54    8997.002350             2.431337\n",
       "3     01007       2535      58   11319.996430             2.287968\n",
       "4     01009       6415     131   11093.625700             2.042089\n",
       "...     ...        ...     ...            ...                  ...\n",
       "3259  90053       1546       5            NaN             0.323415\n",
       "3260  90054          0       0            NaN                  NaN\n",
       "3261  90055          0       0            NaN                  NaN\n",
       "3262  90056          0       0            NaN                  NaN\n",
       "3263  99999        103       3            NaN             2.912621\n",
       "\n",
       "[3264 rows x 5 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#########\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-2021-04-07.csv'\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-05-20-2021.csv'\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-04-07-2021.csv'\n",
    "path = 'DataMerge/Data/H-JHU-covid/jhu-03-27-2021.csv'\n",
    "#########\n",
    "\n",
    "\n",
    "COVID = pd.read_csv(path)\n",
    "\n",
    "COVID = COVID[- COVID['FIPS'].isna()]\n",
    "\n",
    "COVID['GEOID'] = COVID['FIPS'].apply(convertFIPS)\n",
    "COVID = COVID[['GEOID', 'Confirmed', 'Deaths', 'Incident_Rate', 'Case_Fatality_Ratio']].sort_values(by ='GEOID').reset_index(drop = True)\n",
    "# COVID.rename(columns = {'mortality': 'mortality_rate'})\n",
    "COVID.to_csv('DataMerge/H-COVID-03-27.csv', index = False)\n",
    "\n",
    "COVID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Confirmed</th>\n",
       "      <th>Deaths</th>\n",
       "      <th>Incident_Rate</th>\n",
       "      <th>Case_Fatality_Ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>6763</td>\n",
       "      <td>106</td>\n",
       "      <td>12105.103009</td>\n",
       "      <td>1.567352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>20730</td>\n",
       "      <td>302</td>\n",
       "      <td>9286.219841</td>\n",
       "      <td>1.456826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>2271</td>\n",
       "      <td>55</td>\n",
       "      <td>9199.546302</td>\n",
       "      <td>2.421841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>2567</td>\n",
       "      <td>61</td>\n",
       "      <td>11462.891846</td>\n",
       "      <td>2.376315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>6532</td>\n",
       "      <td>133</td>\n",
       "      <td>11295.956836</td>\n",
       "      <td>2.036130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>90053</td>\n",
       "      <td>1516</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.263852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>90054</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3263</th>\n",
       "      <td>90055</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3264</th>\n",
       "      <td>90056</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3265</th>\n",
       "      <td>99999</td>\n",
       "      <td>103</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.912621</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3266 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  Confirmed  Deaths  Incident_Rate  Case_Fatality_Ratio\n",
       "0     01001       6763     106   12105.103009             1.567352\n",
       "1     01003      20730     302    9286.219841             1.456826\n",
       "2     01005       2271      55    9199.546302             2.421841\n",
       "3     01007       2567      61   11462.891846             2.376315\n",
       "4     01009       6532     133   11295.956836             2.036130\n",
       "...     ...        ...     ...            ...                  ...\n",
       "3261  90053       1516       4            NaN             0.263852\n",
       "3262  90054          0       0            NaN                  NaN\n",
       "3263  90055          0       0            NaN                  NaN\n",
       "3264  90056          0       0            NaN                  NaN\n",
       "3265  99999        103       3            NaN             2.912621\n",
       "\n",
       "[3266 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'DataMerge/Data/H-JHU-covid/jhu-04-19-2021.csv'\n",
    "\n",
    "# path = 'DataMerge/Data/H-JHU-covid/jhu-05-20-2021.csv'\n",
    "\n",
    "\n",
    "COVID = pd.read_csv(path)\n",
    "\n",
    "COVID = COVID[- COVID['FIPS'].isna()]\n",
    "\n",
    "COVID['GEOID'] = COVID['FIPS'].apply(convertFIPS)\n",
    "COVID = COVID[['GEOID', 'Confirmed', 'Deaths', 'Incident_Rate', 'Case_Fatality_Ratio']].sort_values(by ='GEOID').reset_index(drop = True)\n",
    "# COVID.rename(columns = {'mortality': 'mortality_rate'})\n",
    "COVID.to_csv('DataMerge/H-COVID-04-19.csv', index = False)\n",
    "\n",
    "COVID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Internet (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "path = 'DataMerge/Data/I-INTERNET/ACSST5Y2019.S2802_2021-03-20T111825/ACSST5Y2019.S2802_data_with_overlays_2021-03-20T111812.csv'\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS = pd.read_csv(path, skiprows=[0], low_memory=False)\n",
    "\n",
    "cols = list(INTERNET_SUBSCRIPTIONS.columns)\n",
    "cols = [i for i in cols if 'Margin of Error!' not in i]\n",
    "INTERNET_SUBSCRIPTIONS = INTERNET_SUBSCRIPTIONS[cols]\n",
    "\n",
    "\n",
    "new_names = []\n",
    "\n",
    "for t in [i.split('!!') for i in list(INTERNET_SUBSCRIPTIONS.columns)]:\n",
    "    # print(t)\n",
    "    \n",
    "    if len(t) <= 2:\n",
    "        name = t[0]\n",
    "    else:\n",
    "        name = t[1] +'_' + t[-1]\n",
    "    \n",
    "    new_names.append(name)\n",
    "    \n",
    "    \n",
    "new_names_dict = dict(zip(list(INTERNET_SUBSCRIPTIONS.columns), new_names))\n",
    "\n",
    "# [v for k, v in new_names_dict.items()]\n",
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "\n",
    "\n",
    "def getGEOID(idx):\n",
    "    return idx.split('US')[-1]\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS = INTERNET_SUBSCRIPTIONS.rename(columns = new_names_dict)\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS['Geographic Area Name'] = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(remove_string)\n",
    "INTERNET_SUBSCRIPTIONS['County'] = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(get_county)\n",
    "INTERNET_SUBSCRIPTIONS['State']  = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(get_state)\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS['GEOID'] = INTERNET_SUBSCRIPTIONS['id'].apply(getGEOID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [i for i in INTERNET_SUBSCRIPTIONS.columns if 'Broadband' in i ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols_dict = {\n",
    " 'GEOID': 'GEOID',\n",
    "\n",
    " 'Geographic Area Name': 'Geographic Area Name',\n",
    "\n",
    " #  Total Race\n",
    " # 'Total_Total population in households',\n",
    " # 'Total_White alone',\n",
    " # 'Total_Black or African American alone',\n",
    "\n",
    "\n",
    " # Education\n",
    " # 'Total_Household population 25 years and over',\n",
    " # 'Total_Less than high school graduate or equivalency':'HighSchool_WholeNum',\n",
    " # \"Total_High school graduate (includes equivalency), some college or associate's degree\": 'HighSchool_WholeNum',\n",
    " # \"Total_Bachelor's degree or higher\": 'Bachelor_WholeNum',\n",
    "    \n",
    "#  # Employment\n",
    " # 'Total_Civilian population 16 years and over',\n",
    " # 'Total_In labor force',\n",
    " # 'Total_Employed',\n",
    " # 'Total_Unemployed',\n",
    " # 'Total_Not in labor force',\n",
    "\n",
    "    \n",
    " # Race Rate with Internet\n",
    " 'Broadband Internet Subscription_Total population in households':'IT_WholeNum',\n",
    " 'Broadband Internet Subscription_White alone':'IT_WhiteNum',\n",
    " 'Broadband Internet Subscription_White alone, not Hispanic or Latino':'IT_NHWhiteNum',\n",
    " 'Broadband Internet Subscription_Black or African American alone':'IT_BlackNum',\n",
    " # 'Percent  Broadband Internet Subscription_American Indian and Alaska Native alone',\n",
    " #'Percent  Broadband Internet Subscription_Asian alone',\n",
    " \n",
    " 'Percent  Broadband Internet Subscription_Total population in households':'IT_WholeRate',\n",
    " 'Percent  Broadband Internet Subscription_White alone':'IT_WhiteRate',\n",
    " 'Percent  Broadband Internet Subscription_White alone, not Hispanic or Latino':'IT_NHWhiteRate',\n",
    " 'Percent  Broadband Internet Subscription_Black or African American alone':'IT_BlackRate',\n",
    "\n",
    "    \n",
    " # Race Rate without Internet\n",
    "#  'without an Internet Subscription_Total population in households':'CnI_WholeNum',\n",
    "#  'without an Internet Subscription_White alone':'CnI_WhiteNum',\n",
    "#  'without an Internet Subscription_White alone, not Hispanic or Latino':'CnI_NHWhiteNum',\n",
    "#  'without an Internet Subscription_Black or African American alone':'CnI_BlackNum',\n",
    " # 'without an Internet Subscription_American Indian and Alaska Native alone',\n",
    " # 'without an Internet Subscription_Asian alone',\n",
    "\n",
    "\n",
    " # Race Rate without Computer\n",
    "#  'no computer in household_Total population in households':'noC_WholeNum',\n",
    "#  'no computer in household_White alone':'noC_WhiteNum',\n",
    "#  'no computer in household_White alone, not Hispanic or Latino':'noC_NHWhiteNum',\n",
    "#  'no computer in household_Black or African American alone':'noC_BlackNum',\n",
    " # 'Percent no computer in household_American Indian and Alaska Native alone',\n",
    " # 'Percent no computer in household_Asian alone',\n",
    "}\n",
    "\n",
    "# should be empty\n",
    "[i for i in cols_dict if i not in INTERNET_SUBSCRIPTIONS.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3220, 10)\n",
      "Index(['GEOID', 'Geographic Area Name', 'IT_WholeNum', 'IT_WhiteNum',\n",
      "       'IT_NHWhiteNum', 'IT_BlackNum', 'IT_WholeRate', 'IT_WhiteRate',\n",
      "       'IT_NHWhiteRate', 'IT_BlackRate'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>IT_WholeNum</th>\n",
       "      <th>IT_WhiteNum</th>\n",
       "      <th>IT_NHWhiteNum</th>\n",
       "      <th>IT_BlackNum</th>\n",
       "      <th>IT_WholeRate</th>\n",
       "      <th>IT_WhiteRate</th>\n",
       "      <th>IT_NHWhiteRate</th>\n",
       "      <th>IT_BlackRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>Autauga, Alabama</td>\n",
       "      <td>46771</td>\n",
       "      <td>37224</td>\n",
       "      <td>36341</td>\n",
       "      <td>7410</td>\n",
       "      <td>85.3</td>\n",
       "      <td>88.4</td>\n",
       "      <td>88.8</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>Baldwin, Alabama</td>\n",
       "      <td>180022</td>\n",
       "      <td>157637</td>\n",
       "      <td>152004</td>\n",
       "      <td>14072</td>\n",
       "      <td>85.8</td>\n",
       "      <td>86.9</td>\n",
       "      <td>87.0</td>\n",
       "      <td>74.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>Barbour, Alabama</td>\n",
       "      <td>14663</td>\n",
       "      <td>7623</td>\n",
       "      <td>7587</td>\n",
       "      <td>6108</td>\n",
       "      <td>65.2</td>\n",
       "      <td>71.0</td>\n",
       "      <td>71.9</td>\n",
       "      <td>58.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>Bibb, Alabama</td>\n",
       "      <td>15689</td>\n",
       "      <td>13065</td>\n",
       "      <td>12875</td>\n",
       "      <td>2541</td>\n",
       "      <td>76.2</td>\n",
       "      <td>79.6</td>\n",
       "      <td>80.6</td>\n",
       "      <td>62.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>Blount, Alabama</td>\n",
       "      <td>45013</td>\n",
       "      <td>43201</td>\n",
       "      <td>39453</td>\n",
       "      <td>337</td>\n",
       "      <td>78.8</td>\n",
       "      <td>79.0</td>\n",
       "      <td>79.3</td>\n",
       "      <td>44.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3215</th>\n",
       "      <td>72145</td>\n",
       "      <td>Vega Baja Municipio, Puerto Rico</td>\n",
       "      <td>25590</td>\n",
       "      <td>20359</td>\n",
       "      <td>668</td>\n",
       "      <td>1691</td>\n",
       "      <td>49.3</td>\n",
       "      <td>47.1</td>\n",
       "      <td>43.2</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>72147</td>\n",
       "      <td>Vieques Municipio, Puerto Rico</td>\n",
       "      <td>6520</td>\n",
       "      <td>3361</td>\n",
       "      <td>409</td>\n",
       "      <td>438</td>\n",
       "      <td>75.4</td>\n",
       "      <td>71.1</td>\n",
       "      <td>93.4</td>\n",
       "      <td>67.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>72149</td>\n",
       "      <td>Villalba Municipio, Puerto Rico</td>\n",
       "      <td>16977</td>\n",
       "      <td>9378</td>\n",
       "      <td>26</td>\n",
       "      <td>540</td>\n",
       "      <td>76.2</td>\n",
       "      <td>74.2</td>\n",
       "      <td>74.3</td>\n",
       "      <td>70.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3218</th>\n",
       "      <td>72151</td>\n",
       "      <td>Yabucoa Municipio, Puerto Rico</td>\n",
       "      <td>21232</td>\n",
       "      <td>3063</td>\n",
       "      <td>0</td>\n",
       "      <td>17824</td>\n",
       "      <td>63.5</td>\n",
       "      <td>49.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3219</th>\n",
       "      <td>72153</td>\n",
       "      <td>Yauco Municipio, Puerto Rico</td>\n",
       "      <td>20451</td>\n",
       "      <td>15943</td>\n",
       "      <td>72</td>\n",
       "      <td>869</td>\n",
       "      <td>58.1</td>\n",
       "      <td>60.3</td>\n",
       "      <td>64.3</td>\n",
       "      <td>64.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3220 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID              Geographic Area Name  IT_WholeNum  IT_WhiteNum  \\\n",
       "0     01001                  Autauga, Alabama        46771        37224   \n",
       "1     01003                  Baldwin, Alabama       180022       157637   \n",
       "2     01005                  Barbour, Alabama        14663         7623   \n",
       "3     01007                     Bibb, Alabama        15689        13065   \n",
       "4     01009                   Blount, Alabama        45013        43201   \n",
       "...     ...                               ...          ...          ...   \n",
       "3215  72145  Vega Baja Municipio, Puerto Rico        25590        20359   \n",
       "3216  72147    Vieques Municipio, Puerto Rico         6520         3361   \n",
       "3217  72149   Villalba Municipio, Puerto Rico        16977         9378   \n",
       "3218  72151    Yabucoa Municipio, Puerto Rico        21232         3063   \n",
       "3219  72153      Yauco Municipio, Puerto Rico        20451        15943   \n",
       "\n",
       "      IT_NHWhiteNum  IT_BlackNum  IT_WholeRate  IT_WhiteRate  IT_NHWhiteRate  \\\n",
       "0             36341         7410          85.3          88.4            88.8   \n",
       "1            152004        14072          85.8          86.9            87.0   \n",
       "2              7587         6108          65.2          71.0            71.9   \n",
       "3             12875         2541          76.2          79.6            80.6   \n",
       "4             39453          337          78.8          79.0            79.3   \n",
       "...             ...          ...           ...           ...             ...   \n",
       "3215            668         1691          49.3          47.1            43.2   \n",
       "3216            409          438          75.4          71.1            93.4   \n",
       "3217             26          540          76.2          74.2            74.3   \n",
       "3218              0        17824          63.5          49.4             0.0   \n",
       "3219             72          869          58.1          60.3            64.3   \n",
       "\n",
       "      IT_BlackRate  \n",
       "0             71.0  \n",
       "1             74.3  \n",
       "2             58.6  \n",
       "3             62.9  \n",
       "4             44.5  \n",
       "...            ...  \n",
       "3215          66.0  \n",
       "3216          67.3  \n",
       "3217          70.1  \n",
       "3218          67.5  \n",
       "3219          64.9  \n",
       "\n",
       "[3220 rows x 10 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# cols_dict = {name: change_Demo_col_name(name) for name in cols}\n",
    "INTERNET_SUBSCRIPTIONS = INTERNET_SUBSCRIPTIONS[[i for i in cols_dict]].rename(columns = cols_dict)\n",
    "print(INTERNET_SUBSCRIPTIONS.shape)\n",
    "\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS = INTERNET_SUBSCRIPTIONS.replace('-', np.nan)\n",
    "for i in INTERNET_SUBSCRIPTIONS.columns:\n",
    "    if 'Rate' in i:\n",
    "        INTERNET_SUBSCRIPTIONS[i] = INTERNET_SUBSCRIPTIONS[i].astype(float)\n",
    "\n",
    "print(INTERNET_SUBSCRIPTIONS.columns)\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>IT_WholeNum</th>\n",
       "      <th>IT_WhiteNum</th>\n",
       "      <th>IT_NHWhiteNum</th>\n",
       "      <th>IT_BlackNum</th>\n",
       "      <th>IT_WholeRate</th>\n",
       "      <th>IT_WhiteRate</th>\n",
       "      <th>IT_NHWhiteRate</th>\n",
       "      <th>IT_BlackRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>Autauga, Alabama</td>\n",
       "      <td>46771</td>\n",
       "      <td>37224</td>\n",
       "      <td>36341</td>\n",
       "      <td>7410</td>\n",
       "      <td>85.3</td>\n",
       "      <td>88.4</td>\n",
       "      <td>88.8</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>Baldwin, Alabama</td>\n",
       "      <td>180022</td>\n",
       "      <td>157637</td>\n",
       "      <td>152004</td>\n",
       "      <td>14072</td>\n",
       "      <td>85.8</td>\n",
       "      <td>86.9</td>\n",
       "      <td>87.0</td>\n",
       "      <td>74.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>Barbour, Alabama</td>\n",
       "      <td>14663</td>\n",
       "      <td>7623</td>\n",
       "      <td>7587</td>\n",
       "      <td>6108</td>\n",
       "      <td>65.2</td>\n",
       "      <td>71.0</td>\n",
       "      <td>71.9</td>\n",
       "      <td>58.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>Bibb, Alabama</td>\n",
       "      <td>15689</td>\n",
       "      <td>13065</td>\n",
       "      <td>12875</td>\n",
       "      <td>2541</td>\n",
       "      <td>76.2</td>\n",
       "      <td>79.6</td>\n",
       "      <td>80.6</td>\n",
       "      <td>62.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>Blount, Alabama</td>\n",
       "      <td>45013</td>\n",
       "      <td>43201</td>\n",
       "      <td>39453</td>\n",
       "      <td>337</td>\n",
       "      <td>78.8</td>\n",
       "      <td>79.0</td>\n",
       "      <td>79.3</td>\n",
       "      <td>44.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID Geographic Area Name  IT_WholeNum  IT_WhiteNum  IT_NHWhiteNum  \\\n",
       "0  01001     Autauga, Alabama        46771        37224          36341   \n",
       "1  01003     Baldwin, Alabama       180022       157637         152004   \n",
       "2  01005     Barbour, Alabama        14663         7623           7587   \n",
       "3  01007        Bibb, Alabama        15689        13065          12875   \n",
       "4  01009      Blount, Alabama        45013        43201          39453   \n",
       "\n",
       "   IT_BlackNum  IT_WholeRate  IT_WhiteRate  IT_NHWhiteRate  IT_BlackRate  \n",
       "0         7410          85.3          88.4            88.8          71.0  \n",
       "1        14072          85.8          86.9            87.0          74.3  \n",
       "2         6108          65.2          71.0            71.9          58.6  \n",
       "3         2541          76.2          79.6            80.6          62.9  \n",
       "4          337          78.8          79.0            79.3          44.5  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS.to_csv('DataMerge/I-INTERNET_SUBSCRIPTIONS.csv', index = False)\n",
    "\n",
    "INTERNET_SUBSCRIPTIONS.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# J. Employment (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "path = 'DataMerge/Data/J-EMPLOYMENT/ACSST5Y2019.S2301_2021-03-20T122111/ACSST5Y2019.S2301_data_with_overlays_2021-03-20T122103.csv'\n",
    "EMPLOYMENT = pd.read_csv(path, skiprows=[0], low_memory=False)\n",
    "# How to understand total population in household?\n",
    "\n",
    "cols = list(EMPLOYMENT.columns)\n",
    "cols = [i for i in cols if 'Margin of Error!' not in i]\n",
    "EMPLOYMENT = EMPLOYMENT[cols]\n",
    "\n",
    "new_names = []\n",
    "\n",
    "\n",
    "for i in EMPLOYMENT.columns:\n",
    "\n",
    "    t = i.split('!!')\n",
    "    \n",
    "    if len(t) <= 2:\n",
    "        name = t[0]\n",
    "    else:\n",
    "        name = t[1] +'_' + t[-1]\n",
    "    \n",
    "    new_names.append(name)\n",
    "    \n",
    "    \n",
    "new_names_dict = dict(zip(list(EMPLOYMENT.columns), new_names))\n",
    "# new_names_dict\n",
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "\n",
    "\n",
    "EMPLOYMENT = EMPLOYMENT.rename(columns = new_names_dict)\n",
    "\n",
    "EMPLOYMENT['Geographic Area Name'] = EMPLOYMENT['Geographic Area Name'].apply(remove_string)\n",
    "EMPLOYMENT['County'] = EMPLOYMENT['Geographic Area Name'].apply(get_county)\n",
    "EMPLOYMENT['State']  = EMPLOYMENT['Geographic Area Name'].apply(get_state)\n",
    "# EMPLOYMENT['GEOID'] = EMPLOYMENT['id'].apply(getGEOID)\n",
    "# EMPLOYMENT.to_csv('DataMerge/J-EMPLOYMENT.csv', index = False)\n",
    "\n",
    "# EMPLOYMENT.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(EMPLOYMENT.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3220, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>Unempl_WholeRate</th>\n",
       "      <th>Unempl_WhiteRate</th>\n",
       "      <th>Unempl_NHWhiteRate</th>\n",
       "      <th>Unempl_BlackRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Autauga, Alabama</td>\n",
       "      <td>3.7</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>5.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Baldwin, Alabama</td>\n",
       "      <td>4.3</td>\n",
       "      <td>3.8</td>\n",
       "      <td>3.9</td>\n",
       "      <td>8.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Barbour, Alabama</td>\n",
       "      <td>9.2</td>\n",
       "      <td>4.7</td>\n",
       "      <td>4.7</td>\n",
       "      <td>14.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bibb, Alabama</td>\n",
       "      <td>7.3</td>\n",
       "      <td>6.8</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Blount, Alabama</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3215</th>\n",
       "      <td>Vega Baja Municipio, Puerto Rico</td>\n",
       "      <td>17.4</td>\n",
       "      <td>16.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>Vieques Municipio, Puerto Rico</td>\n",
       "      <td>7.5</td>\n",
       "      <td>4.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>Villalba Municipio, Puerto Rico</td>\n",
       "      <td>21.9</td>\n",
       "      <td>21.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3218</th>\n",
       "      <td>Yabucoa Municipio, Puerto Rico</td>\n",
       "      <td>19.8</td>\n",
       "      <td>12.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3219</th>\n",
       "      <td>Yauco Municipio, Puerto Rico</td>\n",
       "      <td>20.2</td>\n",
       "      <td>18.6</td>\n",
       "      <td>62.5</td>\n",
       "      <td>22.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3220 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Geographic Area Name  Unempl_WholeRate  Unempl_WhiteRate  \\\n",
       "0                     Autauga, Alabama               3.7               3.4   \n",
       "1                     Baldwin, Alabama               4.3               3.8   \n",
       "2                     Barbour, Alabama               9.2               4.7   \n",
       "3                        Bibb, Alabama               7.3               6.8   \n",
       "4                      Blount, Alabama               3.4               3.3   \n",
       "...                                ...               ...               ...   \n",
       "3215  Vega Baja Municipio, Puerto Rico              17.4              16.8   \n",
       "3216    Vieques Municipio, Puerto Rico               7.5               4.1   \n",
       "3217   Villalba Municipio, Puerto Rico              21.9              21.6   \n",
       "3218    Yabucoa Municipio, Puerto Rico              19.8              12.9   \n",
       "3219      Yauco Municipio, Puerto Rico              20.2              18.6   \n",
       "\n",
       "      Unempl_NHWhiteRate  Unempl_BlackRate  \n",
       "0                    3.5               5.1  \n",
       "1                    3.9               8.6  \n",
       "2                    4.7              14.8  \n",
       "3                    7.0              10.3  \n",
       "4                    3.4               1.7  \n",
       "...                  ...               ...  \n",
       "3215                 0.0              21.8  \n",
       "3216                 0.0               0.0  \n",
       "3217                 NaN              19.3  \n",
       "3218                 NaN              21.1  \n",
       "3219                62.5              22.4  \n",
       "\n",
       "[3220 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# [i for i in EMPLOYMENT.columns if 'Unemployment' in i]\n",
    "\n",
    "\n",
    "cols_dict = {\n",
    "#     'GEOID': 'GEOID',\n",
    " 'Geographic Area Name': 'Geographic Area Name',\n",
    " 'Unemployment rate_Population 16 years and over':'Unempl_WholeRate',\n",
    " 'Unemployment rate_White alone': 'Unempl_WhiteRate',\n",
    " 'Unemployment rate_White alone, not Hispanic or Latino': 'Unempl_NHWhiteRate',\n",
    " 'Unemployment rate_Black or African American alone': 'Unempl_BlackRate',}\n",
    "\n",
    "# EMPLOYMENT = EMPLOYMENT[cols]\n",
    "# cols_dict = {name: change_Employment_col_name(name) for name in cols}\n",
    "EMPLOYMENT = EMPLOYMENT[[i for i in cols_dict]].rename(columns = cols_dict)\n",
    "print(EMPLOYMENT.shape)\n",
    "\n",
    "\n",
    "EMPLOYMENT = EMPLOYMENT.replace('-', np.nan)\n",
    "for i in EMPLOYMENT.columns[2:]:\n",
    "    EMPLOYMENT[i] = EMPLOYMENT[i].astype(float)\n",
    "\n",
    "    \n",
    "# Disparity\n",
    "# EMPLOYMENT['Unemployment Parity'] = EMPLOYMENT['Rate_Unemployment_White'] - EMPLOYMENT['Rate_Unemployment_Black']\n",
    "\n",
    "EMPLOYMENT.to_csv('DataMerge/J-EMPLOYMENT.csv', index = False)\n",
    "EMPLOYMENT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K. Education (*Disparity)\n",
    "\n",
    "\n",
    "25 Above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "path = 'DataMerge/Data/K-EDUCATION/ACSST5Y2019.S1501_2021-03-20T122521/ACSST5Y2019.S1501_data_with_overlays_2021-03-18T142754.csv'\n",
    "EDUCATION = pd.read_csv(path, skiprows=[0], low_memory=False)\n",
    "# How to understand total population in household?\n",
    "\n",
    "cols = list(EDUCATION.columns)\n",
    "cols = [i for i in cols if 'Margin of Error!' not in i]\n",
    "EDUCATION = EDUCATION[cols]\n",
    "# [i for i in list(EDUCATION.columns) if 'Percent!!' in i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "\n",
    "\n",
    "# EDUCATION = EDUCATION.rename(columns = new_names_dict)\n",
    "\n",
    "EDUCATION['Geographic Area Name'] = EDUCATION['Geographic Area Name'].apply(remove_string)\n",
    "EDUCATION['County'] = EDUCATION['Geographic Area Name'].apply(get_county)\n",
    "EDUCATION['State']  = EDUCATION['Geographic Area Name'].apply(get_state)\n",
    "# EDUCATION.to_csv('DataMerge/K-EDUCATION.csv', index = False)\n",
    "\n",
    "# EDUCATION.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [i for i in EDUCATION.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3220, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>HighSchool_WholeRate</th>\n",
       "      <th>Bachelor_WholeRate</th>\n",
       "      <th>HighSchool_WhiteRate</th>\n",
       "      <th>Bachelor_WhiteRate</th>\n",
       "      <th>HighSchool_NHWhiteRate</th>\n",
       "      <th>Bachelor_NHWhiteRate</th>\n",
       "      <th>HighSchool_BlackRate</th>\n",
       "      <th>Bachelor_BlackRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Autauga, Alabama</td>\n",
       "      <td>88.5</td>\n",
       "      <td>26.6</td>\n",
       "      <td>91.2</td>\n",
       "      <td>29.1</td>\n",
       "      <td>91.1</td>\n",
       "      <td>28.8</td>\n",
       "      <td>78.9</td>\n",
       "      <td>15.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Baldwin, Alabama</td>\n",
       "      <td>90.8</td>\n",
       "      <td>31.9</td>\n",
       "      <td>91.9</td>\n",
       "      <td>33.2</td>\n",
       "      <td>92.2</td>\n",
       "      <td>33.3</td>\n",
       "      <td>83.7</td>\n",
       "      <td>19.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Barbour, Alabama</td>\n",
       "      <td>73.2</td>\n",
       "      <td>11.6</td>\n",
       "      <td>78.6</td>\n",
       "      <td>17.5</td>\n",
       "      <td>79.1</td>\n",
       "      <td>17.7</td>\n",
       "      <td>69.1</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bibb, Alabama</td>\n",
       "      <td>79.1</td>\n",
       "      <td>10.4</td>\n",
       "      <td>84.4</td>\n",
       "      <td>12.4</td>\n",
       "      <td>84.6</td>\n",
       "      <td>12.7</td>\n",
       "      <td>60.8</td>\n",
       "      <td>3.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Blount, Alabama</td>\n",
       "      <td>80.5</td>\n",
       "      <td>13.1</td>\n",
       "      <td>80.9</td>\n",
       "      <td>13.1</td>\n",
       "      <td>83.8</td>\n",
       "      <td>13.9</td>\n",
       "      <td>61.2</td>\n",
       "      <td>4.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3215</th>\n",
       "      <td>Vega Baja Municipio, Puerto Rico</td>\n",
       "      <td>71.6</td>\n",
       "      <td>21.2</td>\n",
       "      <td>70.4</td>\n",
       "      <td>21.0</td>\n",
       "      <td>71.3</td>\n",
       "      <td>30.9</td>\n",
       "      <td>79.9</td>\n",
       "      <td>18.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>Vieques Municipio, Puerto Rico</td>\n",
       "      <td>71.2</td>\n",
       "      <td>18.0</td>\n",
       "      <td>76.5</td>\n",
       "      <td>25.9</td>\n",
       "      <td>89.9</td>\n",
       "      <td>35.9</td>\n",
       "      <td>75.7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>Villalba Municipio, Puerto Rico</td>\n",
       "      <td>78.0</td>\n",
       "      <td>19.9</td>\n",
       "      <td>79.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.1</td>\n",
       "      <td>25.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3218</th>\n",
       "      <td>Yabucoa Municipio, Puerto Rico</td>\n",
       "      <td>71.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>64.3</td>\n",
       "      <td>16.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>73.1</td>\n",
       "      <td>18.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3219</th>\n",
       "      <td>Yauco Municipio, Puerto Rico</td>\n",
       "      <td>73.4</td>\n",
       "      <td>24.6</td>\n",
       "      <td>73.1</td>\n",
       "      <td>25.4</td>\n",
       "      <td>85.3</td>\n",
       "      <td>5.9</td>\n",
       "      <td>72.5</td>\n",
       "      <td>31.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3220 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Geographic Area Name  HighSchool_WholeRate  \\\n",
       "0                     Autauga, Alabama                  88.5   \n",
       "1                     Baldwin, Alabama                  90.8   \n",
       "2                     Barbour, Alabama                  73.2   \n",
       "3                        Bibb, Alabama                  79.1   \n",
       "4                      Blount, Alabama                  80.5   \n",
       "...                                ...                   ...   \n",
       "3215  Vega Baja Municipio, Puerto Rico                  71.6   \n",
       "3216    Vieques Municipio, Puerto Rico                  71.2   \n",
       "3217   Villalba Municipio, Puerto Rico                  78.0   \n",
       "3218    Yabucoa Municipio, Puerto Rico                  71.0   \n",
       "3219      Yauco Municipio, Puerto Rico                  73.4   \n",
       "\n",
       "      Bachelor_WholeRate  HighSchool_WhiteRate  Bachelor_WhiteRate  \\\n",
       "0                   26.6                  91.2                29.1   \n",
       "1                   31.9                  91.9                33.2   \n",
       "2                   11.6                  78.6                17.5   \n",
       "3                   10.4                  84.4                12.4   \n",
       "4                   13.1                  80.9                13.1   \n",
       "...                  ...                   ...                 ...   \n",
       "3215                21.2                  70.4                21.0   \n",
       "3216                18.0                  76.5                25.9   \n",
       "3217                19.9                  79.0                20.0   \n",
       "3218                18.0                  64.3                16.3   \n",
       "3219                24.6                  73.1                25.4   \n",
       "\n",
       "      HighSchool_NHWhiteRate  Bachelor_NHWhiteRate  HighSchool_BlackRate  \\\n",
       "0                       91.1                  28.8                  78.9   \n",
       "1                       92.2                  33.3                  83.7   \n",
       "2                       79.1                  17.7                  69.1   \n",
       "3                       84.6                  12.7                  60.8   \n",
       "4                       83.8                  13.9                  61.2   \n",
       "...                      ...                   ...                   ...   \n",
       "3215                    71.3                  30.9                  79.9   \n",
       "3216                    89.9                  35.9                  75.7   \n",
       "3217                   100.0                  10.0                  93.1   \n",
       "3218                     NaN                   NaN                  73.1   \n",
       "3219                    85.3                   5.9                  72.5   \n",
       "\n",
       "      Bachelor_BlackRate  \n",
       "0                   15.9  \n",
       "1                   19.7  \n",
       "2                    5.0  \n",
       "3                    3.1  \n",
       "4                    4.6  \n",
       "...                  ...  \n",
       "3215                18.9  \n",
       "3216                 0.0  \n",
       "3217                25.3  \n",
       "3218                18.6  \n",
       "3219                31.6  \n",
       "\n",
       "[3220 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "cols_dict = {\n",
    " 'Geographic Area Name':'Geographic Area Name',\n",
    "\n",
    "    \n",
    " 'Estimate!!Percent!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over!!High school graduate or higher':'HighSchool_WholeRate',\n",
    " \"Estimate!!Percent!!AGE BY EDUCATIONAL ATTAINMENT!!Population 25 years and over!!Bachelor's degree or higher\": \"Bachelor_WholeRate\",\n",
    "    \n",
    "    \n",
    " 'Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!White alone!!High school graduate or higher':'HighSchool_WhiteRate',\n",
    " \"Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!White alone!!Bachelor's degree or higher\": 'Bachelor_WhiteRate',\n",
    "    \n",
    "    \n",
    " 'Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!White alone, not Hispanic or Latino!!High school graduate or higher': 'HighSchool_NHWhiteRate',\n",
    " \"Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!White alone, not Hispanic or Latino!!Bachelor's degree or higher\": 'Bachelor_NHWhiteRate',\n",
    "\n",
    "    \n",
    " 'Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!Black alone!!High school graduate or higher':'HighSchool_BlackRate',\n",
    " \"Estimate!!Percent!!RACE AND HISPANIC OR LATINO ORIGIN BY EDUCATIONAL ATTAINMENT!!Black alone!!Bachelor's degree or higher\":'Bachelor_BlackRate',\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "EDUCATION = EDUCATION[cols]\n",
    "# cols_dict = {name: change_Education_col_name(name) for name in cols}\n",
    "\n",
    "EDUCATION = EDUCATION[[i for i in cols_dict ]].rename(columns = cols_dict)\n",
    "print(EDUCATION.shape)\n",
    "\n",
    "\n",
    "EDUCATION = EDUCATION.replace('-', np.nan)\n",
    "for i in EDUCATION.columns[1:]:\n",
    "    EDUCATION[i] = EDUCATION[i].astype(float)#/100\n",
    "\n",
    "EDUCATION\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Geographic Area Name', 'HighSchool_WholeRate', 'Bachelor_WholeRate',\n",
       "       'HighSchool_WhiteRate', 'Bachelor_WhiteRate', 'HighSchool_NHWhiteRate',\n",
       "       'Bachelor_NHWhiteRate', 'HighSchool_BlackRate', 'Bachelor_BlackRate'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "EDUCATION.to_csv('DataMerge/K-EDUCATION.csv', index = False)\n",
    "\n",
    "EDUCATION.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# L. BroadBoard Speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>fixed_avg_d_mbps_wt</th>\n",
       "      <th>fixed_avg_u_mbps_wt</th>\n",
       "      <th>fixed_avg_lat_wt</th>\n",
       "      <th>mobile_avg_d_mbps_wt</th>\n",
       "      <th>mobile_avg_u_mbps_wt</th>\n",
       "      <th>mobile_avg_lat_wt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>155.893636</td>\n",
       "      <td>77.383649</td>\n",
       "      <td>23.550424</td>\n",
       "      <td>39.934262</td>\n",
       "      <td>15.456766</td>\n",
       "      <td>46.469349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>122.081455</td>\n",
       "      <td>53.476077</td>\n",
       "      <td>46.965039</td>\n",
       "      <td>45.170522</td>\n",
       "      <td>9.442240</td>\n",
       "      <td>52.995281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>113.287736</td>\n",
       "      <td>11.731600</td>\n",
       "      <td>62.774059</td>\n",
       "      <td>20.890680</td>\n",
       "      <td>3.842048</td>\n",
       "      <td>56.840000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>46.402450</td>\n",
       "      <td>33.444540</td>\n",
       "      <td>103.175520</td>\n",
       "      <td>35.700959</td>\n",
       "      <td>6.787694</td>\n",
       "      <td>59.570588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>83.130446</td>\n",
       "      <td>19.635431</td>\n",
       "      <td>38.061094</td>\n",
       "      <td>30.791686</td>\n",
       "      <td>8.734358</td>\n",
       "      <td>60.794805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3197</th>\n",
       "      <td>72151</td>\n",
       "      <td>34.647772</td>\n",
       "      <td>7.279017</td>\n",
       "      <td>35.837315</td>\n",
       "      <td>41.740484</td>\n",
       "      <td>7.582627</td>\n",
       "      <td>41.437799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3198</th>\n",
       "      <td>72153</td>\n",
       "      <td>46.942987</td>\n",
       "      <td>7.725302</td>\n",
       "      <td>34.334703</td>\n",
       "      <td>25.031864</td>\n",
       "      <td>7.452362</td>\n",
       "      <td>49.892696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3199</th>\n",
       "      <td>78010</td>\n",
       "      <td>27.823811</td>\n",
       "      <td>8.965607</td>\n",
       "      <td>20.848733</td>\n",
       "      <td>31.461869</td>\n",
       "      <td>9.440546</td>\n",
       "      <td>62.838462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3200</th>\n",
       "      <td>78020</td>\n",
       "      <td>22.055864</td>\n",
       "      <td>8.717722</td>\n",
       "      <td>26.484169</td>\n",
       "      <td>20.498675</td>\n",
       "      <td>8.731175</td>\n",
       "      <td>73.475000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3201</th>\n",
       "      <td>78030</td>\n",
       "      <td>30.624871</td>\n",
       "      <td>6.874817</td>\n",
       "      <td>18.069549</td>\n",
       "      <td>28.145640</td>\n",
       "      <td>9.654556</td>\n",
       "      <td>65.875549</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3202 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  fixed_avg_d_mbps_wt  fixed_avg_u_mbps_wt  fixed_avg_lat_wt  \\\n",
       "0     01001           155.893636            77.383649         23.550424   \n",
       "1     01003           122.081455            53.476077         46.965039   \n",
       "2     01005           113.287736            11.731600         62.774059   \n",
       "3     01007            46.402450            33.444540        103.175520   \n",
       "4     01009            83.130446            19.635431         38.061094   \n",
       "...     ...                  ...                  ...               ...   \n",
       "3197  72151            34.647772             7.279017         35.837315   \n",
       "3198  72153            46.942987             7.725302         34.334703   \n",
       "3199  78010            27.823811             8.965607         20.848733   \n",
       "3200  78020            22.055864             8.717722         26.484169   \n",
       "3201  78030            30.624871             6.874817         18.069549   \n",
       "\n",
       "      mobile_avg_d_mbps_wt  mobile_avg_u_mbps_wt  mobile_avg_lat_wt  \n",
       "0                39.934262             15.456766          46.469349  \n",
       "1                45.170522              9.442240          52.995281  \n",
       "2                20.890680              3.842048          56.840000  \n",
       "3                35.700959              6.787694          59.570588  \n",
       "4                30.791686              8.734358          60.794805  \n",
       "...                    ...                   ...                ...  \n",
       "3197             41.740484              7.582627          41.437799  \n",
       "3198             25.031864              7.452362          49.892696  \n",
       "3199             31.461869              9.440546          62.838462  \n",
       "3200             20.498675              8.731175          73.475000  \n",
       "3201             28.145640              9.654556          65.875549  \n",
       "\n",
       "[3202 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path1 = 'DataMerge/Data/L-Broadband/fixed_internet_speed.xlsx'\n",
    "path2 = 'DataMerge/Data/L-Broadband/mobile_internet_speed.xlsx'\n",
    "\n",
    "BroadBoardSpeed = pd.merge(pd.read_excel(path1), pd.read_excel(path2), on = 'GEOID')\n",
    "\n",
    "BroadBoardSpeed = BroadBoardSpeed[['GEOID', 'fixed_avg_d_mbps_wt',\n",
    "       'fixed_avg_u_mbps_wt', 'fixed_avg_lat_wt',\n",
    "       'mobile_avg_d_mbps_wt', 'mobile_avg_u_mbps_wt', 'mobile_avg_lat_wt']]\n",
    "\n",
    "\n",
    "BroadBoardSpeed['GEOID'] = BroadBoardSpeed['GEOID'].apply(modifyGEOID)\n",
    "BroadBoardSpeed.to_csv('DataMerge/L-BroadBoardSpeed.csv', index = False)\n",
    "BroadBoardSpeed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# M. Social Vulnerability Index\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AREA_SQMI</th>\n",
       "      <th>DENSITY</th>\n",
       "      <th>EP_POV</th>\n",
       "      <th>EP_AGE65</th>\n",
       "      <th>EP_DISABL</th>\n",
       "      <th>EP_MINRTY</th>\n",
       "      <th>EP_LIMENG</th>\n",
       "      <th>EP_NOVEH</th>\n",
       "      <th>RPL_THEME1</th>\n",
       "      <th>RPL_THEME2</th>\n",
       "      <th>RPL_THEME3</th>\n",
       "      <th>RPL_THEME4</th>\n",
       "      <th>RPL_THEMES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3141.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3141.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3141.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1124.319749</td>\n",
       "      <td>270.678668</td>\n",
       "      <td>0.156012</td>\n",
       "      <td>0.183652</td>\n",
       "      <td>0.159217</td>\n",
       "      <td>0.235055</td>\n",
       "      <td>0.017006</td>\n",
       "      <td>0.063515</td>\n",
       "      <td>0.499983</td>\n",
       "      <td>0.499959</td>\n",
       "      <td>0.499843</td>\n",
       "      <td>0.499970</td>\n",
       "      <td>0.499993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3614.348182</td>\n",
       "      <td>1803.519578</td>\n",
       "      <td>0.064785</td>\n",
       "      <td>0.045861</td>\n",
       "      <td>0.044053</td>\n",
       "      <td>0.201833</td>\n",
       "      <td>0.027858</td>\n",
       "      <td>0.044762</td>\n",
       "      <td>0.288813</td>\n",
       "      <td>0.288810</td>\n",
       "      <td>0.288924</td>\n",
       "      <td>0.288812</td>\n",
       "      <td>0.288814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.046443</td>\n",
       "      <td>0.037198</td>\n",
       "      <td>0.023000</td>\n",
       "      <td>0.038000</td>\n",
       "      <td>0.038000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>431.120769</td>\n",
       "      <td>16.689431</td>\n",
       "      <td>0.110000</td>\n",
       "      <td>0.154250</td>\n",
       "      <td>0.129000</td>\n",
       "      <td>0.073000</td>\n",
       "      <td>0.003000</td>\n",
       "      <td>0.042000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.249975</td>\n",
       "      <td>0.249900</td>\n",
       "      <td>0.249975</td>\n",
       "      <td>0.249700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>616.479521</td>\n",
       "      <td>44.719178</td>\n",
       "      <td>0.147000</td>\n",
       "      <td>0.180000</td>\n",
       "      <td>0.154000</td>\n",
       "      <td>0.161000</td>\n",
       "      <td>0.007000</td>\n",
       "      <td>0.056500</td>\n",
       "      <td>0.499700</td>\n",
       "      <td>0.499800</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.499800</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>924.005428</td>\n",
       "      <td>116.710171</td>\n",
       "      <td>0.191000</td>\n",
       "      <td>0.208000</td>\n",
       "      <td>0.185000</td>\n",
       "      <td>0.352000</td>\n",
       "      <td>0.019000</td>\n",
       "      <td>0.076000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750025</td>\n",
       "      <td>0.749925</td>\n",
       "      <td>0.749800</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>145573.832761</td>\n",
       "      <td>72052.993377</td>\n",
       "      <td>0.551000</td>\n",
       "      <td>0.556000</td>\n",
       "      <td>0.337000</td>\n",
       "      <td>0.993000</td>\n",
       "      <td>0.304000</td>\n",
       "      <td>0.878000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           AREA_SQMI       DENSITY       EP_POV     EP_AGE65    EP_DISABL  \\\n",
       "count    3142.000000   3142.000000  3141.000000  3142.000000  3142.000000   \n",
       "mean     1124.319749    270.678668     0.156012     0.183652     0.159217   \n",
       "std      3614.348182   1803.519578     0.064785     0.045861     0.044053   \n",
       "min         2.046443      0.037198     0.023000     0.038000     0.038000   \n",
       "25%       431.120769     16.689431     0.110000     0.154250     0.129000   \n",
       "50%       616.479521     44.719178     0.147000     0.180000     0.154000   \n",
       "75%       924.005428    116.710171     0.191000     0.208000     0.185000   \n",
       "max    145573.832761  72052.993377     0.551000     0.556000     0.337000   \n",
       "\n",
       "         EP_MINRTY    EP_LIMENG     EP_NOVEH   RPL_THEME1   RPL_THEME2  \\\n",
       "count  3142.000000  3142.000000  3142.000000  3141.000000  3142.000000   \n",
       "mean      0.235055     0.017006     0.063515     0.499983     0.499959   \n",
       "std       0.201833     0.027858     0.044762     0.288813     0.288810   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.073000     0.003000     0.042000     0.250000     0.249975   \n",
       "50%       0.161000     0.007000     0.056500     0.499700     0.499800   \n",
       "75%       0.352000     0.019000     0.076000     0.750000     0.750025   \n",
       "max       0.993000     0.304000     0.878000     1.000000     1.000000   \n",
       "\n",
       "        RPL_THEME3   RPL_THEME4   RPL_THEMES  \n",
       "count  3142.000000  3142.000000  3141.000000  \n",
       "mean      0.499843     0.499970     0.499993  \n",
       "std       0.288924     0.288812     0.288814  \n",
       "min       0.000000     0.000000     0.000000  \n",
       "25%       0.249900     0.249975     0.249700  \n",
       "50%       0.500000     0.499800     0.500000  \n",
       "75%       0.749925     0.749800     0.750000  \n",
       "max       1.000000     1.000000     1.000000  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Leave it for now\n",
    "\n",
    "\n",
    "path = 'DataMerge/Data/M-SocialVulnerabilityIndex/SVI2018_US_COUNTY.csv'\n",
    "\n",
    "SocialVulnerabilityIndex = pd.read_csv(path)\n",
    "\n",
    "SocialVulnerabilityIndex['GEOID'] = SocialVulnerabilityIndex['FIPS'].apply(modifyGEOID)\n",
    "\n",
    "\n",
    "SocialVulnerabilityIndex = SocialVulnerabilityIndex.replace([-999], np.nan)\n",
    "\n",
    "\n",
    "cols = [\n",
    "    'GEOID',\n",
    "    'AREA_SQMI', \n",
    "    'DENSITY',\n",
    "    'EP_POV', # poverty\n",
    "     'EP_AGE65',\n",
    "     'EP_DISABL', # disablity\n",
    "     'EP_MINRTY', # minority\n",
    "     'EP_LIMENG', # Limited English \n",
    "     'EP_NOVEH',  # No Vechile\n",
    "    \n",
    "\n",
    "     'RPL_THEME1',\n",
    "     'RPL_THEME2',\n",
    "     'RPL_THEME3',\n",
    "     'RPL_THEME4',\n",
    "     'RPL_THEMES',\n",
    " ]\n",
    "\n",
    "# 'LOCATION': 'Autauga County, Alabama',\n",
    "# 'AREA_SQMI': 594.4434589,\n",
    "\n",
    "\n",
    "SocialVulnerabilityIndex['DENSITY'] = SocialVulnerabilityIndex['E_TOTPOP'] / SocialVulnerabilityIndex['AREA_SQMI']\n",
    "\n",
    "\n",
    "\n",
    "SocialVulnerabilityIndex = SocialVulnerabilityIndex.sort_values(by = 'GEOID')[cols].reset_index(drop = True)\n",
    "for i in ['EP_POV', # poverty\n",
    "         'EP_AGE65',\n",
    "         'EP_DISABL', # disablity\n",
    "         'EP_MINRTY', # minority\n",
    "         'EP_LIMENG', # Limited English \n",
    "         'EP_NOVEH',  # No Vechile\n",
    "    ]:\n",
    "    SocialVulnerabilityIndex[i] = SocialVulnerabilityIndex[i].astype(float)/100\n",
    "\n",
    "    \n",
    "SocialVulnerabilityIndex.to_csv('DataMerge/M-SocialVulnerabilityIndex.csv', index = False)\n",
    "\n",
    "SocialVulnerabilityIndex.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>AREA_SQMI</th>\n",
       "      <th>DENSITY</th>\n",
       "      <th>EP_POV</th>\n",
       "      <th>EP_AGE65</th>\n",
       "      <th>EP_DISABL</th>\n",
       "      <th>EP_MINRTY</th>\n",
       "      <th>EP_LIMENG</th>\n",
       "      <th>EP_NOVEH</th>\n",
       "      <th>RPL_THEME1</th>\n",
       "      <th>RPL_THEME2</th>\n",
       "      <th>RPL_THEME3</th>\n",
       "      <th>RPL_THEME4</th>\n",
       "      <th>RPL_THEMES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>594.443459</td>\n",
       "      <td>92.859967</td>\n",
       "      <td>0.154</td>\n",
       "      <td>0.146</td>\n",
       "      <td>0.193</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.3631</td>\n",
       "      <td>0.5810</td>\n",
       "      <td>0.5947</td>\n",
       "      <td>0.3741</td>\n",
       "      <td>0.4354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>1589.793007</td>\n",
       "      <td>130.901947</td>\n",
       "      <td>0.106</td>\n",
       "      <td>0.195</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.170</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.2232</td>\n",
       "      <td>0.1990</td>\n",
       "      <td>0.4358</td>\n",
       "      <td>0.3359</td>\n",
       "      <td>0.2162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>885.001636</td>\n",
       "      <td>29.132150</td>\n",
       "      <td>0.289</td>\n",
       "      <td>0.180</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.539</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.092</td>\n",
       "      <td>0.9780</td>\n",
       "      <td>0.9153</td>\n",
       "      <td>0.8558</td>\n",
       "      <td>0.9889</td>\n",
       "      <td>0.9959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>622.461089</td>\n",
       "      <td>36.190214</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.163</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.254</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.060</td>\n",
       "      <td>0.7694</td>\n",
       "      <td>0.1203</td>\n",
       "      <td>0.4323</td>\n",
       "      <td>0.7189</td>\n",
       "      <td>0.6003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>644.830460</td>\n",
       "      <td>89.395591</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.178</td>\n",
       "      <td>0.142</td>\n",
       "      <td>0.129</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.6143</td>\n",
       "      <td>0.3187</td>\n",
       "      <td>0.5915</td>\n",
       "      <td>0.1741</td>\n",
       "      <td>0.4242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID    AREA_SQMI     DENSITY  EP_POV  EP_AGE65  EP_DISABL  EP_MINRTY  \\\n",
       "0  01001   594.443459   92.859967   0.154     0.146      0.193      0.250   \n",
       "1  01003  1589.793007  130.901947   0.106     0.195      0.140      0.170   \n",
       "2  01005   885.001636   29.132150   0.289     0.180      0.222      0.539   \n",
       "3  01007   622.461089   36.190214   0.140     0.163      0.167      0.254   \n",
       "4  01009   644.830460   89.395591   0.144     0.178      0.142      0.129   \n",
       "\n",
       "   EP_LIMENG  EP_NOVEH  RPL_THEME1  RPL_THEME2  RPL_THEME3  RPL_THEME4  \\\n",
       "0      0.008     0.056      0.3631      0.5810      0.5947      0.3741   \n",
       "1      0.005     0.034      0.2232      0.1990      0.4358      0.3359   \n",
       "2      0.016     0.092      0.9780      0.9153      0.8558      0.9889   \n",
       "3      0.003     0.060      0.7694      0.1203      0.4323      0.7189   \n",
       "4      0.017     0.042      0.6143      0.3187      0.5915      0.1741   \n",
       "\n",
       "   RPL_THEMES  \n",
       "0      0.4354  \n",
       "1      0.2162  \n",
       "2      0.9959  \n",
       "3      0.6003  \n",
       "4      0.4242  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SocialVulnerabilityIndex.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N. TrafficAllTransit\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'District of Columbia, District Of Columbia'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>TCI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>New York, New York</td>\n",
       "      <td>52.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kings, New York</td>\n",
       "      <td>34.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bronx, New York</td>\n",
       "      <td>29.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>San Francisco, California</td>\n",
       "      <td>31.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Queens, New York</td>\n",
       "      <td>23.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3137</th>\n",
       "      <td>Lincoln, Kansas</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3138</th>\n",
       "      <td>Lincoln, Kentucky</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3139</th>\n",
       "      <td>Lincoln, Maine</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3140</th>\n",
       "      <td>Lincoln, Minnesota</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3141</th>\n",
       "      <td>Lincoln, Missouri</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3142 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Geographic Area Name   TCI\n",
       "0            New York, New York  52.6\n",
       "1               Kings, New York  34.5\n",
       "2               Bronx, New York  29.5\n",
       "3     San Francisco, California  31.7\n",
       "4              Queens, New York  23.8\n",
       "...                         ...   ...\n",
       "3137            Lincoln, Kansas   0.0\n",
       "3138          Lincoln, Kentucky   0.0\n",
       "3139             Lincoln, Maine   0.0\n",
       "3140         Lincoln, Minnesota   0.0\n",
       "3141          Lincoln, Missouri   0.0\n",
       "\n",
       "[3142 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "path = 'DataMerge/Data/N-TrafficAllTransit/AllTransit.xlsx'\n",
    "DATA = pd.read_excel(path)\n",
    "DATA.head()\n",
    "\n",
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "\n",
    "def change_state(s):\n",
    "    State_Ab = {'NM': 'New Mexico', 'AL': 'Alabama', 'AK': 'Alaska', 'AZ': 'Arizona', 'AR': 'Arkansas', 'CA': 'California', 'CO': 'Colorado', 'CT': 'Connecticut', 'DE': 'Delaware', 'FL': 'Florida', 'GA': 'Georgia', 'ID': 'Idaho', 'IL': 'Illinois', 'IN': 'Indiana', 'IA': 'Iowa', 'KS': 'Kansas', 'KY': 'Kentucky', 'LA': 'Louisiana', 'ME': 'Maine', 'MD': 'Maryland', 'MA': 'Massachusetts', 'MI': 'Michigan', 'MN': 'Minnesota', 'MS': 'Mississippi', 'MO': 'Missouri', 'MT': 'Montana', 'NE': 'Nebraska', 'NV': 'Nevada', 'NH': 'New Hampshire', 'NJ': 'New Jersey', 'NY': 'New York', 'NC': 'North Carolina', 'ND': 'North Dakota', 'OH': 'Ohio', 'OK': 'Oklahoma', 'OR': 'Oregon', 'PA': 'Pennsylvania', 'RI': 'Rhode Island', 'SC': 'South Carolina', 'SD': 'South Dakota', 'TN': 'Tennessee', 'TX': 'Texas', 'UT': 'Utah', 'VT': 'Vermont', 'VA': 'Virginia', 'WA': 'Washington', 'WV': 'West Virginia', 'WI': 'Wisconsin', 'WY': 'Wyoming', 'HI': 'Hawaii', 'DC': 'District Of Columbia'}\n",
    "    \n",
    "    return State_Ab[s.strip()]\n",
    "\n",
    "# DATA = DATA.rename(columns = new_names_dict)\n",
    "\n",
    "# DATA['Geographic Area Name'] = DATA['Geographic Area Name'].apply(remove_string)\n",
    "DATA['County'] = DATA['County'].apply(remove_string)\n",
    "DATA['State'] = DATA['State'].apply(change_state)\n",
    "# DATA['State']  = DATA['Geographic Area Name'].apply(get_state)\n",
    "\n",
    "DATA['Geographic Area Name'] =  DATA['County'] + ', ' +   DATA['State']\n",
    "Traffic = DATA\n",
    "Traffic = Traffic[['Geographic Area Name', 'TCI+']]\n",
    "\n",
    "\n",
    "set1 = set(DATA ['Geographic Area Name'].unique())\n",
    "\n",
    "set2 = set(EDUCATION['Geographic Area Name'].unique())\n",
    "\n",
    "print(set1 - set2)\n",
    "\n",
    "Traffic = Traffic.rename(columns = {'TCI+': 'TCI'})\n",
    "\n",
    "Traffic.to_csv('DataMerge/N-Traffic.csv', index = False)\n",
    "\n",
    "Traffic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# O. Vaccine Coverage Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Th1_Historic_Undervaccination</th>\n",
       "      <th>Th2_Sociodemographic_Barriers</th>\n",
       "      <th>Th3_Resource_constrained_Healthcare_System</th>\n",
       "      <th>Th4_Healthcare_Accessibility_Barriers</th>\n",
       "      <th>Th5_Irregular_Care_Seeking_Behavior</th>\n",
       "      <th>CVAC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>0.419612</td>\n",
       "      <td>0.231455</td>\n",
       "      <td>0.827762</td>\n",
       "      <td>0.484241</td>\n",
       "      <td>0.830309</td>\n",
       "      <td>0.607768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>0.228590</td>\n",
       "      <td>0.150271</td>\n",
       "      <td>0.170965</td>\n",
       "      <td>0.391277</td>\n",
       "      <td>0.726839</td>\n",
       "      <td>0.227953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>0.505571</td>\n",
       "      <td>0.991404</td>\n",
       "      <td>0.621777</td>\n",
       "      <td>0.966253</td>\n",
       "      <td>0.694683</td>\n",
       "      <td>0.885068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>0.410060</td>\n",
       "      <td>0.888252</td>\n",
       "      <td>0.335880</td>\n",
       "      <td>0.842089</td>\n",
       "      <td>0.834448</td>\n",
       "      <td>0.764406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>0.440306</td>\n",
       "      <td>0.716014</td>\n",
       "      <td>0.850684</td>\n",
       "      <td>0.580388</td>\n",
       "      <td>0.837950</td>\n",
       "      <td>0.795288</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  Th1_Historic_Undervaccination  Th2_Sociodemographic_Barriers  \\\n",
       "0  01001                       0.419612                       0.231455   \n",
       "1  01003                       0.228590                       0.150271   \n",
       "2  01005                       0.505571                       0.991404   \n",
       "3  01007                       0.410060                       0.888252   \n",
       "4  01009                       0.440306                       0.716014   \n",
       "\n",
       "   Th3_Resource_constrained_Healthcare_System  \\\n",
       "0                                    0.827762   \n",
       "1                                    0.170965   \n",
       "2                                    0.621777   \n",
       "3                                    0.335880   \n",
       "4                                    0.850684   \n",
       "\n",
       "   Th4_Healthcare_Accessibility_Barriers  Th5_Irregular_Care_Seeking_Behavior  \\\n",
       "0                               0.484241                             0.830309   \n",
       "1                               0.391277                             0.726839   \n",
       "2                               0.966253                             0.694683   \n",
       "3                               0.842089                             0.834448   \n",
       "4                               0.580388                             0.837950   \n",
       "\n",
       "       CVAC  \n",
       "0  0.607768  \n",
       "1  0.227953  \n",
       "2  0.885068  \n",
       "3  0.764406  \n",
       "4  0.795288  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'DataMerge/Data/O-Vaccine Coverage Index/CVAC_data_Feb2021.xlsx'\n",
    "\n",
    "VaccineCoverageIndex = pd.read_excel(path, sheet_name = 'County_CVAC')\n",
    "\n",
    "VaccineCoverageIndex['GEOID'] = VaccineCoverageIndex['FIPS'].apply(modifyGEOID)\n",
    "VaccineCoverageIndex = VaccineCoverageIndex[['GEOID',\n",
    "                      'Th1_Historic_Undervaccination',\n",
    "                      'Th2_Sociodemographic_Barriers',\n",
    "                      'Th3_Resource_constrained_Healthcare_System',\n",
    "                      'Th4_Healthcare_Accessibility_Barriers',\n",
    "                      'Th5_Irregular_Care_Seeking_Behavior', \n",
    "                      'CVAC']]\n",
    "\n",
    "\n",
    "VaccineCoverageIndex.to_csv('DataMerge/O-VaccineCoverageIndex.csv', index = False)\n",
    "VaccineCoverageIndex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Th1_Historic_Undervaccination</th>\n",
       "      <th>Th2_Sociodemographic_Barriers</th>\n",
       "      <th>Th3_Resource_constrained_Healthcare_System</th>\n",
       "      <th>Th4_Healthcare_Accessibility_Barriers</th>\n",
       "      <th>Th5_Irregular_Care_Seeking_Behavior</th>\n",
       "      <th>CVAC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "      <td>3142.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.499942</td>\n",
       "      <td>0.499938</td>\n",
       "      <td>0.499924</td>\n",
       "      <td>0.499921</td>\n",
       "      <td>0.499913</td>\n",
       "      <td>0.499966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.288812</td>\n",
       "      <td>0.288810</td>\n",
       "      <td>0.288807</td>\n",
       "      <td>0.288815</td>\n",
       "      <td>0.288811</td>\n",
       "      <td>0.288815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.249761</td>\n",
       "      <td>0.249761</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.499841</td>\n",
       "      <td>0.499841</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.499841</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.749920</td>\n",
       "      <td>0.749920</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Th1_Historic_Undervaccination  Th2_Sociodemographic_Barriers  \\\n",
       "count                    3142.000000                    3142.000000   \n",
       "mean                        0.499942                       0.499938   \n",
       "std                         0.288812                       0.288810   \n",
       "min                         0.000000                       0.000000   \n",
       "25%                         0.250000                       0.250000   \n",
       "50%                         0.500000                       0.499841   \n",
       "75%                         0.750000                       0.750000   \n",
       "max                         1.000000                       1.000000   \n",
       "\n",
       "       Th3_Resource_constrained_Healthcare_System  \\\n",
       "count                                 3142.000000   \n",
       "mean                                     0.499924   \n",
       "std                                      0.288807   \n",
       "min                                      0.000000   \n",
       "25%                                      0.249761   \n",
       "50%                                      0.499841   \n",
       "75%                                      0.749920   \n",
       "max                                      1.000000   \n",
       "\n",
       "       Th4_Healthcare_Accessibility_Barriers  \\\n",
       "count                            3142.000000   \n",
       "mean                                0.499921   \n",
       "std                                 0.288815   \n",
       "min                                 0.000000   \n",
       "25%                                 0.249761   \n",
       "50%                                 0.500000   \n",
       "75%                                 0.749920   \n",
       "max                                 1.000000   \n",
       "\n",
       "       Th5_Irregular_Care_Seeking_Behavior         CVAC  \n",
       "count                          3142.000000  3142.000000  \n",
       "mean                              0.499913     0.499966  \n",
       "std                               0.288811     0.288815  \n",
       "min                               0.000000     0.000000  \n",
       "25%                               0.250000     0.250000  \n",
       "50%                               0.499841     0.500000  \n",
       "75%                               0.750000     0.750000  \n",
       "max                               1.000000     1.000000  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VaccineCoverageIndex.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# P. Vaccine Faclility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>FQHC Count</th>\n",
       "      <th>FacDensity10K</th>\n",
       "      <th>HOPD count</th>\n",
       "      <th>PHMCY Count</th>\n",
       "      <th>RUHC Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>1</td>\n",
       "      <td>2.1479</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>5</td>\n",
       "      <td>2.5086</td>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>2</td>\n",
       "      <td>3.2407</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>6</td>\n",
       "      <td>4.9120</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>1</td>\n",
       "      <td>2.9399</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  FQHC Count  FacDensity10K  HOPD count  PHMCY Count  RUHC Count\n",
       "0  01001           1         2.1479           1           10           0\n",
       "1  01003           5         2.5086           3           48           0\n",
       "2  01005           2         3.2407           1            5           0\n",
       "3  01007           6         4.9120           1            4           0\n",
       "4  01009           1         2.9399           0            8           8"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'DataMerge/Data/P-VaccineFaclility/Racial Disparities Cont US.xlsx'\n",
    "\n",
    "VaccineFaclility = pd.read_excel(path)\n",
    "\n",
    "\n",
    "VaccineFaclility['GEOID'] = VaccineFaclility['fips Calc'].apply(modifyGEOID)\n",
    "VaccineFaclility = VaccineFaclility.sort_values(by = 'GEOID')\n",
    "\n",
    "cols = ['GEOID',\n",
    "        'FQHC Count',\n",
    "        'FacDensity10K', \n",
    "        'HOPD count', \n",
    "        'PHMCY Count', \n",
    "        'RUHC Count', ]\n",
    "\n",
    "VaccineFaclility = VaccineFaclility[cols].reset_index(drop = True)\n",
    "VaccineFaclility.to_csv('DataMerge/P-VaccineFaclility.csv', index = False)\n",
    "VaccineFaclility.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FQHC Count</th>\n",
       "      <th>FacDensity10K</th>\n",
       "      <th>HOPD count</th>\n",
       "      <th>PHMCY Count</th>\n",
       "      <th>RUHC Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3108.000000</td>\n",
       "      <td>3108.000000</td>\n",
       "      <td>3108.000000</td>\n",
       "      <td>3108.000000</td>\n",
       "      <td>3108.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.925997</td>\n",
       "      <td>2.694558</td>\n",
       "      <td>1.020592</td>\n",
       "      <td>16.386100</td>\n",
       "      <td>0.406692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>13.078346</td>\n",
       "      <td>1.908029</td>\n",
       "      <td>2.628278</td>\n",
       "      <td>44.999985</td>\n",
       "      <td>1.042382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.705525</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.293550</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.261475</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>419.000000</td>\n",
       "      <td>30.030000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>1055.000000</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        FQHC Count  FacDensity10K   HOPD count  PHMCY Count   RUHC Count\n",
       "count  3108.000000    3108.000000  3108.000000  3108.000000  3108.000000\n",
       "mean      3.925997       2.694558     1.020592    16.386100     0.406692\n",
       "std      13.078346       1.908029     2.628278    44.999985     1.042382\n",
       "min       0.000000       0.000000     0.000000     0.000000     0.000000\n",
       "25%       0.000000       1.705525     0.000000     2.000000     0.000000\n",
       "50%       1.000000       2.293550     0.000000     4.000000     0.000000\n",
       "75%       3.000000       3.261475     1.000000    11.000000     0.000000\n",
       "max     419.000000      30.030000    73.000000  1055.000000    14.000000"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VaccineFaclility.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Q. FCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Rate_poor_fair_health</th>\n",
       "      <th>provcount_b</th>\n",
       "      <th>provcount_c</th>\n",
       "      <th>injury_deaths_per_100000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>01001</td>\n",
       "      <td>0.228</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>73.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>01003</td>\n",
       "      <td>0.127</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>74.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2434</th>\n",
       "      <td>01005</td>\n",
       "      <td>0.234</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>56.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3115</th>\n",
       "      <td>01007</td>\n",
       "      <td>0.179</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>80.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2787</th>\n",
       "      <td>01009</td>\n",
       "      <td>0.242</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>91.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  Rate_poor_fair_health  provcount_b  provcount_c  \\\n",
       "612   01001                  0.228           10            9   \n",
       "785   01003                  0.127           20           13   \n",
       "2434  01005                  0.234           13            9   \n",
       "3115  01007                  0.179            7            6   \n",
       "2787  01009                  0.242           12           11   \n",
       "\n",
       "      injury_deaths_per_100000  \n",
       "612                       73.3  \n",
       "785                       74.5  \n",
       "2434                      56.1  \n",
       "3115                      80.8  \n",
       "2787                      91.9  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Health\n",
    "FCC = pd.read_csv('DataMerge/Data/Q-FCC/c2hgis_county.csv')\n",
    "\n",
    "\n",
    "FCC = FCC.sort_values(by = 'geography_id')\n",
    "FCC['GEOID'] = FCC['geography_id'].apply(modifyGEOID)\n",
    "FCC['Rate_poor_fair_health'] = FCC['poor_fair_health_pct'].astype(float)/100\n",
    "FCC = FCC[['GEOID', 'Rate_poor_fair_health', 'provcount_b', 'provcount_c', 'injury_deaths_per_100000']]\n",
    "\n",
    "FCC.to_csv('DataMerge/Q-FCC.csv', index = False)\n",
    "\n",
    "FCC.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R. Health Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alabama DataMerge/Data/R-CountyHealthRanking/Alabama.xls\n",
      "North Dakota DataMerge/Data/R-CountyHealthRanking/North Dakota.xls\n",
      "West Virginia DataMerge/Data/R-CountyHealthRanking/West Virginia.xls\n",
      "Wyoming DataMerge/Data/R-CountyHealthRanking/Wyoming.xls\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from DataMerge import us_popu\n",
    "\n",
    "# path_cc  = 'DataMerge/Data/C-AnnualCountyPopulation/cc-est2019-alldata.csv'\n",
    "# path_acs = 'DataMerge/Data/I-INTERNET/ACSST5Y2019.S2802_2021-03-20T111825/ACSST5Y2019.S2802_data_with_overlays_2021-03-20T111812.csv'\n",
    "# US_COUNTY_POPULATION = us_popu.get_US_COUNTY_POPULATION(path_cc, path_acs)\n",
    "\n",
    "path = 'DataMerge/Data/C-AnnualCountyPopulation/US_COUNTY_POPULATION.csv'\n",
    "\n",
    "US_COUNTY_POPULATION = pd.read_csv(path)\n",
    "States = US_COUNTY_POPULATION['State'].unique()\n",
    "\n",
    "\n",
    "\n",
    "import requests\n",
    "\n",
    "def get_url(state):\n",
    "    \n",
    "    url = 'https://www.countyhealthrankings.org/sites/default/files/media/document/2020%20County%20Health%20Rankings%20'\n",
    "    url = url + state.replace(' ', '%20') + '%20Data%20-%20v1_0.xlsx'\n",
    "    return url\n",
    "\n",
    "header = {\n",
    "  \"User-Agent\": \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36\",\n",
    "  \"X-Requested-With\": \"XMLHttpRequest\"\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "D = {}\n",
    "for state in States:\n",
    "\n",
    "    tmp_path = 'DataMerge/Data/R-CountyHealthRanking/' + state + '.xls'\n",
    "    D[state] = tmp_path\n",
    "    \n",
    "#     url = get_url(state)\n",
    "#     resp = requests.get(url, headers=header)\n",
    "#     output = open(tmp_path, 'wb')\n",
    "#     output.write(resp.content)\n",
    "#     output.close()\n",
    "\n",
    "def get_heatlth_ranking(state, D):\n",
    "    tmp_path = D[state]\n",
    "\n",
    "    data = pd.read_excel(tmp_path, sheet_name ='Additional Measure Data', skiprows=1)[['FIPS', 'Segregation index', 'Average Traffic Volume per Meter of Major Roadways']]\n",
    "\n",
    "    data1 = data.rename(columns = {'FIPS': 'GEOID'})\n",
    "    data1\n",
    "    \n",
    "    cols = ['FIPS',\n",
    "     '% Vaccinated',\n",
    "     '% Vaccinated (Asian)',\n",
    "     '% Vaccinated (Black)',\n",
    "     '% Vaccinated (White)',\n",
    "     'Violent Crime Rate'\n",
    "    ]\n",
    "    data = pd.read_excel(tmp_path, sheet_name ='Ranked Measure Data', skiprows=1)[cols]\n",
    "\n",
    "\n",
    "    data2 = data.rename(columns = {'FIPS': 'GEOID'})\n",
    "    data2\n",
    "\n",
    "    data = pd.merge(data1, data2,  on = 'GEOID')\n",
    "    return data\n",
    "\n",
    "\n",
    "D2 = {}\n",
    "for state in States:\n",
    "    try:\n",
    "        D2[state] = get_heatlth_ranking(state, D)\n",
    "    except:\n",
    "        print(state, D[state])\n",
    "\n",
    "    \n",
    "        \n",
    "tmp_path = 'DataMerge/Data/R-CountyHealthRanking/2020 County Health Rankings West Virginia Data - v1.xlsx'\n",
    "\n",
    "data = pd.read_excel(tmp_path, sheet_name ='Additional Measure Data', skiprows=1)[['FIPS', 'Segregation index', 'Average Traffic Volume per Meter of Major Roadways']]\n",
    "\n",
    "data1 = data.rename(columns = {'FIPS': 'GEOID'})\n",
    "data1\n",
    "\n",
    "cols = ['FIPS',\n",
    " '% Vaccinated',\n",
    " '% Vaccinated (Asian)',\n",
    " '% Vaccinated (Black)',\n",
    " '% Vaccinated (White)',\n",
    " 'Violent Crime Rate'\n",
    "]\n",
    "data = pd.read_excel(tmp_path, sheet_name ='Ranked Measure Data', skiprows=1)[cols]\n",
    "\n",
    "\n",
    "data2 = data.rename(columns = {'FIPS': 'GEOID'})\n",
    "# data2\n",
    "\n",
    "data = pd.merge(data1, data2,  on = 'GEOID')\n",
    "D2['West Virginia'] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>Segregation index</th>\n",
       "      <th>Average Traffic Volume per Meter of Major Roadways</th>\n",
       "      <th>% Vaccinated</th>\n",
       "      <th>% Vaccinated (Asian)</th>\n",
       "      <th>% Vaccinated (Black)</th>\n",
       "      <th>% Vaccinated (White)</th>\n",
       "      <th>Violent Crime Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02000</td>\n",
       "      <td>51.512262</td>\n",
       "      <td>217.596161</td>\n",
       "      <td>35.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>720.263271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02016</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>294.617564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02020</td>\n",
       "      <td>38.379522</td>\n",
       "      <td>462.572067</td>\n",
       "      <td>38.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1006.657195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>02050</td>\n",
       "      <td>13.872255</td>\n",
       "      <td>4.727448</td>\n",
       "      <td>10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>54101</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>54103</td>\n",
       "      <td>61.871617</td>\n",
       "      <td>91.509904</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>89.333572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>54105</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.065980</td>\n",
       "      <td>39.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>245.741886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54107</td>\n",
       "      <td>54.575574</td>\n",
       "      <td>69.036711</td>\n",
       "      <td>49.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>300.438163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>54109</td>\n",
       "      <td>47.866961</td>\n",
       "      <td>16.666769</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>625.689156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3047 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    GEOID  Segregation index  \\\n",
       "0   02000          51.512262   \n",
       "1   02013                NaN   \n",
       "2   02016                NaN   \n",
       "3   02020          38.379522   \n",
       "4   02050          13.872255   \n",
       "..    ...                ...   \n",
       "51  54101                NaN   \n",
       "52  54103          61.871617   \n",
       "53  54105                NaN   \n",
       "54  54107          54.575574   \n",
       "55  54109          47.866961   \n",
       "\n",
       "    Average Traffic Volume per Meter of Major Roadways  % Vaccinated  \\\n",
       "0                                          217.596161           35.0   \n",
       "1                                            0.000000            NaN   \n",
       "2                                            0.000000            NaN   \n",
       "3                                          462.572067           38.0   \n",
       "4                                            4.727448           10.0   \n",
       "..                                                ...            ...   \n",
       "51                                           0.000000           26.0   \n",
       "52                                          91.509904           40.0   \n",
       "53                                           0.065980           39.0   \n",
       "54                                          69.036711           49.0   \n",
       "55                                          16.666769           29.0   \n",
       "\n",
       "    % Vaccinated (Asian)  % Vaccinated (Black)  % Vaccinated (White)  \\\n",
       "0                    NaN                   NaN                   NaN   \n",
       "1                    NaN                   NaN                   NaN   \n",
       "2                    NaN                   NaN                   NaN   \n",
       "3                   44.0                  26.0                  39.0   \n",
       "4                    NaN                   NaN                   NaN   \n",
       "..                   ...                   ...                   ...   \n",
       "51                   NaN                   NaN                   NaN   \n",
       "52                   NaN                   NaN                   NaN   \n",
       "53                   NaN                   NaN                   NaN   \n",
       "54                  31.0                  44.0                  49.0   \n",
       "55                   NaN                  36.0                  30.0   \n",
       "\n",
       "    Violent Crime Rate  \n",
       "0           720.263271  \n",
       "1                  NaN  \n",
       "2           294.617564  \n",
       "3          1006.657195  \n",
       "4                  NaN  \n",
       "..                 ...  \n",
       "51                 NaN  \n",
       "52           89.333572  \n",
       "53          245.741886  \n",
       "54          300.438163  \n",
       "55          625.689156  \n",
       "\n",
       "[3047 rows x 8 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HealthRanking = pd.concat([v for k,v in D2.items()])\n",
    "HealthRanking['GEOID'] = HealthRanking['GEOID'].apply(modifyGEOID)\n",
    "HealthRanking.to_csv('DataMerge/R-HealthRanking.csv', index = False)\n",
    "HealthRanking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# S. ADI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ADIMean</th>\n",
       "      <th>ADIStd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3212.000000</td>\n",
       "      <td>3171.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>67.024803</td>\n",
       "      <td>13.859699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>17.779891</td>\n",
       "      <td>5.012005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.909091</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>57.210714</td>\n",
       "      <td>10.639848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>70.782353</td>\n",
       "      <td>13.901507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>80.045767</td>\n",
       "      <td>17.020868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>99.444444</td>\n",
       "      <td>36.773926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ADIMean       ADIStd\n",
       "count  3212.000000  3171.000000\n",
       "mean     67.024803    13.859699\n",
       "std      17.779891     5.012005\n",
       "min       2.909091     0.000000\n",
       "25%      57.210714    10.639848\n",
       "50%      70.782353    13.901507\n",
       "75%      80.045767    17.020868\n",
       "max      99.444444    36.773926"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "adi = pd.read_csv(\"DataMerge/Data/S-Area Deprivation Index/us_bg.txt\", sep=',', thousands = ',')\n",
    "\n",
    "def county_fips(x):\n",
    "    if len(str(x)) == 11:\n",
    "        fips = str(0) + str(x)[0:4]\n",
    "    else:\n",
    "        fips = str(x)[0:5]\n",
    "    return fips\n",
    "\n",
    "adi['GEOID'] = adi['FIPS'].apply(lambda x: county_fips(x))\n",
    "\n",
    "\n",
    "def convert_int(x):\n",
    "    try:\n",
    "        return float(x)\n",
    "    except:\n",
    "        return np.nan\n",
    "    \n",
    "adi['ADI_NATRANK'] = adi['ADI_NATRANK'].apply(lambda x: convert_int(x))\n",
    "\n",
    "\n",
    "\n",
    "L = []\n",
    "error = []\n",
    "for county, df in adi.groupby('GEOID'):\n",
    "    \n",
    "    d =  {}\n",
    "    d['GEOID'] = county\n",
    "    d['ADIMean'] = df['ADI_NATRANK'].mean()\n",
    "    d['ADIStd'] = df['ADI_NATRANK'].std()\n",
    "    if np.isnan(d['ADIStd']):\n",
    "        error.append(df)\n",
    "    L.append(d)\n",
    "\n",
    "ADI = pd.DataFrame(L)\n",
    "\n",
    "ADI.to_csv('DataMerge/S-ADI.csv', index = False)\n",
    "ADI.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>ADIMean</th>\n",
       "      <th>ADIStd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>64.375000</td>\n",
       "      <td>19.410257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>52.923913</td>\n",
       "      <td>19.098492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>83.714286</td>\n",
       "      <td>13.187657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>79.500000</td>\n",
       "      <td>14.097190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>69.857143</td>\n",
       "      <td>13.237964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID    ADIMean     ADIStd\n",
       "0  01001  64.375000  19.410257\n",
       "1  01003  52.923913  19.098492\n",
       "2  01005  83.714286  13.187657\n",
       "3  01007  79.500000  14.097190\n",
       "4  01009  69.857143  13.237964"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ADI.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T. SDI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['county', 'population', 'sdi_score', 'fpl_100_score',\n",
      "       'sing_parent_fam_score', 'black_score', 'dropout_score', 'no_car_score',\n",
      "       'rent_occup_score', 'crowding_score', 'nonemp_score', 'unemp_score',\n",
      "       'highneeds_score', 'hisp_score', 'foreignb_score', 'lingisol_score',\n",
      "       'percnt_ltfpl100', 'percnt_singlparntfly', 'percnt_black',\n",
      "       'percnt_dropout', 'percnt_hhnocar', 'percnt_rentoccup',\n",
      "       'percnt_crowding', 'percnt_nonemp', 'percnt_unemp', 'percnt_highneeds',\n",
      "       'percnt_hispanic', 'percnt_frgnborn', 'percnt_lingisol', 'GEOID'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>sdi_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID  sdi_score\n",
       "0  01001         22\n",
       "1  01003         15\n",
       "2  01005         88\n",
       "3  01007         31\n",
       "4  01009         36"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path ='DataMerge/Data/T-SDI/ACS2015_countyallvars.xlsx'\n",
    "\n",
    "SDI = pd.read_excel(path)\n",
    "\n",
    "SDI['GEOID'] = SDI['county'].apply(modifyGEOID)\n",
    "print(SDI.columns)\n",
    "SDI = SDI[['GEOID', 'sdi_score']]\n",
    "\n",
    "\n",
    "SDI.to_csv('DataMerge/T-SDI.csv', index = False)\n",
    "SDI.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sdi_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3142.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>40.687460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>26.932835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>17.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>39.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sdi_score\n",
       "count  3142.000000\n",
       "mean     40.687460\n",
       "std      26.932835\n",
       "min       1.000000\n",
       "25%      17.000000\n",
       "50%      39.000000\n",
       "75%      60.000000\n",
       "max     100.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SDI.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# U. Flu Vaccination (*Disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>urban</th>\n",
       "      <th>FluVax_BlackRate</th>\n",
       "      <th>FluVax_NHWhiteRate</th>\n",
       "      <th>FluVax_WholeRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>1</td>\n",
       "      <td>27.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>1</td>\n",
       "      <td>19.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>1</td>\n",
       "      <td>25.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3215</th>\n",
       "      <td>72145</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>72147</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55.0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>72149</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3218</th>\n",
       "      <td>72151</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3219</th>\n",
       "      <td>72153</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3220 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      GEOID  urban  FluVax_BlackRate  FluVax_NHWhiteRate  FluVax_WholeRate\n",
       "0     01001      1              27.0                44.0                42\n",
       "1     01003      0              27.0                48.0                47\n",
       "2     01005      0              29.0                38.0                35\n",
       "3     01007      1              19.0                33.0                31\n",
       "4     01009      1              25.0                39.0                39\n",
       "...     ...    ...               ...                 ...               ...\n",
       "3215  72145      1               NaN                 NaN                15\n",
       "3216  72147      0               NaN                55.0                18\n",
       "3217  72149      1               NaN                 NaN                11\n",
       "3218  72151      1               NaN                 NaN                17\n",
       "3219  72153      1               NaN                 NaN                16\n",
       "\n",
       "[3220 rows x 5 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "path = 'DataMerge/Data/U-FluVax/FluVax_Black.csv'\n",
    "FluBlack = pd.read_csv(path)[['fips', 'analysis_value']].rename(columns = {'analysis_value':'FluVax_BlackRate'})\n",
    "\n",
    "path = 'DataMerge/Data/U-FluVax/FluVax_White.csv'\n",
    "FluWhite = pd.read_csv(path)[['fips', 'analysis_value']].rename(columns = {'analysis_value':'FluVax_NHWhiteRate'})\n",
    "\n",
    "path = 'DataMerge/Data/U-FluVax/FluVax_Whole.csv'\n",
    "FluWhole = pd.read_csv(path)[['fips', 'urban', 'analysis_value']].rename(columns = {'analysis_value':'FluVax_WholeRate'})\n",
    "\n",
    "\n",
    "# path = 'DataMerge/Data/U-FluVax/FluVaxAA_Black.csv'\n",
    "# FluAABlack = pd.read_csv(path)[['fips', 'analysis_value']].rename(columns = {'analysis_value':'FluAABlack'})\n",
    "\n",
    "# path = 'DataMerge/Data/U-FluVax/FluVaxAA_White.csv'\n",
    "# FluAAWhite = pd.read_csv(path)[['fips', 'analysis_value']].rename(columns = {'analysis_value':'FluAAWhite'})\n",
    "\n",
    "# path = 'DataMerge/Data/U-FluVax/FluVaxAA_Whole.csv'\n",
    "# FluAAWhole = pd.read_csv(path)[['fips', 'analysis_value']].rename(columns = {'analysis_value':'FluAAWhole'})\n",
    "\n",
    "\n",
    "df = pd.merge(FluWhole, FluWhite, on = 'fips', how = 'left')\n",
    "df = pd.merge(df, FluBlack, on = 'fips', how = 'left')\n",
    "# df = pd.merge(df, FluAAWhole, on = 'fips', how = 'left')\n",
    "# df = pd.merge(df, FluAAWhite, on = 'fips', how = 'left')\n",
    "# df = pd.merge(df, FluAABlack, on = 'fips', how = 'left')\n",
    "\n",
    "for i in ['FluVax_BlackRate', 'FluVax_NHWhiteRate','FluVax_WholeRate']:\n",
    "    df[i] = df[i]\n",
    "    \n",
    "df['GEOID'] = df['fips'].apply(convertFIPS)\n",
    "\n",
    "FluVax = df[['GEOID', 'urban',  'FluVax_BlackRate', 'FluVax_NHWhiteRate','FluVax_WholeRate']]\n",
    "FluVax = FluVax.reset_index(drop = True)\n",
    "FluVax.loc[FluVax.index, 'urban'] = (FluVax['urban'] == 'Urban').astype(int)\n",
    "FluVax.to_csv('DataMerge/U-FluVax.csv', index = False)\n",
    "\n",
    "FluVax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# V. Race_Population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_county_id(v):\n",
    "    v = str(v)\n",
    "    if len(v) == 1:\n",
    "        return '00' + str(v)\n",
    "    elif len(v) == 2:\n",
    "        return '0' + str(v)\n",
    "    elif len(v) == 3:\n",
    "        return v\n",
    "    else:\n",
    "        raise ValueError('Error')\n",
    "    \n",
    "def get_state_id(v):\n",
    "    v = str(v)\n",
    "    if len(v) == 1:\n",
    "        return '0' + str(v)\n",
    "    elif len(v) == 2:\n",
    "        return v\n",
    "    else:\n",
    "        raise ValueError('Error')\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "def remove_string(s):\n",
    "    return s.replace(' County', '')\n",
    "\n",
    "def get_county(s):\n",
    "    return s.split(', ')[0]\n",
    "\n",
    "def get_state(s):\n",
    "    return s.split(', ')[1]\n",
    "\n",
    "def getGEOID(idx):\n",
    "    return idx.split('US')[-1]\n",
    "\n",
    "\n",
    "def change_col_name(name):\n",
    "    d = {}\n",
    "    \n",
    "    d['TOT_POP'] = 'WholeNum'\n",
    "    d['WA'] = 'WhiteNum'\n",
    "    d['BA'] = 'BlackNum'\n",
    "    d['NHWA'] = 'NHWhiteNum'\n",
    "    d['NHBA'] = 'NHBlackNum'\n",
    "    # d['IA'] = 'Total_AIndA'\n",
    "    # d['AA'] = 'Total_Asian'\n",
    "    \n",
    "    for k, v in d.items():\n",
    "        if k in name:\n",
    "            name = name.replace(k, v)\n",
    "    \n",
    "    return name\n",
    "\n",
    "\n",
    "\n",
    "def get_US_COUNTY_POPULATION(path_cc, path_acs):\n",
    "    # path_cc = 'Data/cc-est2019-alldata.csv'\n",
    "\n",
    "    CCALLDATA = pd.read_csv(path_cc)\n",
    "\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['YEAR'] == 12]  # 12: 2019-7-1\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['AGEGRP'] == 0] #  0: Age Group Total\n",
    "    CCALLDATA = CCALLDATA[CCALLDATA['SUMLEV'] == 50] #  50: County and/or Statistical Equivalent\n",
    "    CCALLDATA = CCALLDATA.reset_index(drop = True)\n",
    "\n",
    "\n",
    "    CCALLDATA['GEOID'] = CCALLDATA['STATE'].apply(get_state_id) + CCALLDATA['COUNTY'].apply(get_county_id)\n",
    "    # POPULATION\n",
    "\n",
    "    PrefixCols = ['GEOID', 'TOT_POP'] \n",
    "    RaceCols = ['WA', 'BA', 'NHWA', 'NHBA']\n",
    "\n",
    "    for i in RaceCols:\n",
    "\n",
    "        CCALLDATA[i] = CCALLDATA[i + '_MALE'] + CCALLDATA[i +'_FEMALE']\n",
    "\n",
    "\n",
    "    CCALLDATA = CCALLDATA[PrefixCols + RaceCols]\n",
    "\n",
    "    # print(CCALLDATA.shape)\n",
    "    # CCALLDATA.head()\n",
    "    \n",
    "    INTERNET_SUBSCRIPTIONS = pd.read_csv(path_acs, skiprows=[0], low_memory=False)[['id', 'Geographic Area Name'] ]\n",
    "    # How to understand total population in household?\n",
    "\n",
    "\n",
    "    INTERNET_SUBSCRIPTIONS['Geographic Area Name'] = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(remove_string)\n",
    "    INTERNET_SUBSCRIPTIONS['County'] = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(get_county)\n",
    "    INTERNET_SUBSCRIPTIONS['State']  = INTERNET_SUBSCRIPTIONS['Geographic Area Name'].apply(get_state)\n",
    "\n",
    "\n",
    "\n",
    "    INTERNET_SUBSCRIPTIONS['GEOID'] = INTERNET_SUBSCRIPTIONS['id'].apply(getGEOID)\n",
    "    INTERNET_SUBSCRIPTIONS = INTERNET_SUBSCRIPTIONS[['GEOID', 'State', 'County', 'Geographic Area Name', ]]\n",
    "    # print(INTERNET_SUBSCRIPTIONS.shape)\n",
    "    # INTERNET_SUBSCRIPTIONS.head()\n",
    "\n",
    "\n",
    "\n",
    "    US_COUNTY_POPULATION = pd.merge(INTERNET_SUBSCRIPTIONS, CCALLDATA, on = 'GEOID')\n",
    "\n",
    "    cols_dict = {name: change_col_name(name) for name in US_COUNTY_POPULATION.columns}\n",
    "\n",
    "    US_COUNTY_POPULATION = US_COUNTY_POPULATION.rename(columns = cols_dict)\n",
    "\n",
    "\n",
    "    # print(US_COUNTY_POPULATION.shape)\n",
    "    # US_COUNTY_POPULATION.head()\n",
    "    \n",
    "    \n",
    "    return US_COUNTY_POPULATION\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_cc = 'DataMerge/Data/C-AnnualCountyPopulation/cc-est2019-alldata.csv'\n",
    "path_acs = 'DataMerge/Data/I-INTERNET/ACSST5Y2019.S2802_2021-03-20T111825/ACSST5Y2019.S2802_data_with_overlays_2021-03-20T111812.csv'\n",
    "\n",
    "\n",
    "US_COUNTY_POPULATION = get_US_COUNTY_POPULATION(path_cc, path_acs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GEOID</th>\n",
       "      <th>State</th>\n",
       "      <th>County</th>\n",
       "      <th>Geographic Area Name</th>\n",
       "      <th>WholeNum</th>\n",
       "      <th>WhiteNum</th>\n",
       "      <th>BlackNum</th>\n",
       "      <th>NHWhiteNum</th>\n",
       "      <th>NHBlackNum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>Autauga</td>\n",
       "      <td>Autauga, Alabama</td>\n",
       "      <td>55869</td>\n",
       "      <td>42607</td>\n",
       "      <td>11237</td>\n",
       "      <td>41215</td>\n",
       "      <td>11098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01003</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>Baldwin</td>\n",
       "      <td>Baldwin, Alabama</td>\n",
       "      <td>223234</td>\n",
       "      <td>195198</td>\n",
       "      <td>19593</td>\n",
       "      <td>185747</td>\n",
       "      <td>19215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01005</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>Barbour</td>\n",
       "      <td>Barbour, Alabama</td>\n",
       "      <td>24686</td>\n",
       "      <td>12134</td>\n",
       "      <td>11906</td>\n",
       "      <td>11235</td>\n",
       "      <td>11807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01007</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>Bibb</td>\n",
       "      <td>Bibb, Alabama</td>\n",
       "      <td>22394</td>\n",
       "      <td>17191</td>\n",
       "      <td>4763</td>\n",
       "      <td>16663</td>\n",
       "      <td>4719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01009</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>Blount</td>\n",
       "      <td>Blount, Alabama</td>\n",
       "      <td>57826</td>\n",
       "      <td>55412</td>\n",
       "      <td>978</td>\n",
       "      <td>50176</td>\n",
       "      <td>872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   GEOID    State   County Geographic Area Name  WholeNum  WhiteNum  BlackNum  \\\n",
       "0  01001  Alabama  Autauga     Autauga, Alabama     55869     42607     11237   \n",
       "1  01003  Alabama  Baldwin     Baldwin, Alabama    223234    195198     19593   \n",
       "2  01005  Alabama  Barbour     Barbour, Alabama     24686     12134     11906   \n",
       "3  01007  Alabama     Bibb        Bibb, Alabama     22394     17191      4763   \n",
       "4  01009  Alabama   Blount      Blount, Alabama     57826     55412       978   \n",
       "\n",
       "   NHWhiteNum  NHBlackNum  \n",
       "0       41215       11098  \n",
       "1      185747       19215  \n",
       "2       11235       11807  \n",
       "3       16663        4719  \n",
       "4       50176         872  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "US_COUNTY_POPULATION.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "AdultPopu = AgeDisparity[['GEOID'] + [i for i in AgeDisparity.columns if ('Above15' in i or 'Above20' in i) and 'Num' in i]]\n",
    "US_COUNTY_POPULATION = US_COUNTY_POPULATION.merge(AdultPopu, on = 'GEOID')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['GEOID', 'State', 'County', 'Geographic Area Name', 'WholeNum',\n",
       "       'WhiteNum', 'BlackNum', 'NHWhiteNum', 'NHBlackNum', 'Above15_WholeNum',\n",
       "       'Above15_WhiteNum', 'Above15_NHWhiteNum', 'Above15_BlackNum',\n",
       "       'Above15_NHBlackNum', 'Above20_WholeNum', 'Above20_WhiteNum',\n",
       "       'Above20_NHWhiteNum', 'Above20_BlackNum', 'Above20_NHBlackNum'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "US_COUNTY_POPULATION.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'DataMerge/V-US_COUNTY_RACE_POPULATION.csv'\n",
    "\n",
    "US_COUNTY_POPULATION.to_csv(path, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}