Weather Crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import requests\n",
    "import itertools\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "#url = 'http://13.54.173.115/brokenapp.php' + '?grppwd=lol'\n",
    "url = 'https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019'\n",
    "url = 'https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190405'\n",
    "response = requests.get(url)\n",
    "soup = BeautifulSoup(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190401\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190402\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190403\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190404\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190405\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190406\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190407\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190408\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190409\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190410\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190411\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190412\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190413\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190414\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190415\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190416\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190417\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190418\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190419\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190420\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190421\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190422\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190423\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190424\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190425\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190426\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190427\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190428\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190429\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=20190430\n"
     ]
    }
   ],
   "source": [
    "main_arr = []\n",
    "for i in range(1,31):\n",
    "    current_date = '201904' + str(i).zfill(2)\n",
    "    #print(current_date)\n",
    "    url = 'https://www.timeanddate.com/weather/singapore/singapore/historic?month=4&year=2019&hd=' + current_date\n",
    "    print(url)\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content)\n",
    "    \n",
    "    table = soup.find( \"table\", {\"id\":\"wt-his\"} )\n",
    "    rows=list()\n",
    "\n",
    "    hourly_time = ['1:00',\n",
    "                   '2:00',\n",
    "                   '3:00',\n",
    "                   '4:00',\n",
    "                   '5:00',\n",
    "                   '6:00',\n",
    "                   '7:00',\n",
    "                   '8:00',\n",
    "                   '9:00',\n",
    "                   '10:00',\n",
    "                   '11:00',\n",
    "                   '12:00',\n",
    "                   '13:00',\n",
    "                   '14:00',\n",
    "                   '15:00',\n",
    "                   '16:00',\n",
    "                   '17:00',\n",
    "                   '18:00',\n",
    "                   '19:00',\n",
    "                   '20:00',\n",
    "                   '21:00',\n",
    "                   '22:00',\n",
    "                   '23:00']\n",
    "\n",
    "    # To skip headers\n",
    "    header_counter = 2\n",
    "    counter = 0\n",
    "    for row in table.findAll(\"tr\"):\n",
    "        #print(row)\n",
    "        try:\n",
    "            if counter > header_counter:\n",
    "                time = row.findAll('th')[0]\n",
    "                time = str(time)\n",
    "                time = time.replace('<th>', '')\n",
    "                time = time.replace('.</th>', '')\n",
    "                #print(time)\n",
    "                if time in hourly_time:\n",
    "                    #print(row)\n",
    "                    temp_arr = []\n",
    "                    weather_string = row.findAll('td')[2]\n",
    "                    weather_string = str(weather_string).replace('<td class=\"small\">','')\n",
    "                    weather_string = weather_string.replace('</td>','')\n",
    "                    #print(time,weather_string)\n",
    "                    temp_arr.append(current_date)\n",
    "                    temp_arr.append(time)\n",
    "                    temp_arr.append(weather_string)\n",
    "                    main_arr.append(temp_arr)\n",
    "                    #print('')\n",
    "        except:\n",
    "            pass\n",
    "        counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1:00 Passing clouds.\n",
      "\n",
      "2:00 Partly cloudy.\n",
      "\n",
      "3:00 Partly cloudy.\n",
      "\n",
      "4:00 Passing clouds.\n",
      "\n",
      "5:00 Passing clouds.\n",
      "\n",
      "6:00 Partly cloudy.\n",
      "\n",
      "7:00 Passing clouds.\n",
      "\n",
      "8:00 Partly sunny.\n",
      "\n",
      "9:00 Partly sunny.\n",
      "\n",
      "10:00 Partly sunny.\n",
      "\n",
      "11:00 Partly sunny.\n",
      "\n",
      "12:00 Broken clouds.\n",
      "\n",
      "13:00 Broken clouds.\n",
      "\n",
      "14:00 Scattered clouds.\n",
      "\n",
      "15:00 Scattered showers. Scattered clouds.\n",
      "\n",
      "16:00 Thundershowers. Scattered clouds.\n",
      "\n",
      "17:00 Scattered clouds.\n",
      "\n",
      "18:00 Scattered clouds.\n",
      "\n",
      "19:00 Partly sunny.\n",
      "\n",
      "20:00 Passing clouds.\n",
      "\n",
      "21:00 Passing clouds.\n",
      "\n",
      "22:00 Passing clouds.\n",
      "\n",
      "23:00 Passing clouds.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = soup.find( \"table\", {\"id\":\"wt-his\"} )\n",
    "rows=list()\n",
    "\n",
    "hourly_time = ['1:00',\n",
    "               '2:00',\n",
    "               '3:00',\n",
    "               '4:00',\n",
    "               '5:00',\n",
    "               '6:00',\n",
    "               '7:00',\n",
    "               '8:00',\n",
    "               '9:00',\n",
    "               '10:00',\n",
    "               '11:00',\n",
    "               '12:00',\n",
    "               '13:00',\n",
    "               '14:00',\n",
    "               '15:00',\n",
    "               '16:00',\n",
    "               '17:00',\n",
    "               '18:00',\n",
    "               '19:00',\n",
    "               '20:00',\n",
    "               '21:00',\n",
    "               '22:00',\n",
    "               '23:00']\n",
    "\n",
    "# To skip headers\n",
    "header_counter = 2\n",
    "counter = 0\n",
    "for row in table.findAll(\"tr\"):\n",
    "    #print(row)\n",
    "    try:\n",
    "        if counter > header_counter:\n",
    "            time = row.findAll('th')[0]\n",
    "            time = str(time)\n",
    "            time = time.replace('<th>', '')\n",
    "            time = time.replace('.</th>', '')\n",
    "            #print(time)\n",
    "            if time in hourly_time:\n",
    "                #print(row)\n",
    "                weather_string = row.findAll('td')[2]\n",
    "                weather_string = str(weather_string).replace('<td class=\"small\">','')\n",
    "                weather_string = weather_string.replace('</td>','')\n",
    "                print(time,weather_string)\n",
    "\n",
    "                print('')\n",
    "    except:\n",
    "        pass\n",
    "    counter += 1\n",
    "        \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20190401</td>\n",
       "      <td>1:00</td>\n",
       "      <td>Passing clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20190401</td>\n",
       "      <td>2:00</td>\n",
       "      <td>Thundershowers. Partly cloudy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20190401</td>\n",
       "      <td>3:00</td>\n",
       "      <td>Thunderstorms. Passing clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20190401</td>\n",
       "      <td>4:00</td>\n",
       "      <td>Thunderstorms. Passing clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20190401</td>\n",
       "      <td>5:00</td>\n",
       "      <td>Thunderstorms. Passing clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>20190430</td>\n",
       "      <td>19:00</td>\n",
       "      <td>Broken clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>20190430</td>\n",
       "      <td>20:00</td>\n",
       "      <td>Partly cloudy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>684</th>\n",
       "      <td>20190430</td>\n",
       "      <td>21:00</td>\n",
       "      <td>Partly cloudy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>20190430</td>\n",
       "      <td>22:00</td>\n",
       "      <td>Passing clouds.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>20190430</td>\n",
       "      <td>23:00</td>\n",
       "      <td>Passing clouds.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>687 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0      1                               2\n",
       "0    20190401   1:00                 Passing clouds.\n",
       "1    20190401   2:00  Thundershowers. Partly cloudy.\n",
       "2    20190401   3:00  Thunderstorms. Passing clouds.\n",
       "3    20190401   4:00  Thunderstorms. Passing clouds.\n",
       "4    20190401   5:00  Thunderstorms. Passing clouds.\n",
       "..        ...    ...                             ...\n",
       "682  20190430  19:00                  Broken clouds.\n",
       "683  20190430  20:00                  Partly cloudy.\n",
       "684  20190430  21:00                  Partly cloudy.\n",
       "685  20190430  22:00                 Passing clouds.\n",
       "686  20190430  23:00                 Passing clouds.\n",
       "\n",
       "[687 rows x 3 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(main_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "apr_weather = pd.DataFrame(main_arr)\n",
    "apr_weather.columns = ['date', 'time','weather_string']\n",
    "apr_weather['weather_string'] = apr_weather['weather_string'].str.lower()\n",
    "apr_weather['rain'] = apr_weather['weather_string'].str.contains('showers') | apr_weather['weather_string'].str.contains('rain') | apr_weather['weather_string'].str.contains('storm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>weather_string</th>\n",
       "      <th>rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>20190430</td>\n",
       "      <td>19:00</td>\n",
       "      <td>broken clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>20190430</td>\n",
       "      <td>20:00</td>\n",
       "      <td>partly cloudy.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>684</th>\n",
       "      <td>20190430</td>\n",
       "      <td>21:00</td>\n",
       "      <td>partly cloudy.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>20190430</td>\n",
       "      <td>22:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>20190430</td>\n",
       "      <td>23:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date   time   weather_string   rain\n",
       "682  20190430  19:00   broken clouds.  False\n",
       "683  20190430  20:00   partly cloudy.  False\n",
       "684  20190430  21:00   partly cloudy.  False\n",
       "685  20190430  22:00  passing clouds.  False\n",
       "686  20190430  23:00  passing clouds.  False"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_weather.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "687"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(apr_weather)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "apr_weather.to_csv('APR_SG_hourly_weather.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190501\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190502\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190503\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190504\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190505\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190506\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190507\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190508\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190509\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190510\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190511\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190512\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190513\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190514\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190515\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190516\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190517\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190518\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190519\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190520\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190521\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190522\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190523\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190524\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190525\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190526\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190527\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190528\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190529\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190530\n",
      "https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=20190531\n"
     ]
    }
   ],
   "source": [
    "may_main_arr = []\n",
    "for i in range(1,32):\n",
    "    current_date = '201905' + str(i).zfill(2)\n",
    "    #print(current_date)\n",
    "    url = 'https://www.timeanddate.com/weather/singapore/singapore/historic?month=5&year=2019&hd=' + current_date\n",
    "    print(url)\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content)\n",
    "    \n",
    "    table = soup.find( \"table\", {\"id\":\"wt-his\"} )\n",
    "    rows=list()\n",
    "\n",
    "    hourly_time = ['1:00',\n",
    "                   '2:00',\n",
    "                   '3:00',\n",
    "                   '4:00',\n",
    "                   '5:00',\n",
    "                   '6:00',\n",
    "                   '7:00',\n",
    "                   '8:00',\n",
    "                   '9:00',\n",
    "                   '10:00',\n",
    "                   '11:00',\n",
    "                   '12:00',\n",
    "                   '13:00',\n",
    "                   '14:00',\n",
    "                   '15:00',\n",
    "                   '16:00',\n",
    "                   '17:00',\n",
    "                   '18:00',\n",
    "                   '19:00',\n",
    "                   '20:00',\n",
    "                   '21:00',\n",
    "                   '22:00',\n",
    "                   '23:00']\n",
    "\n",
    "    # To skip headers\n",
    "    header_counter = 2\n",
    "    counter = 0\n",
    "    for row in table.findAll(\"tr\"):\n",
    "        #print(row)\n",
    "        try:\n",
    "            if counter > header_counter:\n",
    "                time = row.findAll('th')[0]\n",
    "                time = str(time)\n",
    "                time = time.replace('<th>', '')\n",
    "                time = time.replace('.</th>', '')\n",
    "                #print(time)\n",
    "                if time in hourly_time:\n",
    "                    #print(row)\n",
    "                    temp_arr = []\n",
    "                    weather_string = row.findAll('td')[2]\n",
    "                    weather_string = str(weather_string).replace('<td class=\"small\">','')\n",
    "                    weather_string = weather_string.replace('</td>','')\n",
    "                    #print(time,weather_string)\n",
    "                    temp_arr.append(current_date)\n",
    "                    temp_arr.append(time)\n",
    "                    temp_arr.append(weather_string)\n",
    "                    may_main_arr.append(temp_arr)\n",
    "                    #print('')\n",
    "        except:\n",
    "            pass\n",
    "        counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "may_weather = pd.DataFrame(may_main_arr)\n",
    "may_weather.columns = ['date', 'time','weather_string']\n",
    "may_weather['weather_string'] = may_weather['weather_string'].str.lower()\n",
    "may_weather['rain'] = may_weather['weather_string'].str.contains('showers') | may_weather['weather_string'].str.contains('rain') | may_weather['weather_string'].str.contains('storm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>weather_string</th>\n",
       "      <th>rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20190501</td>\n",
       "      <td>1:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20190501</td>\n",
       "      <td>2:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20190501</td>\n",
       "      <td>3:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20190501</td>\n",
       "      <td>4:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20190501</td>\n",
       "      <td>5:00</td>\n",
       "      <td>passing clouds.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       date  time   weather_string   rain\n",
       "0  20190501  1:00  passing clouds.  False\n",
       "1  20190501  2:00  passing clouds.  False\n",
       "2  20190501  3:00  passing clouds.  False\n",
       "3  20190501  4:00  passing clouds.  False\n",
       "4  20190501  5:00  passing clouds.  False"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may_weather.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "may_weather.to_csv('MAY_SG_hourly_weather.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}