Seatgeek_to_S3_nonscheduled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SeatGeek Events Uploaded to S3 9_13_2018\n",
      "SeatGeek Venues Uploaded to S3 9_13_2018\n",
      "SeatGeek Prices Uploaded to S3 9_13_2018\n"
     ]
    }
   ],
   "source": [
    "from io import StringIO\n",
    "import boto3\n",
    "from botocore.client import Config\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "df = pd.DataFrame()\n",
    "\n",
    "city_list = ['San Francisco', 'Oakland', 'Berkeley', 'San Jose',\n",
    "            'New York', 'Brooklyn', 'Bronx', 'Flushing', 'East Rutherford',\n",
    "           'Washington, DC', 'Vienna',\n",
    "            'Chicago', 'Rosemont', 'Evanston',\n",
    "           'Los Angeles', 'Hollywood', 'West Hollywood', 'Pasadena',\n",
    "           'Boston', 'Medford']\n",
    "\n",
    "for i in range(1,11):\n",
    "    responseEvents = requests.get(f'https://api.seatgeek.com/events?per_page=5000&page={i}&client_id=MTI5NzU0MTd8MTUzNTkwMTU5Mi45Nw')\n",
    "\n",
    "    eventsDict = json.loads(responseEvents.content.decode(\"utf-8\")) #dictionary\n",
    "\n",
    "    eventsList = eventsDict['events'] #list of ten dictionaries\n",
    "\n",
    "    local_date = [eventsList[i]['datetime_local'] for i in range(len(eventsList))]\n",
    "    utc_date = [eventsList[i]['datetime_utc'] for i in range(len(eventsList))]\n",
    "    eventID = [eventsList[i]['id'] for i in range(len(eventsList))]\n",
    "    score = [eventsList[i]['score'] for i in range(len(eventsList))]\n",
    "    title = [eventsList[i]['short_title'] for i in range(len(eventsList))]\n",
    "    avgPrice = [eventsList[i]['stats']['average_price'] for i in range(len(eventsList))]\n",
    "    hiPrice = [eventsList[i]['stats']['highest_price'] for i in range(len(eventsList))]\n",
    "    listCount = [eventsList[i]['stats']['listing_count'] for i in range(len(eventsList))]\n",
    "    lowPrice = [eventsList[i]['stats']['lowest_price'] for i in range(len(eventsList))]\n",
    "    lowPriceDeals = [eventsList[i]['stats']['lowest_price_good_deals'] for i in range(len(eventsList))]\n",
    "    Type = [eventsList[i]['type'] for i in range(len(eventsList))]\n",
    "    lat = [eventsList[i]['venue']['location']['lat'] for i in range(len(eventsList))]\n",
    "    lon = [eventsList[i]['venue']['location']['lon'] for i in range(len(eventsList))]\n",
    "    postal = [eventsList[i]['venue']['postal_code'] for i in range(len(eventsList))]\n",
    "    url = [eventsList[i]['url'] for i in range(len(eventsList))]\n",
    "    performer = [eventsList[i]['performers'][0]['name'] for i in range(len(eventsList))]\n",
    "    venueCity = [eventsList[i]['venue']['city'] for i in range(len(eventsList))]\n",
    "    venueState = [eventsList[i]['venue']['state'] for i in range(len(eventsList))]\n",
    "    venueName = [eventsList[i]['venue']['name'] for i in range(len(eventsList))]\n",
    "    venueID = [eventsList[i]['venue']['id'] for i in range(len(eventsList))]\n",
    "\n",
    "    dfNew = pd.DataFrame({\"local_date\": local_date, \"utc_date\": utc_date, \"eventID\": eventID, \"score\": score, \"title\":title, \n",
    "        \"avgPrice\": avgPrice, \"hiPrice\": hiPrice, \"listCount\": listCount, \"lowPrice\": lowPrice, \n",
    "        \"lowPriceDeals\": lowPriceDeals, \"Type\":Type, \"lat\": lat, \"lon\": lon, \"postal\":postal, \n",
    "        \"url\": url, \"performer\": performer, \"venueCity\": venueCity, \"venueState\": venueState,\n",
    "        \"venueName\": venueName, \"venueID\": venueID})\n",
    "    \n",
    "    df = pd.concat([df, dfNew])\n",
    "\n",
    "today = datetime.date.today()\n",
    "\n",
    "df = df.loc[df.venueCity.apply(lambda x: x in city_list)] #filter by city list\n",
    "df = df.loc[df.Type == \"concert\"] #filter by Type == concert\n",
    "\n",
    "eventsDF = df[['eventID','local_date','performer','title','utc_date', 'venueID']].drop_duplicates(subset = 'eventID')\n",
    "\n",
    "venueDF = df[['venueID','venueName','venueCity','venueState','postal','lat','lon']].drop_duplicates(subset = 'venueID')\n",
    "\n",
    "pricesDF = df[['eventID','avgPrice','hiPrice','listCount',\n",
    "               'lowPrice','lowPriceDeals','score']].groupby('eventID').mean()\n",
    "\n",
    "pricesDF = pricesDF[np.sum(pricesDF.isnull(), axis = 1) != 5].reset_index()\n",
    "pricesDF['accessDate'] = today\n",
    "pricesDF['accessTime'] = pd.Timestamp.now()\n",
    "eventsDF['accessTime'] = pd.Timestamp.now()\n",
    "venueDF['accessTime'] = pd.Timestamp.now()\n",
    "\n",
    "#if we want to get a csv without pushing to s3\n",
    "#eventsDF.to_csv(f'SeatGeek_eventsDF_{today.month}_{today.day}_{today.year}.csv', index = False) #accessTime\n",
    "#venueDF.to_csv(f'SeatGeek_venuesDF_{today.month}_{today.day}_{today.year}.csv', index = False) #accessTime\n",
    "#pricesDF.to_csv(f'SeatGeek_pricesDF_{today.month}_{today.day}_{today.year}.csv', index = False) #accessTime\n",
    "\n",
    "s3_credentials = open('s3_credentials.txt', 'r')\n",
    "ACCESS_SECRET = s3_credentials.readlines()\n",
    "ACCESS_KEY = ACCESS_SECRET[1][20:-1]\n",
    "SECRET_KEY = ACCESS_SECRET[2][24:]\n",
    "s3_credentials.close()\n",
    "\n",
    "BUCKET_NAME = 'nycdsa.ta-am'\n",
    "\n",
    "s3 = boto3.resource(\n",
    "    's3',\n",
    "    aws_access_key_id=ACCESS_KEY,\n",
    "    aws_secret_access_key=SECRET_KEY,\n",
    "    config=Config(signature_version='s3v4')\n",
    ")\n",
    "\n",
    "#################################### PUSH EVENTS TO S3 #############################################################\n",
    "eventsDF_buffer = StringIO()\n",
    "eventsDF.to_csv(eventsDF_buffer, index=False)\n",
    "\n",
    "s3.Bucket(BUCKET_NAME).put_object(Key=f'SeatGeek_eventsDF_{today.month}_{today.day}_{today.year}.csv', Body=eventsDF_buffer.getvalue())\n",
    "\n",
    "print(f'SeatGeek Events Uploaded to S3 {today.month}_{today.day}_{today.year}')\n",
    "\n",
    "\n",
    "#################################### PUSH VENUES TO S3 #############################################################\n",
    "venueDF_buffer = StringIO()\n",
    "venueDF.to_csv(venueDF_buffer, index=False)\n",
    "\n",
    "s3.Bucket(BUCKET_NAME).put_object(Key=f'SeatGeek_venueDF_{today.month}_{today.day}_{today.year}.csv', Body=venueDF_buffer.getvalue())\n",
    "\n",
    "print(f'SeatGeek Venues Uploaded to S3 {today.month}_{today.day}_{today.year}')\n",
    "\n",
    "#################################### PUSH PRICES TO S3 #############################################################\n",
    "pricesDF_buffer = StringIO()\n",
    "pricesDF.to_csv(pricesDF_buffer, index=False)\n",
    "\n",
    "s3.Bucket(BUCKET_NAME).put_object(Key=f'SeatGeek_pricesDF_{today.month}_{today.day}_{today.year}.csv', Body=pricesDF_buffer.getvalue())\n",
    "\n",
    "print(f'SeatGeek Prices Uploaded to S3 {today.month}_{today.day}_{today.year}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}