dl.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('dmenv': conda)",
   "metadata": {
    "interpreter": {
     "hash": "7443be6333979a5671edb97a6208c12f43c7c42bc49d43d9a0706d3198065d4b"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path,listdir\n",
    "from datetime import datetime,timedelta\n",
    "import time\n",
    "import json\n",
    "import requests\n",
    "from stem import Signal\n",
    "from stem.control import Controller\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import pytz\n",
    "from tqdm import tqdm\n",
    "local_tz = 'Asia/Almaty'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1=pd.read_csv('data/op/matches_done.csv', index_col=None)\n",
    "df1.drop_duplicates(subset='link').to_csv('data/op/matches_done1.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(329958, 31) (8668, 33) (338626, 33)\n"
     ]
    }
   ],
   "source": [
    "df3=pd.concat([df1, df2], axis=0)\n",
    "print(df1.shape,df2.shape,df3.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.to_csv('data/sofa/statistics.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('data/fbref/matches.csv')\n",
    "df['id']=df.link.apply(lambda x: x.split('/')[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "raw/fbref/matches/22a24ffa.htm 201030\n",
      "raw/fbref/matches/67e63bcf.htm 209407\n",
      "raw/fbref/matches/fd457ac5.htm 203377\n",
      "raw/fbref/matches/20a3bbe2.htm 203534\n",
      "raw/fbref/matches/e5b88fa8.htm 199517\n",
      "raw/fbref/matches/e0516d1f.htm 212733\n",
      "raw/fbref/matches/e6c44aaf.htm 206716\n",
      "raw/fbref/matches/b9e082c3.htm 211194\n",
      "raw/fbref/matches/3e2548f0.htm 209397\n",
      "raw/fbref/matches/ffb3031c.htm 206483\n",
      "raw/fbref/matches/ee52969d.htm 191758\n",
      "raw/fbref/matches/f41fce84.htm 199907\n",
      "raw/fbref/matches/e7837e18.htm 210982\n",
      "raw/fbref/matches/a02fd3af.htm 207772\n",
      "raw/fbref/matches/a3446c57.htm 203220\n",
      "raw/fbref/matches/157b6509.htm 202152\n",
      "raw/fbref/matches/a02fbb1d.htm 201590\n",
      "raw/fbref/matches/5bb581a0.htm 201066\n",
      "raw/fbref/matches/0a2b1965.htm 209134\n",
      "raw/fbref/matches/f22ae0bc.htm 202024\n",
      "raw/fbref/matches/62169af8.htm 212048\n",
      "raw/fbref/matches/709a9f1c.htm 215233\n",
      "raw/fbref/matches/00f2921c.htm 202406\n",
      "raw/fbref/matches/41d2a28a.htm 210281\n",
      "raw/fbref/matches/c421a4b4.htm 191917\n",
      "raw/fbref/matches/4b61bf4c.htm 208978\n",
      "raw/fbref/matches/17aa4a64.htm 199965\n",
      "raw/fbref/matches/5a8c985a.htm 211806\n",
      "raw/fbref/matches/c3294ed3.htm 207824\n",
      "raw/fbref/matches/bc3aca30.htm 212726\n",
      "raw/fbref/matches/9724e9af.htm 131884\n",
      "raw/fbref/matches/7191bed5.htm 208046\n",
      "raw/fbref/matches/03901566.htm 208210\n",
      "raw/fbref/matches/faa59a57.htm 187710\n",
      "raw/fbref/matches/a6698389.htm 210752\n",
      "raw/fbref/matches/316f3a9a.htm 197985\n",
      "raw/fbref/matches/3060bd26.htm 211347\n"
     ]
    }
   ],
   "source": [
    "folder='raw/fbref/matches/'\n",
    "for row in df.itertuples():\n",
    "    fn=folder+row.id+'.htm'\n",
    "    with open(fn, 'r', encoding='utf8') as f:\n",
    "        txt=f.read()\n",
    "        print(fn,len(txt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfd=pd.read_csv('data/op/matches_done.csv')\n",
    "#df=pd.read_csv('data/op/matches.csv')\n",
    "dfd0=dfd[dfd.done==0]\n",
    "dfd1=dfd[dfd.done==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfd0.to_csv('data/op/matches.csv', index=False)\n",
    "dfd1.to_csv('data/op/matches_done.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                ds country                   champ              t1  \\\n",
       "102151  2020-02-08      au                A-League        Brisbane   \n",
       "103389  2017-11-24     NaN          WCQ — UEFA (W)         Hungary   \n",
       "103407  2017-04-05      ve        Primera División          Zamora   \n",
       "103410  2015-10-30      bg                 A Group    Slavia Sofia   \n",
       "103411  2019-08-10     eng                  FA Cup       Bishop AL   \n",
       "122918  2015-10-20     eng              League One   Sheffield Utd   \n",
       "122925  2017-10-07     NaN          Friendlies (M)        Tanzania   \n",
       "122933  2016-11-06      nl        Dutch Eredivisie      AZ Alkmaar   \n",
       "122936  2018-11-17      it                 Serie A         AS Roma   \n",
       "122937  2020-10-17      se          Damallsvenskan      Djurgården   \n",
       "122947  2016-10-02      ru  Russian Premier League       Krasnodar   \n",
       "122951  2016-07-05     NaN           Europa League        Chikhura   \n",
       "122955  2016-09-09      hr                  1. HNL  Inter Zaprešić   \n",
       "122958  2018-05-19      fr     Division 1 Féminine          Soyaux   \n",
       "122962  2015-01-18      es        Segunda División  Sporting Gijón   \n",
       "\n",
       "                     t2  sc1  sc2  \\\n",
       "102151         Adelaide  2.0  1.0   \n",
       "103389          Ukraine  0.0  1.0   \n",
       "103407   Atlé Venezuela  4.0  0.0   \n",
       "103410    Botev Plovdiv  2.0  0.0   \n",
       "103411         Thornaby  0.0  2.0   \n",
       "122918   Fleetwood Town  3.0  0.0   \n",
       "122925           Malawi  1.0  1.0   \n",
       "122933             Ajax  2.0  2.0   \n",
       "122936     ChievoVerona  7.0  1.0   \n",
       "122937        Linköping  0.0  3.0   \n",
       "122947      Rubin Kazan  1.0  0.0   \n",
       "122951  Zimbru Chișinău  2.0  3.0   \n",
       "122955     Hajduk Split  1.0  1.0   \n",
       "122958         Bordeaux  1.0  0.0   \n",
       "122962            Betis  1.0  2.0   \n",
       "\n",
       "                                                     link  done  \n",
       "102151  /en/matches/08016b59/Brisbane-Roar-Adelaide-Un...     0  \n",
       "103389  /en/matches/408f5dcd/Hungary-Ukraine-November-...     0  \n",
       "103407  /en/matches/403d8b5a/Zamora-Atletico-Venezuela...     0  \n",
       "103410  /en/matches/a6ac4472/Slavia-Sofia-Botev-Plovdi...     0  \n",
       "103411  /en/matches/97eb5d25/Bishop-AL-Thornaby-August...     0  \n",
       "122918  /en/matches/5bce5855/Sheffield-United-Fleetwoo...     0  \n",
       "122925  /en/matches/cef8af19/Tanzania-Malawi-October-7...     0  \n",
       "122933  /en/matches/5d6b3601/AZ-Alkmaar-Ajax-November-...     0  \n",
       "122936  /en/matches/07da3f53/AS-Roma-ChievoVerona-Nove...     0  \n",
       "122937  /en/matches/76950db8/Djurgarden-Linkoping-Octo...     0  \n",
       "122947  /en/matches/14427644/Krasnodar-Rubin-Kazan-Oct...     0  \n",
       "122951  /en/matches/e849cec7/Chikhura-Sachkhere-Zimbru...     0  \n",
       "122955  /en/matches/b77964bc/Inter-Zapresic-Hajduk-Spl...     0  \n",
       "122958  /en/matches/c4f419d8/Soyaux-Bordeaux-May-19-20...     0  \n",
       "122962  /en/matches/351f08f5/Sporting-Gijon-Real-Betis...     0  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ds</th>\n      <th>country</th>\n      <th>champ</th>\n      <th>t1</th>\n      <th>t2</th>\n      <th>sc1</th>\n      <th>sc2</th>\n      <th>link</th>\n      <th>done</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>102151</th>\n      <td>2020-02-08</td>\n      <td>au</td>\n      <td>A-League</td>\n      <td>Brisbane</td>\n      <td>Adelaide</td>\n      <td>2.0</td>\n      <td>1.0</td>\n      <td>/en/matches/08016b59/Brisbane-Roar-Adelaide-Un...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>103389</th>\n      <td>2017-11-24</td>\n      <td>NaN</td>\n      <td>WCQ — UEFA (W)</td>\n      <td>Hungary</td>\n      <td>Ukraine</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>/en/matches/408f5dcd/Hungary-Ukraine-November-...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>103407</th>\n      <td>2017-04-05</td>\n      <td>ve</td>\n      <td>Primera División</td>\n      <td>Zamora</td>\n      <td>Atlé Venezuela</td>\n      <td>4.0</td>\n      <td>0.0</td>\n      <td>/en/matches/403d8b5a/Zamora-Atletico-Venezuela...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>103410</th>\n      <td>2015-10-30</td>\n      <td>bg</td>\n      <td>A Group</td>\n      <td>Slavia Sofia</td>\n      <td>Botev Plovdiv</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>/en/matches/a6ac4472/Slavia-Sofia-Botev-Plovdi...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>103411</th>\n      <td>2019-08-10</td>\n      <td>eng</td>\n      <td>FA Cup</td>\n      <td>Bishop AL</td>\n      <td>Thornaby</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>/en/matches/97eb5d25/Bishop-AL-Thornaby-August...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122918</th>\n      <td>2015-10-20</td>\n      <td>eng</td>\n      <td>League One</td>\n      <td>Sheffield Utd</td>\n      <td>Fleetwood Town</td>\n      <td>3.0</td>\n      <td>0.0</td>\n      <td>/en/matches/5bce5855/Sheffield-United-Fleetwoo...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122925</th>\n      <td>2017-10-07</td>\n      <td>NaN</td>\n      <td>Friendlies (M)</td>\n      <td>Tanzania</td>\n      <td>Malawi</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>/en/matches/cef8af19/Tanzania-Malawi-October-7...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122933</th>\n      <td>2016-11-06</td>\n      <td>nl</td>\n      <td>Dutch Eredivisie</td>\n      <td>AZ Alkmaar</td>\n      <td>Ajax</td>\n      <td>2.0</td>\n      <td>2.0</td>\n      <td>/en/matches/5d6b3601/AZ-Alkmaar-Ajax-November-...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122936</th>\n      <td>2018-11-17</td>\n      <td>it</td>\n      <td>Serie A</td>\n      <td>AS Roma</td>\n      <td>ChievoVerona</td>\n      <td>7.0</td>\n      <td>1.0</td>\n      <td>/en/matches/07da3f53/AS-Roma-ChievoVerona-Nove...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122937</th>\n      <td>2020-10-17</td>\n      <td>se</td>\n      <td>Damallsvenskan</td>\n      <td>Djurgården</td>\n      <td>Linköping</td>\n      <td>0.0</td>\n      <td>3.0</td>\n      <td>/en/matches/76950db8/Djurgarden-Linkoping-Octo...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122947</th>\n      <td>2016-10-02</td>\n      <td>ru</td>\n      <td>Russian Premier League</td>\n      <td>Krasnodar</td>\n      <td>Rubin Kazan</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>/en/matches/14427644/Krasnodar-Rubin-Kazan-Oct...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122951</th>\n      <td>2016-07-05</td>\n      <td>NaN</td>\n      <td>Europa League</td>\n      <td>Chikhura</td>\n      <td>Zimbru Chișinău</td>\n      <td>2.0</td>\n      <td>3.0</td>\n      <td>/en/matches/e849cec7/Chikhura-Sachkhere-Zimbru...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122955</th>\n      <td>2016-09-09</td>\n      <td>hr</td>\n      <td>1. HNL</td>\n      <td>Inter Zaprešić</td>\n      <td>Hajduk Split</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>/en/matches/b77964bc/Inter-Zapresic-Hajduk-Spl...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122958</th>\n      <td>2018-05-19</td>\n      <td>fr</td>\n      <td>Division 1 Féminine</td>\n      <td>Soyaux</td>\n      <td>Bordeaux</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>/en/matches/c4f419d8/Soyaux-Bordeaux-May-19-20...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>122962</th>\n      <td>2015-01-18</td>\n      <td>es</td>\n      <td>Segunda División</td>\n      <td>Sporting Gijón</td>\n      <td>Betis</td>\n      <td>1.0</td>\n      <td>2.0</td>\n      <td>/en/matches/351f08f5/Sporting-Gijon-Real-Betis...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "dfd0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getone(pattern, string):\n",
    "    m = pattern.search(string)\n",
    "    res = m.group(1) if m else ''\n",
    "    return res.strip()\n",
    "\n",
    "def clean(txt):\n",
    "    return txt.replace('&','').replace('\\'','').strip()\n",
    "\n",
    "def get_cookie_value(name):\n",
    "    return [x['value'] for x in cookies if x['name']==name][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url='https://sofascore.com/'\n",
    "api_url='https://api.sofascore.com/api/v1/'\n",
    "\n",
    "def generate_headers(referer):\n",
    "    return {\n",
    "            \"Accept\": \"*/*\",\n",
    "            \"Accept-Encoding\": \"gzip, deflate, br\",\n",
    "            \"Accept-Language\": \"en-US,en;q=0.5\",\n",
    "            \"Connection\": \"keep-alive\",\n",
    "            \"Host\": \"api.sofascore.com\",\n",
    "            \"Origin\": \"https://www.sofascore.com\",\n",
    "            \"Referer\": referer,\n",
    "            \"TE\": \"Trailers\",\n",
    "            \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0\"\n",
    "        }\n",
    "\n",
    "def load_match_info(did, isft):\n",
    "    load_json('lineups',did, headers)\n",
    "    load_json('votes',did, headers, isft=isft)\n",
    "    if isft==1:\n",
    "        load_json('incidents',did, headers)\n",
    "        load_json('graph',did, headers)\n",
    "    #load_json('best-players',did, headers)\n",
    "    #load_json('event',did, headers)\n",
    "    #load_json('statistics',did, headers)\n",
    "    #load_json('h2h',did, headers)\n",
    "    #load_json('pregame-form',did, headers)\n",
    "    #load_json('winning-odds',did, headers)\n",
    "\n",
    "#\n",
    "\n",
    "def load_json(fn, did, headers, isft=0):\n",
    "    file_name='raw/{}_{}_{:%Y-%m-%d-%H%M}.json'.format(fn, did, datetime.now()) if fn=='votes' else f'raw/{fn}_{did}.json'\n",
    "    if not path.exists(file_name) or (fn=='votes' and isft==0):\n",
    "        script='' if fn=='event' else '/provider/1/'+fn if fn=='winning-odds' else '/'+fn\n",
    "        link=f'{api_url}event/{did}{script}'\n",
    "        r = requests.get(link, headers=headers)\n",
    "        if r.status_code==200:\n",
    "            with open(file_name, 'w+', encoding='utf8') as f:\n",
    "                f.write(r.text)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name='data/matches.csv'\n",
    "df_matches=pd.read_csv(file_name, index_col=None)\n",
    "#SERVER_TZ = 'UTC'\n",
    "#df_matches['ts']=pd.DatetimeIndex(pd.to_datetime(df_matches['startTimestamp'], unit='s')).tz_localize(SERVER_TZ)\n",
    "df_matches=df_matches.drop_duplicates(keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matches_done=df_matches.loc[df_matches['done']==1]\n",
    "df_matches=df_matches.loc[df_matches['done']<1]\n",
    "df_matches_done.to_csv('data/matches_done.csv', index=False)\n",
    "df_matches.to_csv('data/matches1.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude=['india', 'peru', 'south-africa', 'germany-amateur', 'saudi-arabia', 'united-arab-emirates']\n",
    "df_matches=df_matches.loc[~df_matches['country'].isin(exclude)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matches.to_csv('data/matches1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matches_new=pd.read_csv('data/matches1.csv', index_col=None).set_index('id')\n",
    "df_matches=pd.read_csv('data/matches.csv', index_comatches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matches.update(df_matches_new[['awayScoreHT','awayScoreET','homeScoreET','awayScoreFT','homeScoreFT','status','winnerCode','homeScoreHT']])\n",
    "df_matches['id'] = df_matches.index\n",
    "df_matches_new['id'] = df_matches_new.index\n",
    "matchespd.concat([df_matches,df_matches_new]).drop_duplicates(subset='id', keep='first').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOCAL_TZ = 'Asia/Almaty'\n",
    "df_matches['ts'].apply(lambda d: d.astimezone(pytz.timezone(LOCAL_TZ)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avoid_countries=['saudi-arabia', 'chile']\n",
    "PROXY = \"127.0.0.1:9051\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seleniumwire import webdriver\n",
    "#from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "#webdriver.DesiredCapabilities.FIREFOX['proxy'] = {\n",
    "#    \"socksProxy\": PROXY,\n",
    "#    \"socksVersion\": 5,\n",
    "#    \"proxyType\": \"MANUAL\",\n",
    "#}\n",
    "\n",
    "#options = webdriver.ChromeOptions()\n",
    "#options.add_argument('--proxy-server=socks5://' + PROXY)\n",
    "#chrome = webdriver.Chrome(executable_path=r'../lib/chromedriver.exe', options=options)\n",
    "firefox = webdriver.Firefox(executable_path=r'../lib/geckodriver.exe')\n",
    "firefox.scopes = [\n",
    "    'fb.oddsportal.com/feed/match/*'\n",
    "]\n",
    "firefox.get(\"https://www.oddsportal.com/soccer/argentina/superliga/estudiantes-l-p-racing-club-C2xXOviA/\")\n",
    "#chrome.get(\"https://www.sofascore.com/\")\n",
    "\n",
    "#referer=firefox.current_url\n",
    "#cookies=firefox.get_cookies()\n",
    "#html = firefox.page_source\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "request = firefox.requests[0]\n",
    "parts=request.url.split('/')\n",
    "id=parts[5].split('-')[2]\n",
    "fn=f'raw/op/matches/{parts[4]}_{id}.json'\n",
    "print(\n",
    "    request.url,\n",
    "    fn\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(fn, 'w+', encoding='utf8') as f:\n",
    "    f.write(str(request.response.body))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file_name, 'w+', encoding='utf8') as f:\n",
    "    f.write(firefox.page_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_link(file_name, link):\n",
    "    if path.exists(file_name):\n",
    "        print(file_name, ' exists!')\n",
    "        with open(file_name, 'r', encoding='utf8') as f:\n",
    "            html=f.read()\n",
    "    else:\n",
    "        print(f'loading {link}...', end='')\n",
    "        firefox.get('https://www.oddsportal.com/'+link)\n",
    "        html = firefox.page_source\n",
    "        time.sleep(random.uniform(0, 1))\n",
    "        with open(file_name, 'w+', encoding='utf8') as f:\n",
    "            f.write(html)\n",
    "        print(f'done {len(html)} bytes')\n",
    "    return html\n",
    "\n",
    "def clear_html(file_name):\n",
    "    html=''\n",
    "    if path.exists(file_name):\n",
    "        with open(file_name, 'r', encoding='utf8') as f:\n",
    "            html=f.read()\n",
    "        if '<!-- PAGE BODY -->' in html:\n",
    "            html=html.split('<!-- PAGE BODY -->')[1]\n",
    "            html=html.split('<!--  END PAGE BODY -->')[0]\n",
    "            with open(file_name, 'w', encoding='utf8') as f:\n",
    "                f.write(html)\n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "BASE_PATH='raw/op/days/'\n",
    "pYears=r'<strong><a href=\"/soccer/([^/]+)/([^/]+)/results/\">([^<]+)</a></strong>'\n",
    "pNumbers=r'#/page/(\\d+)/'\n",
    "df_champs=pd.read_csv('data/op/champs.csv', index_col=None)\n",
    "for row in df_champs[df_champs['active']==1].itertuples():\n",
    "    link=row.link\n",
    "    country=link.split('/')[2]\n",
    "    champ=link.split('/')[3]\n",
    "    file_name='{}{}_{}_{}.htm'.format(BASE_PATH,country,champ,1)\n",
    "    html=load_link(file_name,link)\n",
    "    \n",
    "    mm=re.findall(pNumbers, html)\n",
    "    if len(mm)>0:\n",
    "        for x in mm:\n",
    "            n=x[0]\n",
    "            file_name='{}{}_{}_{}.htm'.format(BASE_PATH,country,champ,n)\n",
    "            load_link(file_name,link+f'#/page/{n}/')\n",
    "    mmy=re.findall(pYears, html)\n",
    "    if len(mmy)>0:\n",
    "        for y in mmy:\n",
    "            country,champ,season=y\n",
    "            if '2010' in season:\n",
    "                break\n",
    "            link=f'/soccer/{country}/{champ}/results/'\n",
    "            file_name='{}{}_{}_{}.htm'.format(BASE_PATH,country,champ,1)\n",
    "            html=load_link(file_name,link)\n",
    "            mm=re.findall(pNumbers, html)\n",
    "            if len(mm)>0:\n",
    "                for x in mm:\n",
    "                    n=x[0]\n",
    "                    file_name='{}{}_{}_{}.htm'.format(BASE_PATH,country,champ,n)\n",
    "                    load_link(file_name,link+f'#/page/{n}/')\n",
    "\n",
    "    print(f' done {file_name}')\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "DAYS_RAW_PATH='raw/op/days/'\n",
    "for f in tqdm(listdir(DAYS_RAW_PATH)):\n",
    "    print(f)\n",
    "    fn=DAYS_RAW_PATH+f\n",
    "    html=clear_html(fn)\n",
    "    #break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d=datetime.strptime('2020-10-31', '%Y-%m-%d')\n",
    "dstr='{:%Y-%m-%d}'.format(d)\n",
    "matches['ts']=matches['match_time'].apply(lambda x: pytz.timezone(local_tz).localize(datetime.strptime(f'{dstr} {x}', '%Y-%m-%d %H:%M')))\n",
    "matches.to_csv(f'raw/matches_{dstr}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_matches(arr):\n",
    "    l=len(arr)\n",
    "    if l>0:\n",
    "        did,isft=arr[0]\n",
    "        print(f'Loading {did},{isft} from {l}...', end='')\n",
    "        load_match_info(did, isft)\n",
    "        print(' done!')\n",
    "        time.sleep(random.uniform(1, 5))\n",
    "        load_matches(np.delete(arr, 0, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_match_info(1399303, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers=generate_headers(referer)\n",
    "load_matches(dids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datetime.utcfromtimestamp(1583791200)"
   ]
  },
  {
   "source": [
    "# Days"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tor_new_identity():\n",
    "    with Controller.from_port(port=9052) as controller:\n",
    "        controller.authenticate()\n",
    "        #controller.authenticate(password=\"1234\")\n",
    "        controller.signal(Signal.NEWNYM)\n",
    "        controller.close()\n",
    "\n",
    "def load_days(test=False,c=0,n=0,max_load=random.randint(50, 100)):\n",
    "    global dates,r,dstr,proxies\n",
    "    l=len(dates)\n",
    "    if l>0:\n",
    "        dstr='{:%Y-%m-%d}'.format(dates[0])\n",
    "        link=f'https://api.sofascore.com/api/v1/sport/football/scheduled-events/{dstr}'\n",
    "        referer=f'https://www.sofascore.com/football/{dstr}'\n",
    "        print(f'Loading {dstr} from {l}...', end='')\n",
    "        headers=generate_headers(referer)\n",
    "        r = requests.get(link, headers=headers, proxies=proxies)\n",
    "        if test:\n",
    "            print(r.status_code)\n",
    "        if r.status_code==200:\n",
    "            c=0\n",
    "            with open(f'raw/{dstr}.json', 'w+', encoding='utf8') as f:\n",
    "                f.write(r.text)\n",
    "            n+=1\n",
    "            if n>max_load:\n",
    "                tor_new_identity()\n",
    "                time.sleep(random.uniform(10, 20))\n",
    "                load_days()\n",
    "        else:\n",
    "            print(f'ERROR {r.status_code}!!! Obtaining new identity. Try # {c}.')\n",
    "            tor_new_identity()\n",
    "            time.sleep(random.uniform(10, 20))\n",
    "            c+=1\n",
    "            if c>10:\n",
    "                return \n",
    "            load_days(c=c)\n",
    "        c=0\n",
    "        print(f' done #{n} from the batch of {max_load}!')\n",
    "        time.sleep(random.uniform(1, 5))\n",
    "        dates=np.delete(dates, 0, 0)\n",
    "        if not test:\n",
    "            load_days(c=c,n=n,max_load=max_load)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "proxies = {'https': f'socks5://{PROXY}'}\n",
    "\n",
    "dates=[]\n",
    "d= datetime(2020, 10, 29)\n",
    "end_date= datetime(2020, 11, 30)\n",
    "\n",
    "while d<=end_date:\n",
    "    dates.append(d)\n",
    "    d+=timedelta(days=1)\n",
    "dates=np.array(dates) \n",
    "np.random.shuffle(dates)\n",
    "\n",
    "load_days(test=False)\n",
    "#len(dates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from data_provider import DataProvider\n",
    "ds='2014-01-01'\n",
    "de='2020-10-31'\n",
    "\n",
    "#ds='2020-04-05'\n",
    "#de='2020-04-05'\n",
    "\n",
    "dp=DataProvider()\n",
    "#dp.load_days(ds, de)\n",
    "#dp.load_days() # Process al files\n",
    "dp.load_matches()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#matches=[x for x in dp.DATA['events'] if 'coverage' in x.keys()]\n",
    "dp.DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[datetime.strptime(f.replace('.json', ''), '%Y-%m-%d') for f in listdir('raw/')] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name='data/matches1.csv'\n",
    "dp.df_matches.to_csv(file_name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dp.COUNTER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "file_name='raw/2016-08-28.json'\n",
    "with open(file_name, 'r', encoding='utf8') as f:\n",
    "    data=json.load(f)\n",
    "matches=[x for x in data['events'] if 'coverage' in x.keys()]\n",
    "matches=[x for x in matches if x['coverage']>-1]\n",
    "matches=[x for x in matches if not x['status']['code'] in [60,70]]\n",
    "len(matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[x for x in matches if not x['status']['code']==100 and not x['status']['code']==110 and not x['status']['code']==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches=[x for x in dp.data['events'] if 'coverage' in x.keys()]\n",
    "matches=[x for x in matches if x['coverage']>-1]\n",
    "matches=[x for x in matches if x['status']['code']==100 or x['status']['code']==110]\n",
    "len(matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=[  {'tournament' : x['tournament']['slug'],\n",
    "    'country' : x['tournament']['category']['slug'],\n",
    "    'round' : x['roundInfo']['round'] if 'roundInfo' in x.keys() else np.NaN,\n",
    "    'status' : x['status']['code'],\n",
    "    'homeTeam' : x['homeTeam']['name'],\n",
    "    'homeTeamShort' : x['homeTeam']['shortName'],\n",
    "    'awayTeam' : x['awayTeam']['name'],\n",
    "    'awayTeamShort' : x['awayTeam']['shortName'],\n",
    "    'homeScoreFT' : x['homeScore']['normaltime'] if 'normaltime' in x.keys() else np.NaN,\n",
    "    'homeScoreET' : x['homeScore']['current'] if 'current' in x.keys() else np.NaN,\n",
    "    'homeScoreHT' : x['homeScore']['period1'] if 'period1' in x.keys() else np.NaN,\n",
    "    'awayScoreFT' : x['awayScore']['normaltime'] if 'normaltime' in x.keys() else np.NaN,\n",
    "    'awayScoreET' : x['awayScore']['current'] if 'current' in x.keys() else np.NaN,\n",
    "    'awayScoreHT' : x['awayScore']['period1'] if 'period1' in x.keys() else np.NaN,\n",
    "    'id' : x['id'],\n",
    "    'startTimestamp' : x['startTimestamp'],\n",
    "    'coverage' : x['coverage'],\n",
    "    'winnerCode' : x['winnerCode']} for x in matches]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(10):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_json('best-players',did, headers)"
   ]
  },
  {
   "source": [
    "# Fbref"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from fbref_parser import FbrefParser\n",
    "\n",
    "fp=FbrefParser()\n",
    "fp.parse_days()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp.DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "https://fbref.com/en/matches/2020-12-01 raw/fbref/days/2020-12-01.htm\n",
      "raw/fbref/days/2020-12-01.htm  exists!\n"
     ]
    }
   ],
   "source": [
    "from data_provider import DataProvider\n",
    "\n",
    "dp=DataProvider()\n",
    "#dp.load_fbref_matches()\n",
    "df=pd.read_csv('data/sofa/matches_done.csv')\n",
    "ds=df.ts.max()[:10]\n",
    "de='{:%Y-%m-%d}'.format(datetime.today()-timedelta(days=1))\n",
    "dp.load_fbref_days(ds, de)"
   ]
  },
  {
   "source": [
    "## Days"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('data/sofa/matches_done.csv')\n",
    "ds=df.ts.max()[:10]\n",
    "de='{:%Y-%m-%d}'.format(datetime.today()-timedelta(days=1))\n",
    "d = datetime.strptime(ds, '%Y-%m-%d')\n",
    "de = datetime.strptime(de, '%Y-%m-%d')\n",
    "dates=[]\n",
    "while d<=de:\n",
    "    dates.append(d)\n",
    "    d+=timedelta(days=1)\n",
    "\n",
    "# https://fbref.com/en/matches/2021-01-04"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[datetime.datetime(2020, 12, 1, 0, 0),\n",
       " datetime.datetime(2020, 12, 2, 0, 0),\n",
       " datetime.datetime(2020, 12, 3, 0, 0),\n",
       " datetime.datetime(2020, 12, 4, 0, 0),\n",
       " datetime.datetime(2020, 12, 5, 0, 0),\n",
       " datetime.datetime(2020, 12, 6, 0, 0),\n",
       " datetime.datetime(2020, 12, 7, 0, 0),\n",
       " datetime.datetime(2020, 12, 8, 0, 0),\n",
       " datetime.datetime(2020, 12, 9, 0, 0),\n",
       " datetime.datetime(2020, 12, 10, 0, 0),\n",
       " datetime.datetime(2020, 12, 11, 0, 0),\n",
       " datetime.datetime(2020, 12, 12, 0, 0),\n",
       " datetime.datetime(2020, 12, 13, 0, 0),\n",
       " datetime.datetime(2020, 12, 14, 0, 0),\n",
       " datetime.datetime(2020, 12, 15, 0, 0),\n",
       " datetime.datetime(2020, 12, 16, 0, 0),\n",
       " datetime.datetime(2020, 12, 17, 0, 0),\n",
       " datetime.datetime(2020, 12, 18, 0, 0),\n",
       " datetime.datetime(2020, 12, 19, 0, 0),\n",
       " datetime.datetime(2020, 12, 20, 0, 0),\n",
       " datetime.datetime(2020, 12, 21, 0, 0),\n",
       " datetime.datetime(2020, 12, 22, 0, 0),\n",
       " datetime.datetime(2020, 12, 23, 0, 0),\n",
       " datetime.datetime(2020, 12, 24, 0, 0),\n",
       " datetime.datetime(2020, 12, 25, 0, 0),\n",
       " datetime.datetime(2020, 12, 26, 0, 0),\n",
       " datetime.datetime(2020, 12, 27, 0, 0),\n",
       " datetime.datetime(2020, 12, 28, 0, 0),\n",
       " datetime.datetime(2020, 12, 29, 0, 0),\n",
       " datetime.datetime(2020, 12, 30, 0, 0),\n",
       " datetime.datetime(2020, 12, 31, 0, 0),\n",
       " datetime.datetime(2021, 1, 1, 0, 0),\n",
       " datetime.datetime(2021, 1, 2, 0, 0),\n",
       " datetime.datetime(2021, 1, 3, 0, 0),\n",
       " datetime.datetime(2021, 1, 4, 0, 0)]"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "for data in self.DATA:\n",
    "    self._load_data(data)"
   ]
  },
  {
   "source": [
    "## Match"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getone(pattern, string):\n",
    "    m=re.search(pattern, string)\n",
    "    return '' if not m else m.groups()[0].replace('&nbsp;&nbsp;',' ').replace('&nbsp;',' ').strip()\n",
    "\n",
    "def inbetween(txt,st,end):\n",
    "    print(st,end)\n",
    "    return txt.split(st)[1].split(end)[0]\n",
    "\n",
    "def gk(matches, tid):\n",
    "    pid, pname, nationality,age,minutes,shots_on_target_against,goals_against_gk,saves= m.groups()\n",
    "    return {\n",
    "        'tid':tid2, \n",
    "        'pid':pid,\n",
    "        'pname':pname,\n",
    "        'nationality':nationality,\n",
    "        'age':age,\n",
    "        'minutes':minutes,\n",
    "        'shots_on_target_against':shots_on_target_against,\n",
    "        'goals_against_gk':goals_against_gk,\n",
    "        'saves':saves\n",
    "    }\n",
    "def pl(matches, tid):\n",
    "    players=[]\n",
    "    for x in matches:\n",
    "        players.append({\n",
    "            'tid':tid, \n",
    "            'pid':x[0],\n",
    "            'nameen':x[1],\n",
    "            'name':x[2],\n",
    "            'position':x[3],\n",
    "            'nationality':x[4],\n",
    "            'age':x[5],\n",
    "            'goals':x[6],\n",
    "            'assists':x[7],\n",
    "            'pens_made':x[8],\n",
    "            'pens_att':x[9],\n",
    "            'shots_total':x[10],\n",
    "            'shots_on_target':x[11],\n",
    "            'cards_yellow':x[12],\n",
    "            'cards_red':x[13],\n",
    "            'fouls':x[14],\n",
    "            'fouled':x[15],\n",
    "            'offsides':x[16],\n",
    "            'crosses':x[17],\n",
    "            'tackles_won':x[18],\n",
    "            'interceptions':x[19],\n",
    "            'own_goals':x[20],\n",
    "            'pens_won':x[21],\n",
    "            'pens_conceded':x[22]\n",
    "        })\n",
    "    return players\n",
    "\n",
    "pKeeper='<a href=\"/en/players/([^/]+)/\">([^<]+)</a></th><td class=\"left poptip\" data-stat=\"nationality\" data-tip=\"birth\" ><a[^>]+><span style=\"white-space: nowrap\"><span[^>]+>[^<]+</span>([^<]+)</span></a></td><td class=\"center \" data-stat=\"age\" >([^<]+)</td><td class=\"right \" data-stat=\"minutes\"[^>]+>(\\d*)</td><td class=\"right iz group_start\" data-stat=\"shots_on_target_against\" >(\\d*)</td><td class=\"right iz\" data-stat=\"goals_against_gk\" >(\\d*)</td><td class=\"right iz\" data-stat=\"saves\" >(\\d*)</td>'\n",
    "\n",
    "pPlayer='<a href=\"/en/players/([^/]+)/([^\"]+)\">([^<]+)</a></th><td class=\"right[^\"]*\" data-stat=\"shirtnumber\" >[^<]+</td><td class=\"left poptip\" data-stat=\"nationality\"[^>]+><a[^>]+><span style=\"white-space: nowrap\"><span[^>]+>[^<]+</span>([^<]+)</span></a></td><td class=\"center \" data-stat=\"position\" >([^<]+)</td><td class=\"center[^\"]+\" data-stat=\"age\" >([^<]+)</td><td class=\"right[^\"]*\" data-stat=\"minutes\" >([^<]+)</td><td class=\"right[^\"]*\" data-stat=\"goals\" >([^<]+)</td><td class=\"right[^\"]*\" data-stat=\"assists\" >([^<]+)</td><td class=\"right[^\"]*\" data-stat=\"pens_made\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"pens_att\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"shots_total\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"shots_on_target\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"cards_yellow\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"cards_red\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"fouls\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"fouled\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"offsides\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"crosses\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"tackles_won\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"interceptions\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"own_goals\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"pens_won\" >([^<]*)</td><td class=\"right[^\"]*\" data-stat=\"pens_conceded\" >([^<]*)</td></tr>'"
   ]
  },
  {
   "source": [
    "matches=[]\n",
    "players=[]\n",
    "goalkeepers=[]\n",
    "match={}\n",
    "file_name='html/0000ec37.htm'\n",
    "match['mid']=file_name.split('/')[1].replace('.htm','')\n",
    "with open(file_name, 'r', encoding='utf8') as f:\n",
    "    html=f.read()\n",
    "html=html.replace('&nbsp;&nbsp;&nbsp;','')\n",
    "html=inbetween(html,'<div id=\"content\" role=\"main\" class=\"box\">','<h4>About FBref.com</h4>')\n",
    "scoreboards=inbetween(html,'<div class=\"scorebox\">','<div class=\"scorebox_meta\">')\n",
    "parts=scoreboards.split('<div itemscope=\"\" itemprop=\"performer\"')[1:]\n",
    "match['manager1']=getone(r'<strong>Manager</strong>: ([^<]+)</div>', parts[0])\n",
    "match['manager2']=getone(r'<strong>Manager</strong>: ([^<]+)</div>', parts[1])\n",
    "match['squad_link1']=getone(r'<a href=\"([^\"]+)\" itemprop=\"name\">', parts[0])\n",
    "match['tid1']=match['squad_link1'].split('/')[3]\n",
    "match['squad_link2']=getone(r'<a href=\"([^\"]+)\" itemprop=\"name\">', parts[1])\n",
    "match['tid2']=match['squad_link2'].split('/')[3]\n",
    "match['team1']=getone(r' itemprop=\"name\">([^<]+)</a>', parts[0])\n",
    "match['team2']=getone(r' itemprop=\"name\">([^<]+)</a>', parts[1])\n",
    "\n",
    "match['sc1']=getone(r'<div class=\"score\">([^<]+)</div>', parts[0])\n",
    "match['sc2']=getone(r'<div class=\"score\">([^<]+)</div>', parts[1])\n",
    "\n",
    "match['form1']=getone(r'</div><div>([^<]+)</div>', parts[0])\n",
    "match['form2']=getone(r'</div><div>([^<]+)</div>', parts[1])\n",
    "\n",
    "scorebox=inbetween(html,'<div class=\"scorebox_meta\">','<div class=\"event\" id=\"a\">')\n",
    "match['ds_venue']=getone(r'data-venue-epoch=\"(\\d+)\">', scorebox)\n",
    "match['competition']=getone(r'<a href=\"/en/comps/\\d+/\\d+/([^\"]+)\">', scorebox)\n",
    "match['attendance']=getone(r'<small>Attendance</small></strong>: <small>(\\d+)</small>', scorebox)\n",
    "match['venue']=getone(r'<small>Venue</small></strong>: <small>([^<]+)', scorebox)\n",
    "match['referee']=getone(r'<span style=\"display:inline-block\">([^\\()]+) \\(Referee\\)</span>', scorebox)\n",
    "match['ar1']=getone(r'<span style=\"display:inline-block\">([^\\()]+) \\(AR1\\)</span>', scorebox)\n",
    "match['ar2']=getone(r'<span style=\"display:inline-block\">([^\\()]+) \\(AR2\\)</span>', scorebox)\n",
    "match['ar3']=getone(r'<span style=\"display:inline-block\">([^\\()]+) \\(4th\\)</span>', scorebox)\n",
    "\n",
    "matches.append(match)\n",
    "\n",
    "tstats=inbetween(html,'all_stats_'+match['tid1'],'all_stats_'+match['tid2'])\n",
    "tstats,keeper=tstats.split('<div id=\"all_keeper_stats')\n",
    "\n",
    "mm=re.findall(pPlayer, tstats)\n",
    "if len(mm)>0:\n",
    "    players+=pl(mm,tid1)\n",
    "m=re.search(pKeeper, keeper)\n",
    "if m:\n",
    "    goalkeepers.append(gk(m,tid1))\n",
    "tstats=inbetween(html,'all_stats_'+match['tid2'],'</tbody></table>')\n",
    "tstats,keeper=tstats.split('<div id=\"all_keeper_stats')\n",
    "mm=re.findall(pPlayer, tstats)\n",
    "if len(mm)>0:\n",
    "    players+=pl(mm,tid2)\n",
    "m=re.search(pKeeper, keeper)\n",
    "if m:\n",
    "    goalkeepers.append(gk(m,tid2))"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 82,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<div id=\"content\" role=\"main\" class=\"box\"> <h4>About FBref.com</h4>\n<div class=\"scorebox\"> <div class=\"scorebox_meta\">\n<div class=\"scorebox_meta\"> <div class=\"event\" id=\"a\">\nall_stats_4792ab67 all_stats_38154f92\nall_stats_38154f92 </tbody></table>\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "        mid    manager1     manager2  \\\n",
       "0  0000ec37  John Askey  Billy Heath   \n",
       "\n",
       "                                         squad_link1      tid1  \\\n",
       "0  /en/squads/4792ab67/2017-2018/Macclesfield-Tow...  4792ab67   \n",
       "\n",
       "                                         squad_link2      tid2  \\\n",
       "0  /en/squads/38154f92/2017-2018/FC-Halifax-Town-...  38154f92   \n",
       "\n",
       "               team1            team2 sc1  ...   form1   form2    ds_venue  \\\n",
       "0  Macclesfield Town  FC Halifax Town   2  ...  14-6-6  8-8-10  1514300400   \n",
       "\n",
       "                       competition attendance                           venue  \\\n",
       "0  2017-2018-National-League-Stats             Moss Rose Ground, Macclesfield   \n",
       "\n",
       "         referee ar1 ar2 ar3  \n",
       "0  Peter Gibbons              \n",
       "\n",
       "[1 rows x 21 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mid</th>\n      <th>manager1</th>\n      <th>manager2</th>\n      <th>squad_link1</th>\n      <th>tid1</th>\n      <th>squad_link2</th>\n      <th>tid2</th>\n      <th>team1</th>\n      <th>team2</th>\n      <th>sc1</th>\n      <th>...</th>\n      <th>form1</th>\n      <th>form2</th>\n      <th>ds_venue</th>\n      <th>competition</th>\n      <th>attendance</th>\n      <th>venue</th>\n      <th>referee</th>\n      <th>ar1</th>\n      <th>ar2</th>\n      <th>ar3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0000ec37</td>\n      <td>John Askey</td>\n      <td>Billy Heath</td>\n      <td>/en/squads/4792ab67/2017-2018/Macclesfield-Tow...</td>\n      <td>4792ab67</td>\n      <td>/en/squads/38154f92/2017-2018/FC-Halifax-Town-...</td>\n      <td>38154f92</td>\n      <td>Macclesfield Town</td>\n      <td>FC Halifax Town</td>\n      <td>2</td>\n      <td>...</td>\n      <td>14-6-6</td>\n      <td>8-8-10</td>\n      <td>1514300400</td>\n      <td>2017-2018-National-League-Stats</td>\n      <td></td>\n      <td>Moss Rose Ground, Macclesfield</td>\n      <td>Peter Gibbons</td>\n      <td></td>\n      <td></td>\n      <td></td>\n    </tr>\n  </tbody>\n</table>\n<p>1 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 83
    }
   ],
   "source": [
    "pd.DataFrame(matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "  0%|          | 1/89914 [00:00<53:56, 27.78it/s]00012c13.htm\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from fbref_parser import FbrefParser\n",
    "\n",
    "fp=FbrefParser()\n",
    "fp.parse_matches()"
   ]
  },
  {
   "source": [
    "# OP"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "from op_parser import OpParser\n",
    "\n",
    "op=OpParser()\n",
    "op.parse_days()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "roup-g',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Gladiator',\n",
       "  't2': 'Nocerina',\n",
       "  'sc1': '2',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.46',\n",
       "  'oddsdraw': '3.09',\n",
       "  'odds2': '2.61',\n",
       "  'bn': '2',\n",
       "  'link': '/soccer/italy/serie-d-group-g/san-felice-gladiator-nocerina-dIdZRVFs/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-d-group-g',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Torres',\n",
       "  't2': 'Vis Artena',\n",
       "  'sc1': '0',\n",
       "  'sc2': '0',\n",
       "  'odds1': '2.30',\n",
       "  'oddsdraw': '3.30',\n",
       "  'odds2': '2.70',\n",
       "  'bn': '3',\n",
       "  'link': '/soccer/italy/serie-d-group-g/sassari-torres-vis-artena-QcljOmF6/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-d-group-h',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Puteolana',\n",
       "  't2': 'Altamura',\n",
       "  'sc1': '1',\n",
       "  'sc2': '3',\n",
       "  'odds1': '2.75',\n",
       "  'oddsdraw': '3.30',\n",
       "  'odds2': '2.25',\n",
       "  'bn': '1',\n",
       "  'link': '/soccer/italy/serie-d-group-h/puteolana-internapoli-altamura-WIunKoCQ/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-d-group-h',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Portici 1906',\n",
       "  't2': 'Gravina',\n",
       "  'sc1': '1',\n",
       "  'sc2': '3',\n",
       "  'odds1': '2.28',\n",
       "  'oddsdraw': '3.40',\n",
       "  'odds2': '2.73',\n",
       "  'bn': '9',\n",
       "  'link': '/soccer/italy/serie-d-group-h/portici-1906-gravina-MoTlHIen/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-d-group-i',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'ACR Messina',\n",
       "  't2': 'Gelbison Cilento',\n",
       "  'sc1': '2',\n",
       "  'sc2': '2',\n",
       "  'odds1': '1.70',\n",
       "  'oddsdraw': '3.54',\n",
       "  'odds2': '4.26',\n",
       "  'bn': '8',\n",
       "  'link': '/soccer/italy/serie-d-group-i/messina-gelbison-cilento-zBxzPS98/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'liberia',\n",
       "  'liga': 'lfa-first-division',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'LPRC Oiler',\n",
       "  't2': 'Nimba Kwado',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.14',\n",
       "  'oddsdraw': '3.07',\n",
       "  'odds2': '3.27',\n",
       "  'bn': '3',\n",
       "  'link': '/soccer/liberia/lfa-first-division/lprc-oiler-nimba-kwado-YV1vxYeP/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'montenegro',\n",
       "  'liga': 'prva-crnogorska-liga',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'OFK Petrovac',\n",
       "  't2': 'Zeta',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.70',\n",
       "  'oddsdraw': '2.79',\n",
       "  'odds2': '2.79',\n",
       "  'bn': '12',\n",
       "  'link': '/soccer/montenegro/prva-crnogorska-liga/ofk-petrovac-zeta-n5zT4jCa/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 30),\n",
       "  'country': 'russia',\n",
       "  'liga': 'premier-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Dynamo Moscow',\n",
       "  't2': 'Arsenal Tula',\n",
       "  'sc1': '1',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.61',\n",
       "  'oddsdraw': '3.83',\n",
       "  'odds2': '5.93',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/russia/premier-league/dynamo-moscow-arsenal-tula-tpggow7a/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 13, 45),\n",
       "  'country': 'france',\n",
       "  'liga': 'division-1-women',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Paris SG W',\n",
       "  't2': 'Paris FC W',\n",
       "  'sc1': '4',\n",
       "  'sc2': '1',\n",
       "  'odds1': '1.06',\n",
       "  'oddsdraw': '9.61',\n",
       "  'odds2': '24.12',\n",
       "  'bn': '10',\n",
       "  'link': '/soccer/france/division-1-women/paris-sg-paris-fc-lAWkrjQ8/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'bosnia-and-herzegovina',\n",
       "  'liga': 'premier-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Zeljeznicar',\n",
       "  't2': 'Siroki Brijeg',\n",
       "  'sc1': '2',\n",
       "  'sc2': '3',\n",
       "  'odds1': '1.64',\n",
       "  'oddsdraw': '3.59',\n",
       "  'odds2': '4.91',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/bosnia-and-herzegovina/premier-league/zeljeznicar-siroki-brijeg-IsAaowNo/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'burundi',\n",
       "  'liga': 'primus-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Atletico Olympic',\n",
       "  't2': 'Aigle Noir',\n",
       "  'sc1': '0',\n",
       "  'sc2': '0',\n",
       "  'odds1': '3.14',\n",
       "  'oddsdraw': '3.39',\n",
       "  'odds2': '2.01',\n",
       "  'bn': '5',\n",
       "  'link': '/soccer/burundi/primus-league/atletico-olympic-aigle-noir-zyUiIoon/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'denmark',\n",
       "  'liga': '1st-division',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Vendsyssel',\n",
       "  't2': 'Silkeborg',\n",
       "  'sc1': '2',\n",
       "  'sc2': '1',\n",
       "  'odds1': '6.05',\n",
       "  'oddsdraw': '4.60',\n",
       "  'odds2': '1.46',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/denmark/1st-division/vendsyssel-ff-silkeborg-0GzHU7G2/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'england',\n",
       "  'liga': 'women-s-championship',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Leicester W',\n",
       "  't2': 'London Bees W',\n",
       "  'sc1': '3',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.21',\n",
       "  'oddsdraw': '6.18',\n",
       "  'odds2': '9.17',\n",
       "  'bn': '8',\n",
       "  'link': '/soccer/england/women-s-championship/leicester-london-bees-j7j1lsws/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'england',\n",
       "  'liga': 'women-s-championship',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Lewes W',\n",
       "  't2': 'Charlton W',\n",
       "  'sc1': '2',\n",
       "  'sc2': '1',\n",
       "  'odds1': '1.78',\n",
       "  'oddsdraw': '3.67',\n",
       "  'odds2': '3.72',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/england/women-s-championship/lewes-charlton-Uck5m1hm/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'england',\n",
       "  'liga': 'women-s-championship',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Liverpool W',\n",
       "  't2': 'Crystal Palace W',\n",
       "  'sc1': '4',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.30',\n",
       "  'oddsdraw': '5.01',\n",
       "  'odds2': '7.65',\n",
       "  'bn': '8',\n",
       "  'link': '/soccer/england/women-s-championship/liverpool-crystal-palace-nZqAnL7g/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'england',\n",
       "  'liga': 'women-s-championship',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'London City Lionesses W',\n",
       "  't2': 'Coventry United W',\n",
       "  'sc1': '2',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.43',\n",
       "  'oddsdraw': '4.51',\n",
       "  'odds2': '5.34',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/england/women-s-championship/london-city-lionesses-coventry-united-QPrEouNa/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'germany',\n",
       "  'liga': 'dfb-pokal-women',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Koln W',\n",
       "  't2': 'Hoffenheim W',\n",
       "  'sc1': '1',\n",
       "  'sc2': '6',\n",
       "  'odds1': '12.41',\n",
       "  'oddsdraw': '6.96',\n",
       "  'odds2': '1.16',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/germany/dfb-pokal-women/koln-hoffenheim-KpiOBwl4/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'germany',\n",
       "  'liga': 'regionalliga-west',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Dusseldorf II',\n",
       "  't2': 'Schalke II',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '1.76',\n",
       "  'oddsdraw': '3.64',\n",
       "  'odds2': '4.22',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/germany/regionalliga-west/dusseldorf-schalke-C4ed4aqc/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'AlbinoLeffe',\n",
       "  't2': 'Olbia',\n",
       "  'sc1': '0',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.96',\n",
       "  'oddsdraw': '3.03',\n",
       "  'odds2': '4.02',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/albinoleffe-olbia-h2R3cKu4/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Carrarese',\n",
       "  't2': 'Pergolettese',\n",
       "  'sc1': '1',\n",
       "  'sc2': '2',\n",
       "  'odds1': '1.51',\n",
       "  'oddsdraw': '3.84',\n",
       "  'odds2': '5.83',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/carrarese-pergolettese-WhQ7dveA/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Grosseto',\n",
       "  't2': 'Novara',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.59',\n",
       "  'oddsdraw': '2.91',\n",
       "  'odds2': '2.76',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/grosseto-novara-0SUBebAG/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Juventus U23',\n",
       "  't2': 'Pro Patria',\n",
       "  'sc1': '3',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.44',\n",
       "  'oddsdraw': '2.80',\n",
       "  'odds2': '3.08',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/juventus-pro-patria-vJTFfIPM/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Lecco',\n",
       "  't2': 'Pistoiese',\n",
       "  'sc1': '4',\n",
       "  'sc2': '1',\n",
       "  'odds1': '1.87',\n",
       "  'oddsdraw': '3.05',\n",
       "  'odds2': '4.44',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/lecco-pistoiese-MsJKgxuT/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Lucchese',\n",
       "  't2': 'Alessandria',\n",
       "  'sc1': '0',\n",
       "  'sc2': '2',\n",
       "  'odds1': '4.90',\n",
       "  'oddsdraw': '3.38',\n",
       "  'odds2': '1.69',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/lucchese-alessandria-dzAnkdmp/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Piacenza',\n",
       "  't2': 'Pro Sesto',\n",
       "  'sc1': '6',\n",
       "  'sc2': '0',\n",
       "  'odds1': '2.57',\n",
       "  'oddsdraw': '2.96',\n",
       "  'odds2': '2.75',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/piacenza-pro-sesto-jc9jlG2j/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-a',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Pontedera',\n",
       "  't2': 'Como',\n",
       "  'sc1': '2',\n",
       "  'sc2': '2',\n",
       "  'odds1': '2.55',\n",
       "  'oddsdraw': '2.87',\n",
       "  'odds2': '2.86',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-a/us-pontedera-como-Um8fmzId/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-b',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Arezzo',\n",
       "  't2': 'Sudtirol',\n",
       "  'sc1': '0',\n",
       "  'sc2': '4',\n",
       "  'odds1': '4.11',\n",
       "  'oddsdraw': '3.06',\n",
       "  'odds2': '1.93',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-b/arezzo-sudtirol-EuYGQ1Fl/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-b',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Ravenna',\n",
       "  't2': 'Padova',\n",
       "  'sc1': '1',\n",
       "  'sc2': '3',\n",
       "  'odds1': '5.13',\n",
       "  'oddsdraw': '3.49',\n",
       "  'odds2': '1.65',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-b/ravenna-padova-ARHJpN6K/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-b',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Triestina',\n",
       "  't2': 'Sambenedettese',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.42',\n",
       "  'oddsdraw': '2.89',\n",
       "  'odds2': '3.02',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-b/triestina-sambenedettese-hj7OqsMQ/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-c',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Catania',\n",
       "  't2': 'Cavese',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '1.54',\n",
       "  'oddsdraw': '3.58',\n",
       "  'odds2': '6.01',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-c/catania-cavese-bkwtAyog/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-c',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Potenza',\n",
       "  't2': 'Viterbese',\n",
       "  'sc1': '2',\n",
       "  'sc2': '3',\n",
       "  'odds1': '2.48',\n",
       "  'oddsdraw': '2.98',\n",
       "  'odds2': '2.86',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-c/potenza-viterbese-lvWh7ZUB/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-c',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Teramo',\n",
       "  't2': 'Vibonese',\n",
       "  'sc1': '2',\n",
       "  'sc2': '2',\n",
       "  'odds1': '1.71',\n",
       "  'oddsdraw': '3.28',\n",
       "  'odds2': '4.96',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-c/teramo-vibonese-2szc6gpI/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'italy',\n",
       "  'liga': 'serie-c-group-c',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Ternana',\n",
       "  't2': 'Bisceglie',\n",
       "  'sc1': '3',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.23',\n",
       "  'oddsdraw': '5.42',\n",
       "  'odds2': '11.15',\n",
       "  'bn': '13',\n",
       "  'link': '/soccer/italy/serie-c-group-c/ternana-bisceglie-KEhOOEPd/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'northern-ireland',\n",
       "  'liga': 'premiership-women',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Linfield W',\n",
       "  't2': 'Sion Swifts W',\n",
       "  'sc1': '4',\n",
       "  'sc2': '2',\n",
       "  'odds1': '2.76',\n",
       "  'oddsdraw': '4.37',\n",
       "  'odds2': '1.96',\n",
       "  'bn': '6',\n",
       "  'link': '/soccer/northern-ireland/premiership-women/linfield-sion-swifts-zqU5d0SP/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'norway',\n",
       "  'liga': 'obos-ligaen',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Ham-Kam',\n",
       "  't2': 'Lillestrom',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '3.22',\n",
       "  'oddsdraw': '3.36',\n",
       "  'odds2': '2.16',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/norway/obos-ligaen/ham-kam-lillestrom-Cd7g7nZL/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 0),\n",
       "  'country': 'tunisia',\n",
       "  'liga': 'ligue-professionnelle-1',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Esperance Tunis',\n",
       "  't2': 'Slimane',\n",
       "  'sc1': '2',\n",
       "  'sc2': '2',\n",
       "  'odds1': '1.25',\n",
       "  'oddsdraw': '4.90',\n",
       "  'odds2': '11.63',\n",
       "  'bn': '11',\n",
       "  'link': '/soccer/tunisia/ligue-professionnelle-1/esperance-tunis-slimane-bHJYOgC0/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 15),\n",
       "  'country': 'malta',\n",
       "  'liga': 'division-1',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Fgura',\n",
       "  't2': 'Naxxar Lions',\n",
       "  'sc1': '2',\n",
       "  'sc2': '0',\n",
       "  'odds1': '2.91',\n",
       "  'oddsdraw': '3.56',\n",
       "  'odds2': '2.07',\n",
       "  'bn': '6',\n",
       "  'link': '/soccer/malta/division-1/fgura-naxxar-lions-hAy2kP5m/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 25),\n",
       "  'country': 'saudi-arabia',\n",
       "  'liga': 'saudi-professional-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Al Qadisiya',\n",
       "  't2': 'Al-Shabab',\n",
       "  'sc1': '2',\n",
       "  'sc2': '1',\n",
       "  'odds1': '4.08',\n",
       "  'oddsdraw': '3.60',\n",
       "  'odds2': '1.78',\n",
       "  'bn': '12',\n",
       "  'link': '/soccer/saudi-arabia/saudi-professional-league/al-qadisiya-al-shabab-AN5CrCF0/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 30),\n",
       "  'country': 'montenegro',\n",
       "  'liga': 'prva-crnogorska-liga',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Titograd',\n",
       "  't2': 'Decic',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '3.20',\n",
       "  'oddsdraw': '2.62',\n",
       "  'odds2': '2.52',\n",
       "  'bn': '12',\n",
       "  'link': '/soccer/montenegro/prva-crnogorska-liga/ofk-titograd-decic-4tVX3AR5/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 30),\n",
       "  'country': 'morocco',\n",
       "  'liga': 'botola-pro',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Hassania Agadir',\n",
       "  't2': 'IR Tanger',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.38',\n",
       "  'oddsdraw': '2.70',\n",
       "  'odds2': '3.38',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/morocco/botola-pro/hassania-agadir-ir-tanger-ALK1BNQf/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 30),\n",
       "  'country': 'qatar',\n",
       "  'liga': 'division-2',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Al-Shahaniya',\n",
       "  't2': 'Shamal',\n",
       "  'sc1': '1',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.24',\n",
       "  'oddsdraw': '3.31',\n",
       "  'odds2': '2.77',\n",
       "  'bn': '6',\n",
       "  'link': '/soccer/qatar/division-2/al-shahaniya-shamal-QNgUYPgd/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 14, 30),\n",
       "  'country': 'spain',\n",
       "  'liga': 'segunda-division-b-group-1',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Valladolid Promesas',\n",
       "  't2': 'Lealtad',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.38',\n",
       "  'oddsdraw': '2.87',\n",
       "  'odds2': '3.12',\n",
       "  'bn': '12',\n",
       "  'link': '/soccer/spain/segunda-division-b-group-1/valladolid-promesas-lealtad-I5YR2nne/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'belgium',\n",
       "  'liga': 'proximus-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Westerlo',\n",
       "  't2': 'Lommel SK',\n",
       "  'sc1': '2',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.32',\n",
       "  'oddsdraw': '3.30',\n",
       "  'odds2': '2.92',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/belgium/proximus-league/westerlo-lommel-sk-GhIXq0rM/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'denmark',\n",
       "  'liga': 'superliga',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'FC Copenhagen',\n",
       "  't2': 'Horsens',\n",
       "  'sc1': '2',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.55',\n",
       "  'oddsdraw': '4.20',\n",
       "  'odds2': '5.80',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/denmark/superliga/fc-copenhagen-horsens-hOz6MGm9/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-4',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'San Ignacio',\n",
       "  't2': 'Pasaia',\n",
       "  'sc1': '1',\n",
       "  'sc2': '3',\n",
       "  'odds1': '2.05',\n",
       "  'oddsdraw': '2.79',\n",
       "  'odds2': '3.90',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/spain/tercera-division-group-4/san-ignacio-pasaia-K8P0Jc10/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-6',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Elche CF Ilicitano B',\n",
       "  't2': 'Novelda CF',\n",
       "  'sc1': '1',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.47',\n",
       "  'oddsdraw': '3.79',\n",
       "  'odds2': '6.50',\n",
       "  'bn': '8',\n",
       "  'link': '/soccer/spain/tercera-division-group-6/elche-cf-ilicitano-novelda-cf-SYvE9a1f/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-7',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'RSD Alcala',\n",
       "  't2': 'Complutense',\n",
       "  'sc1': '2',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.63',\n",
       "  'oddsdraw': '3.37',\n",
       "  'odds2': '5.32',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/spain/tercera-division-group-7/rsd-alcala-ad-alcala-UcVXKvuD/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'switzerland',\n",
       "  'liga': 'super-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Luzern',\n",
       "  't2': 'Young Boys',\n",
       "  'sc1': '2',\n",
       "  'sc2': '3',\n",
       "  'odds1': '4.51',\n",
       "  'oddsdraw': '3.84',\n",
       "  'odds2': '1.73',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/switzerland/super-league/luzern-young-boys-lWHStzg3/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 0),\n",
       "  'country': 'switzerland',\n",
       "  'liga': 'super-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Zurich',\n",
       "  't2': 'Lausanne',\n",
       "  'sc1': '4',\n",
       "  'sc2': '0',\n",
       "  'odds1': '2.52',\n",
       "  'oddsdraw': '3.52',\n",
       "  'odds2': '2.64',\n",
       "  'bn': '14',\n",
       "  'link': '/soccer/switzerland/super-league/zurich-lausanne-I5MWuf89/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 15),\n",
       "  'country': 'spain',\n",
       "  'liga': 'primera-division-women',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Real Sociedad W',\n",
       "  't2': 'Espanyol W',\n",
       "  'sc1': '1',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.33',\n",
       "  'oddsdraw': '4.74',\n",
       "  'odds2': '7.85',\n",
       "  'bn': '10',\n",
       "  'link': '/soccer/spain/primera-division-women/real-sociedad-espanyol-h0nGVh60/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 30),\n",
       "  'country': 'burkina-faso',\n",
       "  'liga': 'premier-league',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'ASF Dioulasso',\n",
       "  't2': 'Vitesse',\n",
       "  'sc1': '3',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.68',\n",
       "  'oddsdraw': '2.65',\n",
       "  'odds2': '2.84',\n",
       "  'bn': '3',\n",
       "  'link': '/soccer/burkina-faso/premier-league/asf-dioulasso-vitesse-468EujT1/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 30),\n",
       "  'country': 'spain',\n",
       "  'liga': 'segunda-division-b-group-2',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Calahorra',\n",
       "  't2': 'Ejea',\n",
       "  'sc1': '1',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.76',\n",
       "  'oddsdraw': '3.28',\n",
       "  'odds2': '4.43',\n",
       "  'bn': '12',\n",
       "  'link': '/soccer/spain/segunda-division-b-group-2/cd-calahorra-sd-ejea-jwXixHDf/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 30),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-13',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'CAP Ciudad de Murcia',\n",
       "  't2': 'Muleno',\n",
       "  'sc1': '0',\n",
       "  'sc2': '2',\n",
       "  'odds1': '2.54',\n",
       "  'oddsdraw': '3.17',\n",
       "  'odds2': '2.52',\n",
       "  'bn': '8',\n",
       "  'link': '/soccer/spain/tercera-division-group-13/ciudad-de-murcia-muleno-cf-x2cb3f2a/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 30),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-18',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Madridejos',\n",
       "  't2': 'Villacanas',\n",
       "  'sc1': '0',\n",
       "  'sc2': '1',\n",
       "  'odds1': '2.63',\n",
       "  'oddsdraw': '3.18',\n",
       "  'odds2': '2.49',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/spain/tercera-division-group-18/madridejos-villacanas-Wjum1fI1/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 30),\n",
       "  'country': 'spain',\n",
       "  'liga': 'tercera-division-group-8',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Atl. Astorga',\n",
       "  't2': 'La Baneza',\n",
       "  'sc1': '3',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.21',\n",
       "  'oddsdraw': '5.80',\n",
       "  'odds2': '9.79',\n",
       "  'bn': '7',\n",
       "  'link': '/soccer/spain/tercera-division-group-8/atletico-astorga-la-baneza-td7nBMN1/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 45),\n",
       "  'country': 'united-arab-emirates',\n",
       "  'liga': 'presidents-cup',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Shabab Al-Ahli Dubai',\n",
       "  't2': 'Hatta',\n",
       "  'sc1': '3',\n",
       "  'sc2': '0',\n",
       "  'odds1': '1.17',\n",
       "  'oddsdraw': '6.69',\n",
       "  'odds2': '10.93',\n",
       "  'bn': '11',\n",
       "  'link': '/soccer/united-arab-emirates/presidents-cup/shabab-al-ahli-dubai-hatta-h0vpsjsE/'},\n",
       " {'ds': datetime.datetime(2020, 12, 6, 15, 45),\n",
       "  'country': 'united-arab-emirates',\n",
       "  'liga': 'presidents-cup',\n",
       "  'season': '2020/2021',\n",
       "  't1': 'Al Dhafra',\n",
       "  't2': 'Al Jazira',\n",
       "  'sc1': '1',\n",
       "  'sc2': '0',\n",
       "  'odds1': '5.96',\n",
       "  'oddsdraw': '4.56',\n",
       "  'odds2': '1.41',\n",
       "  'bn': '11',\n",
       "  'link': '/soccer/united-arab-emirates/presidents-cup/al-dhafra-al-jazira-ba6HOsRn/'},\n",
       " ...]"
      ]
     },
     "metadata": {},
     "execution_count": 46
    }
   ],
   "source": [
    "op.DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(op.DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fbref=pd.read_csv('data/fbref/matches.csv', index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_provider import DataProvider\n",
    "dp=DataProvider()\n",
    "dp.load_op_matches()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fbref.country.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fbref[df_fbref.champ=='Premier League 2 — Division 1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries={'Netherlands' : 'nl','France' : 'fr','Bolivia' : 'bo','England' : 'eng','Ukraine' : 'ua','Russia' : 'ru','USA' : 'us','Spain' : 'es','Belgium' : 'be','Germany' : 'de','Sweden' : 'se','Portugal' : 'pt','Argentina' : 'ar','Finland' : 'fi','India' : 'in','Denmark' : 'dk','Uruguay' : 'uy','Venezuela' : 've','Scotland' : 'sco','Italy' : 'it','Mexico' : 'mx','Peru' : 'pe','Japan' : 'jp','Brazil' : 'br','Saudi Arabia' : 'sa','Serbia' : 'rs','Poland' : 'pl','South Korea' : 'kr','Czech Republic' : 'cz','Ecuador' : 'ec','Croatia' : 'hr','Turkey' : 'tr','Paraguay' : 'py','Romania' : 'ro','Chile' : 'cl','Colombia' : 'co','Australia' : 'au','Austria' : 'at','China' : 'cn','Greece' : 'gr','Norway' : 'no','Bulgaria' : 'bg','Switzerland' : 'ch','Ireland' : 'ir','Hungary' : 'hu','Zambia' : 'za','Canada' : 'ca'}\n",
    "\n",
    "avoid_countries=['Albania', 'Algeria', 'Andorra', 'Angola','Armenia', 'Aruba', 'Azerbaijan', 'Bahrain', 'Bangladesh', 'Belarus','Benin', 'Bosnia and Herzegovina','Botswana', 'Burkina Faso', 'Burundi','Cambodia', 'Cameroon','Costa Rica', 'Curaçao', 'Cyprus','DR Congo', 'Dominican Republic', 'Egypt','El Salvador', 'Estonia', 'Ethiopia','Faroe Islands', 'Fiji', 'Gabon', 'Gambia','Georgia', 'Ghana', 'Gibraltar', 'Guatemala','Haiti', 'Honduras', 'Hong Kong', 'Iceland','Indonesia', 'Iraq', 'Israel','Ivory Coast', 'Jamaica', 'Jordan', 'Kazakhstan', 'Kenya','Kosovo', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon','Lesotho', 'Liberia', 'Liechtenstein', 'Lithuania', 'Luxembourg','Macao', 'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta','Mauritania', 'Moldova', 'Mongolia', 'Montenegro','Morocco', 'Mozambique', 'Myanmar', 'Namibia','New Zealand', 'Nicaragua', 'Nigeria','North Macedonia', 'Northern Ireland', 'Oman','Pakistan', 'Palestine', 'Panama','Philippines', 'Qatar','Republic of the Congo', 'Rwanda','San Marino', 'Senegal','Serbia and Montenegro', 'Singapore', 'Slovakia', 'Slovenia','South Africa', 'Sri Lanka', 'Sudan', 'Syria', 'Taiwan','Tajikistan', 'Tanzania', 'Thailand', 'Togo','Trinidad and Tobago', 'Tunisia', 'Turkmenistan','Uganda', 'United Arab Emirates','Uzbekistan', 'Vietnam', 'Wales', 'Yemen','Zimbabwe']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "champs=[]\n",
    "pCountry=r'</span>([^<]+)</a></th>'\n",
    "pChamp=r'<a foo=\"f\" href=\"([^\"]+)\">([^<]+)</a>'\n",
    "with open('html/op.html', 'r', encoding='utf8') as f:\n",
    "    html=f.read()\n",
    "parts=html.split('<th class=\"first2 tl\" colspan=\"2\">')[1:]\n",
    "for part in parts:\n",
    "    m=re.search(pCountry, part)\n",
    "    if m:\n",
    "        country=m[1]\n",
    "    mm=re.findall(pChamp, part)\n",
    "    if len(mm)>0:\n",
    "        for x in mm:\n",
    "            champs.append({\n",
    "                'country':country,\n",
    "                'champ':x[1],\n",
    "                'link':x[0]\n",
    "            })\n",
    "df_champs=pd.DataFrame(champs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs=df_champs.loc[~df_champs.country.isin(avoid_countries)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs.to_csv('data/op/champs1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs=pd.read_csv('data/op/champs.csv', index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs[df_champs['active']==1].to_csv('data/op/champs1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs['active']=df_champs['country'].apply(lambda x: 1 if '#' in x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_champs['country']=df_champs['country'].apply(lambda x: x.replace('#',''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('html/bookies.json', 'r', encoding='utf8') as f:\n",
    "    txt=f.read()\n",
    "jb=json.loads(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "bookies=[{\n",
    "    'bid':int(jb[b]['idProvider']),\n",
    "    'name':jb[b]['WebName'],\n",
    "    'url':jb[b]['WebUrl'],\n",
    "    'link':jb[b]['Url'],\n",
    "    'isbookie':jb[b]['IsBookmaker']=='y'\n",
    "    } for b in jb]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bookies=pd.DataFrame(bookies)\n",
    "df_bookies.to_csv('data/op/bookies.csv', index=False)"
   ]
  },
  {
   "source": [
    "## Match"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name='html/EFEkIb54.json'\n",
    "with open(file_name, 'r', encoding='utf8') as f:\n",
    "    html=f.read()\n",
    "js=json.loads(html)\n",
    "mid=file_name.split('/')[1].replace('.json','')\n",
    "# 0 - w1\n",
    "# 1 - x\n",
    "# 2 - w2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "odds=js['d']['oddsdata']['back']['E-1-2-0-0-0']['odds']\n",
    "movement=js['d']['oddsdata']['back']['E-1-2-0-0-0']['movement']\n",
    "opening_odds=js['d']['oddsdata']['back']['E-1-2-0-0-0']['opening_odds']\n",
    "opening_change_time=js['d']['oddsdata']['back']['E-1-2-0-0-0']['opening_change_time']\n",
    "opening_volume=js['d']['oddsdata']['back']['E-1-2-0-0-0']['opening_volume']\n",
    "volume=js['d']['oddsdata']['back']['E-1-2-0-0-0']['volume']\n",
    "change_time=js['d']['oddsdata']['back']['E-1-2-0-0-0']['change_time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "bookies={}\n",
    "for x in odds:\n",
    "    bookies[x]={\n",
    "        'mid':mid,\n",
    "        'bid':x,\n",
    "        'w1':odds[x]['0'],\n",
    "        'wx':odds[x]['1'],\n",
    "        'w2':odds[x]['2']\n",
    "    }\n",
    "\n",
    "for x in movement:\n",
    "    bookies[x]['move_1']=str(movement[x]['0'])[0].upper()\n",
    "    bookies[x]['move_x']=str(movement[x]['1'])[0].upper()\n",
    "    bookies[x]['move_2']=str(movement[x]['2'])[0].upper()\n",
    "\n",
    "for x in opening_odds:\n",
    "    bookies[x]['open_1']=opening_odds[x]['0']\n",
    "    bookies[x]['open_x']=opening_odds[x]['1']\n",
    "    bookies[x]['open_2']=opening_odds[x]['2']\n",
    "\n",
    "for x in opening_change_time:\n",
    "    bookies[x]['time_open']=max([opening_change_time[x]['0'],opening_change_time[x]['1'],opening_change_time[x]['2']])\n",
    "\n",
    "for x in change_time:\n",
    "    bookies[x]['time_close']=max([change_time[x]['0'],change_time[x]['1'],change_time[x]['2']])\n"
   ]
  },
  {
   "source": [
    "## ELO\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'http://api.clubelo.com/2015-01-02'"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "d= datetime(2015, 1, 2)\n",
    "'http://api.clubelo.com/{:%Y-%m-%d}'.format(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates=[]\n",
    "d= datetime(2015, 1, 4)\n",
    "end_date= datetime(2021, 1, 10)\n",
    "\n",
    "while d<=end_date:\n",
    "    r = requests.get('http://api.clubelo.com/{:%Y-%m-%d}'.format(d), allow_redirects=True)\n",
    "    open('data/elo/elo_{:%Y-%m-%d}.csv'.format(d), 'wb').write(r.content)\n",
    "    #time.sleep(random.uniform(1, 5))\n",
    "    d+=timedelta(days=1)\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(fn, did, headers, isft=0):\n",
    "    file_name='raw/{}_{}_{:%Y-%m-%d-%H%M}.json'.format(fn, did, datetime.now()) if fn=='votes' else f'raw/{fn}_{did}.json'\n",
    "    if not path.exists(file_name) or (fn=='votes' and isft==0):\n",
    "        script='' if fn=='event' else '/provider/1/'+fn if fn=='winning-odds' else '/'+fn\n",
    "        link=f'{api_url}event/{did}{script}'\n",
    "        r = requests.get(link, headers=headers)\n",
    "        if r.status_code==200:\n",
    "            with open(file_name, 'w+', encoding='utf8') as f:\n",
    "                f.write(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "COUNTRIES=['england', 'france', 'greece', 'spain', 'italy', 'portugal', 'mexico', 'asia', 'scotland', 'netherlands', 'belgium', 'africa',\n",
    "                        'turkey', 'australia', 'argentina', 'germany', 'switzerland', 'poland', 'austria', 'europe', 'south-america', 'denmark',\n",
    "                        'ukraine', 'usa', 'russia', 'japan', 'bulgaria', 'lithuania', 'world', 'sweden', 'norway', 'romania', 'brazil', 'estonia',\n",
    "                        'slovakia', 'north-central-america', 'finland', 'serbia', 'slovenia', 'china', 'hungary', 'czech-republic', 'chile',\n",
    "                        'belarus', 'croatia', 'paraguay', 'cyprus', 'uruguay', 'ireland', 'colombia', 'south-korea', 'ecuador']\n"
   ]
  }
 ]
}