Adding MaxEntry and ConnectionError with Retry object for details scr…

…aping
lcsrodriguez · Jul 9, 2023 · a787629 · a787629
1 parent 4259175
commit a787629
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 11 deletions.
diff --git a/ecocal/main.py b/ecocal/main.py
@@ -7,7 +7,16 @@
 import time
 from threading import Thread
 from tqdm import tqdm
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
 
+retry = Retry(
+    total=5,
+    backoff_factor=1
+)
+adapter = HTTPAdapter(
+    max_retries=retry
+)
 
 class EconomicCalendar:
     __slots__ = (
@@ -183,14 +192,17 @@ def _requestDetails(self, resource_id: str = "", output: dict = {}) -> Union[dic
             raise Exception("Please enter a valid resource id")
         URL = f"{self.SOURCE_URL}/{resource_id}"
 
-        r = requests.get(url=URL,
-                         headers={
-                             "Accept": "application/json",
-                             "Content-Type": "application/json",
-                             "Referer": "https://www.fxstreet.com/",
-                             "Connection": "keep-alive",
-                             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36",
-                         })
+        s = requests.Session()
+        s.mount("https://", adapter)
+        s.mount("http://", adapter)
+        r = s.get(url=URL,
+                  headers={
+                      "Accept": "application/json",
+                      "Content-Type": "application/json",
+                      "Referer": "https://www.fxstreet.com/",
+                      "Connection": "keep-alive",
+                      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36",
+                  })
         if r.status_code == 200:
             output[resource_id] = r.json()
             return output[resource_id]

diff --git a/test/test.py b/test/test.py
@@ -1,9 +1,9 @@
 from ecocal import *
 
 
-ec = EconomicCalendar(startHorizon="2023-01-01",
-                      endHorizon="2023-12-31",
+ec = EconomicCalendar(startHorizon="2020-01-01",
+                      endHorizon="2024-12-31",
                       withDetails=True,
-                      nbThreads=40
+                      nbThreads=200
                       )
 ec.saveCalendar(True)