sleep request

README.md

@@ -92,7 +92,10 @@ optional arguments:
   * default: `8`
 * `HTTP_HEADERS`
   * Headers for HTTP requests
-  * default: None
+  * default: `None`
+* `HTTP_SLEEP`
+  * Sleep after request (in seconds)
+  * default: `0`
 
 ### Write to a JSON file
 

citation_crawler/crawlers/common.py

@@ -6,6 +6,7 @@
 import aiohttp
 import logging
 from aiofile import async_open
+import asyncio
 from asyncio import Semaphore
 
 logger = logging.getLogger("common")
@@ -21,6 +22,16 @@ def getenv_int(key) -> int:
     return None
 
 
+def getenv_float(key) -> float:
+    cache_days = os.getenv(key)
+    if cache_days is not None:
+        try:
+            return float(cache_days)
+        except:
+            pass
+    return None
+
+
 def getenv_headers(key) -> Dict:
     headers = os.getenv(key)
     if headers is not None:
@@ -35,6 +46,7 @@ def getenv_headers(key) -> Dict:
 http_sem = Semaphore(http_concorent if http_concorent is not None else 8)
 file_sem = Semaphore(512)
 http_headers = getenv_headers('HTTP_HEADERS')
+http_sleep = getenv_float('HTTP_SLEEP') or 0
 
 
 def get_cache_datetime(path) -> datetime:
@@ -75,6 +87,8 @@ async def download_item(url: str, path: str, cache_days: int, is_valid: Callable
                     os.makedirs(os.path.dirname(save_path), exist_ok=True)
                     async with async_open(save_path, 'w') as f:
                         await f.write(text)
+                    if http_sleep is not None:
+                        await asyncio.sleep(http_sleep)
                     return text
         except Exception as e:
             logger.error(" down err: %s" % e)

setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name='citation_crawler',
-    version='2.9',
+    version='2.9.1',
     author='yindaheng98',
     author_email='[email protected]',
     url='https://github.com/yindaheng98/citation-crawler',