From 81b1efb0659edf1767dfa0c134cc863da3e5219e Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:24:19 +1200 Subject: [PATCH] Updated timeout to increase on retries Convert config values to expected types Fixed retry logic --- ckanext/invalid_uris/helpers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ckanext/invalid_uris/helpers.py b/ckanext/invalid_uris/helpers.py index 799c475..ba36ba8 100644 --- a/ckanext/invalid_uris/helpers.py +++ b/ckanext/invalid_uris/helpers.py @@ -23,10 +23,12 @@ def valid_uri(uri, retries=0, method='head'): proxies = None headers = None proxy = config.get('ckanext.invalid_uris.proxy') - timeout = config.get('ckanext.invalid_uris.timeout', 10) + timeout = toolkit.asint(config.get('ckanext.invalid_uris.timeout', 10)) + # Increase timeout for each retry attempt. + timeout = timeout * (retries+1) user_agent = config.get('ckanext.invalid_uris.user_agent') - verify_certificate = config.get('ckanext.invalid_uris.verify_certificate', True) - retry_attempts = config.get('ckanext.invalid_uris.retry_attempts', 3) + verify_certificate = toolkit.asbool(config.get('ckanext.invalid_uris.verify_certificate', True)) + retry_attempts = toolkit.asint(config.get('ckanext.invalid_uris.retry_attempts', 3)) try: if proxy: @@ -60,9 +62,9 @@ def valid_uri(uri, retries=0, method='head'): } except (requests.RequestException, requests.ConnectionError, requests.HTTPError) as e: log.error(f'Request exception: {e}') - if retries <= retry_attempts: - log.warning(f'Retry attempt {retries} for uri {uri}') + if retries < retry_attempts: retries = retries+1 + log.warning(f'Retry attempt {retries} for uri {uri}') time.sleep(retries) result = valid_uri(uri, retries) else: