diff --git a/rialto_airflow/harvest/dimensions.py b/rialto_airflow/harvest/dimensions.py index fe69bab..61f1f11 100644 --- a/rialto_airflow/harvest/dimensions.py +++ b/rialto_airflow/harvest/dimensions.py @@ -22,6 +22,7 @@ def dois_from_orcid(orcid): """.format(orcid) # The Dimensions API can flake out sometimes, so try to catch & retry. + # TODO: Consider using retry param in query() instead try_count = 0 while try_count < 20: try_count += 1 @@ -82,7 +83,7 @@ def publications_from_dois(dois: list, batch_size=200): limit 1000 """ - result = dsl().query(q) + result = dsl().query(q, retry=5) for pub in result["publications"]: yield normalize_publication(pub) diff --git a/test/harvest/test_openalex.py b/test/harvest/test_openalex.py index 5812e86..f614db2 100644 --- a/test/harvest/test_openalex.py +++ b/test/harvest/test_openalex.py @@ -121,6 +121,7 @@ def test_pyalex_urlencoding(): ), "we handle url URL encoding DOIs until pyalex does" +@pytest.mark.skip(reason="This record no longer exhibits the problem") def test_pyalex_varnish_bug(): # it seems like this author has a few records that are so big they blow out # OpenAlex's Varnish index. See https://groups.google.com/u/1/g/openalex-community/c/hl09WRF3Naw