From 90bda44c6f2c13fd9953b5b3f02077fa6ff1d9e8 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 21 Feb 2024 14:12:56 +0545 Subject: [PATCH 1/5] Fixes unit test with reformatting of functions and replacing typo --- tests/test_API.py | 242 +++------------------------------------------- 1 file changed, 16 insertions(+), 226 deletions(-) diff --git a/tests/test_API.py b/tests/test_API.py index 24fabe75..fa9c0b9d 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -12,7 +12,7 @@ ## Status -def wait_for_task_completion(track_link, max_attempts=6, interval_seconds=10): +def wait_for_task_completion(track_link, max_attempts=12, interval_seconds=5): """ Waits for a task to complete, polling the task status at specified intervals. @@ -93,7 +93,7 @@ def test_snapshot(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - wait_for_task_completion(client, track_link) + wait_for_task_completion(track_link) def test_snapshot_featurecollection(): @@ -126,24 +126,7 @@ def test_snapshot_featurecollection(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_feature(): @@ -171,24 +154,7 @@ def test_snapshot_feature(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_feature_fgb_wrap_geom(): @@ -218,7 +184,7 @@ def test_snapshot_feature_fgb_wrap_geom(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - wait_for_task_completion(client, track_link) + wait_for_task_completion(track_link) def test_snapshot_centroid(): @@ -243,24 +209,7 @@ def test_snapshot_centroid(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_filters(): @@ -486,24 +435,7 @@ def test_snapshot_filters(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_and_filter(): @@ -554,24 +486,7 @@ def test_snapshot_and_filter(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_authentication_uuid(): @@ -597,24 +512,7 @@ def test_snapshot_authentication_uuid(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_snapshot_bind_zip(): @@ -640,24 +538,7 @@ def test_snapshot_bind_zip(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) ## Snapshot Plain @@ -757,24 +638,7 @@ def test_hdx_submit_normal_iso3(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_hdx_submit_normal_iso3_multiple_format(): @@ -803,24 +667,7 @@ def test_hdx_submit_normal_iso3_multiple_format(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_hdx_submit_normal_custom_polygon(): @@ -866,24 +713,7 @@ def test_hdx_submit_normal_custom_polygon(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_custom_submit_normal_custom_polygon_TM_project(): @@ -976,24 +806,7 @@ def test_custom_submit_normal_custom_polygon_TM_project(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_hdx_submit_normal_custom_polygon_upload(): @@ -1040,24 +853,7 @@ def test_hdx_submit_normal_custom_polygon_upload(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" + wait_for_task_completion(track_link) def test_full_hdx_set_iso(): @@ -1342,13 +1138,7 @@ def test_full_hdx_set_iso(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(60) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - print(res) - check_status = res["status"] - assert check_status == "SUCCESS" + wait_for_task_completion(track_link) # ## Tasks connection From 873b6066315bd39e0e9e921d8d05cb293275f4ab Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 21 Feb 2024 14:32:27 +0545 Subject: [PATCH 2/5] Log response of status if task is failed or unattempted --- tests/test_API.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_API.py b/tests/test_API.py index fa9c0b9d..71b73e5a 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -34,7 +34,7 @@ def wait_for_task_completion(track_link, max_attempts=12, interval_seconds=5): if attempt == max_attempts: raise AssertionError( - f"Task did not complete successfully after {max_attempts} attempts" + f"Task did not complete successfully after {max_attempts} attempts with following response {res}" ) From 212efbe2d3454ed0ec187b37f4d36cc82db34250 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 21 Feb 2024 15:13:27 +0545 Subject: [PATCH 3/5] Add test cases for different fileformats --- tests/test_API.py | 148 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 2 deletions(-) diff --git a/tests/test_API.py b/tests/test_API.py index 71b73e5a..bb55a09a 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -157,11 +157,10 @@ def test_snapshot_feature(): wait_for_task_completion(track_link) -def test_snapshot_feature_fgb_wrap_geom(): +def test_snapshot_feature_fgb(): response = client.post( "/v1/snapshot/", json={ - "fgbWrapGeoms": True, "outputType": "fgb", "geometry": { "type": "Feature", @@ -187,6 +186,151 @@ def test_snapshot_feature_fgb_wrap_geom(): wait_for_task_completion(track_link) +def test_snapshot_feature_shp(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "shp", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_gpkg(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "gpkg", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_kml(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "kml", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_sql(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "sql", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_csv(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "csv", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + def test_snapshot_centroid(): response = client.post( "/v1/snapshot/", From 43c94b0db7830ee13d66f40a7bf8e479a558f756 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 21 Feb 2024 15:29:33 +0545 Subject: [PATCH 4/5] Fix issue , unit test failing when tiles are disabled --- src/app.py | 23 +++++++++++++---------- tests/test_API.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/app.py b/src/app.py index 4177c58f..b6e19f20 100644 --- a/src/app.py +++ b/src/app.py @@ -544,16 +544,6 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params): file.write(query) format_options = { - RawDataOutputType.MBTILES.value: { - "format": "MBTILES", - "extra": ( - "-dsco MINZOOM={} -dsco MAXZOOM={} ".format( - params.min_zoom, params.max_zoom - ) - if params.min_zoom and params.max_zoom - else "-dsco ZOOM_LEVEL_AUTO=YES" - ), - }, RawDataOutputType.FLATGEOBUF.value: { "format": "FLATGEOBUF", "extra": "-lco SPATIAL_INDEX=YES VERIFY_BUFFERS=NO", @@ -580,6 +570,19 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params): }, } + if ENABLE_TILES: + format_options[RawDataOutputType.MBTILES.value] = { + "format": "MBTILES", + "extra": ( + "-dsco MINZOOM={} -dsco MAXZOOM={} ".format( + params.min_zoom, params.max_zoom + ) + if params.min_zoom and params.max_zoom + else "-dsco ZOOM_LEVEL_AUTO=YES" + ), + } + + file_name_option = ( f"-nln {params.file_name if params.file_name else 'raw_export'}" ) diff --git a/tests/test_API.py b/tests/test_API.py index bb55a09a..163cbc00 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -186,6 +186,36 @@ def test_snapshot_feature_fgb(): wait_for_task_completion(track_link) +def test_snapshot_feature_fgb_wrap_geom(): + response = client.post( + "/v1/snapshot/", + json={ + "fgbWrapGeoms": True, + "outputType": "fgb", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + def test_snapshot_feature_shp(): response = client.post( "/v1/snapshot/", From 60db771b09957b0f9884bf4177101a7b682abdbc Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 21 Feb 2024 15:33:56 +0545 Subject: [PATCH 5/5] Reformat app using black formatter --- src/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.py b/src/app.py index b6e19f20..0b40042d 100644 --- a/src/app.py +++ b/src/app.py @@ -582,7 +582,6 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params): ), } - file_name_option = ( f"-nln {params.file_name if params.file_name else 'raw_export'}" )