diff --git a/backend/ecs_tasks/delete_files/main.py b/backend/ecs_tasks/delete_files/main.py index 15b2cb28..0690ee09 100644 --- a/backend/ecs_tasks/delete_files/main.py +++ b/backend/ecs_tasks/delete_files/main.py @@ -116,9 +116,11 @@ def build_matches(cols, manifest_object): map( lambda c: { "MatchIds": matches[ - COMPOSITE_MATCH_TOKEN.join(c["Columns"]) - if "Columns" in c - else c["Column"] + ( + COMPOSITE_MATCH_TOKEN.join(c["Columns"]) + if "Columns" in c + else c["Column"] + ) ], **c, }, diff --git a/backend/ecs_tasks/delete_files/requirements.in b/backend/ecs_tasks/delete_files/requirements.in index ec4da4be..87ea141a 100644 --- a/backend/ecs_tasks/delete_files/requirements.in +++ b/backend/ecs_tasks/delete_files/requirements.in @@ -5,6 +5,6 @@ boto3==1.24.38 s3transfer==0.6.0 numpy==1.22.0 cryptography==42.0.4 -urllib3>=1.26.18 +urllib3>=1.26.19 aws-assume-role-lib>=2.10.0 tenacity==8.0.1 diff --git a/backend/ecs_tasks/delete_files/requirements.txt b/backend/ecs_tasks/delete_files/requirements.txt index b49da503..5b29b1be 100644 --- a/backend/ecs_tasks/delete_files/requirements.txt +++ b/backend/ecs_tasks/delete_files/requirements.txt @@ -5,10 +5,10 @@ # pip-compile --output-file=backend/ecs_tasks/delete_files/requirements.txt backend/ecs_tasks/delete_files/requirements.in # aws-assume-role-lib==2.10.0 - # via -r requirements.in + # via -r backend/ecs_tasks/delete_files/requirements.in boto3==1.24.38 # via - # -r requirements.in + # -r backend/ecs_tasks/delete_files/requirements.in # aws-assume-role-lib botocore==1.27.38 # via @@ -17,20 +17,20 @@ botocore==1.27.38 cffi==1.16.0 # via cryptography cryptography==42.0.4 - # via -r requirements.in + # via -r backend/ecs_tasks/delete_files/requirements.in jmespath==1.0.1 # via # boto3 # botocore numpy==1.22.0 # via - # -r requirements.in + # -r backend/ecs_tasks/delete_files/requirements.in # pandas # pyarrow pandas==1.4.3 - # via -r requirements.in + # via -r backend/ecs_tasks/delete_files/requirements.in pyarrow==14.0.1 - # via -r requirements.in + # via -r backend/ecs_tasks/delete_files/requirements.in pycparser==2.21 # via cffi python-dateutil==2.8.2 @@ -38,18 +38,18 @@ python-dateutil==2.8.2 # botocore # pandas python-snappy==0.6.1 - # via -r requirements.in + # via -r backend/ecs_tasks/delete_files/requirements.in pytz==2023.3.post1 # via pandas s3transfer==0.6.0 # via - # -r requirements.in + # -r backend/ecs_tasks/delete_files/requirements.in # boto3 six==1.16.0 # via python-dateutil tenacity==8.0.1 - # via -r requirements.in -urllib3==1.26.18 + # via -r backend/ecs_tasks/delete_files/requirements.in +urllib3==1.26.19 # via - # -r requirements.in + # -r backend/ecs_tasks/delete_files/requirements.in # botocore diff --git a/backend/ecs_tasks/delete_files/s3.py b/backend/ecs_tasks/delete_files/s3.py index f6f6566e..1d3ac60c 100644 --- a/backend/ecs_tasks/delete_files/s3.py +++ b/backend/ecs_tasks/delete_files/s3.py @@ -109,9 +109,9 @@ def get_requester_payment(client, bucket): return ( remove_none( { - "RequestPayer": "requester" - if request_payer["Payer"] == "Requester" - else None, + "RequestPayer": ( + "requester" if request_payer["Payer"] == "Requester" else None + ), } ), request_payer, diff --git a/backend/lambda_layers/aws_sdk/requirements.in b/backend/lambda_layers/aws_sdk/requirements.in index 58626466..bc2d927c 100644 --- a/backend/lambda_layers/aws_sdk/requirements.in +++ b/backend/lambda_layers/aws_sdk/requirements.in @@ -1,3 +1,3 @@ boto3==1.24.38 -urllib3>=1.26.18 +urllib3>=1.26.19 aws-assume-role-lib>=2.10.0 diff --git a/backend/lambda_layers/aws_sdk/requirements.txt b/backend/lambda_layers/aws_sdk/requirements.txt index 89a69c3a..938c2e9d 100644 --- a/backend/lambda_layers/aws_sdk/requirements.txt +++ b/backend/lambda_layers/aws_sdk/requirements.txt @@ -24,7 +24,7 @@ s3transfer==0.6.0 # via boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.18 +urllib3==1.26.19 # via # -r backend/lambda_layers/aws_sdk/requirements.in # botocore diff --git a/backend/lambda_layers/boto_utils/python/boto_utils.py b/backend/lambda_layers/boto_utils/python/boto_utils.py index 66bbb5d6..4968ba02 100644 --- a/backend/lambda_layers/boto_utils/python/boto_utils.py +++ b/backend/lambda_layers/boto_utils/python/boto_utils.py @@ -255,7 +255,7 @@ def json_lines_iterator(content, include_unparsed=False): for i, line in enumerate(lines): try: parsed = json.loads(line) - except (json.JSONDecodeError) as e: + except json.JSONDecodeError as e: raise ValueError( "Serialization error when parsing JSON lines: {}".format( str(e).replace("line 1", "line {}".format(i + 1)), diff --git a/backend/lambdas/data_mappers/handlers.py b/backend/lambdas/data_mappers/handlers.py index f469714b..1c934b55 100644 --- a/backend/lambdas/data_mappers/handlers.py +++ b/backend/lambdas/data_mappers/handlers.py @@ -1,6 +1,7 @@ """ DataMapper handlers """ + import json import os diff --git a/backend/lambdas/jobs/handlers.py b/backend/lambdas/jobs/handlers.py index 64427d0c..57846e09 100644 --- a/backend/lambdas/jobs/handlers.py +++ b/backend/lambdas/jobs/handlers.py @@ -1,6 +1,7 @@ """ Job handlers """ + import json import os diff --git a/backend/lambdas/jobs/stats_updater.py b/backend/lambdas/jobs/stats_updater.py index 57863800..641b0522 100644 --- a/backend/lambdas/jobs/stats_updater.py +++ b/backend/lambdas/jobs/stats_updater.py @@ -1,6 +1,7 @@ """ Job Stats Updater """ + import json import logging @@ -33,9 +34,9 @@ def _aggregate_stats(events): stats += Counter( { "TotalQueryCount": 1, - "TotalQuerySucceededCount": 1 - if event_name == "QuerySucceeded" - else 0, + "TotalQuerySucceededCount": ( + 1 if event_name == "QuerySucceeded" else 0 + ), "TotalQueryFailedCount": 1 if event_name == "QueryFailed" else 0, "TotalQueryScannedInBytes": event_data.get("Statistics", {}).get( "DataScannedInBytes", 0 @@ -53,18 +54,18 @@ def _aggregate_stats(events): ]: stats += Counter( { - "TotalObjectUpdatedCount": 1 - if event_name == "ObjectUpdated" - else 0, - "TotalObjectUpdateSkippedCount": 1 - if event_name == "ObjectUpdateSkipped" - else 0, - "TotalObjectUpdateFailedCount": 1 - if event_name == "ObjectUpdateFailed" - else 0, - "TotalObjectRollbackFailedCount": 1 - if event_name == "ObjectRollbackFailed" - else 0, + "TotalObjectUpdatedCount": ( + 1 if event_name == "ObjectUpdated" else 0 + ), + "TotalObjectUpdateSkippedCount": ( + 1 if event_name == "ObjectUpdateSkipped" else 0 + ), + "TotalObjectUpdateFailedCount": ( + 1 if event_name == "ObjectUpdateFailed" else 0 + ), + "TotalObjectRollbackFailedCount": ( + 1 if event_name == "ObjectRollbackFailed" else 0 + ), } ) diff --git a/backend/lambdas/jobs/status_updater.py b/backend/lambdas/jobs/status_updater.py index 5a37ab2d..0e540914 100644 --- a/backend/lambdas/jobs/status_updater.py +++ b/backend/lambdas/jobs/status_updater.py @@ -1,6 +1,7 @@ """ Job Status Updater """ + import json import logging import os diff --git a/backend/lambdas/queue/handlers.py b/backend/lambdas/queue/handlers.py index cc6ebdc9..0909be96 100644 --- a/backend/lambdas/queue/handlers.py +++ b/backend/lambdas/queue/handlers.py @@ -1,6 +1,7 @@ """ Queue handlers """ + import random import json import os diff --git a/backend/lambdas/settings/handlers.py b/backend/lambdas/settings/handlers.py index 1a29f16e..02494e46 100644 --- a/backend/lambdas/settings/handlers.py +++ b/backend/lambdas/settings/handlers.py @@ -1,6 +1,7 @@ """ Settings handlers """ + import json import boto3 diff --git a/backend/lambdas/tasks/check_queue_size.py b/backend/lambdas/tasks/check_queue_size.py index b4a61982..26f4a612 100644 --- a/backend/lambdas/tasks/check_queue_size.py +++ b/backend/lambdas/tasks/check_queue_size.py @@ -1,6 +1,7 @@ """ Task to check the SQS Queue Size """ + import boto3 from decorators import with_logging diff --git a/backend/lambdas/tasks/check_task_count.py b/backend/lambdas/tasks/check_task_count.py index 0aed9f5f..411c7fad 100644 --- a/backend/lambdas/tasks/check_task_count.py +++ b/backend/lambdas/tasks/check_task_count.py @@ -1,6 +1,7 @@ """ Task to check the number of running and pending tasks """ + import logging import boto3 diff --git a/backend/lambdas/tasks/emit_event.py b/backend/lambdas/tasks/emit_event.py index dcb86995..389dca49 100644 --- a/backend/lambdas/tasks/emit_event.py +++ b/backend/lambdas/tasks/emit_event.py @@ -1,6 +1,7 @@ """ Task to emit events """ + from uuid import uuid4 from boto_utils import emit_event diff --git a/backend/lambdas/tasks/execute_query.py b/backend/lambdas/tasks/execute_query.py index e9a57262..b7fd2ba8 100644 --- a/backend/lambdas/tasks/execute_query.py +++ b/backend/lambdas/tasks/execute_query.py @@ -110,11 +110,13 @@ def make_query(query_data): queryable_matches = ( cast_as_str.format(escape_column(col["Column"])) if is_simple - else cast_as_str.format(escape_column(col["Columns"][0])) - if len(col["Columns"]) == 1 - else "concat({})".format( - columns_composite_join_token.join( - "t.{0}".format(escape_column(c)) for c in col["Columns"] + else ( + cast_as_str.format(escape_column(col["Columns"][0])) + if len(col["Columns"]) == 1 + else "concat({})".format( + columns_composite_join_token.join( + "t.{0}".format(escape_column(c)) for c in col["Columns"] + ) ) ) ) diff --git a/backend/lambdas/tasks/generate_queries.py b/backend/lambdas/tasks/generate_queries.py index 1c4ad11a..7b75f786 100644 --- a/backend/lambdas/tasks/generate_queries.py +++ b/backend/lambdas/tasks/generate_queries.py @@ -1,6 +1,7 @@ """ Task for generating Athena queries from glue catalog aka Query Planning """ + import json import os import boto3 @@ -434,9 +435,11 @@ def column_mapper(col): result = { "Name": col["Name"], "Type": result_type, - "CanBeIdentifier": col["CanBeIdentifier"] - if "CanBeIdentifier" in col - else result_type in ALLOWED_TYPES or type_is_decimal_with_precision, + "CanBeIdentifier": ( + col["CanBeIdentifier"] + if "CanBeIdentifier" in col + else result_type in ALLOWED_TYPES or type_is_decimal_with_precision + ), } if has_children: diff --git a/backend/lambdas/tasks/orchestrate_ecs_service_scaling.py b/backend/lambdas/tasks/orchestrate_ecs_service_scaling.py index b2e986b5..d5909bbf 100644 --- a/backend/lambdas/tasks/orchestrate_ecs_service_scaling.py +++ b/backend/lambdas/tasks/orchestrate_ecs_service_scaling.py @@ -1,6 +1,7 @@ """ Task to orchestrate scaling for a ECS Service """ + import boto3 from decorators import with_logging diff --git a/backend/lambdas/tasks/purge_queue.py b/backend/lambdas/tasks/purge_queue.py index 0a80eb48..2faa31d5 100644 --- a/backend/lambdas/tasks/purge_queue.py +++ b/backend/lambdas/tasks/purge_queue.py @@ -1,6 +1,7 @@ """ Task to purge an SQS queue """ + import boto3 from decorators import with_logging diff --git a/backend/lambdas/tasks/scan_table.py b/backend/lambdas/tasks/scan_table.py index 3a4c52e8..e7149fef 100644 --- a/backend/lambdas/tasks/scan_table.py +++ b/backend/lambdas/tasks/scan_table.py @@ -1,6 +1,7 @@ """ Task to scan a DynamoDB table """ + import boto3 from boto3.dynamodb.types import TypeDeserializer diff --git a/backend/lambdas/tasks/submit_query_results.py b/backend/lambdas/tasks/submit_query_results.py index 789ced0c..707acd95 100644 --- a/backend/lambdas/tasks/submit_query_results.py +++ b/backend/lambdas/tasks/submit_query_results.py @@ -1,6 +1,7 @@ """ Submits results from Athena queries to the Fargate deletion queue """ + import os import boto3 diff --git a/requirements.txt b/requirements.txt index f9cbc697..6f92aade 100644 --- a/requirements.txt +++ b/requirements.txt @@ -212,7 +212,7 @@ typing-extensions==4.6.0 # aws-sam-translator # black # pydantic -urllib3==1.26.18 +urllib3==1.26.19 # via # -r ./backend/ecs_tasks/delete_files/requirements.txt # -r ./backend/lambda_layers/aws_sdk/requirements.txt diff --git a/tests/unit/ecs_tasks/test_s3.py b/tests/unit/ecs_tasks/test_s3.py index 9704f353..97d1bce5 100644 --- a/tests/unit/ecs_tasks/test_s3.py +++ b/tests/unit/ecs_tasks/test_s3.py @@ -178,7 +178,9 @@ def test_it_gets_tagging_args(): client.get_object_tagging.return_value = { "TagSet": [{"Key": "a", "Value": "b"}, {"Key": "c", "Value": "d"}] } - assert {"Tagging": "a=b&c=d",} == get_object_tags( + assert { + "Tagging": "a=b&c=d", + } == get_object_tags( client, "bucket", "key" )[0]