From 4a100c234b466c71864247349c1d61fe4073fbe6 Mon Sep 17 00:00:00 2001 From: LTrasca Date: Mon, 2 Dec 2024 17:16:45 +0000 Subject: [PATCH 1/4] init --- app/main/routes.py | 6 +- app/main/util/render_utils.py | 125 +++++++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index de3e963a..a124d4da 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -45,6 +45,8 @@ paginate, ) from app.main.util.render_utils import ( + UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES, + UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES, create_presigned_url, generate_breadcrumb_values, generate_image_manifest, @@ -828,9 +830,9 @@ def generate_manifest(record_id: uuid.UUID): filename = file.FileName file_type = filename.split(".")[-1].lower() - if file_type == "pdf": + if file_type in UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES: return generate_pdf_manifest(record_id) - elif file_type in ["png", "jpg", "jpeg"]: + elif file_type in UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES: return generate_image_manifest(s3_file_object, record_id) else: return http_exception(BadRequest()) diff --git a/app/main/util/render_utils.py b/app/main/util/render_utils.py index cab2d187..11d37fc6 100644 --- a/app/main/util/render_utils.py +++ b/app/main/util/render_utils.py @@ -7,11 +7,130 @@ from app.main.db.models import File, db from app.main.db.queries import get_file_metadata +UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES = [ + "png", + "jpg", + "jpeg", + "gif", + "bmp", + "ico", + "tif", + "tiff", + "3fr", + "arw", + "cr2", + "crw", + "dcr", + "dng", + "erf", + "k25", + "kdc", + "mef", + "mos", + "mrw", + "nef", + "orf", + "pef", + "ptx", + "raf", + "raw", + "rw2", + "sr2", + "srf", + "x3f", + "heic", + "heif", + "jpe", +] + +UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES = [ + "pdf", + "docx", + "xlsx", + "pptx", + "odoc", + "osheet", + "oslides", + "txt", + "json", + "md", + "xml", + "html", + "htm", +] + +UNIVERSAL_VIEWER_SUPPORTED_OTHER_TYPES = [ + # text and Code Files + "c", + "cpp", + "cs", + "java", + "py", + "js", + "jsx", + "ts", + "tsx", + "php", + "rb", + "pl", + "sql", + "css", + "scss", + "less", + "sh", + "yaml", + "yml", + "xml", + "xhtml", + "plist", + "properties", + "bat", + "cmd", + "make", + "groovy", + "scala", + "swift", + "diff", + "erl", + "lst", + "out", + "patch", + "sml", + # archives and Compressed Files + "zip", + "rar", + "7z", + "gz", + "bz2", + "tar", + "tgz", + "tbz", + "txz", + # multimedia + "mp3", + "wav", + "ogg", + "mp4", + "webm", + # 3D + "obj", + "stl", + "dae", + "gltf", + "glb", + # other Formats + "ps", + "psd", + "epub", + "djvu", + "csv", +] + def get_file_mimetype(file_type): - if file_type == "pdf": - return "application/pdf" - elif file_type in ["png", "jpg", "jpeg"]: + if file_type in UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES: + return f"application/{file_type}" + elif file_type in UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES: return f"image/{file_type}" From 5557f2695b36edc694604f2ab47861d30d2bae20 Mon Sep 17 00:00:00 2001 From: LTrasca Date: Wed, 22 Jan 2025 23:32:25 +0000 Subject: [PATCH 2/4] extended tests for mime type util --- app/main/util/render_utils.py | 6 +++++- app/tests/test_render_utils.py | 25 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/app/main/util/render_utils.py b/app/main/util/render_utils.py index 11d37fc6..6fe32cb8 100644 --- a/app/main/util/render_utils.py +++ b/app/main/util/render_utils.py @@ -128,10 +128,14 @@ def get_file_mimetype(file_type): - if file_type in UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES: + if ( + file_type in UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES + or file_type in UNIVERSAL_VIEWER_SUPPORTED_OTHER_TYPES + ): return f"application/{file_type}" elif file_type in UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES: return f"image/{file_type}" + return None def get_file_details(file): diff --git a/app/tests/test_render_utils.py b/app/tests/test_render_utils.py index e9e56898..473aad73 100644 --- a/app/tests/test_render_utils.py +++ b/app/tests/test_render_utils.py @@ -1,5 +1,6 @@ from unittest.mock import Mock, patch +import pytest from flask import Flask from app.main.util.render_utils import ( @@ -11,11 +12,25 @@ ) -def test_get_file_mimetype(): - assert get_file_mimetype("pdf") == "application/pdf" - assert get_file_mimetype("png") == "image/png" - assert get_file_mimetype("jpg") == "image/jpg" - assert get_file_mimetype("jpeg") == "image/jpeg" +@pytest.mark.parametrize( + "file_type, expected_mimetype", + [ + # test a few supported document types + ("pdf", "application/pdf"), + ("docx", "application/docx"), + ("txt", "application/txt"), + # test a few supported image types + ("png", "image/png"), + ("jpg", "image/jpg"), + ("gif", "image/gif"), + # test a few other types (code files, archives, multimedia) + ("py", "application/py"), + ("zip", "application/zip"), + ("mp3", "application/mp3"), + ], +) +def test_get_file_mimetype(file_type, expected_mimetype): + assert get_file_mimetype(file_type) == expected_mimetype @patch("app.main.util.render_utils.get_file_metadata") From 0766d0d2cd872c67327f90094fbc870fb35c6484 Mon Sep 17 00:00:00 2001 From: LTrasca Date: Thu, 23 Jan 2025 10:59:38 +0000 Subject: [PATCH 3/4] added test for unsupported type --- app/main/util/render_utils.py | 2 +- app/tests/test_render_utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/main/util/render_utils.py b/app/main/util/render_utils.py index 6fe32cb8..92066f7f 100644 --- a/app/main/util/render_utils.py +++ b/app/main/util/render_utils.py @@ -135,7 +135,7 @@ def get_file_mimetype(file_type): return f"application/{file_type}" elif file_type in UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES: return f"image/{file_type}" - return None + return "application/octet-stream" def get_file_details(file): diff --git a/app/tests/test_render_utils.py b/app/tests/test_render_utils.py index 473aad73..4cdf5f1b 100644 --- a/app/tests/test_render_utils.py +++ b/app/tests/test_render_utils.py @@ -27,6 +27,8 @@ ("py", "application/py"), ("zip", "application/zip"), ("mp3", "application/mp3"), + # unsupported type + ("foobar", "application/octet-stream"), ], ) def test_get_file_mimetype(file_type, expected_mimetype): From b012fb9bcc981092a1013deae2cc1939a750c2ae Mon Sep 17 00:00:00 2001 From: LTrasca Date: Mon, 3 Feb 2025 08:41:48 +0000 Subject: [PATCH 4/4] added tests for generate manifest route --- app/main/routes.py | 3 ++ app/tests/test_routes.py | 102 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/app/main/routes.py b/app/main/routes.py index e4c0fc5d..83f61c96 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -823,4 +823,7 @@ def generate_manifest(record_id: uuid.UUID): elif file_type in UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES: return generate_image_manifest(s3_file_object, record_id) else: + current_app.app_logger.error( + f"Failed to create manifest for file with ID {file.FileId}" + ) return http_exception(BadRequest()) diff --git a/app/tests/test_routes.py b/app/tests/test_routes.py index 0e40d19b..9b254590 100644 --- a/app/tests/test_routes.py +++ b/app/tests/test_routes.py @@ -10,6 +10,11 @@ from moto import mock_aws from PIL import Image +from app.main.util.render_utils import ( + UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES, + UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES, +) + def verify_cookies_header_row(data): soup = BeautifulSoup(data, "html.parser") @@ -355,3 +360,100 @@ def test_search_route_with_various_cases_standard_user( for key, expected_value in expected_params.items(): assert f"{key}={expected_value}" in response.headers["Location"] + + @pytest.mark.parametrize( + "document_format", UNIVERSAL_VIEWER_SUPPORTED_DOCUMENT_TYPES + ) + @mock_aws + @patch("app.main.routes.boto3.client") + @patch("app.main.routes.generate_pdf_manifest") + def test_generate_manifest_pdf( + self, + mock_pdf, + mock_boto_client, + app, + client: FlaskClient, + mock_all_access_user, + record_files, + document_format, + ): + """ + Test that a PDF manifest is successfully generated. + """ + mock_all_access_user(client) + file = record_files[0]["file_object"] + file.FileName = f"document.{document_format}" + bucket_name = "test_bucket" + app.config["RECORD_BUCKET_NAME"] = bucket_name + + s3_mock = mock_boto_client.return_value + s3_mock.get_object.return_value = {"Body": b"file content"} + + mock_pdf.return_value = ({"mock": "pdf_manifest"}, 200) + response = client.get(f"/record/{file.FileId}/manifest") + + mock_pdf.assert_called_once() + assert response.status_code == 200 + assert json.loads(response.text) == {"mock": "pdf_manifest"} + + @pytest.mark.parametrize( + "image_format", UNIVERSAL_VIEWER_SUPPORTED_IMAGE_TYPES + ) + @mock_aws + @patch("app.main.routes.boto3.client") + @patch("app.main.routes.generate_image_manifest") + def test_generate_manifest_image( + self, + mock_image, + mock_boto_client, + app, + client: FlaskClient, + mock_all_access_user, + record_files, + image_format, + ): + """ + Test that a image manifest is successfully generated. + """ + mock_all_access_user(client) + file = record_files[0]["file_object"] + file.FileName = f"image.{image_format}" + bucket_name = "test_bucket" + app.config["RECORD_BUCKET_NAME"] = bucket_name + + s3_mock = mock_boto_client.return_value + s3_mock.get_object.return_value = {"Body": b"file content"} + + mock_image.return_value = ({"mock": "image_manifest"}, 200) + response = client.get(f"/record/{file.FileId}/manifest") + + mock_image.assert_called_once() + assert response.status_code == 200 + assert json.loads(response.text) == {"mock": "image_manifest"} + + @mock_aws + @patch("app.main.routes.boto3.client") + def test_generate_manifest_unsupported( + self, + mock_boto_client, + app, + client: FlaskClient, + mock_all_access_user, + record_files, + caplog, + ): + """ + Test that an unsupported format will return a bad request and log the error + """ + mock_all_access_user(client) + file = record_files[0]["file_object"] + file.FileName = "unsupported.xyz" + bucket_name = "test_bucket" + app.config["RECORD_BUCKET_NAME"] = bucket_name + + s3_mock = mock_boto_client.return_value + s3_mock.get_object.return_value = {"Body": b"file content"} + + response = client.get(f"/record/{file.FileId}/manifest") + assert response.status_code == 400 + assert "Failed to create manifest for file with ID" in caplog.text