Merge pull request #46 from nasa/HARMONY-1866-generate-reformatting-f…

…ilename HARMONY-1866 - Enable _reformatted prefix in output file names. HARMONY-1908 - Modernise packaging to use pyproject.toml.
nasa · Sep 26, 2024 · d0b30a3 · d0b30a3
2 parents b912502 + d5a43bb
commit d0b30a3
Show file tree

Hide file tree

Showing 6 changed files with 131 additions and 92 deletions.
diff --git a/Makefile b/Makefile
@@ -9,8 +9,8 @@ version:
 	sed -i.bak "s/__version__ .*/__version__ = \"$(VERSION)\"/" harmony/__init__.py && rm harmony/__init__.py.bak
 
 build: clean version
-	python -m pip install --upgrade --quiet setuptools wheel twine
-	python setup.py --quiet sdist bdist_wheel
+	python -m pip install --upgrade --quiet setuptools wheel twine build
+	python -m build
 
 publish: build
 	python -m twine check dist/*
@@ -35,5 +35,6 @@ test:
 test-no-warnings:
 	pytest --disable-warnings --cov=harmony tests
 
+# HARMONY-1908 - Ignore jinja2 vulnerability (safety ID 70612)
 cve-check:
-	safety check
+	safety check -i 70612
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -4,12 +4,13 @@ Faker ~= 8.1.3
 flake8 ~= 5.0.4
 ipython ~= 8.10.0
 jedi ~= 0.17.2
+packaging ~= 24.1
 parameterized ~= 0.7
+pycodestyle ~= 2.9.1
 pytest ~= 7.2.0
 pytest-cov ~=2.11
 pytest-mock ~=3.5
 python-language-server ~= 0.35
 responses ~=0.22.0
-safety ~= 2.3.5
-pycodestyle ~= 2.9.1
-setuptools ~= 68.1.2
+safety ~= 3.2.7
+setuptools == 70.0.0
diff --git a/harmony/util.py b/harmony/util.py
@@ -49,6 +49,8 @@
                           alive.
     MAX_DOWNLOAD_RETRIES: Number of times to retry HTTP download calls that fail due to transient errors.
 """
+# Remove this import when Python 3.8 is no longer supported
+from __future__ import annotations
 
 from base64 import b64decode
 from collections import namedtuple
@@ -490,10 +492,17 @@ def nop_decrypter(ciphertext):
     return ciphertext
 
 
-def generate_output_filename(filename, ext=None, variable_subset=None, is_regridded=False, is_subsetted=False):
+def generate_output_filename(
+    filename: str,
+    ext: str = None,
+    variable_subset: list[str] = None,
+    is_regridded: bool = False,
+    is_subsetted: bool = False,
+    is_reformatted: bool = False
+) -> str:
     """
     Return an output filename for the given granules according to our naming conventions:
-    {original filename without suffix}(_{single var})?(_regridded)?(_subsetted)?.<ext>
+    {original filename without suffix}(_{single var})?(_regridded)?(_subsetted)?(_reformatted)?.<ext>
 
     Parameters
     ----------
@@ -507,6 +516,8 @@ def generate_output_filename(filename, ext=None, variable_subset=None, is_regrid
             True if a regridding operation has been performed (default: False)
         is_subsetted : bool, optional
             True if a subsetting operation has been performed (default: False)
+        is_reformatted : bool, optional
+            True if a reformatting operation has been performed (default: False)
 
     Returns
     -------
@@ -536,6 +547,9 @@ def generate_output_filename(filename, ext=None, variable_subset=None, is_regrid
         suffixes.append('_regridded')
     if is_subsetted:
         suffixes.append('_subsetted')
+    if is_reformatted:
+        suffixes.append('_reformatted')
+
     suffixes.append(ext)
 
     result = original_basename

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,37 @@
+# License and classifier list:
+# https://pypi.org/pypi?%3Aaction=list_classifiers
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+authors = [
+    {name = "NASA EOSDIS Harmony Team", email = "[email protected]"}
+]
+classifiers = [
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+]
+description = "A library for Python-based Harmony services to parse incoming message, fetch data, stage data, and call back to Harmony."
+dynamic = ["dependencies", "optional-dependencies", "version"]
+license = { text = "License :: OSI Approved :: Apache Software License" }
+name = "harmony-service-lib"
+readme = "README.md"
+requires-python = ">= 3.8"
+
+[project.scripts]
+harmony-service-lib = "harmony.cli.__main__:main"
+
+[project.urls]
+Homepage = "https://github.com/nasa/harmony-service-lib-py"
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies = {dev = {file = ["dev-requirements.txt"]}}
+# Will read __version__ from harmony.__init__.py
+version = {attr = "harmony.__version__"}
+
+[tool.setuptools.packages.find]
+exclude = ["contrib", "docs", "tests*"]
diff --git a/setup.py b/setup.py
@@ -7,68 +7,9 @@
 # To upload this file to PyPI you must build it then upload it:
 # python setup.py sdist bdist_wheel  # build in 'dist' folder
 # python-m twine upload dist/*  # 'twine' must be installed: 'pip install twine'
+#
+# 2024-09-24 - Updated to use pyproject.toml.
 
-import ast
-import io
-import re
-import os
-from setuptools import find_packages, setup
+from setuptools import setup
 
-DEPENDENCIES = []
-with open("requirements.txt", "r") as f:
-    DEPENDENCIES = f.read().strip().split('\n')
-
-DEV_DEPENDENCIES = []
-with open("dev-requirements.txt", "r") as f:
-    DEV_DEPENDENCIES = f.read().strip().split('\n')
-
-EXCLUDE_FROM_PACKAGES = ["contrib", "docs", "tests*"]
-CURDIR = os.path.abspath(os.path.dirname(__file__))
-
-with io.open(os.path.join(CURDIR, "README.md"), "r", encoding="utf-8") as f:
-    README = f.read()
-
-
-def get_version():
-    main_file = os.path.join(CURDIR, "harmony", "__init__.py")
-    _version_re = re.compile(r"__version__\s+=\s+(?P<version>.*)")
-    with open(main_file, "r", encoding="utf8") as f:
-        match = _version_re.search(f.read())
-        version = match.group("version") if match is not None else '"unknown"'
-    return str(ast.literal_eval(version))
-
-
-setup(
-    name="harmony-service-lib",
-    version=get_version(),
-    author="NASA EOSDIS Harmony Team",
-    author_email="[email protected]",
-    description=("A library for Python-based Harmony services to parse incoming messages, "
-                 "fetch data, stage data, and call back to Harmony"),
-    long_description=README,
-    long_description_content_type="text/markdown",
-    url="https://github.com/nasa/harmony-service-lib-py",
-    packages=find_packages(exclude=EXCLUDE_FROM_PACKAGES),
-    include_package_data=True,
-    keywords=[],
-    scripts=[],
-    entry_points={
-        "console_scripts": ["harmony-service-lib=harmony.cli.__main__:main"]
-    },
-    zip_safe=False,
-    install_requires=DEPENDENCIES,
-    # HARMONY-1188 - uncomment this
-    # extras_require={
-    #     'dev': DEV_DEPENDENCIES
-    # },
-    test_suite="tests",
-    python_requires=">=3.8",
-    # license and classifier list:
-    # https://pypi.org/pypi?%3Aaction=list_classifiers
-    license="License :: OSI Approved :: Apache Software License",
-    classifiers=[
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Operating System :: OS Independent"
-    ],
-)
+setup()
diff --git a/tests/test_util.py b/tests/test_util.py
@@ -220,32 +220,71 @@ def test_when_reading_from_queue_health_update_happens(self, parser, mock_path,
 
 
 class TestGenerateOutputFilename(unittest.TestCase):
-    def test_includes_provided_regridded_subsetted_ext(self):
+    def test_includes_provided_suffixes_ext(self):
+        """Ensure the correct combinations of regridded, subsetted and
+        reformatted are included in the correct order, per the optional
+        arguments to the function."""
         url = 'https://example.com/fake-path/abc.123.nc/?query=true'
         ext = 'zarr'
 
         # Basic cases
         variables = []
-        self.assertEqual(
-            util.generate_output_filename(url, ext),
-            'abc.123.zarr'
-        )
-        self.assertEqual(
-            util.generate_output_filename(url, ext, is_subsetted=True),
-            'abc.123_subsetted.zarr'
-        )
-        self.assertEqual(
-            util.generate_output_filename(url, ext, is_regridded=True),
-            'abc.123_regridded.zarr'
-        )
-        self.assertEqual(
-            util.generate_output_filename(url, ext, is_subsetted=True, is_regridded=True),
-            'abc.123_regridded_subsetted.zarr'
-        )
-        self.assertEqual(
-            util.generate_output_filename(url, ext, variable_subset=variables, is_subsetted=True, is_regridded=True),
-            'abc.123_regridded_subsetted.zarr'
-        )
+        with self.subTest('No suffix options'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext),
+                'abc.123.zarr'
+            )
+        with self.subTest('Only is_subsetted'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_subsetted=True),
+                'abc.123_subsetted.zarr'
+            )
+
+        with self.subTest('Only is_regridded'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_regridded=True),
+                'abc.123_regridded.zarr'
+            )
+
+        with self.subTest('Only is_reformatted'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_reformatted=True),
+                'abc.123_reformatted.zarr'
+            )
+
+        with self.subTest('is_subsetted and is_regridded'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_subsetted=True, is_regridded=True),
+                'abc.123_regridded_subsetted.zarr'
+            )
+
+        with self.subTest('is_subsetted, is_regridded with empty variables list'):
+            self.assertEqual(
+                util.generate_output_filename(
+                    url, ext, variable_subset=variables, is_subsetted=True, is_regridded=True
+                ),
+                'abc.123_regridded_subsetted.zarr'
+            )
+
+        with self.subTest('is_subsetted and is_reformatted'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_subsetted=True, is_reformatted=True),
+                'abc.123_subsetted_reformatted.zarr'
+            )
+
+        with self.subTest('is_regridded and is_reformatted'):
+            self.assertEqual(
+                util.generate_output_filename(url, ext, is_regridded=True, is_reformatted=True),
+                'abc.123_regridded_reformatted.zarr'
+            )
+
+        with self.subTest('is_subsetted, is_regridded and is_reformatted'):
+            self.assertEqual(
+                util.generate_output_filename(
+                    url, ext, is_subsetted=True, is_regridded=True, is_reformatted=True
+                ),
+                'abc.123_regridded_subsetted_reformatted.zarr'
+            )
 
     def test_includes_single_variable_name_replacing_slashes(self):
         url = 'https://example.com/fake-path/abc.123.nc/?query=true'
@@ -332,6 +371,12 @@ def test_includes_single_variable(self):
             util.generate_output_filename(url, ext, variable_subset=variables, is_subsetted=True, is_regridded=True),
             'abc.123_VarA_regridded_subsetted.zarr'
         )
+        self.assertEqual(
+            util.generate_output_filename(
+                url, ext, variable_subset=variables, is_subsetted=True, is_regridded=True, is_reformatted=True,
+            ),
+            'abc.123_VarA_regridded_subsetted_reformatted.zarr'
+        )
 
     def test_excludes_multiple_variable(self):
         url = 'https://example.com/fake-path/abc.123.nc/?query=true'