Merge pull request #85 from podaac/develop

Develop release to Main (1.10.1)

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
+## [1.10.1]
+### Fixed
+- Support for SHA-256 and SHA-512 checksums
+
 ## [1.10.0]
 ### Changed
 - Changed minimum supported python version to 3.7, down from 3.8.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "podaac-data-subscriber"
-version = "1.10.0"
+version = "1.10.1"
 description = "PO.DAAC Data Subscriber Command Line Tool"
 authors = ["PO.DAAC <[email protected]>"]
 readme = "README.md"

subscriber/podaac_access.py

@@ -21,7 +21,7 @@
 import tenacity
 from datetime import datetime
 
-__version__ = "1.10.0"
+__version__ = "1.10.1"
 extensions = [".nc", ".h5", ".zip", ".tar.gz"]
 edl = "urs.earthdata.nasa.gov"
 cmr = "cmr.earthdata.nasa.gov"
@@ -428,7 +428,8 @@ def make_checksum(file_path, algorithm):
     """
     # Based on https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file#answer-3431838
     # with modification to handle multiple algorithms
-    hash_alg = getattr(hashlib, algorithm.lower())()
+    hashlib_algorithm_name = algorithm.lower().replace("-", "")
+    hash_alg = hashlib.new(hashlib_algorithm_name)
 
     with open(file_path, 'rb') as f:
         for chunk in iter(lambda: f.read(4096), b""):

tests/test_downloader_regression.py

@@ -47,3 +47,24 @@ def test_downloader_MUR():
     assert t2 != os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc')
 
     shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2')
+
+
+@pytest.mark.regression
+def test_downloader_GRACE_with_SHA_512(tmpdir):
+    # start with empty directory
+    directory_str = str(tmpdir)
+    assert len( os.listdir(directory_str) ) == 0
+
+    # run the command once -> should download the file. Note the modified time for the file
+    args = create_downloader_args(f"-c GRACEFO_L2_CSR_MONTHLY_0060 -sd 2020-01-01T00:00:00Z -ed 2020-01-02T00:00:01Z -d {str(tmpdir)} --limit 1 --verbose -e 00".split())
+    pdd.run(args)
+    assert len( os.listdir(directory_str) ) > 0
+    filename = directory_str + "/" + os.listdir(directory_str)[0]
+    modified_time_1 = os.path.getmtime(filename)
+    print( modified_time_1 )
+
+    # run the command again -> should not redownload the file. The modified time for the file should not change
+    pdd.run(args)
+    modified_time_2 = os.path.getmtime(filename)
+    print( modified_time_2 )
+    assert modified_time_1 == modified_time_2

tests/test_subscriber_matching_checksums.py

@@ -35,7 +35,7 @@ def test_checksum_does_match__positive_match_sha512(tmpdir):
     checksums = {
         "tmp.nc": {
             "Value": "439de7997fe599d7af6d108534cae418ac95f70f614e3c2fda7a26b03e599211ffbfc85eede5dd933aa7a3c5cfe87d6b3de30ab2d9b4fd45162a5e22b71fffe8",
-            "Algorithm": "SHA512"
+            "Algorithm": "SHA-512"
         }
     }
 
@@ -50,7 +50,37 @@ def test_checksum_does_match__negative_match_sha512(tmpdir):
     checksums = {
         "tmp.nc": {
             "Value": "439de7997fe599d7af6d108534cae418ac95f70f614e3c2fda7a26b03e599211ffbfc85eede5dd933aa7a3c5cfe87d6b3de30ab2d9b4fd45162a5e22b71fffe8",
-            "Algorithm": "SHA512"
+            "Algorithm": "SHA-512"
+        }
+    }
+
+    with open(output_path, 'w') as f:
+        f.write("This is a different temporary test file")
+
+    assert not checksum_does_match(output_path, checksums)
+
+
+def test_checksum_does_match__positive_match_sha256(tmpdir):
+    output_path = str(tmpdir) + '/tmp.nc'
+    checksums = {
+        "tmp.nc": {
+            "Value": "020b00190141a585d214454e3c1c676eaaab12f10c2cb0bf266a0bdb47a78609",
+            "Algorithm": "SHA-256"
+        }
+    }
+
+    with open(output_path, 'w') as f:
+        f.write("This is a temporary test file")
+
+    assert checksum_does_match(output_path, checksums)
+
+
+def test_checksum_does_match__negative_match_sha256(tmpdir):
+    output_path = str(tmpdir) + '/tmp.nc'
+    checksums = {
+        "tmp.nc": {
+            "Value": "020b00190141a585d214454e3c1c676eaaab12f10c2cb0bf266a0bdb47a78609",
+            "Algorithm": "SHA-256"
         }
     }