diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 0c9638c..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: bug -assignees: '' - ---- - -### The bug - - -### To Reproduce - - - ### Additional context - diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 4104538..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: enhancement -assignees: '' - ---- -### Background - - -### Describe the solution you'd like - - -### Alternatives - - - ### Additional context - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3dba626..12fad79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: # Docs: https://github.com/ASFHyP3/actions uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 with: - release_prefix: HyP3 opera-rtc-s1-browse + release_prefix: opera-rtc-s1-browse release_branch: main develop_branch: develop sync_pr_label: actions-bot diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c599122..8beb987 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,3 +16,5 @@ jobs: uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 with: local_package_name: opera_rtc_s1_browse + python_versions: >- + ["3.10", "3.11", "3.12"] diff --git a/.trufflehog.txt b/.trufflehog.txt deleted file mode 100644 index a91912f..0000000 --- a/.trufflehog.txt +++ /dev/null @@ -1 +0,0 @@ -.*gitleaks.toml$ diff --git a/CHANGELOG.md b/CHANGELOG.md index fce278f..e0b08a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,5 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0](opera-rtc-s1-browse/compare/v0.0.0...v0.1.0) -- unreleased +## [0.1.0] +### Added +* RTC granule download functionality +* Browse image creation functionality + +### Removed +* Unused functionality created during HyP3-Cookiecutter setup diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 1b33785..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -global-exclude *.py[cod] __pycache__ *.so diff --git a/README.md b/README.md index 949a7a7..5e4e68a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,56 @@ # opera-rtc-s1-browse -A tool for create OPERA RTC S1 browse images +A tool for creating OPERA RTC Sentinel-1 browse images for [NASA Worldview](https://worldview.earthdata.nasa.gov). + +## Usage +Once installed (see below for details) you can run the tool using the command: +```bash +python -m opera_rtc_s1_browse OPERA_L2_RTC-S1_T035-073251-IW2_20240113T020816Z_20240113T113128Z_S1A_30_v1.0 +``` +Where you replace `OPERA_L2_RTC-S1_T035-073251-IW2_20240113T020816Z_20240113T113128Z_S1A_30_v1.0` with the name of OPERA RTC S1 product you want to create a browse image for. + +To explore the available options, run: +```bash +python -m opera_rtc_s1_browse --help +``` +These options allow you to specify an AWS S3 bucket path where the browse image will be uploaded. + +## Setup +### Installation +1. Ensure that conda is installed on your system (we recommend using [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) to reduce setup times). +2. Download a local version of the `opera-s1-rtc-browse` repository (`git clone https://github.com/ASFHyP3/opera_rtc_s1_browse.git`) +3. In the base directory for this project, call `mamba env create -f environment.yml` to create your Python environment and activate it (`mamba activate opera-s1-rtc-browse`) +4. Finally, install a development version of the package (`python -m pip install -e .`) + +To run all commands in sequence use: +```bash +git clone https://github.com/ASFHyP3/opera-s1-rtc-browse.git +cd opera-s1-rtc-browse +mamba env create -f environment.yml +mamba activate opera-s1-rtc-browse +python -m pip install -e . +``` + +### Credentials +To use `opera_rtc_s1_browse`, you must provide your Earthdata Login credentials via two environment variables (`EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD`), or via your `.netrc` file. + +If you do not already have an Earthdata account, you can sign up [here](https://urs.earthdata.nasa.gov/home). + +If you would like to set up Earthdata Login via your `.netrc` file, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started) to get started. + +To use the S3 upload functionality, you will also need to have [AWS credentials configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) that have permission to write to the specified S3 bucket. + +## License +`opera-rtc-s1-browse` is licensed under the BSD 2-Clause License. See the LICENSE file for more details. + +## Contributing +Contributions to this project are welcome! If you would like to contribute, please submit a pull request on the GitHub repository. + +## Contact Us +Want to talk about `opera-rtc-s1-browse`? We would love to hear from you! + +Found a bug? Want to request a feature? +[open an issue](https://github.com/ASFHyP3/opera-rtc-s1-browse/issues/new) + +General questions? Suggestions? Or just want to talk to the team? +[chat with us on gitter](https://gitter.im/ASFHyP3/community) diff --git a/environment.yml b/environment.yml index d28f489..c1a50ff 100644 --- a/environment.yml +++ b/environment.yml @@ -3,18 +3,12 @@ channels: - conda-forge - nodefaults dependencies: - - python>=3.8 + - python>=3.10 - pip + # For running + - asf_search + - boto3 + - gdal # For packaging, and testing - - flake8 - - flake8-import-order - - flake8-blind-except - - flake8-builtins - - setuptools - - setuptools_scm - - wheel - pytest - - pytest-console-scripts - pytest-cov - # For running - - hyp3lib>=3,<4 diff --git a/pyproject.toml b/pyproject.toml index c124fbc..f039b39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,31 +16,29 @@ classifiers=[ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ - "hyp3lib>=3,<4", - # insert python dependencies as list here + "asf_search", + "boto3", + "gdal", ] dynamic = ["version", "readme"] [project.optional-dependencies] develop = [ - "flake8", - "flake8-import-order", - "flake8-blind-except", - "flake8-builtins", "pytest", "pytest-cov", - "pytest-console-scripts", ] [project.urls] Homepage = "opera-rtc-s1-browse" -Documentation = "https://hyp3-docs.asf.alaska.edu" +Documentation = "https://github.com/asfhyp3/opera-rtc-s1-browse#opera-rtc-s1-browse" + +[project.scripts] +create_browse = "opera_rtc_s1_browse.create_browse:main" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/opera_rtc_s1_browse/__init__.py b/src/opera_rtc_s1_browse/__init__.py index f90395b..e69de29 100644 --- a/src/opera_rtc_s1_browse/__init__.py +++ b/src/opera_rtc_s1_browse/__init__.py @@ -1,9 +0,0 @@ -"""A tool for create OPERA RTC S1 browse images""" - -from importlib.metadata import version - -__version__ = version(__name__) - -__all__ = [ - '__version__', -] diff --git a/src/opera_rtc_s1_browse/__main__.py b/src/opera_rtc_s1_browse/__main__.py deleted file mode 100644 index a49678b..0000000 --- a/src/opera_rtc_s1_browse/__main__.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -opera-rtc-s1-browse processing -""" -import logging -from argparse import ArgumentParser - -from hyp3lib.aws import upload_file_to_s3 -from hyp3lib.image import create_thumbnail - -from opera_rtc_s1_browse.process import process_opera_rtc_s1_browse - - -def main(): - """ - Entrypoint for opera_rtc_s1_browse - """ - parser = ArgumentParser() - parser.add_argument('--bucket', help='AWS S3 bucket for uploading the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - - # TODO: Your arguments here - parser.add_argument('--greeting', default='Hello world!', help='Write this greeting to a product file') - - args = parser.parse_args() - - logging.basicConfig( - format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO - ) - - product_file = process_opera_rtc_s1_browse( - greeting=args.greeting, - ) - - if args.bucket: - upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) - browse_images = product_file.with_suffix('.png') - for browse in browse_images: - thumbnail = create_thumbnail(browse) - upload_file_to_s3(browse, args.bucket, args.bucket_prefix) - upload_file_to_s3(thumbnail, args.bucket, args.bucket_prefix) - - -if __name__ == '__main__': - main() diff --git a/src/opera_rtc_s1_browse/auth.py b/src/opera_rtc_s1_browse/auth.py new file mode 100644 index 0000000..8355d39 --- /dev/null +++ b/src/opera_rtc_s1_browse/auth.py @@ -0,0 +1,77 @@ +import netrc +import os +from pathlib import Path +from platform import system +from typing import Tuple + + +EARTHDATA_HOST = 'urs.earthdata.nasa.gov' + + +def get_netrc() -> Path: + """Get the location of the netrc file. + + Returns: + Path to the netrc file + """ + netrc_name = '_netrc' if system().lower() == 'windows' else '.netrc' + netrc_file = Path.home() / netrc_name + return netrc_file + + +def find_creds_in_env(username_name, password_name) -> Tuple[str, str]: + """Find credentials for a service in the environment. + + Args: + username_name: Name of the environment variable for the username + password_name: Name of the environment variable for the password + + Returns: + Tuple of the username and password found in the environment + """ + if username_name in os.environ and password_name in os.environ: + username = os.environ[username_name] + password = os.environ[password_name] + return username, password + + return None, None + + +def find_creds_in_netrc(service) -> Tuple[str, str]: + """Find credentials for a service in the netrc file. + + Args: + service: Service to find credentials for + + Returns: + Tuple of the username and password found in the netrc file + """ + netrc_file = get_netrc() + if netrc_file.exists(): + netrc_credentials = netrc.netrc(netrc_file) + if service in netrc_credentials.hosts: + username = netrc_credentials.hosts[service][0] + password = netrc_credentials.hosts[service][2] + return username, password + + return None, None + + +def get_earthdata_credentials() -> Tuple[str, str]: + """Get NASA EarthData credentials from the environment or netrc file. + + Returns: + Tuple of the NASA EarthData username and password + """ + username, password = find_creds_in_env('EARTHDATA_USERNAME', 'EARTHDATA_PASSWORD') + if username and password: + return username, password + + username, password = find_creds_in_netrc(EARTHDATA_HOST) + if username and password: + return username, password + + raise ValueError( + 'Please provide NASA Earthdata credentials via the ' + 'EARTHDATA_USERNAME and EARTHDATA_PASSWORD environment variables, or your netrc file.' + ) diff --git a/src/opera_rtc_s1_browse/create_browse.py b/src/opera_rtc_s1_browse/create_browse.py new file mode 100644 index 0000000..e4af388 --- /dev/null +++ b/src/opera_rtc_s1_browse/create_browse.py @@ -0,0 +1,199 @@ +""" +opera-rtc-s1-browse processing +""" + +import argparse +import logging +from pathlib import Path + +import asf_search +import boto3 +import numpy as np +from osgeo import gdal + +from opera_rtc_s1_browse.auth import get_earthdata_credentials + + +log = logging.getLogger(__name__) +gdal.UseExceptions() +s3 = boto3.client('s3') + + +def download_data(granule: str, working_dir: Path) -> tuple[Path, Path]: + """Download co-pol and cross-pol images for an OPERA S1 RTC granule. + + Args: + granule: The granule to download data for. + working_dir: Working directory to store the downloaded files. + + Returns: + Path to the co-pol and cross-pol images. + """ + result = asf_search.granule_search([granule])[0] + urls = result.properties['additionalUrls'] + urls.append(result.properties['url']) + + co_pol = [x for x in urls if 'VV' in x] + if not co_pol: + raise ValueError('No co-pol found in granule.') + co_pol = co_pol[0] + + cross_pol = [x for x in urls if 'VH' in x] + if not cross_pol: + raise ValueError('No cross-pol found in granule.') + cross_pol = cross_pol[0] + + co_pol_path = working_dir / Path(co_pol).name + cross_pol_path = working_dir / Path(cross_pol).name + if co_pol_path.exists() and cross_pol_path.exists(): + return co_pol_path, cross_pol_path + + username, password = get_earthdata_credentials() + session = asf_search.ASFSession().auth_with_creds(username, password) + asf_search.download_urls(urls=[co_pol, cross_pol], path=working_dir, session=session) + return co_pol_path, cross_pol_path + + +def normalize_image_array( + input_array: np.ndarray, vmin: float | None = None, vmax: float | None = None +) -> np.ndarray: + """Function to normalize a browse image band. + Modified from OPERA-ADT/RTC. + + Args: + input_array: The array to normalize. + vmin: The minimum value to normalize to. + vmax: The maximum value to normalize to. + + Returns + The normalized array. + """ + input_array = input_array.astype(float) + + if vmin is None: + vmin = np.nanpercentile(input_array, 3) + + if vmax is None: + vmax = np.nanpercentile(input_array, 97) + + # gamma correction: 0.5 + is_not_negative = input_array - vmin >= 0 + is_negative = input_array - vmin < 0 + input_array[is_not_negative] = np.sqrt((input_array[is_not_negative] - vmin) / (vmax - vmin)) + input_array[is_negative] = 0 + input_array[np.isnan(input_array)] = 0 + normalized_array = np.round(np.clip(input_array, 0, 1) * 255).astype(np.uint8) + return normalized_array + + +def create_browse_array(co_pol_array: np.ndarray, cross_pol_array: np.ndarray) -> np.ndarray: + """Create a browse image array for an OPERA S1 RTC granule. + Bands are normalized and follow the format: [co-pol, cross-pol, co-pol, no-data]. + + Args: + co_pol_array: Co-pol image array. + cross_pol_array: Cross-pol image array. + + Returns: + Browse image array. + """ + co_pol_nodata = ~np.isnan(co_pol_array) + co_pol = normalize_image_array(co_pol_array, 0, 0.15) + + cross_pol_nodata = ~np.isnan(cross_pol_array) + cross_pol = normalize_image_array(cross_pol_array, 0, 0.025) + + no_data = (np.logical_and(co_pol_nodata, cross_pol_nodata) * 255).astype(np.uint8) + browse_image = np.stack([co_pol, cross_pol, co_pol, no_data], axis=-1) + return browse_image + + +def create_browse_image(co_pol_path: Path, cross_pol_path: Path, working_dir: Path) -> Path: + """Create a browse image for an OPERA S1 RTC granule meeting GIBS requirements. + + Args: + co_pol_path: Path to the co-pol image. + cross_pol_path: Path to the cross-pol image. + working_dir: Working directory to store intermediate files. + + Returns: + Path to the created browse image. + """ + co_pol_ds = gdal.Open(str(co_pol_path)) + co_pol = co_pol_ds.GetRasterBand(1).ReadAsArray() + + cross_pol_ds = gdal.Open(str(cross_pol_path)) + cross_pol = cross_pol_ds.GetRasterBand(1).ReadAsArray() + + browse_array = create_browse_array(co_pol, cross_pol) + + tmp_browse_path = working_dir / 'tmp.tif' + driver = gdal.GetDriverByName('GTiff') + browse_ds = driver.Create(str(tmp_browse_path), browse_array.shape[1], browse_array.shape[0], 4, gdal.GDT_Byte) + browse_ds.SetGeoTransform(co_pol_ds.GetGeoTransform()) + browse_ds.SetProjection(co_pol_ds.GetProjection()) + for i in range(4): + browse_ds.GetRasterBand(i + 1).WriteArray(browse_array[:, :, i]) + + co_pol_ds = None + cross_pol_ds = None + browse_ds = None + + browse_path = working_dir / f'{co_pol_path.stem[:-3]}_rgb.tif' + gdal.Warp( + browse_path, + tmp_browse_path, + dstSRS='+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', + xRes=2.74658203125e-4, + yRes=2.74658203125e-4, + format='GTiff', + creationOptions=['COMPRESS=LZW', 'TILED=YES'], + ) + tmp_browse_path.unlink() + return browse_path + + +def create_browse_and_upload( + granule: str, + bucket: str = None, + bucket_prefix: str = '', + working_dir: Path | None = None, +) -> None: + """Create browse images for an OPERA S1 RTC granule. + + Args: + granule: The granule to create browse images for. + bucket: AWS S3 bucket for upload the final product(s). + bucket_prefix: Add a bucket prefix to product(s). + working_dir: Working directory to store intermediate files. + """ + if working_dir is None: + working_dir = Path.cwd() + + co_pol_path, cross_pol_path = download_data(granule, working_dir) + browse_path = create_browse_image(co_pol_path, cross_pol_path, working_dir) + co_pol_path.unlink() + cross_pol_path.unlink() + + if bucket: + key = str(Path(bucket_prefix) / browse_path.name) + s3.upload_file(browse_path, bucket, key) + + +def main(): + """opera_rtc_s1_browse entrypoint + + Example: + create_browse OPERA_L2_RTC-S1_T035-073251-IW2_20240113T020816Z_20240113T113128Z_S1A_30_v1.0 + """ + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--bucket', help='AWS S3 bucket for uploading the final product') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix for product') + parser.add_argument('granule', type=str, help='OPERA S1 RTC granule to create a browse image for.') + args = parser.parse_args() + + create_browse_and_upload(**args.__dict__) + + +if __name__ == '__main__': + main() diff --git a/src/opera_rtc_s1_browse/process.py b/src/opera_rtc_s1_browse/process.py deleted file mode 100644 index 43b27b2..0000000 --- a/src/opera_rtc_s1_browse/process.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -opera-rtc-s1-browse processing -""" - -import argparse -import logging -from pathlib import Path - -from opera_rtc_s1_browse import __version__ - - -log = logging.getLogger(__name__) - - -def process_opera_rtc_s1_browse(greeting: str = 'Hello world!') -> Path: - """Create a greeting product - - Args: - greeting: Write this greeting to a product file (Default: "Hello world!" ) - """ - log.debug(f'Greeting: {greeting}') - product_file = Path('greeting.txt') - product_file.write_text(greeting) - return product_file - - -def main(): - """process_opera_rtc_s1_browse entrypoint""" - parser = argparse.ArgumentParser( - prog='process_opera_rtc_s1_browse', - description=__doc__, - ) - parser.add_argument('--greeting', default='Hello world!', help='Write this greeting to a product file') - parser.add_argument('--version', action='version', version=f'%(prog)s {__version__}') - args = parser.parse_args() - - process_opera_rtc_s1_browse(**args.__dict__) - - -if __name__ == '__main__': - main() diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..2fbd047 --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,37 @@ +from pathlib import Path + +from opera_rtc_s1_browse import auth + + +def test_get_netrc(monkeypatch): + with monkeypatch.context() as m: + m.setattr(auth, 'system', lambda: 'Windows') + assert auth.get_netrc() == Path.home() / '_netrc' + + with monkeypatch.context() as m: + m.setattr(auth, 'system', lambda: 'Linux') + assert auth.get_netrc() == Path.home() / '.netrc' + + +def test_find_creds_in_env(monkeypatch): + with monkeypatch.context() as m: + m.setenv('TEST_USERNAME', 'foo') + m.setenv('TEST_PASSWORD', 'bar') + assert auth.find_creds_in_env('TEST_USERNAME', 'TEST_PASSWORD') == ('foo', 'bar') + + with monkeypatch.context() as m: + m.delenv('TEST_USERNAME', raising=False) + m.delenv('TEST_PASSWORD', raising=False) + assert auth.find_creds_in_env('TEST_USERNAME', 'TEST_PASSWORD') == (None, None) + + +def test_find_creds_in_netrc(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.setattr(auth, 'get_netrc', lambda: tmp_path / '.netrc') + (tmp_path / '.netrc').write_text('machine test login foo password bar') + assert auth.find_creds_in_netrc('test') == ('foo', 'bar') + + with monkeypatch.context() as m: + m.setattr(auth, 'get_netrc', lambda: tmp_path / '.netrc') + (tmp_path / '.netrc').write_text('') + assert auth.find_creds_in_netrc('test') == (None, None) diff --git a/tests/test_create_browse.py b/tests/test_create_browse.py new file mode 100644 index 0000000..3778f91 --- /dev/null +++ b/tests/test_create_browse.py @@ -0,0 +1,28 @@ +import numpy as np + +from opera_rtc_s1_browse import create_browse + + +def test_normalize_image_array(): + input_array = np.arange(0, 10) + golden_array = np.array([0, 75, 115, 145, 169, 191, 210, 227, 244, 255]) + output_array = create_browse.normalize_image_array(input_array) + assert np.array_equal(output_array, golden_array) + + input_array = np.append(input_array.astype(float), np.nan) + golden_array = np.append(golden_array, 0) + output_array = create_browse.normalize_image_array(input_array) + assert np.array_equal(output_array, golden_array) + + +def test_create_browse_array(): + vv_min, vv_max = 0, 0.15 + test_vv = np.array([[0, vv_min], [(vv_min + vv_max) / 2, vv_max], [np.nan, np.nan]]) + vh_min, vh_max = 0, 0.025 + test_vh = np.array([[np.nan, np.nan], [(vh_min + vh_max) / 2, vh_max], [0, vh_min]]) + output_array = create_browse.create_browse_array(test_vv, test_vh) + assert output_array.shape == (3, 2, 4) + assert np.array_equal(output_array[:, :, 0], np.array([[0, 0], [180, 255], [0, 0]])) + assert np.array_equal(output_array[:, :, 1], np.array([[0, 0], [180, 255], [0, 0]])) + assert np.array_equal(output_array[:, :, 0], output_array[:, :, 2]) + assert np.array_equal(output_array[:, :, 3], np.array([[0, 0], [255, 255], [0, 0]])) diff --git a/tests/test_entrypoints.py b/tests/test_entrypoints.py deleted file mode 100644 index 7fcd2be..0000000 --- a/tests/test_entrypoints.py +++ /dev/null @@ -1,3 +0,0 @@ -def test_opera_rtc_s1_browse(script_runner): - ret = script_runner.run('python', '-m', 'opera_rtc_s1_browse', '-h') - assert ret.success