ASFHyP3 · jacquelynsmale · Dec 14, 2023 · Nov 14, 2023 · Nov 14, 2023 · Nov 14, 2023
diff --git a/image_server/environment.yml b/image_server/environment.yml
@@ -5,7 +5,6 @@ channels:
 dependencies:
   - python=3.7
   - arcgis-server-py3=10.9.1
-  - asf_search>=6.7.0
   - boto3
   - gdal>=2.4.4,<3
   - lxml

diff --git a/image_services/opera/config/rtc_vh_urls.json b/image_services/opera/config/rtc_vh_urls.json
@@ -1,11 +1,11 @@
 {
   "project_name": "OPERA",
   "raster_store": "/home/arcgis/raster_store/",
-  "bucket": "hyp3-testing",
+  "bucket": "asf-cumulus-prod-opera-products",
   "overview_path": "/vsis3/gis-service-overviews/overviews/",
   "s3_prefix": "opera-rtc-image-service-na/VH",
   "s3_suffix": "_VH.tif",
-  "dataset_name": "RTC_VH_URLS",
+  "dataset_name": "RTC_VH",
   "raster_function_templates": [
     "Sentinel1_RTC_Power.rft.xml",
     "Sentinel1_RTC_Amplitude.rft.xml",
@@ -14,7 +14,7 @@
   ],
   "default_raster_function_template": "Sentinel1_RTC_dB_Stretch.rft.xml",
   "service_folder": "OPERA",
-  "service_name": "OPERA_RTC_VH",
+  "service_name": "OPERA_RTC_VH_URLS",
   "service_definition_overrides": {
     "Type": "esriServiceDefinitionType_Replacement",
     "Configurations/SVCConfiguration/Definition/Description": "OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VH polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL).",

diff --git a/image_services/opera/config/rtc_vv_urls.json b/image_services/opera/config/rtc_vv_urls.json
@@ -0,0 +1,40 @@
+{
+  "project_name": "OPERA",
+  "raster_store": "/home/arcgis/raster_store/",
+  "bucket": "asf-cumulus-prod-opera-products",
+  "overview_path": "/vsis3/gis-service-overviews/overviews/",
+  "s3_suffix": "_VV.tif",
+  "dataset_name": "RTC_VV",
+  "raster_function_templates": [
+    "Sentinel1_RTC_Power.rft.xml",
+    "Sentinel1_RTC_Amplitude.rft.xml",
+    "Sentinel1_RTC_dB.rft.xml",
+    "Sentinel1_RTC_dB_Stretch.rft.xml"
+  ],
+  "default_raster_function_template": "Sentinel1_RTC_dB_Stretch.rft.xml",
+  "service_folder": "OPERA",
+  "service_name": "OPERA_RTC_VV_URLS",
+  "service_definition_overrides": {
+    "Type": "esriServiceDefinitionType_Replacement",
+    "Configurations/SVCConfiguration/Definition/Description": "OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL).",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='description']/Value": "OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the VV polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='copyright']/Value": "OPERA Level-2 Radiometric Terrain Corrected from Sentinel-1. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. Contains modified Copernicus Sentinel data, processed by ESA.",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='idleTimeout']/Value": "3600",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='minInstances']/Value": "1",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='maxInstances']/Value": "9",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='usageTimeout']/Value": "1200",
+    "Configurations/SVCConfiguration/Definition/ConfigurationProperties/PropertyArray/PropertySetProperty[Key='waitTimeout']/Value": "120",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='IdleTimeout']/Value": "3600",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='keepAliveInterval']/Value": "600",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='MinInstances']/Value": "1",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='MaxInstances']/Value": "9",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='UsageTimeout']/Value": "1200",
+    "Configurations/SVCConfiguration/Definition/Props/PropertyArray/PropertySetProperty[Key='WaitTimeout']/Value": "120",
+    "ItemInfo/Title": "OPERA RTC VV",
+    "ItemInfo/Credits": "OPERA Level-2 Radiometric Terrain Corrected products from Sentinel-1. Contains modified Copernicus Sentinel data, processed by ESA.",
+    "ItemInfo/Description": "OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the VV polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.",
+    "ItemInfo/Snippet": "OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL).",
+    "Configurations/SVCConfiguration/Definition/Extensions/SVCExtension[TypeName='WMSServer']/Props/PropertyArray/PropertySetProperty[Key='title']/Value": "OPERA_RTC_VV_WMS",
+    "Configurations/SVCConfiguration/Definition/Extensions/SVCExtension[TypeName='WMSServer']/Props/PropertyArray/PropertySetProperty[Key='abstract']/Value": "WMS service for Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL)."
+  }
+}
diff --git a/image_services/opera/make_opera_services.py b/image_services/opera/make_opera_services.py
@@ -10,32 +10,20 @@
 from typing import List
 
 import arcpy
-import asf_search
 from arcgis.gis.server import Server
 from lxml import etree
 from osgeo import gdal, osr
 from tenacity import Retrying, before_sleep_log, stop_after_attempt, wait_fixed
 
+gdal.UseExceptions()
+gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
 
-def get_rasters():
-    # TODO: get rid of or update the interesects with option
-    options = {
-        'intersectsWith': 'POLYGON((-78.5937 37.5232,-74.494 37.5232,-74.494 39.8807,-78.5937 39.8807,-78.5937 '
-                          '37.5232))',
-        'dataset': 'OPERA-S1',
-        'start': '2023-10-25T08:00:00Z',
-        'processingLevel': 'RTC',
-        'polarization': 'VH',
-    }
-    results = asf_search.search(**options)
 
-    urls = []
-    for result in results:
-        for item in result.umm['RelatedUrls']:
-            url = item['URL']
-            if url.startswith('s3://') and url.endswith('VH.tif'):
-                urls.append(url.replace('s3://', '/vsis3/'))
-    return urls
+def get_rasters(dataset_name):
+    filename = f'{dataset_name}_vsis3_urls.csv'
+    with open(filename) as urlfile:
+        records = urlfile.read().splitlines()[:-1]
+    return [f'{record}' for record in records]
 
 
 def get_pixel_type(data_type: str) -> int:
@@ -56,11 +44,10 @@ def remove_prefix(raster_path, prefix):
     return raster_path[len(prefix):]
 
 
-def get_raster_metadata(raster_path: str, bucket: str, s3_prefix: str) -> dict:
+def get_raster_metadata(raster_path: str, bucket: str) -> dict:
     assert raster_path.startswith(f'/vsis3/{bucket}/')
-    key = remove_prefix(raster_path, f'/vsis3/{bucket}/')
-    download_url = f'https://datapool.asf.alaska.edu/RTC/{key}'
     name = Path(raster_path).stem
+    download_url = f'https://datapool.asf.alaska.edu/RTC/OPERA-S1/{name}.tif'
     acquisition_date = \
         name[36:38] + '/' + name[38:40] + '/' + name[32:36] + ' ' + name[41:43] + ':' + name[43:45] + ':' + name[45:47]
     info = gdal.Info(raster_path, format='json')
@@ -84,7 +71,7 @@ def get_raster_metadata(raster_path: str, bucket: str, s3_prefix: str) -> dict:
     }
 
 
-def update_csv(csv_file: str, rasters: List[str], bucket: str, s3_prefix: str):
+def update_csv(csv_file: str, rasters: List[str], bucket: str):
     if os.path.isfile(csv_file):
         with open(csv_file) as f:
             records = [record for record in csv.DictReader(f)]
@@ -97,13 +84,13 @@ def update_csv(csv_file: str, rasters: List[str], bucket: str, s3_prefix: str):
     logging.info(f'Adding {len(new_rasters)} new items to {csv_file}')
 
     if new_rasters:
-        header_record = get_raster_metadata(next(iter(new_rasters)), bucket, s3_prefix)
+        header_record = get_raster_metadata(next(iter(new_rasters)), bucket)
         with open(csv_file, 'a', newline='') as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=header_record.keys(), lineterminator=os.linesep)
             if not existing_rasters:
                 writer.writeheader()
             for raster in new_rasters:
-                record = get_raster_metadata(raster, bucket, s3_prefix)
+                record = get_raster_metadata(raster, bucket)
                 writer.writerow(record)
 
     with open(csv_file) as f:
@@ -208,10 +195,6 @@ def main():
     with open(args.config_file) as f:
         config = json.load(f)
 
-    cookie_file = Path.home() / 'cookies.txt'
-    os.environ['GDAL_HTTP_COOKIEFILE'] = str(cookie_file)
-    os.environ['GDAL_HTTP_COOKIEJAR'] = str(cookie_file)
-
     csv_file = os.path.join(args.working_directory, f'{config["project_name"]}_{config["dataset_name"]}.csv')
 
     raster_function_template = ''.join([f'{template_directory / template};'
@@ -224,8 +207,8 @@ def main():
     arcpy.env.parallelProcessingFactor = '75%'
 
     try:
-        rasters = get_rasters()
-        update_csv(csv_file, rasters, config['bucket'], config['s3_prefix'])
+        rasters = get_rasters(csv_file.replace(".csv", ""))
+        update_csv(csv_file, rasters, config['bucket'])
 
         for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(60), reraise=True,
                                 before_sleep=before_sleep_log(logging, logging.WARNING)):
@@ -343,6 +326,12 @@ def main():
                     out_rasterdataset=local_overview,
                 )
 
+            del os.environ['AWS_ACCESS_KEY_ID']
+            del os.environ['AWS_SECRET_ACCESS_KEY']
+
+            os.environ['AWS_DEFAULT_PROFILE'] = 'hyp3'
+            os.environ['AWS_PROFILE'] = 'hyp3'
+
             logging.info(f'Moving CRF to {s3_overview}')
             subprocess.run(['aws', 's3', 'cp', local_overview, s3_overview.replace('/vsis3/', 's3://'), '--recursive'])
 

diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_CA_VH_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_CA_VH_metadata.txt
@@ -0,0 +1,25 @@
+Title:
+OPERA L2 RTC-S1 V1 CA VH
+
+Tags:
+ASF, Alaska Satellite Facility, Synthetic Aperture Radar, SAR, Sentinel-1, OPERA, Radiometric Terrain Correction, RTC
+
+Summary:
+OPERA Level-2 Radiometric Terrain Corrected (RTC) SAR Backscatter products for an area in Northern California from 07-Oct-2023 to 03-Dec-2023 in VH polarization generated from Sentinel-1 (S1) SAR imagery, processed by the OPERA project at JPL.
+
+Description:
+OPERA Level-2 Radiometric Terrain Corrected (RTC) SAR Backscatter products in VH polarization generated from Sentinel-1 (S1) SAR imagery, processed by the OPERA project at JPL. Values for the VH (cross-polarized) polarization are generally driven by volume scattering, with more complex volume scatterers (such as dense vegetation) returning higher backscatter values. Surface water generally appears very dark, as it is predominantly a surface scatterer; most returns remain in the primary polarization.
+
+OPERA RTC-S1 data is available over all landmasses (excluding Antarctica) for Sentinel-1 acquisitions starting October 4, 2023. This is a test service that only includes the data available for a region in Northern California, acquired between October 7 and December 3, 2023. We are working towards providing services that include the full global extent of the dataset, but in incremental developments.
+
+The source rasters referenced in this service are hosted by the Alaska Satellite Facility in NASA's Earthdata Cloud. More information about the dataset is available here: [https://asf.alaska.edu/datasets/daac/opera/](https://asf.alaska.edu/datasets/daac/opera/)
+
+OPERA includes partners from NASA’s Goddard Space Flight Center, the U.S. Geological Survey (USGS), the University of Maryland College Park, the University of Alaska Fairbanks, Southern Methodist University, and EarthScope. Refer to [OPERA’s website](https://www.jpl.nasa.gov/go/opera) for more details.
+
+<b>Data Use:</b> Please cite the use of this data set using the following:
+OPERA. 2023. OPERA Level-2 Radiometric Terrain Corrected from Sentinel-1. Fairbanks, Alaska USA. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. [Date Accessed]
+
+<b>Acknowledgements:</b> This service is provided by [Earthdata GIS (EGIS)](https://gis.earthdata.nasa.gov) supported by [Earth Observing System Data and Information System (EOSDIS)](https://www.earthdata.nasa.gov/). The OPERA project is supported by the Satellite Needs Working Group.
+
+Credits:
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1, processed by the OPERA project at JPL. Contains modified Copernicus Sentinel data, processed by ESA.
diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_CA_VV_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_CA_VV_metadata.txt
@@ -0,0 +1,25 @@
+Title:
+OPERA L2 RTC-S1 V1 CA VV
+
+Tags:
+ASF, Alaska Satellite Facility, Synthetic Aperture Radar, SAR, Sentinel-1, OPERA, Radiometric Terrain Correction, RTC
+
+Summary:
+OPERA Level-2 Radiometric Terrain Corrected (RTC) SAR Backscatter products for an area in Northern California from 07-Oct-2023 to 03-Dec-2023 in VV polarization generated from Sentinel-1 (S1) SAR imagery, processed by the OPERA project at JPL.
+
+Description:
+OPERA Level-2 Radiometric Terrain Corrected (RTC) SAR Backscatter products in VV polarization generated from Sentinel-1 (S1) SAR imagery, processed by the OPERA project at JPL. Values for the VV polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.
+
+OPERA RTC-S1 data is available over all landmasses (excluding Antarctica) for Sentinel-1 acquisitions starting October 4, 2023. This is a test service that only includes the data available for a region in Northern California, acquired between October 7 and December 3, 2023. We are working towards providing services that include the full global extent of the dataset, but in incremental developments.
+
+The source rasters referenced in this service are hosted by the Alaska Satellite Facility in NASA's Earthdata Cloud. More information about the dataset is available here: [https://asf.alaska.edu/datasets/daac/opera/](https://asf.alaska.edu/datasets/daac/opera/)
+
+OPERA includes partners from NASA’s Goddard Space Flight Center, the U.S. Geological Survey (USGS), the University of Maryland College Park, the University of Alaska Fairbanks, Southern Methodist University, and EarthScope. Refer to [OPERA’s website](https://www.jpl.nasa.gov/go/opera) for more details.
+
+<b>Data Use:</b> Please cite the use of this data set using the following:
+OPERA. 2023. OPERA Level-2 Radiometric Terrain Corrected from Sentinel-1. Fairbanks, Alaska USA. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. [Date Accessed]
+
+<b>Acknowledgements:</b> This service is provided by [Earthdata GIS (EGIS)](https://gis.earthdata.nasa.gov) supported by [Earth Observing System Data and Information System (EOSDIS)](https://www.earthdata.nasa.gov/). The OPERA project is supported by the Satellite Needs Working Group.
+
+Credits:
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1, processed by the OPERA project at JPL. Contains modified Copernicus Sentinel data, processed by ESA.
diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_HH_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_HH_metadata.txt
@@ -11,4 +11,4 @@ Description:
 OPERA Level-2 Radiometric Terrain Corrected (RTC) products in HH polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the HH polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.
 
 Credits:
-OPERA Level-2 Radiometric Terrain Corrected Static Layers from Sentinel-1. Contains modified Copernicus Sentinel data, processed by ESA.
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. Contains modified Copernicus Sentinel data, processed by ESA.
diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_HV_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_HV_metadata.txt
@@ -11,4 +11,4 @@ Description:
 OPERA Level-2 Radiometric Terrain Corrected (RTC) products in HV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the HV polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.
 
 Credits:
-OPERA Level-2 Radiometric Terrain Corrected Static Layers from Sentinel-1. Contains modified Copernicus Sentinel data, processed by ESA.
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. Contains modified Copernicus Sentinel data, processed by ESA.
diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_VH_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_VH_metadata.txt
@@ -8,7 +8,7 @@ Summary:
 OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VH polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL)
 
 Description:
-OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VH polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the VH polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.
+OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VH polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the VH (cross-polarized) polarization are generally driven by volume scattering, with more complex volume scatterers (such as dense vegetation) returning higher backscatter values. Surface water generally appears very dark, as it is predominantly a surface scatterer; most returns remain in the primary polarization.
 
 Credits:
-OPERA Level-2 Radiometric Terrain Corrected Static Layers from Sentinel-1. Contains modified Copernicus Sentinel data, processed by ESA.
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. Contains modified Copernicus Sentinel data, processed by ESA.
diff --git a/image_services/opera/metadata/metadata_text/OPERA_RTC_VV_metadata.txt b/image_services/opera/metadata/metadata_text/OPERA_RTC_VV_metadata.txt
@@ -11,4 +11,4 @@ Description:
 OPERA Level-2 Radiometric Terrain Corrected (RTC) products in VV polarization generated from Sentinel-1 SAR imagery, processed by OPERA (JPL). Values for the VV polarization are commonly driven by surface roughness and/or soil moisture, with rougher surfaces and higher soil moisture returning higher backscatter values. Surface water appears very dark under calm conditions, as the signal bounces off the surface away from the sensor.
 
 Credits:
-OPERA Level-2 Radiometric Terrain Corrected Static Layers from Sentinel-1. Contains modified Copernicus Sentinel data, processed by ESA.
+OPERA Level-2 Radiometric Terrain Corrected SAR Backscatter from Sentinel-1. NASA Alaska Satellite Facility Distributed Active Archive Center. doi: https://doi.org/10.5067/SNWG/OPERA_L2_RTC-S1_V1. Contains modified Copernicus Sentinel data, processed by ESA.