Reprocessing historical wave data

AODN/AODN-WAVE-DM/BOM-WAVE-DM/bom_wave_library/bom_wave_parser.py

@@ -41,13 +41,12 @@ def metadata_info(station_path):
             'site_code': site_code,
             'latitude': df.loc[site_code]['latitude'],
             'longitude': df.loc[site_code]['longitude'],
-            'timezone': timezone.hour + timezone.minute/60.0,
+            'timezone': timezone.hour + timezone.minute / 60.0,
             'title': "Waverider Buoy observations at {site_name}".format(site_name=df.loc[site_code]['site_name']),
             'instrument': df.loc[site_code]['instrument'],
             'wave_buoy_type': df.loc[site_code]['wave_buoy_type'],
             'water_depth': df.loc[site_code]['water_depth'],
-            'water_depth_units': 'meters',
-            # 'wmo_id': df.loc[site_code]['wmo_id']
+            'water_depth_units': 'm'
             }
 
 
@@ -198,11 +197,11 @@ def gen_nc_bom_wave_dm_deployment(filepath, metadata, output_path):
 
     var_mapping = param_mapping_parser(BOM_WAVE_PARAMETER_MAPPING)
     site_code = metadata['site_code']
-    site_name = metadata['site_name']
-    nc_file_name = 'BOM_W_{date_start}_{site_name}_WAVE-PARAMETERS_END-{date_end}.nc'.format(
-        date_start=wave_df.datetime.dt.strftime('%Y%m%dT%H%M%SZ').values.min(),
-        site_name=site_name,
-        date_end=wave_df.datetime.dt.strftime('%Y%m%dT%H%M%SZ').values.max()
+    site_name = metadata['site_name'].replace(' ', '-')
+    nc_file_name = 'BOM_{date_start}_{site_name}_DM_WAVE-PARAMETERS_END-{date_end}.nc'.format(
+        date_start=wave_df.datetime.dt.strftime('%Y%m%d').values.min(),
+        site_name=str.upper(site_name),
+        date_end=wave_df.datetime.dt.strftime('%Y%m%d').values.max()
     )
 
     temp_dir = tempfile.mkdtemp()
@@ -211,32 +210,42 @@ def gen_nc_bom_wave_dm_deployment(filepath, metadata, output_path):
     try:
         with Dataset(nc_file_path, 'w', format='NETCDF4') as nc_file_obj:
             nc_file_obj.createDimension("TIME", wave_df.datetime.shape[0])
+            nc_file_obj.createDimension("timeSeries", 1)
 
             nc_file_obj.createVariable("LATITUDE", "d", fill_value=99999.)
             nc_file_obj.createVariable("LONGITUDE", "d", fill_value=99999.)
-            # nc_file_obj.createVariable("WAVE_quality_control", "b", fill_value=127)
+            nc_file_obj.createVariable("WAVE_quality_control", "b", "TIME", fill_value=np.int8(-127))
 
             nc_file_obj["LATITUDE"][:] = metadata['latitude']
             nc_file_obj["LONGITUDE"][:] = metadata['longitude']
 
             var_time = nc_file_obj.createVariable("TIME", "d", "TIME")
+            var_timeseries = nc_file_obj.createVariable("timeSeries", "i", "timeSeries")
 
             # add gatts and variable attributes as stored in config files
             generate_netcdf_att(nc_file_obj, NC_ATT_CONFIG, conf_file_point_of_truth=True)
 
             time_val_dateobj = date2num(wave_df.datetime.dt.to_pydatetime(), var_time.units, var_time.calendar)
 
             var_time[:] = time_val_dateobj
+
             qc_flag = [1 for i in range(wave_df.datetime.shape[0])]
-            wave_df['WAVE_quality_control'] = qc_flag
+            flag_values = [1, 2, 3, 4, 9]
+            setattr(nc_file_obj["WAVE_quality_control"], 'flag_values', np.int8(flag_values))
+            nc_file_obj["WAVE_quality_control"][:] = np.int8(qc_flag)
 
             df_varname_ls = list(wave_df[list(wave_df.keys())].columns.values)
             df_varname_ls.remove("datetime")
-            df_varname_ls.remove("Hrms")
-            df_varname_ls.remove("Tc")
-            df_varname_ls.remove("EPS")
-            df_varname_ls.remove("EPS fd")
-            df_varname_ls.remove("Hrms fd")
+            if "Hrms" in df_varname_ls:
+                df_varname_ls.remove("Hrms")
+            if "Tc" in df_varname_ls:
+                df_varname_ls.remove("Tc")
+            if "EPS" in df_varname_ls:
+                df_varname_ls.remove("EPS")
+            if "EPS fd" in df_varname_ls:
+                df_varname_ls.remove("EPS fd")
+            if "Hrms fd" in df_varname_ls:
+                df_varname_ls.remove("Hrms fd")
 
             for df_varname in df_varname_ls:
                 df_varname_mapped_equivalent = df_varname
@@ -248,11 +257,10 @@ def gen_nc_bom_wave_dm_deployment(filepath, metadata, output_path):
                 else:
                     dtype = np.dtype('f')
 
-                nc_file_obj.createVariable(mapped_varname, dtype, "TIME")
+                nc_file_obj.createVariable(mapped_varname, 'd', "TIME")
                 set_var_attr(nc_file_obj, var_mapping, mapped_varname, df_varname_mapped_equivalent, dtype)
                 setattr(nc_file_obj[mapped_varname], 'coordinates', "TIME LATITUDE LONGITUDE")
                 setattr(nc_file_obj[mapped_varname], 'ancillary_variable', "WAVE_quality_control")
-
                 nc_file_obj[mapped_varname][:] = wave_df[df_varname].values
 
             set_glob_attr(nc_file_obj, wave_df, metadata)

AODN/AODN-WAVE-DM/BOM-WAVE-DM/bom_wave_library/buoys_metadata.csv

@@ -1,3 +1,3 @@
 site_name,site_code,owner,state,wave_buoy_type,longitude,latitude,timezone,frequency,date_of_installation,water_depth,instrument
-Cape Du Couedic,COUEDIC,Bureau of Meteorology,SA,Non-Directional,136.62,-36.07,09:30:00,10 minutes,,80,Datawell Waverider FL
-Cape Sorell,SORELL,Bureau of Meteorology,TAS,Non-Directional,145.03,-42.12,10:00:00,10 minutes,,100,Datawell Waverider FL
+Cape Du Couedic,COUEDIC,Bureau of Meteorology,SA,Non-Directional,136.62,-36.07,09:30:00,10 minutes,,80.,Datawell Waverider FL
+Cape Sorell,SORELL,Bureau of Meteorology,TAS,Non-Directional,145.03,-42.12,10:00:00,10 minutes,,100.,Datawell Waverider FL

AODN/AODN-WAVE-DM/BOM-WAVE-DM/bom_wave_library/common.py

@@ -7,24 +7,21 @@
 from python.util import get_git_revision_script_url
 logger = logging.getLogger(__name__)
 
-METHOD_COMMENT = """
-Waverider buoys contain an accelerometer to measure the vertical acceleration as the buoy moves up and
-down with the water surface. By integrating this acceleration with time received from an internal clock, the
-Waverider buoy provides an instantaneous reading of relative water level around a 2000cm mean. Similarly,
-in a Directional Waverider buoy, separate accelerometers are used to measure the horizontal accelerations
-as the buoy moves sideways with the waves.
-As wave periods increase, the acceleration caused by a given wave height becomes lower. This decreased
-acceleration makes it more difficult for the accelerometer to accurately measure the acceleration, and
-therefore the instantaneous water level change caused by longer period waves. Due to this, there is a
-natural drop off in the response (accuracy) of a Waverider buoy as wave periods increase, particularly
-noticeable with periods greater than 20 seconds.
-Conversely, the Waverider buoy in water has a natural frequency around 1 second, causing the buoy to
-overestimate the instantaneous wave caused accelerations around this period, and therefore the associated
-instantaneous water level changes. This is generally not a major problem as waves in coastal and estuarine
-areas usually quickly develop a period of at least 2 seconds.
-"""
-METADATA_FILE = os.path.join(os.path.dirname(__file__), 'buoys_metadata.csv')
+ABSTRACT = """ Waverider buoys contain an accelerometer to measure the vertical acceleration as the buoy moves\
+ up and down with the water surface. By integrating this acceleration with time received from an internal clock, the\
+ Waverider buoy provides an instantaneous reading of relative water level around a 2000cm mean. Similarly,\
+ in a Directional Waverider buoy, separate accelerometers are used to measure the horizontal accelerations\
+ as the buoy moves sideways with the waves. As wave periods increase, the acceleration caused by a given wave height\
+ becomes lower. This decreased acceleration makes it more difficult for the accelerometer to accurately measure the \
+ acceleration, and therefore the instantaneous water level change caused by longer period waves. Due to this, there is\
+ a natural drop off in the response (accuracy) of a Waverider buoy as wave periods increase, particularly noticeable\
+ with periods greater than 20 seconds. Conversely, the Waverider buoy in water has a natural frequency around 1 second,\
+ causing the buoy to overestimate the instantaneous wave caused accelerations around this period, and therefore the\
+ associated instantaneous water level changes. This is generally not a major problem as waves in coastal and estuarine\
+ areas usually quickly develop a period of at least 2 seconds."""
 
+METADATA_FILE = os.path.join(os.path.dirname(__file__), 'buoys_metadata.csv')
+github_comment = 'Product created with %s' % get_git_revision_script_url(os.path.realpath(__file__))
 
 def read_metadata_file():
     """
@@ -88,11 +85,8 @@ def set_glob_attr(nc_file_obj, data, metadata):
     setattr(nc_file_obj, 'time_coverage_end',
             data.datetime.dt.strftime('%Y-%m-%dT%H:%M:%SZ').values.max())
     setattr(nc_file_obj, 'date_created', datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ"))
-    setattr(nc_file_obj, 'abstract', METHOD_COMMENT)
-    # setattr(nc_file_obj, 'original_filename', metadata['original_filename'])
-
-    github_comment = 'Product created with %s' % get_git_revision_script_url(os.path.realpath(__file__))
-    # nc_file_obj.lineage = ('%s %s' % (getattr(nc_file_obj, 'lineage', ''), github_comment))
+    setattr(nc_file_obj, 'abstract', ABSTRACT +
+            ' The original filename was ' + metadata['original_filename'] + '. ' + github_comment)
 
 
 def set_var_attr(nc_file_obj, var_mapping, nc_varname, df_varname_mapped_equivalent, dtype):

AODN/AODN-WAVE-DM/BOM-WAVE-DM/bom_wave_library/generate_nc_file_att

@@ -1,15 +1,14 @@
 [global_attributes]
 abstract                 =    
-acknowledgement          = These data were collected and Quality controlled by the Australian Bureau of Meteorology. These data were made freely available by the AODN - a data facility of the Integrated Marine Observing System [IMOS]. IMOS is supported by the Australian Government.
+acknowledgement          = These data were collected and Quality controlled by the Australian Bureau of Meteorology. These data were made freely available by the AODN - a data facility of the Integrated Marine Observing System [IMOS]. IMOS is enabled by the National Collaborative Research Infrastructure Strategy (NCRIS).
 author                   = Besnard, Laurent
-buoy_specification_url   = http://content.aodn.org.au/Documents/AODN/Waves/Instruments_manuals/datawell_brochure_dwr4_acm_b-38-09.pdf
+cdm_data_type            = Station
 citation                 = The citation to be used in publications using the dataset should follow the format: "Australian Bureau of Meteorology  (date), [Title], Downloaded from [url] on [date-of-download]."
 Conventions              = CF-1.6
 data_centre              = Australian Ocean Data Network (AODN)
 data_centre_email        = [email protected]
 date_created             =
 disclaimer               = Data, products and services from AODN are provided "as is" without any warranty as to fitness for a particular purpose.
-firmware_version         =
 geospatial_lat_units     = degrees_north
 geospatial_lon_units     = degrees_east
 hull_serial_number       =
@@ -23,23 +22,22 @@ license                  = http://creativecommons.org/licenses/by/4.0/
 platform                 = moored surface buoy
 principal_investigator   = Bureau of Meteorology (BoM)
 principal_investigator_email = [email protected]
-project                  = Australian Bureau of Meteorology (BoM)
-source                   = 
+project                  =
 standard_name_vocabulary = NetCDF Climate and Forecast (CF) Metadata Convention CF standard name table v78
 site_name                =
-time_coverage_duration   =
 time_coverage_end        =
 time_coverage_start      =
-title                    = 
-watch_circle             = 
-water_depth              = 
-water_depth_reference    = The Australian Height Datum (AHD)
-water_depth_source       = chart
+title                    =
+water_depth              =
 water_depth_units        = m
 wave_buoy_type           = 
 wave_motion_sensor_type  = 
 wave_sensor_serial_number = 
 
+[timeSeries]
+long_name = Unique identifier for each feature instance
+cf_role = timeseries_id
+
 [TIME]
 standard_name = time
 long_name = time
@@ -70,9 +68,9 @@ reference_datum = WGS84 coordinate reference system; EPSG:4326
 comment =
 
 [WAVE_quality_control]
-long_name = "primary Quality Control flag for wave variables" ;
-valid_min = 1b ;
-valid_max = 9b ;
-flag_values = 1b, 2b, 3b, 4b, 9b ;
-flag_meanings = "good not_evaluated questionable bad missing" ;
-quality_control_convention = "Ocean Data Standards, UNESCO 2013 - IOC Manuals and Guides, 54, Volume 3 Version 1" ;
+long_name = primary Quality Control flag for wave variables
+valid_min = 1
+valid_max = 9
+flag_values = 1, 2, 3, 4, 9
+flag_meanings = good not_evaluated questionable bad missing
+quality_control_convention = Ocean Data Standards, UNESCO 2013 - IOC Manuals and Guides, 54, Volume 3 Version 1