diff --git a/act/io/armfiles.py b/act/io/armfiles.py index e0ad5f1858..75b08296ff 100644 --- a/act/io/armfiles.py +++ b/act/io/armfiles.py @@ -18,7 +18,9 @@ import numpy as np import xarray as xr +import datetime as dt +import act import act.utils as utils from act.config import DEFAULT_DATASTREAM_NAME from act.utils.io_utils import unpack_tar, unpack_gzip, cleanup_files, is_gunzip_file @@ -653,6 +655,12 @@ def write_netcdf( except KeyError: pass + for var_name in list(write_ds.keys()): + if 'string' in list(write_ds[var_name].attrs.keys()): + att = write_ds[var_name].attrs['string'] + write_ds[var_name].attrs[var_name + '_string'] = att + del write_ds[var_name].attrs['string'] + # If requested update global attributes and variables attributes for required # CF attributes. if cf_compliant: @@ -740,12 +748,15 @@ def write_netcdf( # Reorder global attributes to ensure history is last try: - global_attrs = write_ds.attrs - history = copy.copy(global_attrs['history']) - del global_attrs['history'] - global_attrs['history'] = history + history = copy.copy(write_ds.attrs['history']) + del write_ds.attrs['history'] + write_ds.attrs['history'] = history except KeyError: pass + current_time = dt.datetime.now().replace(microsecond=0) + if 'history' in list(write_ds.attrs.keys()): + write_ds.attrs['history'] += ''.join(['\n', str(current_time), ' created by ACT ', str(act.__version__), + ' act.io.write.write_netcdf']) write_ds.to_netcdf(encoding=encoding, **kwargs) diff --git a/act/qc/arm.py b/act/qc/arm.py index de54a65bba..2f2de06af0 100644 --- a/act/qc/arm.py +++ b/act/qc/arm.py @@ -19,6 +19,7 @@ def add_dqr_to_qc( include=None, normalize_assessment=True, cleanup_qc=True, + dqr_link=False, ): """ Function to query the ARM DQR web service for reports and @@ -60,6 +61,8 @@ def add_dqr_to_qc( Call clean.cleanup() method to convert to standardized ancillary quality control variables. Has a little bit of overhead so if the Dataset has already been cleaned up, no need to run. + dqr_link : boolean + Prints out a link for each DQR to read the full DQR. Defaults to False Returns ------- @@ -154,7 +157,9 @@ def add_dqr_to_qc( 'test_assessment': line[3], 'test_meaning': ': '.join([dqr_no, line[-1]]), } - + if dqr_link: + print_url = 'https://adc.arm.gov/ArchiveServices/DQRService?dqrid=' + str(dqr_no) + print(dqr_no, '-', line[3], ':', print_url) for key, value in dqr_results.items(): try: ds.qcfilter.add_test( diff --git a/act/qc/qcfilter.py b/act/qc/qcfilter.py index cfb2e149ef..877060de09 100644 --- a/act/qc/qcfilter.py +++ b/act/qc/qcfilter.py @@ -1050,6 +1050,43 @@ def datafilter( except AttributeError: self._ds[var_name].values = data + # Adding information on filtering to history attribute + flag_masks = None + flag_assessments = None + flag_meanings = None + try: + flag_assessments = list(self._ds[qc_var_name].attrs['flag_assessments']) + flag_masks = list(self._ds[qc_var_name].attrs['flag_masks']) + flag_meanings = list(self._ds[qc_var_name].attrs['flag_meanings']) + except KeyError: + pass + + # Add comment to history for each test that's filtered out + if isinstance(rm_tests, int): + rm_tests = [rm_tests] + if rm_tests is not None: + for test in list(rm_tests): + if test in flag_masks: + index = flag_masks.index(test) + comment = ''.join(['act.qc.datafilter: ', flag_meanings[index]]) + if 'history' in self._ds[var_name].attrs.keys(): + self._ds[var_name].attrs['history'] += '\n' + comment + else: + self._ds[var_name].attrs['history'] = comment + + if isinstance(rm_assessments, str): + rm_assessments = [rm_assessments] + if rm_assessments is not None: + for assessment in rm_assessments: + if assessment in flag_assessments: + index = [i for i, e in enumerate(flag_assessments) if e == assessment] + for ind in index: + comment = ''.join(['act.qc.datafilter: ', flag_meanings[ind]]) + if 'history' in self._ds[var_name].attrs.keys(): + self._ds[var_name].attrs['history'] += '\n' + comment + else: + self._ds[var_name].attrs['history'] = comment + # If requested delete quality control variable if del_qc_var: del self._ds[qc_var_name] diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py index d1f031b03e..f59c08e38f 100644 --- a/act/tests/test_qc.py +++ b/act/tests/test_qc.py @@ -754,6 +754,7 @@ def test_datafilter(): assert np.isclose(ds_1[var_name].values, 98.86, atol=0.01) assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01) assert isinstance(ds_1[var_name].data, da.core.Array) + assert 'act.qc.datafilter' in ds_filtered[var_name].attrs['history'] ds_filtered = copy.deepcopy(ds) ds_filtered.qcfilter.datafilter(rm_assessments='Bad', variables=var_name) diff --git a/act/tests/test_retrievals.py b/act/tests/test_retrievals.py index 700e8a3314..fb521840f7 100644 --- a/act/tests/test_retrievals.py +++ b/act/tests/test_retrievals.py @@ -27,7 +27,7 @@ def test_get_stability_indices(): rtol=1e-5, ) assert sonde_ds['parcel_temperature'].attrs['units'] == 'kelvin' - np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 1.6223, decimal=2) + np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 0.96, decimal=2) assert sonde_ds['surface_based_cape'].attrs['units'] == 'J/kg' assert sonde_ds['surface_based_cape'].attrs['long_name'] == 'Surface-based CAPE' np.testing.assert_almost_equal(sonde_ds['surface_based_cin'], 0.000, decimal=3)