QC Updates (ARM-DOE#675)

* ENH: AirNow API must have updating some algorithms.  What it's displaying now is what I would have assumed would be the closed reported station.

* ENH: Updates to 547 and 548 along with some additions to writing out file to add to the history attribute

* ENH: Update to the stability tests for latest errors and removing print statement

* ENH: Bug fix for when it's a single int passed to rm_tests

act/io/armfiles.py

@@ -18,7 +18,9 @@
 
 import numpy as np
 import xarray as xr
+import datetime as dt
 
+import act
 import act.utils as utils
 from act.config import DEFAULT_DATASTREAM_NAME
 from act.utils.io_utils import unpack_tar, unpack_gzip, cleanup_files, is_gunzip_file
@@ -653,6 +655,12 @@ def write_netcdf(
                 except KeyError:
                     pass
 
+        for var_name in list(write_ds.keys()):
+            if 'string' in list(write_ds[var_name].attrs.keys()):
+                att = write_ds[var_name].attrs['string']
+                write_ds[var_name].attrs[var_name + '_string'] = att
+                del write_ds[var_name].attrs['string']
+
         # If requested update global attributes and variables attributes for required
         # CF attributes.
         if cf_compliant:
@@ -740,12 +748,15 @@ def write_netcdf(
 
             # Reorder global attributes to ensure history is last
             try:
-                global_attrs = write_ds.attrs
-                history = copy.copy(global_attrs['history'])
-                del global_attrs['history']
-                global_attrs['history'] = history
+                history = copy.copy(write_ds.attrs['history'])
+                del write_ds.attrs['history']
+                write_ds.attrs['history'] = history
             except KeyError:
                 pass
+        current_time = dt.datetime.now().replace(microsecond=0)
+        if 'history' in list(write_ds.attrs.keys()):
+            write_ds.attrs['history'] += ''.join(['\n', str(current_time), ' created by ACT ', str(act.__version__),
+                                                   ' act.io.write.write_netcdf'])
 
         write_ds.to_netcdf(encoding=encoding, **kwargs)
 

act/qc/arm.py

@@ -19,6 +19,7 @@ def add_dqr_to_qc(
     include=None,
     normalize_assessment=True,
     cleanup_qc=True,
+    dqr_link=False,
 ):
     """
     Function to query the ARM DQR web service for reports and
@@ -60,6 +61,8 @@ def add_dqr_to_qc(
         Call clean.cleanup() method to convert to standardized ancillary
         quality control variables. Has a little bit of overhead so
         if the Dataset has already been cleaned up, no need to run.
+    dqr_link : boolean
+        Prints out a link for each DQR to read the full DQR.  Defaults to False
 
     Returns
     -------
@@ -154,7 +157,9 @@ def add_dqr_to_qc(
                     'test_assessment': line[3],
                     'test_meaning': ': '.join([dqr_no, line[-1]]),
                 }
-
+                if dqr_link:
+                    print_url = 'https://adc.arm.gov/ArchiveServices/DQRService?dqrid=' + str(dqr_no)
+                    print(dqr_no, '-', line[3], ':', print_url)
         for key, value in dqr_results.items():
             try:
                 ds.qcfilter.add_test(

act/qc/qcfilter.py

@@ -1050,6 +1050,43 @@ def datafilter(
             except AttributeError:
                 self._ds[var_name].values = data
 
+            # Adding information on filtering to history attribute
+            flag_masks = None
+            flag_assessments = None
+            flag_meanings = None
+            try:
+                flag_assessments = list(self._ds[qc_var_name].attrs['flag_assessments'])
+                flag_masks = list(self._ds[qc_var_name].attrs['flag_masks'])
+                flag_meanings = list(self._ds[qc_var_name].attrs['flag_meanings'])
+            except KeyError:
+                pass
+
+            # Add comment to history for each test that's filtered out
+            if isinstance(rm_tests, int):
+                rm_tests = [rm_tests]
+            if rm_tests is not None:
+                for test in list(rm_tests):
+                    if test in flag_masks:
+                        index = flag_masks.index(test)
+                        comment = ''.join(['act.qc.datafilter: ', flag_meanings[index]])
+                        if 'history' in self._ds[var_name].attrs.keys():
+                            self._ds[var_name].attrs['history'] += '\n' + comment
+                        else:
+                            self._ds[var_name].attrs['history'] = comment
+
+            if isinstance(rm_assessments, str):
+                rm_assessments = [rm_assessments]
+            if rm_assessments is not None:
+                for assessment in rm_assessments:
+                    if assessment in flag_assessments:
+                        index = [i for i, e in enumerate(flag_assessments) if e == assessment]
+                        for ind in index:
+                            comment = ''.join(['act.qc.datafilter: ', flag_meanings[ind]])
+                            if 'history' in self._ds[var_name].attrs.keys():
+                                self._ds[var_name].attrs['history'] += '\n' + comment
+                            else:
+                                self._ds[var_name].attrs['history'] = comment
+
             # If requested delete quality control variable
             if del_qc_var:
                 del self._ds[qc_var_name]

act/tests/test_qc.py

@@ -754,6 +754,7 @@ def test_datafilter():
     assert np.isclose(ds_1[var_name].values, 98.86, atol=0.01)
     assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01)
     assert isinstance(ds_1[var_name].data, da.core.Array)
+    assert 'act.qc.datafilter' in ds_filtered[var_name].attrs['history']
 
     ds_filtered = copy.deepcopy(ds)
     ds_filtered.qcfilter.datafilter(rm_assessments='Bad', variables=var_name)

act/tests/test_retrievals.py

@@ -27,7 +27,7 @@ def test_get_stability_indices():
         rtol=1e-5,
     )
     assert sonde_ds['parcel_temperature'].attrs['units'] == 'kelvin'
-    np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 1.6223, decimal=2)
+    np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 0.96, decimal=2)
     assert sonde_ds['surface_based_cape'].attrs['units'] == 'J/kg'
     assert sonde_ds['surface_based_cape'].attrs['long_name'] == 'Surface-based CAPE'
     np.testing.assert_almost_equal(sonde_ds['surface_based_cin'], 0.000, decimal=3)