203 simplify ddlpy to hatyan and assume already flattened qc array (#204

)

* fixed cmems example

* avoid additional flattening

* added testcase (disabled if ddlpy not installed)

hatyan/getonlinedata.py

@@ -326,12 +326,13 @@ def ddlpy_to_hatyan(ddlpy_meas):
         #alfanumeric values for 'Typering.Code':'GETETTPE' #DDL IMPROVEMENT: also include numeric values for getijtype. Also, it is quite complex to get this data in the first place, would be convenient if it would be a column when retrieving 'Groepering.Code':'GETETM2' or 'GETETBRKD2'
         key_numericvalues = 'Meetwaarde.Waarde_Alfanumeriek'
         isnumeric = False
+
     ts_pd = pd.DataFrame({'values':ddlpy_meas[key_numericvalues].values,
-                         'QC':pd.to_numeric(ddlpy_meas['WaarnemingMetadata.KwaliteitswaardecodeLijst'].str[0],downcast='integer').values, # DDL IMPROVEMENT: should be possible with .astype(int), but pd.to_numeric() is necessary for HARVT10 (eg 2019-09-01 to 2019-11-01) since QC contains None values that cannot be ints (in that case array of floats with some nans is returned) >> replace None with int code
-                         'Status':ddlpy_meas['WaarnemingMetadata.StatuswaardeLijst'].str[0].values,
-                         #'Bemonsteringshoogte':measurements_wathte['WaarnemingMetadata.BemonsteringshoogteLijst'].str[0].astype(int).values, 
-                         #'Referentievlak':measurements_wathte['WaarnemingMetadata.ReferentievlakLijst'].str[0].values,
-                         #'OpdrachtgevendeInstantie':measurements_wathte['WaarnemingMetadata.OpdrachtgevendeInstantieLijst'].str[0].values,
+                         'QC':pd.to_numeric(ddlpy_meas['WaarnemingMetadata.KwaliteitswaardecodeLijst'].values,downcast='integer'), # DDL IMPROVEMENT: should be possible with .astype(int), but pd.to_numeric() is necessary for HARVT10 (eg 2019-09-01 to 2019-11-01) since QC contains None values that cannot be ints (in that case array of floats with some nans is returned) >> replace None with int code
+                         'Status':ddlpy_meas['WaarnemingMetadata.StatuswaardeLijst'].values,
+                         #'Bemonsteringshoogte':measurements_wathte['WaarnemingMetadata.BemonsteringshoogteLijst'].astype(int).values, 
+                         #'Referentievlak':measurements_wathte['WaarnemingMetadata.ReferentievlakLijst'].values,
+                         #'OpdrachtgevendeInstantie':measurements_wathte['WaarnemingMetadata.OpdrachtgevendeInstantieLijst'].values,
                          },
                         index=pd.to_datetime(ddlpy_meas['Tijdstip']))
 

tests/examples/get_onlinedata_DDL.py

@@ -7,7 +7,6 @@
 
 import ddlpy # TODO: we require ddlpy from main/master branch (>0.1.0) >> pip install git+https://github.com/openearth/ddlpy
 import datetime as dt
-import pandas as pd
 import matplotlib.pyplot as plt
 plt.close("all")
 
@@ -129,20 +128,16 @@
         # filter measured waterlevels (drop waterlevel extremes)
         meas_wathte_ts = meas_wathte.loc[meas_wathte['Groepering.Code'].isin(['NVT'])]
 
-        # flatten quality code
-        ts_measwl = pd.DataFrame({'waterlevels':meas_wathte_ts['Meetwaarde.Waarde_Numeriek'].values,
-                                  'QC':pd.to_numeric(meas_wathte_ts['WaarnemingMetadata.KwaliteitswaardecodeLijst'].str[0],downcast='integer').values,
-                                  'Status':meas_wathte_ts['WaarnemingMetadata.StatuswaardeLijst'].str[0].values}, 
-                                  index=pd.to_datetime(meas_wathte_ts['Tijdstip']))
+        ts_measwl = meas_wathte_ts.set_index("t")
 
         # sort on time values # TODO: do this in ddlpy or in ddl
         ts_measwl = ts_measwl.sort_index()
 
         stat_name = locs_wathte_one.iloc[0]["Naam"]
         stat_code = current_station
         fig, (ax1,ax2) = plt.subplots(2,1, figsize=(8,6), sharex=True)
-        ax1.plot(ts_measwl["waterlevels"])
-        ax2.plot(ts_measwl["QC"])
+        ax1.plot(ts_measwl["Meetwaarde.Waarde_Numeriek"])
+        ax2.plot(ts_measwl["WaarnemingMetadata.KwaliteitswaardecodeLijst"].astype(int))
         ax1.set_title(f'waterlevels for {stat_code} ({stat_name})')
         ax2.set_title(f'QC for {stat_code} ({stat_name})')
         fig.tight_layout()

tests/test_hatyan.py

@@ -13,6 +13,7 @@
 import datetime as dt
 from netCDF4 import Dataset, num2date
 import hatyan
+import pandas.api.types as ptypes
 
 dir_tests = os.path.dirname(__file__) #F9 doesnt work, only F5 (F5 also only method to reload external definition scripts)
 dir_testdata = os.path.join(dir_tests,'data_unitsystemtests')
@@ -861,3 +862,38 @@ def test_DDL_QCvalues():
     assert uniqueQC.dtype=='int8'
     assert (uniqueQC == np.array([ 0, 25])).all()
 
+
+@pytest.mark.unittest
+def test_ddlpy_to_hatyan():
+    try:
+        import ddlpy # TODO: we require ddlpy from main/master branch (>0.1.0) >> pip install git+https://github.com/openearth/ddlpy
+
+        # input parameters
+        tstart_dt = dt.datetime(2019,12,24) #period begins with Gecontroleerd and ends with Ongecontroleerd for HOEKVHLD
+        tstop_dt = dt.datetime(2020,1,5)
+
+        locations = ddlpy.locations()
+
+        bool_station = locations.index.isin(['HOEKVHLD'])
+        bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP'])
+        bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE'])
+        locs_wathte = locations.loc[bool_station & bool_grootheid & bool_hoedanigheid]
+
+        meas_wathte = ddlpy.measurements(locs_wathte.iloc[0], start_date=tstart_dt, end_date=tstop_dt)
+
+        # TODO: rename lowercase code to uppercase Code
+        meas_wathte.columns = [x.replace(".code",".Code") for x in meas_wathte.columns]
+
+        # timeseries
+        meas_wathte_ts = meas_wathte.loc[meas_wathte['Groepering.Code'].isin(['NVT'])]
+        ts_measwl = hatyan.ddlpy_to_hatyan(meas_wathte_ts)
+
+        assert ts_measwl.columns.tolist() == ['values', 'QC', 'Status']
+        assert ts_measwl.index.name == 'Tijdstip'
+
+        assert ptypes.is_float_dtype(ts_measwl["values"])
+        assert ptypes.is_integer_dtype(ts_measwl["QC"])
+        assert ptypes.is_object_dtype(ts_measwl["Status"])
+
+    except ImportError:
+        print("ddlpy is required for this functionality, install with 'pip install rws-ddlpy'")