ENH: Updating IO module for V2.0.0 standards and all the subsequent c…

…hanges.

CONTRIBUTING.rst

@@ -198,7 +198,7 @@ An example:
 
 .. code-block:: python
 
-    def read_netcdf(filenames, variables=None):
+    def read_arm_netcdf(filenames, variables=None):
 
         """
         Returns `xarray.Dataset` with stored data and metadata from a
@@ -226,7 +226,7 @@ An example:
 
             import act
 
-            the_ds, the_flag = act.io.armfiles.read_netcdf(
+            the_ds, the_flag = act.io.arm.read_arm_netcdf(
                 act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
             print(the_ds.act.datastream)
         """

act/io/__init__.py

@@ -8,17 +8,17 @@
 __getattr__, __dir__, __all__ = lazy.attach(
     __name__,
 
-    submodules=['armfiles', 'csvfiles', 'icartt', 'mpl', 'neon', 'noaagml', 'noaapsl', 'pysp2'],
+    submodules=['arm', 'csv', 'icartt', 'mpl', 'neon', 'noaagml', 'noaapsl', 'pysp2'],
     submod_attrs={
-        'armfiles': [
+        'arm': [
             'WriteDataset',
             'check_arm_standards',
             'create_ds_from_arm_dod',
-            'read_netcdf',
+            'read_arm_netcdf',
             'check_if_tar_gz_file',
-            'read_mmcr',
+            'read_arm_mmcr',
         ],
-        'csvfiles': ['read_csv'],
+        'csv': ['read_csv'],
         'icartt': ['read_icartt'],
         'mpl': ['proc_sigma_mplv5_read', 'read_sigma_mplv5'],
         'neon': ['read_neon_csv'],

act/io/arm.py

@@ -99,7 +99,7 @@ def read_arm_netcdf(
     .. code-block :: python
 
         import act
-        ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
+        ds = act.io.arm.read_arm_netcdf(act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
         print(ds)
 
     """
@@ -312,7 +312,7 @@ def keep_variables_to_drop_variables(
 
         import act
         filename = '/data/datastream/hou/houkasacrcfrM1.a1/houkasacrcfrM1.a1.20220404.*.nc'
-        drop_vars = act.io.armfiles.keep_variables_to_drop_variables(
+        drop_vars = act.io.arm.keep_variables_to_drop_variables(
             filename, ['lat','lon','alt','crosspolar_differential_phase'],
             drop_variables='variable_name_that_only_exists_in_last_file_of_the_day')
 
@@ -436,7 +436,7 @@ def create_ds_from_arm_dod(proc, set_dims, version='', fill_value=-9999.0, scala
     .. code-block :: python
 
         dims = {'time': 1440, 'drop_diameter': 50}
-        ds = act.io.armfiles.create_ds_from_arm_dod(
+        ds = act.io.arm.create_ds_from_arm_dod(
             'vdis.b1', dims, version='1.2', scalar_fill_dim='time')
 
     """
@@ -798,7 +798,7 @@ def check_if_tar_gz_file(filenames):
     return filenames, cleanup
 
 
-def read_mmcr(filenames):
+def read_arm_mmcr(filenames):
     """
 
     Reads in ARM MMCR files and splits up the variables into specific

act/io/csv.py

@@ -4,10 +4,8 @@
 """
 
 import pathlib
-
 import pandas as pd
-
-from .armfiles import check_arm_standards
+from act.io.arm import check_arm_standards
 
 
 def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0, ignore_index=True, **kwargs):
@@ -48,7 +46,7 @@ def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0
 
         import act
 
-        ds = act.io.csvfiles.read(act.tests.sample_files.EXAMPLE_CSV_WILDCARD)
+        ds = act.io.csv.read(act.tests.sample_files.EXAMPLE_CSV_WILDCARD)
 
     """
 

act/io/mpl.py

@@ -7,11 +7,9 @@
 import shutil
 import subprocess
 import tempfile
-
 import dask
 import xarray as xr
-
-from act.io.armfiles import check_arm_standards
+from act.io.arm import check_arm_standards
 
 if shutil.which('mpl2nc') is not None:
     MPLIMPORT = True

act/io/neon.py

@@ -6,7 +6,7 @@
 import xarray as xr
 import datetime as dt
 import numpy as np
-from act.io.csvfiles import read_csv
+from act.io.csv import read_csv
 
 
 def read_neon_csv(files, variable_files=None, position_files=None):

act/io/noaagml.py

@@ -241,7 +241,7 @@ def read_gml_halo(filename, **kwargs):
                 break
             header += 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', header=header,
         na_values=['Nan', 'NaN', 'nan', 'NAN'], **kwargs)
     var_names = list(ds.data_vars)
@@ -418,7 +418,7 @@ def read_gml_co2(filename=None, convert_missing=True, **kwargs):
     with open(test_filename) as fc:
         skiprows = int(fc.readline().strip().split()[-1]) - 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', skiprows=skiprows, **kwargs)
 
     timestamp = np.full(ds['year'].size, np.nan, dtype="datetime64[ns]")
@@ -538,7 +538,7 @@ def read_gml_ozone(filename=None, **kwargs):
                 pass
             skiprows += 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', skiprows=skiprows, **kwargs)
     ds.attrs['station'] = str(ds['STN'].values[0]).lower()
 
@@ -772,7 +772,7 @@ def read_gml_radiation(filename=None, convert_missing=True,
         names.insert(ii + num, 'qc_' + name)
         num += 1
 
-    ds = act.io.csvfiles.read_csv(filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs)
+    ds = act.io.csv.read_csv(filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs)
 
     if isinstance(filename, (list, tuple)):
         filename = filename[0]
@@ -994,7 +994,7 @@ def read_gml_met(filename=None, convert_missing=True, **kwargs):
         minutes = False
         del column_names['minute']
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', header=None,
         column_names=column_names.keys(), **kwargs)
 

act/io/noaapsl.py

@@ -13,7 +13,7 @@
 import xarray as xr
 import datetime as dt
 
-from act.io.csvfiles import read_csv
+from act.io.csv import read_csv
 
 
 def read_psl_wind_profiler(filepath, transpose=True):

act/plotting/distributiondisplay.py

@@ -20,7 +20,7 @@ class DistributionDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.DistsributionDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The DistributionDisplay constructor takes in the same keyword arguments as

act/plotting/histogramdisplay.py

@@ -20,7 +20,7 @@ class HistogramDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.HistogramDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The HistogramDisplay constructor takes in the same keyword arguments as

act/plotting/skewtdisplay.py

@@ -43,7 +43,7 @@ class and has therefore has the same attributes as that class.
 
     .. code-block :: python
 
-        sonde_ds = act.io.armfiles.read_netcdf(
+        sonde_ds = act.io.arm.read_arm_netcdf(
            act.tests.sample_files.EXAMPLE_SONDE1)
 
         skewt = act.plotting.SkewTDisplay(sonde_ds)

act/plotting/timeseriesdisplay.py

@@ -39,7 +39,7 @@ class TimeSeriesDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.TimeSeriesDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The TimeSeriesDisplay constructor takes in the same keyword arguments as
@@ -800,7 +800,7 @@ def plot_barbs_from_spd_dir(
         --------
         ..code-block :: python
 
-            sonde_ds = act.io.armfiles.read_netcdf(
+            sonde_ds = act.io.arm.read_arm_netcdf(
                 act.tests.sample_files.EXAMPLE_TWP_SONDE_WILDCARD)
             BarbDisplay = act.plotting.TimeSeriesDisplay(
                 {'sonde_darwin': sonde_ds}, figsize=(10,5))

act/plotting/windrosedisplay.py

@@ -30,7 +30,7 @@ class and has therefore has the same attributes as that class.
 
     .. code-block :: python
 
-        sonde_ds = act.io.armfiles.read_netcdf('sonde_data.nc')
+        sonde_ds = act.io.arm.read_arm_netcdf('sonde_data.nc')
         WindDisplay = act.plotting.WindRoseDisplay(sonde_ds, figsize=(8,10))
 
     """

act/qc/add_supplemental_qc.py

@@ -113,9 +113,9 @@ def read_yaml_supplemental_qc(
     .. code-block:: python
 
         from act.tests import EXAPLE_MET_YAML, EXAMPLE_MET1
-        from act.io.armfiles import read_netcdf
+        from act.io.arm import read_arm_netcdf
         from act.qc.add_supplemental_qc import read_yaml_supplemental_qc
-        ds = read_netcdf(EXAMPLE_MET1, cleanup_qc=True)
+        ds = read_arm_netcdf(EXAMPLE_MET1, cleanup_qc=True)
         result = read_yaml_supplemental_qc(ds, EXAPLE_MET_YAML,
                                      variables=['rh_mean'], assessments='Bad')
         print(result)
@@ -263,9 +263,9 @@ def apply_supplemental_qc(
     .. code-block:: python
 
         from act.tests import EXAPLE_MET_YAML, EXAMPLE_MET1
-        from act.io.armfiles import read_netcdf
+        from act.io.arm import read_arm_netcdf
         from act.qc.add_supplemental_qc import apply_supplemental_qc
-        ds = read_netcdf(EXAMPLE_MET1, cleanup_qc=True)
+        ds = read_arm_netcdf(EXAMPLE_MET1, cleanup_qc=True)
         apply_supplemental_qc(ds, EXAPLE_MET_YAML, apply_all=False)
         print(ds['qc_temp_mean'].attrs['flag_meanings'])
 

act/qc/bsrn_tests.py

@@ -174,7 +174,7 @@ def bsrn_limits_test(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
                 ds.qcfilter.bsrn_limits_test(
                     gbl_SW_dn_name='down_short_hemisp',
                     glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
@@ -404,7 +404,7 @@ def bsrn_comparison_tests(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
                 ds.qcfilter.bsrn_comparison_tests(
                     gbl_SW_dn_name='down_short_hemisp',
                     glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',

act/qc/clean.py

@@ -127,7 +127,7 @@ def cleanup(
             .. code-block:: python
 
                 files = act.tests.sample_files.EXAMPLE_MET1
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.cleanup()
 
         """
@@ -750,12 +750,12 @@ def normalize_assessment(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.normalize_assessment(variables='temp_mean')
 
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(files, cleanup_qc=True)
                 ds.clean.normalize_assessment(qc_lookup={'Bad': 'Incorrect', 'Indeterminate': 'Suspect'})
 
         """
@@ -818,12 +818,12 @@ def clean_cf_qc(self, variables=None, sep='__', **kwargs):
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.clean_cf_qc(variables='temp_mean')
 
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(files, cleanup_qc=True)
 
         """
 

act/qc/qcfilter.py

@@ -70,8 +70,8 @@ def check_for_ancillary_qc(
             .. code-block:: python
 
                 from act.tests import EXAMPLE_METE40
-                from act.io.armfiles import read_netcdf
-                ds = read_netcdf(EXAMPLE_METE40, cleanup_qc=True)
+                from act.io.arm import read_arm_netcdf
+                ds = read_arm_netcdf(EXAMPLE_METE40, cleanup_qc=True)
                 qc_var_name = ds.qcfilter.check_for_ancillary_qc('atmos_pressure')
                 print(f'qc_var_name: {qc_var_name}')
                 qc_var_name = ds.qcfilter.check_for_ancillary_qc('the_greatest_variable_ever',
@@ -155,8 +155,8 @@ def create_qc_variable(
             .. code-block:: python
 
                 from act.tests import EXAMPLE_AOSMET
-                from act.io.armfiles import read_netcdf
-                ds = read_netcdf(EXAMPLE_AOSMET)
+                from act.io.arm import read_arm_netcdf
+                ds = read_arm_netcdf(EXAMPLE_AOSMET)
                 qc_var_name = ds.qcfilter.create_qc_variable('temperature_ambient')
                 print(qc_var_name)
                 print(ds[qc_var_name])
@@ -242,8 +242,8 @@ def update_ancillary_variable(self, var_name, qc_var_name=None):
             .. code-block:: python
 
                 from act.tests import EXAMPLE_AOSMET
-                from act.io.armfiles import read_netcdf
-                ds = read_netcdf(EXAMPLE_AOSMET)
+                from act.io.arm import read_arm_netcdf
+                ds = read_arm_netcdf(EXAMPLE_AOSMET)
                 var_name = 'temperature_ambient'
                 qc_var_name = ds.qcfilter.create_qc_variable(var_name)
                 del ds[var_name].attrs['ancillary_variables']
@@ -647,8 +647,8 @@ def available_bit(self, qc_var_name, recycle=False):
             .. code-block:: python
 
                 from act.tests import EXAMPLE_METE40
-                from act.io.armfiles import read_netcdf
-                ds = read_netcdf(EXAMPLE_METE40, cleanup_qc=True)
+                from act.io.arm import read_arm_netcdf
+                ds = read_arm_netcdf(EXAMPLE_METE40, cleanup_qc=True)
                 test_number = ds.qcfilter.available_bit('qc_atmos_pressure')
                 print(test_number)
 
@@ -727,10 +727,10 @@ def get_qc_test_mask(
         --------
             .. code-block:: python
 
-                from act.io.armfiles import read_netcdf
+                from act.io.arm import read_arm_netcdf
                 from act.tests import EXAMPLE_IRT25m20s
 
-                ds = read_netcdf(EXAMPLE_IRT25m20s)
+                ds = read_arm_netcdf(EXAMPLE_IRT25m20s)
                 var_name = "inst_up_long_dome_resist"
                 result = ds.qcfilter.add_test(
                     var_name, index=[0, 1, 2], test_meaning="Birds!"
@@ -846,10 +846,10 @@ def get_masked_data(
         --------
             .. code-block:: python
 
-                from act.io.armfiles import read_netcdf
+                from act.io.arm import read_arm_netcdf
                 from act.tests import EXAMPLE_IRT25m20s
 
-                ds = read_netcdf(EXAMPLE_IRT25m20s)
+                ds = read_arm_netcdf(EXAMPLE_IRT25m20s)
                 var_name = "inst_up_long_dome_resist"
                 result = ds.qcfilter.add_test(
                     var_name, index=[0, 1, 2], test_meaning="Birds!"
@@ -987,10 +987,10 @@ def datafilter(
         --------
             .. code-block:: python
 
-                from act.io.armfiles import read_netcdf
+                from act.io.arm import read_arm_netcdf
                 from act.tests import EXAMPLE_MET1
 
-                ds = read_netcdf(EXAMPLE_MET1)
+                ds = read_arm_netcdf(EXAMPLE_MET1)
                 ds.clean.cleanup()
 
                 var_name = "atmos_pressure"