removed detach table for collectors and instead enable direct use lik…

…e for insitu collectors and readers

tests/test_satellite_module.py

@@ -20,7 +20,6 @@ def test_collectors_cmems_L3(tmpdir):
               if '.nc' in filelist[i]]
     assert len(nclist) >= 1
 
-
 @pytest.mark.need_credentials
 def test_collectors_cci_v3_20Hz(tmpdir):
     sco = sc(sd='2020-2-1 12', ed='2020-2-1 12',
@@ -32,6 +31,27 @@ def test_collectors_cci_v3_20Hz(tmpdir):
               if '.nc' in filelist[i]]
     assert len(nclist) >= 1
 
+@pytest.mark.need_credentials
+def test_collectors_cci_v1_01Hz(tmpdir):
+    sco = sc(sd='2018-1-1', ed='2018-1-1',
+             nID='CCIv1_L3', name='multi')
+    sco.download(path=tmpdir, nproc=8)
+    # check if files were download to tmp directory
+    filelist = os.listdir(tmpdir)
+    nclist = [i for i in range(len(filelist))
+              if '.nc' in filelist[i]]
+    assert len(nclist) >= 1
+
+#@pytest.mark.need_credentials
+#def test_collectors_aviso(tmpdir):
+#    sco = sc(sd='2020-2-1 12', ed='2020-2-1 12',
+#             nID='L2_20Hz_s3a', name='s3a')
+#    sco.download(path=tmpdir, nproc=8)
+#    # check if files were download to tmp directory
+#    filelist = os.listdir(tmpdir)
+#    nclist = [i for i in range(len(filelist))
+#              if '.nc' in filelist[i]]
+#    assert len(nclist) >= 1
 
 def test_manually_specified_reader(test_data):
     sd = "2022-2-1 12"

wavy/config/model_cfg.yaml.default

@@ -114,6 +114,53 @@ ww3_4km:
     # optional, to ease grouping
     tags:
 
+ww3_4km_bm1p5_sd2024011300:
+    # not mandatory for models
+    name: ww3_4km
+    # mandatory when downloading
+    # where to store downloaded data
+    download:
+    # optional: where to read from
+    #           can be defined directly when calling wavy
+    wavy_input:
+        src_tmplt: "/lustre/storeB/project/fou/om/patrikb/\
+                    op_test/ww3_betamax/BM1p50/"
+        fl_tmplt: ww3_%Y%m%dT%HZ.nc
+    # optional: where to write to
+    #           can be defined directly when calling wavy
+    wavy_output:
+    # optional, if not defined the class default is used
+    reader: read_ww3_4km
+    collector:
+    # optional, needs to be defined if not cf and in variable_info.yaml
+    vardef:
+        Hs: hs
+        time: time
+        Mdir: dir
+        Pdir: dp
+        Fp: fp
+        Tm01: t01
+        Tm02: t02
+        lons: longitude
+        lats: latitude
+        U: ff
+    coords:
+    # optional, info that can be used by class functions
+    misc:
+        init_times: [0,6,12,18]
+        init_step: 6
+        leadtimes: [0,6,12,18,24,36,48,60]
+        _FillValue: 9.96921e+36
+        proj4: "+proj=ob_tran +o_proj=longlat +lon_0=-40
+                +o_lat_p=22 +R=6.371e+06 +no_defs"
+        grid_date: 2021-11-16 00:00:00
+        convention: meteorological
+        date_incr_unit: h
+        date_incr: 1
+    # optional, to ease grouping
+    tags:
+
+
 ww3_unstr:
     name:
     download:

wavy/config/satellite_cfg.yaml.default

@@ -212,7 +212,8 @@ CCIv1_L3:
         file_date_incr: 1
     # optional, if not defined the default is used
     reader: read_local_ncfiles
-    collector: get_remote_files_cci
+    #collector: get_remote_files_cci
+    collector: get_remote_files_cmems
     # optional, needs to be defined if not cf and in variable_info.yaml
     vardef: # varalias: varname
         Hs: swh_denoised # time
@@ -263,7 +264,7 @@ CCIv3_L3:
         file_date_incr: 1
     # optional, if not defined the default is used
     reader: read_local_ncfiles
-    collector: get_remote_files_cci
+    collector:
     # optional, needs to be defined if not cf and in variable_info.yaml
     vardef: # varalias: varname
         Hs: swh_denoised
@@ -280,8 +281,6 @@ CCIv3_L3:
     # optional, to ease grouping
     tags:
 
-
-
 s3a_sla_5Hz:
     # mandatory
     name:

wavy/satellite_collectors.py

@@ -79,7 +79,9 @@ def get_remote_files_cmems(**kwargs):
     twin = int(np.max([kwargs.get('twin', 30), 30]))
     nproc = kwargs.get('nproc', 1)
     name = kwargs.get('name', 's3a')
-    dict_for_sub = kwargs.get('dict_for_sub')
+    #dict_for_sub = kwargs.get('dict_for_sub')
+    dict_for_sub = kwargs
+
     # define path
     path = kwargs.get('path', None)
     # check if search str template
@@ -88,6 +90,7 @@ def get_remote_files_cmems(**kwargs):
         file_search_template = \
             satellite_dict[product]['download']['ftp']\
             .get('search_str', '%Y%m%dT%H')
+
     # credentials
     server = satellite_dict[product]['download']['ftp']['server']
     user, pw = get_credentials(remoteHostName=server)
@@ -265,14 +268,19 @@ def get_remote_files_cci(**kwargs):
     Download swath files from CCI and store them at defined
     location.
     '''
-    product = kwargs.get('product')
-    sdate = kwargs.get('sdate')
-    edate = kwargs.get('edate')
+    product = kwargs.get('nID')
+    sdate = kwargs.get('sd')
+    edate = kwargs.get('ed')
     twin = kwargs.get('twin', 30)
     nproc = kwargs.get('nproc', 1)
     name = kwargs.get('name', 'multi')
-    path_local = kwargs.get('path_local')
-    dict_for_sub = kwargs.get('dict_for_sub')
+    #path_local = kwargs.get('path_local')
+    #dict_for_sub = kwargs.get('dict_for_sub')
+    dict_for_sub = kwargs
+
+    # define path
+    path_local = kwargs.get('path', None)
+
     # credentials
     server = satellite_dict[product]['src']['server']
     level = satellite_dict[product]['processing_level']
@@ -353,28 +361,3 @@ def get_remote_files_cci(**kwargs):
                    if os.path.isfile(os.path.join(path_local, f))]
         sort_files(path_local, filelst, product, name)
     print('Files downloaded to: \n', path_local)
-
-
-def get_remote_files(**kwargs):
-    '''
-    Download swath files and store them at defined location.
-    It is currently possible to download L3 altimeter data from
-    CMEMS, L3 and L2P from CEDA CCI, and L2 from EUMETSAT,
-    as well as L2P from aviso+ for cfosat swim data.
-    '''
-    dispatch_collector = {
-                'cmems_L3_NRT': get_remote_files_cmems,
-                'cmems_L3_s6a': get_remote_files_cmems,
-                'cmems_L3_MY': get_remote_files_cmems,
-                'cfo_swim_L2P': get_remote_files_aviso,
-                'cci_L2P': get_remote_files_cci,
-                'CCIv1_L3': get_remote_files_cci,
-                }
-    product = kwargs.get('product')
-    # check if product available in dispatcher
-    if product in dispatch_collector.keys():
-        pass
-    else:
-        product = 'cmems_L3_NRT'
-
-    dispatch_collector[product](**kwargs)

wavy/satellite_module.py

@@ -50,8 +50,6 @@
 
 from wavy.filtermod import filter_class as fc
 
-from wavy.satellite_collectors import get_remote_files
-
 from wavy.quicklookmod import quicklook_class_sat as qls
 
 from wavy.writermod import writer_class as wc
@@ -168,17 +166,9 @@ def download(self, path=None, nproc=1, **kwargs):
         print('')
 
         print("Downloading files ...")
-        get_remote_files(
-                        path=path,
-                        nproc=nproc,
-                        twin=kwargs.get('twin', self.twin),
-                        sd=kwargs.get('sd', self.sd),
-                        ed=kwargs.get('ed', self.ed),
-                        nID=self.nID,
-                        name=self.name,
-                        dict_for_sub=vars(self)
-                        )
-        # self.collector(**(vars(self)))
+        #kwargs_in = vars(self)
+        #kwargs_in['path'] = path
+        self.collector(nproc=nproc, path=path, **vars(self))
 
 
     def _get_files(self, dict_for_sub=None, path=None, wavy_path=None):