From 86ffdc8eb1f2d5474ce662bbef33bf15fbc8b0c8 Mon Sep 17 00:00:00 2001 From: Patrik Bohlinger Date: Sun, 28 Jan 2024 20:37:48 +0100 Subject: [PATCH] removed detach table for collectors and instead enable direct use like for insitu collectors and readers --- tests/test_satellite_module.py | 22 +++++++++++- wavy/config/model_cfg.yaml.default | 47 ++++++++++++++++++++++++++ wavy/config/satellite_cfg.yaml.default | 7 ++-- wavy/satellite_collectors.py | 45 ++++++++---------------- wavy/satellite_module.py | 16 ++------- 5 files changed, 88 insertions(+), 49 deletions(-) diff --git a/tests/test_satellite_module.py b/tests/test_satellite_module.py index 92197023..5547249d 100644 --- a/tests/test_satellite_module.py +++ b/tests/test_satellite_module.py @@ -20,7 +20,6 @@ def test_collectors_cmems_L3(tmpdir): if '.nc' in filelist[i]] assert len(nclist) >= 1 - @pytest.mark.need_credentials def test_collectors_cci_v3_20Hz(tmpdir): sco = sc(sd='2020-2-1 12', ed='2020-2-1 12', @@ -32,6 +31,27 @@ def test_collectors_cci_v3_20Hz(tmpdir): if '.nc' in filelist[i]] assert len(nclist) >= 1 +@pytest.mark.need_credentials +def test_collectors_cci_v1_01Hz(tmpdir): + sco = sc(sd='2018-1-1', ed='2018-1-1', + nID='CCIv1_L3', name='multi') + sco.download(path=tmpdir, nproc=8) + # check if files were download to tmp directory + filelist = os.listdir(tmpdir) + nclist = [i for i in range(len(filelist)) + if '.nc' in filelist[i]] + assert len(nclist) >= 1 + +#@pytest.mark.need_credentials +#def test_collectors_aviso(tmpdir): +# sco = sc(sd='2020-2-1 12', ed='2020-2-1 12', +# nID='L2_20Hz_s3a', name='s3a') +# sco.download(path=tmpdir, nproc=8) +# # check if files were download to tmp directory +# filelist = os.listdir(tmpdir) +# nclist = [i for i in range(len(filelist)) +# if '.nc' in filelist[i]] +# assert len(nclist) >= 1 def test_manually_specified_reader(test_data): sd = "2022-2-1 12" diff --git a/wavy/config/model_cfg.yaml.default b/wavy/config/model_cfg.yaml.default index 86b506ce..3b65b426 100644 --- a/wavy/config/model_cfg.yaml.default +++ b/wavy/config/model_cfg.yaml.default @@ -114,6 +114,53 @@ ww3_4km: # optional, to ease grouping tags: +ww3_4km_bm1p5_sd2024011300: + # not mandatory for models + name: ww3_4km + # mandatory when downloading + # where to store downloaded data + download: + # optional: where to read from + # can be defined directly when calling wavy + wavy_input: + src_tmplt: "/lustre/storeB/project/fou/om/patrikb/\ + op_test/ww3_betamax/BM1p50/" + fl_tmplt: ww3_%Y%m%dT%HZ.nc + # optional: where to write to + # can be defined directly when calling wavy + wavy_output: + # optional, if not defined the class default is used + reader: read_ww3_4km + collector: + # optional, needs to be defined if not cf and in variable_info.yaml + vardef: + Hs: hs + time: time + Mdir: dir + Pdir: dp + Fp: fp + Tm01: t01 + Tm02: t02 + lons: longitude + lats: latitude + U: ff + coords: + # optional, info that can be used by class functions + misc: + init_times: [0,6,12,18] + init_step: 6 + leadtimes: [0,6,12,18,24,36,48,60] + _FillValue: 9.96921e+36 + proj4: "+proj=ob_tran +o_proj=longlat +lon_0=-40 + +o_lat_p=22 +R=6.371e+06 +no_defs" + grid_date: 2021-11-16 00:00:00 + convention: meteorological + date_incr_unit: h + date_incr: 1 + # optional, to ease grouping + tags: + + ww3_unstr: name: download: diff --git a/wavy/config/satellite_cfg.yaml.default b/wavy/config/satellite_cfg.yaml.default index af783351..ad6c4a25 100644 --- a/wavy/config/satellite_cfg.yaml.default +++ b/wavy/config/satellite_cfg.yaml.default @@ -212,7 +212,8 @@ CCIv1_L3: file_date_incr: 1 # optional, if not defined the default is used reader: read_local_ncfiles - collector: get_remote_files_cci + #collector: get_remote_files_cci + collector: get_remote_files_cmems # optional, needs to be defined if not cf and in variable_info.yaml vardef: # varalias: varname Hs: swh_denoised # time @@ -263,7 +264,7 @@ CCIv3_L3: file_date_incr: 1 # optional, if not defined the default is used reader: read_local_ncfiles - collector: get_remote_files_cci + collector: # optional, needs to be defined if not cf and in variable_info.yaml vardef: # varalias: varname Hs: swh_denoised @@ -280,8 +281,6 @@ CCIv3_L3: # optional, to ease grouping tags: - - s3a_sla_5Hz: # mandatory name: diff --git a/wavy/satellite_collectors.py b/wavy/satellite_collectors.py index 132b8010..9414ed29 100755 --- a/wavy/satellite_collectors.py +++ b/wavy/satellite_collectors.py @@ -79,7 +79,9 @@ def get_remote_files_cmems(**kwargs): twin = int(np.max([kwargs.get('twin', 30), 30])) nproc = kwargs.get('nproc', 1) name = kwargs.get('name', 's3a') - dict_for_sub = kwargs.get('dict_for_sub') + #dict_for_sub = kwargs.get('dict_for_sub') + dict_for_sub = kwargs + # define path path = kwargs.get('path', None) # check if search str template @@ -88,6 +90,7 @@ def get_remote_files_cmems(**kwargs): file_search_template = \ satellite_dict[product]['download']['ftp']\ .get('search_str', '%Y%m%dT%H') + # credentials server = satellite_dict[product]['download']['ftp']['server'] user, pw = get_credentials(remoteHostName=server) @@ -265,14 +268,19 @@ def get_remote_files_cci(**kwargs): Download swath files from CCI and store them at defined location. ''' - product = kwargs.get('product') - sdate = kwargs.get('sdate') - edate = kwargs.get('edate') + product = kwargs.get('nID') + sdate = kwargs.get('sd') + edate = kwargs.get('ed') twin = kwargs.get('twin', 30) nproc = kwargs.get('nproc', 1) name = kwargs.get('name', 'multi') - path_local = kwargs.get('path_local') - dict_for_sub = kwargs.get('dict_for_sub') + #path_local = kwargs.get('path_local') + #dict_for_sub = kwargs.get('dict_for_sub') + dict_for_sub = kwargs + + # define path + path_local = kwargs.get('path', None) + # credentials server = satellite_dict[product]['src']['server'] level = satellite_dict[product]['processing_level'] @@ -353,28 +361,3 @@ def get_remote_files_cci(**kwargs): if os.path.isfile(os.path.join(path_local, f))] sort_files(path_local, filelst, product, name) print('Files downloaded to: \n', path_local) - - -def get_remote_files(**kwargs): - ''' - Download swath files and store them at defined location. - It is currently possible to download L3 altimeter data from - CMEMS, L3 and L2P from CEDA CCI, and L2 from EUMETSAT, - as well as L2P from aviso+ for cfosat swim data. - ''' - dispatch_collector = { - 'cmems_L3_NRT': get_remote_files_cmems, - 'cmems_L3_s6a': get_remote_files_cmems, - 'cmems_L3_MY': get_remote_files_cmems, - 'cfo_swim_L2P': get_remote_files_aviso, - 'cci_L2P': get_remote_files_cci, - 'CCIv1_L3': get_remote_files_cci, - } - product = kwargs.get('product') - # check if product available in dispatcher - if product in dispatch_collector.keys(): - pass - else: - product = 'cmems_L3_NRT' - - dispatch_collector[product](**kwargs) diff --git a/wavy/satellite_module.py b/wavy/satellite_module.py index d383c5da..4418bf8b 100755 --- a/wavy/satellite_module.py +++ b/wavy/satellite_module.py @@ -50,8 +50,6 @@ from wavy.filtermod import filter_class as fc -from wavy.satellite_collectors import get_remote_files - from wavy.quicklookmod import quicklook_class_sat as qls from wavy.writermod import writer_class as wc @@ -168,17 +166,9 @@ def download(self, path=None, nproc=1, **kwargs): print('') print("Downloading files ...") - get_remote_files( - path=path, - nproc=nproc, - twin=kwargs.get('twin', self.twin), - sd=kwargs.get('sd', self.sd), - ed=kwargs.get('ed', self.ed), - nID=self.nID, - name=self.name, - dict_for_sub=vars(self) - ) - # self.collector(**(vars(self))) + #kwargs_in = vars(self) + #kwargs_in['path'] = path + self.collector(nproc=nproc, path=path, **vars(self)) def _get_files(self, dict_for_sub=None, path=None, wavy_path=None):