Merge pull request #60 from bohlinger/tmp

Tmp

apps/standalone/wavyDownload.py

@@ -71,7 +71,7 @@ def main(sd, ed, nid, name, path, nproc):
 
     # read yaml config files:
     satellite_dict = load_or_default('satellite_cfg.yaml')
-    print(satellite_dict)
+
     # settings
     now = datetime.now()
 

docs/workshop_latest.rst

@@ -1,10 +1,10 @@
-**wavy** workshop 2024
-======================
+Olso **wavy** workshop 2024
+===========================
 
 The following examples are tailored to the **wavy** Workshop. This workshop will focus on some simple examples that can be used as python code snippets in your workflow.
 
 0. checklist **wavy** installation
-##############################
+##################################
 
     * did you add your **wavy** root directory to $PYTHONPATH?
     * is **wavy** activated? (conda activate wavy)
@@ -55,7 +55,7 @@ Where your *.env*-file needs to point to this config folder like in the followin
         WAVY_CONFIG=/home/patrikb/ws24_wavy/config/
         WAVY_DIR=/home/patrikb/wavy/
 
-If you want to download data, the same .env file has to be copied to the wavy root directory, ~/wavy. 
+Finally, the same .env file has to be copied to the wavy root directory, ~/wavy. 
 
 
 2. Download L3 satellite altimetry data
@@ -541,3 +541,50 @@ with the *sco.filter_runmean* method:
 .. code-block:: python3
 
    >>> sco = sco.filter_runmean(window=11, chunk_min=5, sampling_rate_Hz=20)
+   
+10. Saving data to netcdf
+#########################
+It is possible to save the data from the different wavy objects to .nc files. If
+we take again the first example we used for the satellite data:  
+
+.. code-block:: python3
+
+   >>> from wavy.satellite_module import satellite_class as sc
+   >>> # settings
+   >>> region = 'global'
+   >>> varalias = 'Hs'  # default
+   >>> name = 's3a'
+   >>> nID = 'cmems_L3_NRT'
+   >>> twin = 30  # default
+   >>> sd = "2023-2-1 11"  # can also be datetime object
+   >>> ed = "2023-2-1 12"  # not necessary if twin is specified
+   >>> # retrieval
+   >>> sco = sc(sd=sd, ed=ed, region=region, nID=nID, name=name)
+   >>> sco = sco.populate()
+
+Then we can save the data contained in *sco.vars* as follows: 
+
+.. code-block:: python3
+
+   >>> sco.vars.to_netcdf('/home/patrikb/ws24_wavy/test_dump.nc')
+   
+This way, you can directly reimport the data by initializing a new satellite_class
+object and populate it giving the path to the netcdf file created earlier in 
+the *sco.populate()* method using *wavy_path* argument as follows: 
+
+.. code-block:: python3
+
+   >>> from wavy.satellite_module import satellite_class as sc
+   >>> # settings
+   >>> region = 'global'
+   >>> varalias = 'Hs'  # default
+   >>> name = 's3a'
+   >>> nID = 'cmems_L3_NRT'
+   >>> twin = 30  # default
+   >>> sd = "2023-2-1 11"  # can also be datetime object
+   >>> ed = "2023-2-1 12"  # not necessary if twin is specified
+   >>> # retrieval
+   >>> sco = sc(sd=sd, ed=ed, region=region, nID=nID, name=name)
+   >>> sco = sco.populate(wavy_path='/home/patrikb/ws24_wavy/test_dump.nc')
+   
+Note that this works with insitu_class and model_class object as well. 

wavy/config/satellite_cfg.yaml.default

@@ -88,6 +88,14 @@ cmems_L3_MY:
             search_str: '%Y%m%dT'
             strsub: ['name']
             server: "my.cmems-du.eu"
+        copernicus: 
+            dataset_id: cci_obs-wave_glo_phy-swh_my_name-l3_PT1S
+            trgt_tmplt: /home/patrikb/tmp_altimeter/L3/MY/name/%Y/%m
+            path_date_incr_unit: 'm'
+            path_date_incr: 1
+            strsub: ['name']
+            server: "my.cmems-du.eu"
+            time_incr: 'h' # 'h', 'd', 'm'
     # optional: where to read from
     #           can be defined directly when calling wavy
     wavy_input:

wavy/satellite_collectors.py

@@ -18,6 +18,7 @@
 from dateutil.relativedelta import relativedelta
 from joblib import Parallel, delayed
 import logging
+import copernicusmarine as cmc
 #logging.basicConfig(level=logging.DEBUG)
 logging.basicConfig(level=30)
 logger = logging.getLogger(__name__)
@@ -178,6 +179,117 @@ def get_remote_files_cmems(**kwargs):
     print('Files downloaded to: \n', path_local)
 
 
+def get_remote_files_copernicusmarine(**kwargs):
+    '''
+    Download swath files from CMEMS using copernicusmarine parckage
+    and store them at defined location. Time stamps in file name stand for:
+
+    from, to, creation
+    '''
+    product = kwargs.get('nID')
+    sdate = kwargs.get('sd')
+    edate = kwargs.get('ed')
+    nproc = kwargs.get('nproc', 1)
+    name = kwargs.get('name', 's3a')
+    dict_for_sub = kwargs
+    # define path
+    path = kwargs.get('path', None)
+    # Get time increment
+    time_incr = satellite_dict[product]['download']['copernicus']\
+                .get('time_incr','h')
+
+    # Chose search template for time given time_incr
+    if time_incr=='h':
+        file_search_template = '%Y%m%dT%H'
+    elif time_incr=='d':
+        file_search_template = '%Y%m%dT'
+    elif time_incr=='m':
+        file_search_template = '%Y%m'
+    print('Date search format:', file_search_template)
+
+    # Get dataset_id
+    dataset_id = satellite_dict\
+                            [product]['download']['copernicus']\
+                            ['dataset_id']
+    strsublst_src = satellite_dict[product]['download']\
+                            ['copernicus']['strsub']
+    subdict_src = make_subdict(strsublst_src,
+                               class_object_dict=dict_for_sub)
+
+    # replace name of the mission in dataset_id
+    dataset_id = make_pathtofile(dataset_id,
+                                  strsublst_src, 
+                                  subdict_src)
+
+    # Initialize start date to match original files time increment
+    tmpdate = deepcopy(sdate)
+    tmpdate_end = deepcopy(edate)
+
+    while tmpdate.hour%3 != 0:
+        tmpdate = tmpdate - timedelta(hours=1)
+
+    while tmpdate_end.hour%3 != 0:
+        tmpdate_end = tmpdate_end + timedelta(hours=1)
+
+    try:
+
+        print('# ----- ')
+        print('Chosen source: ')
+        print(name + ' values from ' + product + ': ' + 'copernicusmarine')
+        print('# ----- ')
+
+        while (tmpdate <= tmpdate_end):
+
+            if path is None:
+	        # create local path
+                path_template_dst = satellite_dict[product]['download']\
+	                                ['copernicus']['trgt_tmplt']
+                strsublst_dst = satellite_dict[product]['download']\
+	                                ['copernicus']['strsub']
+                subdict_dst = make_subdict(strsublst_dst,
+	                                   class_object_dict=dict_for_sub)
+                path_local = make_pathtofile(path_template_dst,
+	                                     strsublst_dst, subdict_dst,
+	                                     date=tmpdate)
+            else:
+                path_local = path
+
+            print('* --------------')
+            print('Downloading for date:', tmpdate)
+            print('* --------------')
+
+            # check if download path_local exists if not create
+            if not os.path.exists(path_local):
+                os.makedirs(path_local, exist_ok=True)
+
+            # Create regexp filter
+            tmpdate_str = tmpdate.strftime(file_search_template)
+            regexp_tmp = "*{}*_*_*.nc".format(tmpdate_str)
+            # Fetch data corresponding to tmp date 
+            try:
+                cmc.get(
+                        dataset_id = dataset_id,
+                        filter = regexp_tmp,
+                        no_directories = True,
+                        output_directory = path_local,
+                        force_download=True)
+            except:
+                pass
+
+            if time_incr=='h':
+                tmpdate = tmpdate + timedelta(hours=3)
+            elif time_incr=='d':
+                tmpdate = tmpdate + timedelta(days=1)
+            elif time_incr=='m':
+                tmpdate = tmpdate + relativedelta(months=+1)    
+
+    except Exception as e:
+        logger.exception(e)
+
+    print('# -----------------------------------')
+    print('Files downloaded to: \n', path_local)        
+
+
 def get_remote_files_aviso(**kwargs):
     '''
     Download swath files from AVISO+ and store them at defined