diff --git a/schemas/VFrecovery-schema-location.json b/schemas/VFrecovery-schema-location.json index a6d6663..ac898dd 100644 --- a/schemas/VFrecovery-schema-location.json +++ b/schemas/VFrecovery-schema-location.json @@ -13,7 +13,7 @@ "type": "number", "minimum": -180, "maximum": 180, - "description": "Longitude of the geo-location, [-180-180] convention" + "description": "Longitude of the geo-location, [-180/180] convention" }, "latitude": { "type": "number", diff --git a/schemas/VFrecovery-schema-profile.json b/schemas/VFrecovery-schema-profile.json index 072eba4..fcdd2cb 100644 --- a/schemas/VFrecovery-schema-profile.json +++ b/schemas/VFrecovery-schema-profile.json @@ -39,6 +39,7 @@ "$ref": "https://raw.githubusercontent.com/euroargodev/VirtualFleet_recovery/refactoring-as-a-clean-module-and-cli/schemas/VFrecovery-schema-metrics.json" }, "dependencies": { - "virtual_cycle_number": ["metrics"]} + "virtual_cycle_number": ["metrics"] + } } } diff --git a/schemas/VFrecovery-schema-trajectory.json b/schemas/VFrecovery-schema-trajectory.json new file mode 100644 index 0000000..ab2cdca --- /dev/null +++ b/schemas/VFrecovery-schema-trajectory.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://raw.githubusercontent.com/euroargodev/VirtualFleet_recovery/refactoring-as-a-clean-module-and-cli/schemas/VFrecovery-schema-trajectory.json", + "title": "VirtualFleet-Recovery trajectory", + "description": "Represents two or more VirtualFleet-Recovery locations that share a relationship", + "format_version": { + "const": "0.1" + }, + "required": [ "locations" ], + "type": "object", + "properties": { + "locations": { + "type": "array", + "items": { + "$ref": "https://raw.githubusercontent.com/euroargodev/VirtualFleet_recovery/refactoring-as-a-clean-module-and-cli/schemas/VFrecovery-schema-location.json" + }, + "uniqueItems": false + } + } +} diff --git a/vfrecovery/command_line_interface/group_db.py b/vfrecovery/command_line_interface/group_db.py index 21cb38c..051aa6c 100644 --- a/vfrecovery/command_line_interface/group_db.py +++ b/vfrecovery/command_line_interface/group_db.py @@ -71,6 +71,10 @@ def db( if root_logger.isEnabledFor(logging.DEBUG): root_logger.debug("DEBUG mode activated") + # Validate arguments: + if action.lower() not in ["read", "info", "drop"]: + raise ValueError("The first argument ACTION must be one in ['read', 'info', 'drop']") + if action == 'read': df = DB.read_data() if index is not None: @@ -82,8 +86,11 @@ def db( click.secho("Row index #%i:" % irow, fg='green') click.echo(row.T.to_string()) - if action == 'drop': + elif action == 'drop': DB.clear() - if action == 'info': + elif action == 'info': click.echo(DB.info()) + + else: + raise click.BadParameter("Unknown DB action '%s'" % action) diff --git a/vfrecovery/command_line_interface/group_describe.py b/vfrecovery/command_line_interface/group_describe.py index caf9652..b62d505 100644 --- a/vfrecovery/command_line_interface/group_describe.py +++ b/vfrecovery/command_line_interface/group_describe.py @@ -3,6 +3,7 @@ from argopy.utils import is_wmo, is_cyc, check_cyc, check_wmo import argopy.plot as argoplot from argopy import ArgoIndex +from pathlib import Path from vfrecovery.utils.misc import list_float_simulation_folders from vfrecovery.core.db import DB @@ -85,6 +86,9 @@ def describe( elif target == 'run': describe_run(wmo, cyc) + else: + raise click.BadParameter("Unknown describe target '%s'" % target) + def describe_run(wmo, cyc): partial_data = {'wmo': wmo} @@ -94,14 +98,13 @@ def describe_run(wmo, cyc): def describe_velocity(wmo, cyc): + cyc = cyc[0] if len(cyc) > 0 else None - # List folders to examine: - plist = list_float_simulation_folders(wmo, cyc) + for ii, item in DB.from_dict({'wmo': wmo, 'cyc': cyc}).items: + p = Path(item.path_root).joinpath(item.path_obj.velocity) - # List all available velocity files: - for c in plist.keys(): - p = plist[c] - click.secho("Velocity data for WMO=%s / CYC=%s:" % (wmo, c), fg='blue') + click.secho("Velocity data for WMO=%s / CYC=%s / DOMAIN-SIZE=%0.2f / DOWNLOAD-DATE=%s" + % (item.wmo, item.cyc, item.velocity['domain_size'], item.velocity['download']), fg='blue') click.secho("\tNetcdf files:") vlist = sorted(p.glob("velocity_*.nc")) diff --git a/vfrecovery/core/db.py b/vfrecovery/core/db.py index accfe13..6adc9c7 100644 --- a/vfrecovery/core/db.py +++ b/vfrecovery/core/db.py @@ -21,7 +21,7 @@ This first implementation relies on a simple local pickle file with a panda dataframe """ -from typing import List, Dict +from typing import List, Dict, Iterable, Hashable from virtualargofleet import FloatConfiguration from pathlib import Path import pandas as pd @@ -167,7 +167,14 @@ class DB: >>> DB.isconnected() >>> DB.read_data() # Return db content as :class:`pd.DataFrame` - >>> data = {'wmo': 6903091, 'cyc': 120, 'n_predictions': 0, 'cfg': FloatConfiguration('recovery'), 'velocity': {'name': 'GLORYS', 'download': pd.to_datetime('now', utc=True), 'domain_size': 5}, 'path_root': Path('.'), 'swarm_size': 1000} + >>> data = {'wmo': 6903091, 'cyc': 120, 'n_predictions': 0, + >>> 'cfg': FloatConfiguration('recovery'), + >>> 'velocity': {'name': 'GLORYS', + >>> 'download': pd.to_datetime('now', utc=True), + >>> 'domain_size': 5}, + >>> 'path_root': Path('.'), + >>> 'swarm_size': 1000} + >>> >>> DB.from_dict(data).checkin() # save to db >>> DB.from_dict(data).checkout() # delete from db >>> DB.from_dict(data).checked @@ -198,9 +205,6 @@ class DB: "simulations_registry.pkl") def __init__(self, **kwargs): - # for key in self.required: - # if key not in kwargs: - # raise ValueError("Missing '%s' property" % key) for key in kwargs: if key in self.properties: setattr(self, key, kwargs[key]) @@ -252,7 +256,7 @@ def init(cls): return cls @classmethod - def connect(cls): + def connect(cls) -> "DB": """Connect to database and refresh data holder""" if not cls.isconnected(): cls.init() @@ -281,13 +285,14 @@ def connect(cls): # df.apply(has_result_file, axis=1) @classmethod - def read_data(cls): + def read_data(cls) -> pd.DataFrame: """Return database content as a :class:`pd.DataFrame`""" cls.connect() return cls._data @classmethod - def exists(cls, dict_of_values): + def exists(cls, dict_of_values) -> bool: + """Return True if an exact match on all properties is found""" df = cls.read_data() v = df.iloc[:, 0] == df.iloc[:, 0] for key, value in dict_of_values.items(): @@ -319,7 +324,8 @@ def del_data(cls, row): df.to_pickle(cls.dbfile) @classmethod - def get_data(cls, row): + def get_data(cls, row) -> pd.DataFrame: + """Return records matching no-None properties""" df = cls.read_data() mask = df.iloc[:, 0] == df.iloc[:, 0] for key in row: @@ -327,10 +333,6 @@ def get_data(cls, row): mask &= df[key] == row[key] return df[mask] - @classmethod - def info(cls) -> str: - return cls.__repr__(cls) - def __repr__(self): self.connect() summary = [""] @@ -341,6 +343,10 @@ def __repr__(self): return "\n".join(summary) + @classmethod + def info(cls) -> str: + return cls.__repr__(cls) + @staticmethod def from_dict(obj: Dict) -> "DB": return DB(**obj) @@ -367,6 +373,29 @@ def _instance2row(self): return row + @classmethod + def _row2dict(self, row) -> dict: + """Convert a db row to a dictionary input""" + data = {} + data.update({'wmo': row['wmo']}) + data.update({'cyc': row['cyc']}) + data.update({'n_predictions': row['n_predictions']}) + + cfg = FloatConfiguration('recovery') + for key in cfg.mission: + cfg.update(key, row["cfg_%s" % key]) + data.update({'cfg': cfg}) + + vel = {'name': None, 'download': None, 'domain_size': None} + for key in vel: + vel.update({key: row["velocity_%s" % key]}) + data.update({'velocity': vel}) + + data.update({'swarm_size': row['swarm_size']}) + data.update({'path_root': row['path_root']}) + + return data + def checkin(self): """Add one new record to the database""" new_row = self._instance2row() @@ -411,3 +440,8 @@ def path_obj(self): def record(self) -> pd.DataFrame: row = self._instance2row() return self.get_data(row) + + @property + def items(self) -> Iterable[tuple[Hashable, "DB"]]: + for irow, df_row in self.record.iterrows(): + yield irow, DB.from_dict(self._row2dict(df_row)) diff --git a/vfrecovery/core/predict.py b/vfrecovery/core/predict.py index 644c09e..b604a05 100644 --- a/vfrecovery/core/predict.py +++ b/vfrecovery/core/predict.py @@ -115,7 +115,7 @@ def predict_function( output_path.mkdir(parents=True, exist_ok=True) # Set-up simulation logger - templogfile = get_a_log_filename(output_path, name='simulation_') + templogfile = get_a_log_filename(output_path) simlogfile = logging.FileHandler(templogfile, mode='a') simlogfile.setFormatter(logging.Formatter("%(asctime)s | %(levelname)s | %(name)s:%(filename)s | %(message)s", datefmt='%Y/%m/%d %I:%M:%S')) diff --git a/vfrecovery/core/trajfile_handler.py b/vfrecovery/core/trajfile_handler.py index 72a82da..199e8d1 100644 --- a/vfrecovery/core/trajfile_handler.py +++ b/vfrecovery/core/trajfile_handler.py @@ -11,7 +11,7 @@ from vfrecovery.utils.misc import get_cfg_str from vfrecovery.plots.utils import map_add_features, save_figurefile -from vfrecovery.json import Profile +from vfrecovery.json import Profile, Location from vfrecovery.json import Metrics, TrajectoryLengths, PairwiseDistances, PairwiseDistancesState @@ -162,8 +162,9 @@ def to_index(self) -> pd.DataFrame: deploy_lon, deploy_lat = self.obj.isel(obs=0)['lon'].values, self.obj.isel(obs=0)['lat'].values def worker(ds, cyc, x0, y0): - mask = np.logical_and((ds['cycle_number'] == cyc).compute(), - (ds['cycle_phase'] >= 3).compute()) + mask_end_of_cycle = np.logical_or((ds['cycle_phase'] == 3).compute(), (ds['cycle_phase'] == 4).compute()) + + mask = np.logical_and((ds['cycle_number'] == cyc).compute(), mask_end_of_cycle) this_cyc = ds.where(mask, drop=True) if len(this_cyc['time']) > 0: @@ -217,9 +218,13 @@ def index(self): self.get_index() return self._index - def add_distances(self, origin: Profile = None) -> pd.DataFrame: + def add_distances(self, origin: Location = None) -> pd.DataFrame: """Compute profiles distance to some origin + Parameters + ---------- + origin: :class:`Location` + Returns ------- :class:`pandas.dataframe` @@ -235,9 +240,9 @@ def add_distances(self, origin: Profile = None) -> pd.DataFrame: # Simulated cycles: # sim_cyc = np.unique(this_df['cyc']) - df = self._index + df = self.index - x2, y2 = origin.location.longitude, origin.location.latitude # real float initial position + x2, y2 = origin.longitude, origin.latitude # real float initial position df['distance'] = np.nan df['rel_lon'] = np.nan df['rel_lat'] = np.nan @@ -267,7 +272,7 @@ def worker(row): df = df.apply(worker, axis=1) self._index = df - return self._index + return self.index def analyse_pairwise_distances(self, virtual_cycle_number: int = 1, diff --git a/vfrecovery/core/utils.py b/vfrecovery/core/utils.py index 9b92c90..8760ea4 100644 --- a/vfrecovery/core/utils.py +++ b/vfrecovery/core/utils.py @@ -102,7 +102,7 @@ def make_hashable(o): return o -def get_a_log_filename(op, name='simulation'): +def get_a_log_filename(op, name='simulation_'): fname = lambda i: "%s%0.3d.log" % (name, i) i = 1 while op.joinpath(fname(i)).exists(): diff --git a/vfrecovery/json/VFRschema.py b/vfrecovery/json/VFRschema.py index 0a4b0ca..c60e373 100644 --- a/vfrecovery/json/VFRschema.py +++ b/vfrecovery/json/VFRschema.py @@ -70,7 +70,9 @@ def default(self, obj): return obj.isoformat() if isinstance(obj, np.float32): return float(obj) - if getattr(type(obj), '__name__') in ['Location', 'Profile', + if isinstance(obj, np.int64): + return int(obj) + if getattr(type(obj), '__name__') in ['Location', 'Profile', 'Trajectory', 'Metrics', 'TrajectoryLengths', 'PairwiseDistances', 'PairwiseDistancesState', 'SurfaceDrift', 'Transit', 'Location_error', 'MetaDataSystem', 'MetaDataComputation', 'MetaData']: diff --git a/vfrecovery/json/VFRschema_profile.py b/vfrecovery/json/VFRschema_profile.py index 262b3ca..18651f5 100644 --- a/vfrecovery/json/VFRschema_profile.py +++ b/vfrecovery/json/VFRschema_profile.py @@ -1,6 +1,6 @@ import pandas as pd import numpy as np -from typing import List, Dict +from typing import List, Dict, Iterable import argopy.plot as argoplot from .VFRschema import VFvalidators @@ -42,6 +42,16 @@ def __init__(self, **kwargs): def from_dict(obj: Dict) -> 'Location': return Location(**obj) + @staticmethod + def from_tuple(obj: tuple) -> 'Location': + if len(obj) == 2: + d = {'longitude': obj[0], 'latitude': obj[1]} + elif len(obj) == 3: + d = {'longitude': obj[0], 'latitude': obj[1], 'time': obj[2]} + elif len(obj) == 4: + d = {'longitude': obj[0], 'latitude': obj[1], 'time': obj[2], 'description': obj[3]} + return Location(**d) + class Profile(VFvalidators): location: Location @@ -99,3 +109,41 @@ def from_ArgoIndex(df: pd.DataFrame) -> List['Profile']: }) Plist.append(p) return Plist + + +class Trajectory(VFvalidators): + locations: List[Location] + + schema: str = "VFrecovery-schema-trajectory" + description: str = "Represents two or more VirtualFleet-Recovery locations that share a relationship" + required: List = ["locations"] + properties: List = ["locations", "description"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if len(kwargs['locations']) < 2: + raise ValueError("'locations' must a be list with at least 2 elements") + L = [] + for location in kwargs['locations']: + if isinstance(location, dict): + loc = Location.from_dict(location) + elif isinstance(location, Iterable): + loc = Location.from_tuple(location) + else: + raise ValueError("'locations' item must be a dictionary or an Iterable") + L.append(loc) + self.locations = L + + @staticmethod + def from_dict(obj: Dict) -> 'Trajectory': + """ + + Parameters + ---------- + locations: List[Location] + """ + return Trajectory(**obj) + + @staticmethod + def from_tuple(obj: tuple) -> 'Trajectory': + return Trajectory(**{'locations': obj}) diff --git a/vfrecovery/json/__init__.py b/vfrecovery/json/__init__.py index d7153c1..f697ee7 100644 --- a/vfrecovery/json/__init__.py +++ b/vfrecovery/json/__init__.py @@ -1,4 +1,4 @@ -from .VFRschema_profile import Profile, Location +from .VFRschema_profile import Profile, Location, Trajectory from .VFRschema_simulation import Simulation from .VFRschema_meta import MetaData, MetaDataSystem, MetaDataComputation from .VFRschema_metrics import Metrics, TrajectoryLengths, PairwiseDistances, PairwiseDistancesState, Transit, SurfaceDrift, Location_error \ No newline at end of file