From 4ceb2a8b364327124ebbfa96ac42f3532489df4c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 11 Oct 2024 10:36:00 +0200 Subject: [PATCH 1/5] - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo` --- cm/CHANGES.md | 3 +++ cm/cmind/__init__.py | 2 +- cm/cmind/core.py | 3 ++- cm/cmind/repo.py | 11 +++++++++-- cm/cmind/repos.py | 6 ++++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 5db51f54c..79f7b4f53 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,3 +1,6 @@ +## V3.1.0.1 + - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo` + ## V3.1.0 - simplified and changed process_input function API diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index d5996f11a..0038e1259 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -2,7 +2,7 @@ # # Written by Grigori Fursin -__version__ = "3.1.0" +__version__ = "3.1.0.1" from cmind.core import access from cmind.core import x diff --git a/cm/cmind/core.py b/cm/cmind/core.py index 7dfb30fa8..f9e304fad 100644 --- a/cm/cmind/core.py +++ b/cm/cmind/core.py @@ -781,7 +781,8 @@ def _x(self, i, control): # Load info about all CM repositories (to enable search for automations and artifacts) if self.repos == None: repos = Repos(path = self.repos_path, cfg = self.cfg, - path_to_internal_repo = self.path_to_cmind_repo) + path_to_internal_repo = self.path_to_cmind_repo, + cmx = True) r = repos.load() if r['return'] >0 : return r diff --git a/cm/cmind/repo.py b/cm/cmind/repo.py index 3a5ed1c1a..9cf04c037 100644 --- a/cm/cmind/repo.py +++ b/cm/cmind/repo.py @@ -41,7 +41,7 @@ def __init__(self, path, cfg): self.meta = {} ############################################################ - def load(self): + def load(self, cmx = False): """ Load CM repository @@ -71,8 +71,15 @@ def load(self): self.meta = r['meta'] - self.path_prefix = self.meta.get('prefix','') + if cmx and self.meta.get('prefix_cmx', '') != '': + self.path_prefix = self.meta.get('prefix_cmx','') + else: + self.path_prefix = self.meta.get('prefix','') + if self.path_prefix is not None and self.path_prefix.strip() !='': self.path_with_prefix = os.path.join(self.path, self.path_prefix) + if os.path.isdir(self.path) and not os.path.isdir(self.path_with_prefix): + os.makedirs(self.path_with_prefix) + return {'return':0} diff --git a/cm/cmind/repos.py b/cm/cmind/repos.py index dd6c95aab..d25e757b8 100644 --- a/cm/cmind/repos.py +++ b/cm/cmind/repos.py @@ -12,7 +12,7 @@ class Repos: CM repositories class """ - def __init__(self, path, cfg, path_to_internal_repo = ''): + def __init__(self, path, cfg, path_to_internal_repo = '', cmx = False): """ Initialize CM repositories class @@ -49,6 +49,8 @@ def __init__(self, path, cfg, path_to_internal_repo = ''): self.extra_info = {} + self.cmx = cmx + ############################################################ def load(self, init = False): """ @@ -128,7 +130,7 @@ def load(self, init = False): # Load description repo = Repo(full_path_to_repo, self.cfg) - r = repo.load() + r = repo.load(cmx = self.cmx) if r['return']>0 and r['return']!=16: return r # Load only if desc exists From 382ac4bc1ddb6836199351ad030726a6a99501f0 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 13 Oct 2024 11:29:04 +0200 Subject: [PATCH 2/5] - improved CMX logging (-log and -logfile): https://github.com/mlcommons/ck/issues/1317 - print control flags in help (cmx -h | cmx -help): https://github.com/mlcommons/ck/issues/1318 - fail if control flag is not recognized: https://github.com/mlcommons/ck/issues/1315 --- cm/cmind/config.py | 4 +- cm/cmind/core.py | 121 +++++++++++++++++++++++++++++++++++------- cm/cmind/utils.py | 19 +++++++ cm/docs/cmx/README.md | 13 ++++- 4 files changed, 135 insertions(+), 22 deletions(-) diff --git a/cm/cmind/config.py b/cm/cmind/config.py index 36255c138..bf18f7832 100644 --- a/cm/cmind/config.py +++ b/cm/cmind/config.py @@ -38,7 +38,7 @@ def __init__(self, config_file = None): "error_prefix": "CM error:", "info_cli": "cm {action} {automation} {artifact(s)} {flags} @input.yaml @input.json", - "info_clix": "cmx {action} {automation} {artifact(s)} {flags} @input.yaml @input.json", + "info_clix": "cmx {action} {automation} {artifact(s)} {CMX control flags (-)} {CMX automation flags (--)}", "default_home_dir": "CM", @@ -64,7 +64,7 @@ def __init__(self, config_file = None): "cp":"copy" }, - "new_repo_requirements": "cmind >= 3.0.0\n", + "new_repo_requirements": "cmind >= 3.1.0\n", "cmind_automation":"automation", diff --git a/cm/cmind/core.py b/cm/cmind/core.py index f9e304fad..23adc9196 100644 --- a/cm/cmind/core.py +++ b/cm/cmind/core.py @@ -103,7 +103,6 @@ def __init__(self, repos_path = '', debug = False): # Logging self.logger = None - self.log = [] # Index self.index = None @@ -155,24 +154,40 @@ def halt(self, r): sys.exit(r['return']) ############################################################ - def log(self, s): + def log(self, s, t = 'info'): """ Args: - s (string): log string + s (str): log string + t (str): log type - "info" (default) + "debug" + "warning" + "error" Returns: None """ - # Force console - print (s) + logger = self.logger + + if logger != None: + if t == 'debug': + self.logger.debug(s) + elif t == 'warning': + self.logger.warning(s) + elif t == 'error': + self.logger.error(s) + # info + else: + self.logger.info(s) return + ############################################################ def access(self, i, out = None): """ Access CM automation actions in a unified way similar to micro-services. + (Legacy. Further development in the new "x" function). i (dict | str | argv): unified CM input @@ -634,19 +649,23 @@ def x(self, i, out = None): Args: i (dict | str | argv): unified CM input - (action) (str): automation action - (automation (CM object): CM automation in format (alias | UID | alias,UID) + * (action) (str): automation action + * (automation (CM object): CM automation in format (alias | UID | alias,UID) or (repo alias | repo UID | repo alias,UID):(alias | UID | alias,UID) - (artifact) (CM object): CM artifact - (artifacts) (list of CM objects): extra CM artifacts + * (artifact) (CM object): CM artifact + * (artifacts) (list of CM objects): extra CM artifacts - (common) (bool): if True, use common automation action from Automation class - (help) (bool): if True, print CM automation action API + Control flags starting with - : - (ignore_inheritance) (bool): if True, ignore inheritance when searching for artifacts and automations + * (out) (str): if 'con', tell automations and CM to output extra information to console + + * (common) (bool): if True, use common automation action from Automation class + + * (help) (bool): if True, print CM automation action API + + * (ignore_inheritance) (bool): if True, ignore inheritance when searching for artifacts and automations - (out) (str): if 'con', tell automations and CM to output extra information to console Returns: (CM return dict): @@ -654,7 +673,7 @@ def x(self, i, out = None): * return (int): return code == 0 if no error and >0 if error * (error) (str): error string if return>0 - * Output from a CM automation action + * Output from a given CM automation action """ # Check if very first access call @@ -670,10 +689,6 @@ def x(self, i, out = None): if self.cfg['flag_debug'] in i: self.debug = True - # Check if log - if self.logger is None: - self.logger = logging.getLogger("cm") - # Parse as command line if string or list if type(i) == str or type(i) == list: import cmind.cli @@ -694,7 +709,51 @@ def x(self, i, out = None): i['control']['_input'][k] = i[k] control = i['control'] + + # Expose only control flags + control_flags = {} + for flag in control: + if not flag.startswith('_'): + control_flags[flag] = control[flag] + + # Check if unknown flags + unknown_control_flags = [flag for flag in control_flags if flag not in [ + 'h', 'help', 'version', 'out', 'j', 'json', + 'save_to_json_file', 'save_to_yaml_file', 'common', + 'ignore_inheritance', 'log', 'logfile', 'raise']] + + if len(unknown_control_flags)>0: + unknown_control_flags_str = ','.join(unknown_control_flags) + + print (f'Unknown control flag(s): {unknown_control_flags_str}') + print ('') + # Force print help + control['h'] = True + + # Check logging + use_log = control_flags.pop('log', '') + log_level = None + if use_log == True: + log_level = logging.INFO + else: + use_log = use_log.strip().lower() + if use_log == 'debug': + log_level = logging.DEBUG + elif use_log == 'warning': + log_level = logging.WARNING + elif use_log == 'error': + log_level = logging.ERROR + else: + log_level = logging.INFO + log_file = control_flags.pop('logfile', '') + if log_file == '': log_file = None + + # Check if log + if self.logger is None and use_log: + self.logger = logging.getLogger("cmx") + logging.basicConfig(filename = log_file, filemode = 'w', level = log_level) + # Check if force out programmatically (such as from CLI) if 'out' not in control and out is not None: control['out'] = out @@ -731,6 +790,9 @@ def _x(self, i, control): output = control.get('out', '') + if output == True: + output = 'con' + # Check and force json console output if control.get('j', False) or control.get('json', False): output = 'json' @@ -740,6 +802,8 @@ def _x(self, i, control): if self.output is None: self.output = output + control['out'] = output + # Check if console console = (output == 'con') @@ -775,6 +839,25 @@ def _x(self, i, control): if cm_help or extra_help: print_db_actions(self.common_automation, self.cfg['action_substitutions'], '', cmx = True) + + print ('') + print ('Control flags:') + print ('') + print (' -h | -help - print this help') + print (' -version - print version') + print (' -out (default) - output to console') + print (' -out=con (default) - output to console') + print (' -j | -json - print output of the automation action to console as JSON') + print (' -save_to_json_file={file} - save output of the automation action to file as JSON') + print (' -save_to_yaml_file={file} - save output of the automation action to file as YAML') + print (' -common - force call default common CMX automation action') + print (' -ignore_inheritance - ignore CMX meta inheritance') + print (' -log - log internal CMX information to console') + print (' -log={info (default) | debug | warning | error} - log level') + print (' -logfile={path to log file} - record log to file instead of console') + print (' -raise - raise Python error when automation action fails') + print ('') + print ('Check https://github.com/mlcommons/ck/tree/master/cm/docs/cmx for more details.') return {'return':0, 'warning':'no action specified'} @@ -1189,7 +1272,7 @@ def print_db_actions(automation, equivalent_actions, automation_name, cmx = Fals import types print ('') - print ('Common actions to manage CM repositories:') + print ('Common actions to manage CM repositories (use -h | -help to see the API):') print ('') db_actions=[] diff --git a/cm/cmind/utils.py b/cm/cmind/utils.py index 85b204fc6..56c19f2c9 100644 --- a/cm/cmind/utils.py +++ b/cm/cmind/utils.py @@ -1927,3 +1927,22 @@ def convert_dictionary(d, key, sub = True): return dd +############################################################################## +def test_input(i, module): + """ + Test if input has keys and report them as error + """ + + r = {'return':0} + + if len(i)>0: + unknown_keys = i.keys() + unknown_keys_str = ', '.join(unknown_keys) + + r = {'return': 1, + 'error': 'unknown input key(s) "{}" in module {}'.format(unknown_keys_str, module), + 'module': module, + 'unknown_keys': unknown_keys, + 'unknown_keys_str': unknown_keys_str} + + return r diff --git a/cm/docs/cmx/README.md b/cm/docs/cmx/README.md index d3e6e686c..b7880a55e 100644 --- a/cm/docs/cmx/README.md +++ b/cm/docs/cmx/README.md @@ -1,3 +1,14 @@ -# Collective Mind v3 (prototype) +# Collective Mind v3 aka CMX + +We prototype the next generation of CM. + +## Documentation * [Installation (Linux, Windows, MacOS)](install.md) + +TBD + + +## Contacts + +* Contact the author and tech. lead for more details: [Grigori Fursin](https://cKnowledge.org/gfursin) \ No newline at end of file From b8b7df3f8160a66313ec83dfa2e7cae7b419e6db Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 13 Oct 2024 12:27:55 +0200 Subject: [PATCH 3/5] - added -repro flag to record various info to cmx-repro directory: https://github.com/mlcommons/ck/issues/1319 --- cm/CHANGES.md | 10 +++++ cm/cmind/core.py | 107 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 96 insertions(+), 21 deletions(-) diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 79f7b4f53..a2248a19c 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,5 +1,15 @@ ## V3.1.0.1 + - added `utils.test_input` to test if input has keys + and report them as error - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo` + - improved CMX logging (-log and -logfile): + https://github.com/mlcommons/ck/issues/1317 + - print control flags in help (cmx -h | cmx -help): + https://github.com/mlcommons/ck/issues/1318 + - fail if control flag is not recognized: + https://github.com/mlcommons/ck/issues/1315 + - added -repro flag to record various info to cmx-repro directory + https://github.com/mlcommons/ck/issues/1319 ## V3.1.0 - simplified and changed process_input function API diff --git a/cm/cmind/core.py b/cm/cmind/core.py index 23adc9196..b6bc4e9f5 100644 --- a/cm/cmind/core.py +++ b/cm/cmind/core.py @@ -111,8 +111,12 @@ def __init__(self, repos_path = '', debug = False): if os.environ.get(self.cfg['env_index'],'').strip().lower() in ['no','off','false']: self.use_index = False + # Check if CM v3+ was called (to avoid mixing up older versions and make them co-exist) self.x_was_called = False + # Misc state + self.state = {} + ############################################################ def error(self, r): """ @@ -676,10 +680,14 @@ def x(self, i, out = None): * Output from a given CM automation action """ + import copy + # Check if very first access call x_was_called = self.x_was_called self.x_was_called = True + cur_dir = os.getcwd() + # Check the type of input if i is None: i = {} @@ -720,7 +728,7 @@ def x(self, i, out = None): unknown_control_flags = [flag for flag in control_flags if flag not in [ 'h', 'help', 'version', 'out', 'j', 'json', 'save_to_json_file', 'save_to_yaml_file', 'common', - 'ignore_inheritance', 'log', 'logfile', 'raise']] + 'ignore_inheritance', 'log', 'logfile', 'raise', 'repro']] if len(unknown_control_flags)>0: unknown_control_flags_str = ','.join(unknown_control_flags) @@ -730,29 +738,57 @@ def x(self, i, out = None): # Force print help control['h'] = True + # Check repro + use_log = str(control_flags.pop('log', '')).strip().lower() + log_file = control_flags.pop('logfile', '') + + if control.get('repro', '') != '': + if not os.path.isdir('cmx-repro'): + os.mkdir('cmx-repro') + + if log_file == '': + log_file = os.path.join(cur_dir, 'cmx-repro', 'cmx.log') + if use_log == '': + use_log = 'debug' + + ii = copy.deepcopy(i) + ii['control'] = {} + for k in control: + if not k.startswith('_') and k not in ['repro']: + ii['control'][k] = i[k] + + utils.save_json(os.path.join('cmx-repro', 'cmx-input.json'), + meta = ii) + # Check logging - use_log = control_flags.pop('log', '') - log_level = None - if use_log == True: - log_level = logging.INFO - else: - use_log = use_log.strip().lower() - if use_log == 'debug': - log_level = logging.DEBUG - elif use_log == 'warning': - log_level = logging.WARNING - elif use_log == 'error': - log_level = logging.ERROR + if self.logger is None: + log_level = None + + if use_log == "false": + use_log = '' + elif use_log == "true": + use_log = 'info' + + if log_file == '': + log_file = None else: - log_level = logging.INFO - - log_file = control_flags.pop('logfile', '') - if log_file == '': log_file = None + if use_log == '': + use_log = 'debug' + + if use_log != '': + if use_log == 'debug': + log_level = logging.DEBUG + elif use_log == 'warning': + log_level = logging.WARNING + elif use_log == 'error': + log_level = logging.ERROR + else: + # info by default + log_level = logging.INFO - # Check if log - if self.logger is None and use_log: - self.logger = logging.getLogger("cmx") - logging.basicConfig(filename = log_file, filemode = 'w', level = log_level) + # Configure + self.logger = logging.getLogger("cmx") + logging.basicConfig(filename = log_file, filemode = 'w', level = log_level) # Check if force out programmatically (such as from CLI) if 'out' not in control and out is not None: @@ -760,9 +796,28 @@ def x(self, i, out = None): use_raise = control.get('raise', False) + # Log access + recursion = self.state.get('recursion', 0) + self.state['recursion'] = recursion + 1 + + if not self.logger == None: + log_action = i.get('action', '') + log_automation = i.get('automation', '') + log_artifact = i.get('artifact', '') + + spaces = ' ' * recursion + + self.log(f"x log: {spaces} {log_action} {log_automation} {log_artifact}", "info") + self.log(f"x input: {spaces} ({i})", "debug") + # Call access helper r = self._x(i, control) + if not self.logger == None: + self.log(f"x output: {r}", "debug") + + self.state['recursion'] = recursion + if not x_was_called: # Very first call (not recursive) # Check if output to json and save file @@ -770,6 +825,9 @@ def x(self, i, out = None): if self.output == 'json': utils.dump_safe_json(r) + # Restore directory of call + os.chdir(cur_dir) + # Check if save to json if control.get('save_to_json_file', '') != '': utils.save_json(control['save_to_json_file'], meta = r) @@ -777,6 +835,12 @@ def x(self, i, out = None): if control.get('save_to_yaml_file', '') != '': utils.save_yaml(control['save_to_yaml_file'], meta = r) + if control.get('repro', '') != '': + if not os.path.isdir('cmx-repro'): + os.mkdir('cmx-repro') + utils.save_json(os.path.join('cmx-repro', 'cmx-output.json'), + meta = r) + if use_raise and r['return']>0: raise Exception(r['error']) @@ -856,6 +920,7 @@ def _x(self, i, control): print (' -log={info (default) | debug | warning | error} - log level') print (' -logfile={path to log file} - record log to file instead of console') print (' -raise - raise Python error when automation action fails') + print (' -repro - record various info to the cmx-repro directory to replay CMX command') print ('') print ('Check https://github.com/mlcommons/ck/tree/master/cm/docs/cmx for more details.') From 2c75ab11168d59d324872e9481631e9fe03b7512 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 13 Oct 2024 13:50:58 +0200 Subject: [PATCH 4/5] - print call stack when error > 32 to be able to trace error cause: https://github.com/mlcommons/ck/issues/1320 can be combined with -log=debug and -logfile --- cm/CHANGES.md | 3 +++ cm/cmind/core.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/cm/CHANGES.md b/cm/CHANGES.md index a2248a19c..73b957e42 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -10,6 +10,9 @@ https://github.com/mlcommons/ck/issues/1315 - added -repro flag to record various info to cmx-repro directory https://github.com/mlcommons/ck/issues/1319 + - print call stack when error > 32 to be able to trace error cause: + https://github.com/mlcommons/ck/issues/1320 + can be combined with -log=debug and -logfile ## V3.1.0 - simplified and changed process_input function API diff --git a/cm/cmind/core.py b/cm/cmind/core.py index b6bc4e9f5..b448077b7 100644 --- a/cm/cmind/core.py +++ b/cm/cmind/core.py @@ -841,8 +841,21 @@ def x(self, i, out = None): utils.save_json(os.path.join('cmx-repro', 'cmx-output.json'), meta = r) - if use_raise and r['return']>0: - raise Exception(r['error']) + if r['return'] >0: + if r['return'] > 32: + print ('') + print ('CM Error Call Stack:') + + call_stack = self.state['call_stack'] + + for cs in call_stack: + print (f' {cs}') + + self.log(f"x error call stack: {call_stack}", "debug") + self.log(f"x error: {r}", "debug") + + if use_raise: + raise Exception(r['error']) return r @@ -1283,9 +1296,22 @@ def _x(self, i, control): if not k.startswith('_'): ii[k] = control[k] + + # Add call stack + call_stack = self.state.get('call_stack', []) + call_stack.append({'module':automation_full_path, 'func':action}) + self.state['call_stack'] = call_stack + # Call automation action r = action_addr(i) + # Remove from stack if no error + if r['return'] == 0: + call_stack = self.state.get('call_stack', []) + if len(call_stack)>0: + call_stack.pop() + self.state['call_stack'] = call_stack + # Check if need to save index if self.use_index and self.index.updated: rx = self.index.save() From b5e87b7b08be7e6c7c3045a8dc717a4dc163a064 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 13 Oct 2024 13:51:36 +0200 Subject: [PATCH 5/5] V3.2.0: - added `utils.test_input` to test if input has keys and report them as error - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo` - improved CMX logging (-log and -logfile): https://github.com/mlcommons/ck/issues/1317 - print control flags in help (cmx -h | cmx -help): https://github.com/mlcommons/ck/issues/1318 - fail if control flag is not recognized: https://github.com/mlcommons/ck/issues/1315 - added -repro flag to record various info to cmx-repro directory https://github.com/mlcommons/ck/issues/1319 - print call stack when error > 32 to be able to trace error cause: https://github.com/mlcommons/ck/issues/1320 can be combined with -log=debug and -logfile --- cm/CHANGES.md | 2 +- cm/cmind/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 73b957e42..46b9fa2d5 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,4 +1,4 @@ -## V3.1.0.1 +## V3.2.0 - added `utils.test_input` to test if input has keys and report them as error - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo` diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index 0038e1259..cccfdd931 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -2,7 +2,7 @@ # # Written by Grigori Fursin -__version__ = "3.1.0.1" +__version__ = "3.2.0" from cmind.core import access from cmind.core import x