From 76c16c74263be1a193f06d573fcdb8f835bf575e Mon Sep 17 00:00:00 2001 From: Eric Mertens Date: Fri, 4 Aug 2023 11:06:31 -0700 Subject: [PATCH 1/2] Expose cardinality checker via the cli --- cli/rack/__init__.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cli/rack/__init__.py b/cli/rack/__init__.py index 18d0791b..fb5e546f 100755 --- a/cli/rack/__init__.py +++ b/cli/rack/__init__.py @@ -558,6 +558,32 @@ def go() -> None: raise Exception(response['message']) go() +def cardinality_driver( + base_url: Url, + model_graphs: Optional[List[Url]], + data_graphs: Optional[List[Url]], + triple_store: Optional[Url], + triple_store_type: Optional[str], + headers: bool, + export_format: ExportFormat + ) -> None: + """Generate an output table with all cardinality violations in the given datagraphs""" + + if data_graphs is not None: + data_graph = data_graphs[0] + else: + logger.warning("Defaulting data-graph to %s", DEFAULT_DATA_GRAPH) + data_graph = DEFAULT_DATA_GRAPH + + if data_graphs is not None: + extra_data_graphs = data_graphs[1:] + else: + extra_data_graphs = [] + + conn = sparql_connection(base_url, model_graphs, data_graph, extra_data_graphs, triple_store, triple_store_type) + + semtk_table = semtk3.get_cardinality_violations(conn) + print(format_semtk_table(semtk_table, export_format=export_format, headers=headers)) def ingest_data_driver(config_path: Path, base_url: Url, model_graphs: Optional[List[Url]], data_graphs: Optional[List[Url]], triple_store: Optional[Url], triple_store_type: Optional[str], clear: bool) -> None: """Use an import.yaml file to ingest multiple CSV files into the data graph.""" @@ -843,6 +869,11 @@ def dispatch_data_import(args: SimpleNamespace) -> None: cliMethod = CLIMethod.DATA_IMPORT ingest_data_driver(Path(args.config), args.base_url, args.model_graph, args.data_graph, args.triple_store, args.triple_store_type, args.clear) +def dispatch_data_cardinality(args: SimpleNamespace) -> None: + """Implementation of the data cardinality subcommand""" + cliMethod = CLIMethod.DATA_IMPORT + cardinality_driver(args.base_url, args.model_graph, args.data_graph, args.triple_store, args.triple_store_type, export_format=args.format, headers=not args.no_headers) + def dispatch_model_import(args: SimpleNamespace) -> None: """Implementation of the plumbing model subcommand""" cliMethod = CLIMethod.MODEL_IMPORT @@ -899,6 +930,7 @@ def get_argument_parser() -> argparse.ArgumentParser: data_parser = subparsers.add_parser('data', help='Import or export CSV data') data_subparsers = data_parser.add_subparsers(dest='command') data_import_parser = data_subparsers.add_parser('import', help='Import CSV data') + data_cardinality_parser = data_subparsers.add_parser('cardinality', help='Check data cardinality') data_export_parser = data_subparsers.add_parser('export', help='Export query results') data_count_parser = data_subparsers.add_parser('count', help='Count matched query rows') data_clear_parser = data_subparsers.add_parser('clear', help='Clear data graph') @@ -943,6 +975,12 @@ def get_argument_parser() -> argparse.ArgumentParser: data_import_parser.add_argument('--clear', action='store_true', help='Clear data graph before import') data_import_parser.set_defaults(func=dispatch_data_import) + data_cardinality_parser.add_argument('--model-graph', type=str, action='append', help='Model graph URL') + data_cardinality_parser.add_argument('--data-graph', type=str, action='append', help='Data graph URL') + data_cardinality_parser.add_argument('--format', type=ExportFormat, help='Export format', choices=list(ExportFormat), default=ExportFormat.TEXT) + data_cardinality_parser.add_argument('--no-headers', action='store_true', help='Omit header row') + data_cardinality_parser.set_defaults(func=dispatch_data_cardinality) + data_export_parser.add_argument('nodegroup', type=str, help='ID of nodegroup') data_export_parser.add_argument('--model-graph', type=str, action='append', help='Model graph URL') data_export_parser.add_argument('--data-graph', type=str, required=True, action='append', help='Data graph URL') From 9ababcd110ad51cb482d487a198e2fd3afc01ae6 Mon Sep 17 00:00:00 2001 From: Eric Mertens Date: Fri, 4 Aug 2023 15:49:20 -0700 Subject: [PATCH 2/2] Add flags for cardinality checker's optional arguments --- cli/rack/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cli/rack/__init__.py b/cli/rack/__init__.py index fb5e546f..aa09527d 100755 --- a/cli/rack/__init__.py +++ b/cli/rack/__init__.py @@ -565,7 +565,9 @@ def cardinality_driver( triple_store: Optional[Url], triple_store_type: Optional[str], headers: bool, - export_format: ExportFormat + export_format: ExportFormat, + concise: bool, + max_rows: int ) -> None: """Generate an output table with all cardinality violations in the given datagraphs""" @@ -582,7 +584,7 @@ def cardinality_driver( conn = sparql_connection(base_url, model_graphs, data_graph, extra_data_graphs, triple_store, triple_store_type) - semtk_table = semtk3.get_cardinality_violations(conn) + semtk_table = semtk3.get_cardinality_violations(conn, max_rows=max_rows, concise_format=concise) print(format_semtk_table(semtk_table, export_format=export_format, headers=headers)) def ingest_data_driver(config_path: Path, base_url: Url, model_graphs: Optional[List[Url]], data_graphs: Optional[List[Url]], triple_store: Optional[Url], triple_store_type: Optional[str], clear: bool) -> None: @@ -872,7 +874,8 @@ def dispatch_data_import(args: SimpleNamespace) -> None: def dispatch_data_cardinality(args: SimpleNamespace) -> None: """Implementation of the data cardinality subcommand""" cliMethod = CLIMethod.DATA_IMPORT - cardinality_driver(args.base_url, args.model_graph, args.data_graph, args.triple_store, args.triple_store_type, export_format=args.format, headers=not args.no_headers) + cardinality_driver(args.base_url, args.model_graph, args.data_graph, args.triple_store, args.triple_store_type, + export_format=args.format, headers=not args.no_headers, concise=args.concise, max_rows=args.max_rows) def dispatch_model_import(args: SimpleNamespace) -> None: """Implementation of the plumbing model subcommand""" @@ -979,6 +982,8 @@ def get_argument_parser() -> argparse.ArgumentParser: data_cardinality_parser.add_argument('--data-graph', type=str, action='append', help='Data graph URL') data_cardinality_parser.add_argument('--format', type=ExportFormat, help='Export format', choices=list(ExportFormat), default=ExportFormat.TEXT) data_cardinality_parser.add_argument('--no-headers', action='store_true', help='Omit header row') + data_cardinality_parser.add_argument('--max-rows', default=-1, type=int, help='Maximum output rows') + data_cardinality_parser.add_argument('--concise', default=False, action='store_true', help='Use concise output') data_cardinality_parser.set_defaults(func=dispatch_data_cardinality) data_export_parser.add_argument('nodegroup', type=str, help='ID of nodegroup')