From e74f45cae4bad4ee61d7d268f8a493d535f9a21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 26 Apr 2024 11:19:05 -0400 Subject: [PATCH 01/41] refactor(cli/schedules.py): separating schedules functions from pipelines Schedules client created to handle all operations related to schedules --- workflow/cli/main.py | 2 + workflow/cli/pipelines.py | 94 +++----------- workflow/cli/schedules.py | 183 ++++++++++++++++++++++++++++ workflow/http/context.py | 9 ++ workflow/http/pipelines.py | 91 +------------- workflow/http/schedules.py | 157 ++++++++++++++++++++++++ workflow/workspaces/development.yml | 1 + 7 files changed, 376 insertions(+), 161 deletions(-) create mode 100644 workflow/cli/schedules.py create mode 100644 workflow/http/schedules.py diff --git a/workflow/cli/main.py b/workflow/cli/main.py index dac70b8..ca126bb 100755 --- a/workflow/cli/main.py +++ b/workflow/cli/main.py @@ -6,6 +6,7 @@ from workflow.cli.pipelines import pipelines from workflow.cli.results import results from workflow.cli.run import run +from workflow.cli.schedules import schedules from workflow.cli.workspace import workspace @@ -19,6 +20,7 @@ def cli(): # cli.add_command(buckets) cli.add_command(results) cli.add_command(pipelines) +cli.add_command(schedules) cli.add_command(workspace) if __name__ == "__main__": diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index f40c66b..ca93a02 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -62,64 +62,27 @@ def version(): required=False, help="List only Pipelines with provided name.", ) -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def ls(name: Optional[str] = None, schedule: bool = False): +def ls(name: Optional[str] = None): """List all pipelines.""" http = HTTPContext() - objects = ( - http.pipelines.list_pipeline_configs(name) - if not schedule - else http.pipelines.list_schedules(name) - ) - if schedule: - table.title = "Workflow Scheduled Pipelines" - table.add_column("ID", max_width=50, justify="left", style="blue") - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Lives", max_width=50, justify="left") - table.add_column("Has Spawned", max_width=50, justify="left") - table.add_column("Next Time", max_width=50, justify="left") - for schedule_obj in objects: - status = Text( - schedule_obj["status"], style=status_colors[schedule_obj["status"]] - ) - lives = schedule_obj["lives"] - lives_text = Text(str(lives) if lives > -1 else "\u221e") - table.add_row( - schedule_obj["id"], - schedule_obj["pipeline_config"]["name"], - status, - lives_text, - str(schedule_obj["has_spawned"]), - str(schedule_obj["next_time"]), - ) - else: - table.add_column("ID", max_width=50, justify="left", style="blue") - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Stage", max_width=50, justify="left") - for config in objects: - status = Text(config["status"], style=status_colors[config["status"]]) - table.add_row( - config["id"], config["name"], status, str(config["current_stage"]) - ) + objects = http.pipelines.list_pipeline_configs(name) + table.add_column("ID", max_width=50, justify="left", style="blue") + table.add_column("Name", max_width=50, justify="left", style="bright_green") + table.add_column("Status", max_width=50, justify="left") + table.add_column("Stage", max_width=50, justify="left") + for config in objects: + status = Text(config["status"], style=status_colors[config["status"]]) + table.add_row( + config["id"], config["name"], status, str(config["current_stage"]) + ) console.print(table) @pipelines.command("count", help="Count pipeline configurations per collection.") -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def count(schedule: bool): +def count(): """Count pipeline configurations.""" http = HTTPContext() - counts = ( - http.pipelines.count() if not schedule else http.pipelines.count_schedules() - ) - if schedule: - table.title = "Workflow Schedule Pipelines" + counts = http.pipelines.count() table.add_column("Name", max_width=50, justify="left", style="blue") table.add_column("Count", max_width=50, justify="left") total = int() @@ -132,9 +95,6 @@ def count(schedule: bool): @pipelines.command("deploy", help="Deploy a workflow pipeline.") -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) @click.argument( "filename", type=click.Path(exists=True, dir_okay=False, readable=True), @@ -147,16 +107,8 @@ def deploy(filename: click.Path, schedule: bool): data: Dict[str, Any] = {} with open(filepath) as reader: data = yaml.load(reader, Loader=SafeLoader) # type: ignore - if schedule and "schedule" not in data.keys(): - error_text = Text( - "Your configuration file needs a schedule when using the --schedule option", - style="red", - ) - console.print(error_text) - return try: - deploy_result = http.pipelines.deploy(data, schedule) - print(deploy_result) + deploy_result = http.pipelines.deploy(data) except requests.HTTPError as deploy_error: console.print(deploy_error.response.json()["message"]) return @@ -173,28 +125,18 @@ def deploy(filename: click.Path, schedule: bool): @pipelines.command("ps", help="Get pipeline details.") @click.argument("pipeline", type=str, required=True) @click.argument("id", type=str, required=True) -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def ps(pipeline: str, id: str, schedule: bool): +def ps(pipeline: str, id: str): """List a pipeline configuration in detail.""" http = HTTPContext() query: Dict[str, Any] = {"id": id} console_content = None try: - payload = http.pipelines.get_pipeline_config(pipeline, query, schedule) + payload = http.pipelines.get_pipeline_config(pipeline, query) except IndexError: - error_text = Text( - f"No {'Schedule' if schedule else 'PipelineConfig'} were found", style="red" - ) + error_text = Text("No PipelineConfig were found", style="red") console_content = error_text else: - if not schedule: - table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") - else: - table.add_column( - f"Scheduled Pipeline: {pipeline}", max_width=120, justify="left" - ) + table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") text = JSON(json.dumps(payload), indent=2) table.add_row(text) console_content = table diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py new file mode 100644 index 0000000..bc743dc --- /dev/null +++ b/workflow/cli/schedules.py @@ -0,0 +1,183 @@ +"""Manage workflow pipelines schedules.""" + +import json +from typing import Any, Dict, Optional, Tuple + +import click +import requests +import yaml +from rich import pretty +from rich.console import Console +from rich.json import JSON +from rich.table import Table +from rich.text import Text +from yaml.loader import SafeLoader + +from workflow.http.context import HTTPContext + +pretty.install() +console = Console() + +table = Table( + title="\nWorkflow Pipelines - Schedules", + show_header=True, + header_style="magenta", + title_style="bold green", + min_width=50, +) +BASE_URL = "https://frb.chimenet.ca/schedule" +STATUS = ["active", "running", "expired"] +status_colors = { + "active": "bright_blue", + "running": "blue", + "expired": "dark_goldenrod", +} + + +@click.group(name="schedules", help="Manage Workflow Schedules.") +def schedules(): + """Manage workflow Schedules.""" + pass + + +@schedules.command("version", help="Backend version.") +def version(): + """Get version of the pipelines service.""" + http = HTTPContext() + console.print(http.pipelines.info()) + + +@schedules.command("ls", help="List schedules.") +@click.option( + "name", + "--name", + "-n", + type=str, + required=False, + help="List only Schedules with provided name.", +) +def ls(name: Optional[str] = None): + """List schedules. + + Parameters + ---------- + name : Optional[str], optional + Name of specific schedule, by default None + """ + http = HTTPContext() + objects = http.pipelines.list_schedules(name) + table.title = "Workflow Scheduled Pipelines" + table.add_column("ID", max_width=50, justify="left", style="blue") + table.add_column("Name", max_width=50, justify="left", style="bright_green") + table.add_column("Status", max_width=50, justify="left") + table.add_column("Lives", max_width=50, justify="left") + table.add_column("Has Spawned", max_width=50, justify="left") + table.add_column("Next Time", max_width=50, justify="left") + for schedule_obj in objects: + status = Text( + schedule_obj["status"], style=status_colors[schedule_obj["status"]] + ) + lives = schedule_obj["lives"] + lives_text = Text(str(lives) if lives > -1 else "\u221e") + table.add_row( + schedule_obj["id"], + schedule_obj["pipeline_config"]["name"], + status, + lives_text, + str(schedule_obj["has_spawned"]), + str(schedule_obj["next_time"]), + ) + console.print(table) + + +@schedules.command("count", help="Count schedules per collection.") +def count(): + """Count schedules.""" + http = HTTPContext() + counts = http.schedules.count_schedules() + table.title = "Workflow Schedules" + table.add_column("Name", max_width=50, justify="left", style="blue") + table.add_column("Count", max_width=50, justify="left") + total = int() + for k, v in counts.items(): + table.add_row(k, str(v)) + total += v + table.add_section() + table.add_row("Total", str(total)) + console.print(table) + + +@schedules.command("deploy", help="Deploy a scheduled pipeline.") +@click.argument( + "filename", + type=click.Path(exists=True, dir_okay=False, readable=True), + required=True, +) +def deploy(filename: click.Path): + """Deploy a scheduled pipeline.""" + http = HTTPContext() + filepath: str = str(filename) + data: Dict[str, Any] = {} + with open(filepath) as reader: + data = yaml.load(reader, Loader=SafeLoader) # type: ignore + try: + deploy_result = http.schedules.deploy(data) + except requests.HTTPError as deploy_error: + console.print(deploy_error.response.json()["message"]) + return + table.add_column("IDs", max_width=50, justify="left", style="bright_green") + if isinstance(deploy_result, list): + for _id in deploy_result: + table.add_row(_id) + if isinstance(deploy_result, dict): + for v in deploy_result.values(): + table.add_row(v) + console.print(table) + + +@schedules.command("ps", help="Get schedule details.") +@click.argument("id", type=str, required=True) +def ps(id: str): + """Gets schedules details.""" + http = HTTPContext() + query: Dict[str, Any] = {"id": id} + console_content = None + try: + payload = http.schedules.get_schedule(query) + except IndexError: + error_text = Text("No Schedule was found", style="red") + console_content = error_text + else: + table.add_column( + f"Scheduled Pipeline: {payload['pipeline_config']['name']}", + max_width=120, + justify="left", + ) + text = JSON(json.dumps(payload), indent=2) + table.add_row(text) + console_content = table + finally: + console.print(console_content) + + +@schedules.command("rm", help="Remove a schedule.") +@click.argument("id", type=str, required=True) +def rm(id: Tuple[str]): + """Remove a schedule.""" + http = HTTPContext() + content = None + try: + delete_result = http.schedules.remove(id) + if delete_result.status_code == 204: + text = Text("No pipeline configurations were deleted.", style="red") + content = text + except Exception as e: + text = Text( + f"No pipeline configurations were deleted.\nError: {e}", style="red" + ) + content = text + else: + table.add_column("Deleted IDs", max_width=50, justify="left", style="red") + table.add_row(id) + content = table + console.print(content) diff --git a/workflow/http/context.py b/workflow/http/context.py index 402929a..731447a 100644 --- a/workflow/http/context.py +++ b/workflow/http/context.py @@ -9,6 +9,7 @@ from workflow.http.buckets import Buckets from workflow.http.pipelines import Pipelines from workflow.http.results import Results +from workflow.http.schedules import Schedules from workflow.utils import read from workflow.utils.logger import get_logger @@ -83,6 +84,13 @@ class HTTPContext(BaseSettings): exclude=True, ) + schedules: Schedules = Field( + default=None, + validate_default=False, + description="Schedules API Client.", + exclude=True, + ) + @model_validator(mode="after") def create_clients(self) -> "HTTPContext": """Create the HTTP Clients for the Workflow Servers. @@ -94,6 +102,7 @@ def create_clients(self) -> "HTTPContext": "buckets": Buckets, "results": Results, "pipelines": Pipelines, + "schedules": Schedules, } logger.debug(f"creating http clients for {list(clients.keys())}") config: Dict[str, Any] = read.workspace(self.workspace) diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index 39508c0..c4008e8 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -29,15 +29,13 @@ class Pipelines(Client): stop=(stop_after_delay(5) | stop_after_attempt(1)), ) @try_request - def deploy(self, data: Dict[str, Any], schedule: bool = False): + def deploy(self, data: Dict[str, Any]): """Deploys a PipelineConfig from payload data. Parameters ---------- data : Dict[str, Any] YAML data. - schedule : bool - If this function should interact with the Schedule endpoint. Returns ------- @@ -45,11 +43,7 @@ def deploy(self, data: Dict[str, Any], schedule: bool = False): IDs of PipelineConfig objects generated. """ with self.session as session: - url = ( - f"{self.baseurl}/v1/pipelines" - if not schedule - else f"{self.baseurl}/v1/schedule" - ) + url = f"{self.baseurl}/v1/pipelines" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -96,7 +90,7 @@ def list_pipeline_configs( @try_request def get_pipeline_config( - self, collection: str, query: Dict[str, Any], schedule: bool = False + self, collection: str, query: Dict[str, Any] ) -> Dict[str, Any]: """Gets details for one pipeline configuration. @@ -106,8 +100,6 @@ def get_pipeline_config( PipelineConfig name. query : Dict[str, Any] Dictionary with search parameters. - schedule : bool - If this function should interact with the Schedule endpoint. Returns ------- @@ -116,11 +108,7 @@ def get_pipeline_config( """ with self.session as session: params = {"query": dumps(query), "name": collection} - url = ( - f"{self.baseurl}/v1/pipelines?{urlencode(params)}" - if not schedule - else f"{self.baseurl}/v1/schedule?{urlencode(params)}" - ) + url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json()[0] @@ -131,7 +119,7 @@ def get_pipeline_config( stop=(stop_after_delay(5) | stop_after_attempt(1)), ) @try_request - def remove(self, pipeline: str, id: str, schedule: bool) -> Response: + def remove(self, pipeline: str, id: str) -> Response: """Removes a cancelled pipeline configuration. Parameters @@ -140,8 +128,6 @@ def remove(self, pipeline: str, id: str, schedule: bool) -> Response: PipelineConfig name. id : str PipelineConfig ID. - schedule : bool - If this function should interact with the Schedule endpoint. Returns ------- @@ -151,11 +137,7 @@ def remove(self, pipeline: str, id: str, schedule: bool) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": pipeline} - url = ( - f"{self.baseurl}/v1/pipelines?{urlencode(params)}" - if not schedule - else f"{self.baseurl}/v1/schedule?{urlencode(params)}" - ) + url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -202,64 +184,3 @@ def info(self) -> Dict[str, Any]: response.raise_for_status() server_info = response.json() return {"client": client_info, "server": server_info} - - @try_request - def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: - """Gets the list of all schedules. - - Parameters - ---------- - schedule_name : str - Schedule name. - - Returns - ------- - List[Dict[str, Any]] - List of schedule payloads. - """ - with self.session as session: - query = dumps({"pipeline_config.name": schedule_name}) - projection = dumps( - { - "id": True, - "status": True, - "lives": True, - "has_spawned": True, - "next_time": True, - "crontab": True, - "pipeline_config.name": True, - } - ) - url = ( - f"{self.baseurl}/v1/schedule?projection={projection}" - if schedule_name is None - else f"{self.baseurl}/v1/schedule?query={query}&projection={projection}" - ) - response: Response = session.get(url=url) - response.raise_for_status() - return response.json() - - @try_request - def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any]: - """Count schedules per pipeline name. - - Parameters - ---------- - schedule_name : Optional[str], optional - Schedule name, by default None - - Returns - ------- - Dict[str, Any] - Count payload. - """ - with self.session as session: - query = dumps({"name": schedule_name}) - url = ( - f"{self.baseurl}/v1/schedule/count" - if not schedule_name - else f"{self.baseurl}/v1/schedule/count?query={query}" - ) - response: Response = session.get(url=url) - response.raise_for_status() - return response.json() diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py new file mode 100644 index 0000000..f184115 --- /dev/null +++ b/workflow/http/schedules.py @@ -0,0 +1,157 @@ +"""Workflow Schedules API.""" + +from json import dumps +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from requests.models import Response +from tenacity import retry +from tenacity.stop import stop_after_attempt, stop_after_delay +from tenacity.wait import wait_random + +from workflow.http.pipelines import Pipelines +from workflow.utils.decorators import try_request + + +class Schedules(Pipelines): + """HTTP Client for interacting with the Schedules endpoints. + + Args: + Client (workflow.http.client): The base class for interacting with the backend. + + Returns: + Schedules: A client for interacting with the Schedule endpoints. + """ + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + def deploy(self, data: Dict[str, Any]): + """Deploys a Schedule from payload data. + + Parameters + ---------- + data : Dict[str, Any] + YAML data. + + Returns + ------- + List[str] + IDs of Schedule objects generated. + """ + with self.session as session: + url = f"{self.baseurl}/v1/schedule" + response: Response = session.post(url, json=data) + response.raise_for_status() + return response.json() + + @try_request + def get_schedule(self, query: Dict[str, Any]) -> Dict[str, Any]: + """Gets details for one Schedule. + + Parameters + ---------- + query : Dict[str, Any] + Dictionary with search parameters. + + Returns + ------- + Dict[str, Any] + Schedule payload. + """ + with self.session as session: + params = {"query": dumps(query)} + url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + response: Response = session.get(url=url) + response.raise_for_status() + return response.json()[0] + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def remove(self, id: str) -> Response: + """Removes a schedule. + + Parameters + ---------- + id : str + Schedule ID. + + Returns + ------- + List[Dict[str, Any]] + Response payload. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query)} + url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + response: Response = session.delete(url=url) + response.raise_for_status() + return response + + @try_request + def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: + """Gets the list of all schedules. + + Parameters + ---------- + schedule_name : str + Schedule name. + + Returns + ------- + List[Dict[str, Any]] + List of schedule payloads. + """ + with self.session as session: + query = dumps({"pipeline_config.name": schedule_name}) + projection = dumps( + { + "id": True, + "status": True, + "lives": True, + "has_spawned": True, + "next_time": True, + "crontab": True, + "pipeline_config.name": True, + } + ) + url = ( + f"{self.baseurl}/v1/schedule?projection={projection}" + if schedule_name is None + else f"{self.baseurl}/v1/schedule?query={query}&projection={projection}" + ) + response: Response = session.get(url=url) + response.raise_for_status() + return response.json() + + @try_request + def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any]: + """Count schedules per pipeline name. + + Parameters + ---------- + schedule_name : Optional[str], optional + Schedule name, by default None + + Returns + ------- + Dict[str, Any] + Count payload. + """ + with self.session as session: + query = dumps({"name": schedule_name}) + url = ( + f"{self.baseurl}/v1/schedule/count" + if not schedule_name + else f"{self.baseurl}/v1/schedule/count?query={query}" + ) + response: Response = session.get(url=url) + response.raise_for_status() + return response.json() diff --git a/workflow/workspaces/development.yml b/workflow/workspaces/development.yml index d95aec3..f42ca8a 100644 --- a/workflow/workspaces/development.yml +++ b/workflow/workspaces/development.yml @@ -18,6 +18,7 @@ archive: http: baseurls: pipelines: http://localhost:8001 + schedules: http://localhost:8001 buckets: http://localhost:8004 results: http://localhost:8005 # products: http://localhost:8004 From c286e6ebd2760c873bb5f86afd454bb142030e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 30 Apr 2024 14:08:51 -0400 Subject: [PATCH 02/41] feat(cli): improvements to pipelines and schedules commands --- workflow/cli/pipelines.py | 61 ++++++++++++++++--------- workflow/cli/schedules.py | 91 +++++++++++++++++++++++++------------ workflow/http/pipelines.py | 10 +++- workflow/http/schedules.py | 4 +- workflow/utils/variables.py | 21 +++++++++ 5 files changed, 134 insertions(+), 53 deletions(-) create mode 100644 workflow/utils/variables.py diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index ca93a02..7f27452 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,5 +1,6 @@ """Manage workflow pipelines.""" +import datetime as dt import json from typing import Any, Dict, Optional, Tuple @@ -8,12 +9,12 @@ import yaml from rich import pretty from rich.console import Console -from rich.json import JSON from rich.table import Table from rich.text import Text from yaml.loader import SafeLoader from workflow.http.context import HTTPContext +from workflow.utils.variables import status_colors, status_symbols pretty.install() console = Console() @@ -28,16 +29,6 @@ BASE_URL = "https://frb.chimenet.ca/pipelines" STATUS = ["created", "queued", "running", "success", "failure", "cancelled"] -status_colors = { - "active": "bright_blue", - "running": "blue", - "created": "lightblue", - "queued": "yellow", - "success": "green", - "failure": "red", - "cancelled": "dark_goldenrod", - "expired": "dark_goldenrod", -} @click.group(name="pipelines", help="Manage Workflow Pipelines.") @@ -62,19 +53,31 @@ def version(): required=False, help="List only Pipelines with provided name.", ) -def ls(name: Optional[str] = None): +@click.option( + "quiet", + "--quiet", + "-q", + is_flag=True, + required=False, + help="Only show IDs.", +) +def ls(name: Optional[str] = None, quiet: Optional[bool] = False): """List all pipelines.""" http = HTTPContext() objects = http.pipelines.list_pipeline_configs(name) table.add_column("ID", max_width=50, justify="left", style="blue") - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Stage", max_width=50, justify="left") + if not quiet: + table.add_column("Name", max_width=50, justify="left", style="bright_green") + table.add_column("Status", max_width=50, justify="left") + table.add_column("Stage", max_width=50, justify="left") for config in objects: status = Text(config["status"], style=status_colors[config["status"]]) - table.add_row( - config["id"], config["name"], status, str(config["current_stage"]) - ) + if not quiet: + table.add_row( + config["id"], config["name"], status, str(config["current_stage"]) + ) + continue + table.add_row(config["id"]) console.print(table) @@ -100,7 +103,7 @@ def count(): type=click.Path(exists=True, dir_okay=False, readable=True), required=True, ) -def deploy(filename: click.Path, schedule: bool): +def deploy(filename: click.Path): """Deploy a workflow pipeline.""" http = HTTPContext() filepath: str = str(filename) @@ -130,14 +133,30 @@ def ps(pipeline: str, id: str): http = HTTPContext() query: Dict[str, Any] = {"id": id} console_content = None + projection = {"name": False} + time_fields = ["creation", "start", "stop"] try: - payload = http.pipelines.get_pipeline_config(pipeline, query) + payload = http.pipelines.get_pipeline_config(pipeline, query, projection) except IndexError: error_text = Text("No PipelineConfig were found", style="red") console_content = error_text else: table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") - text = JSON(json.dumps(payload), indent=2) + text = Text("") + for k, v in payload.items(): + key_value_text = Text() + if k in time_fields and v: + v = dt.datetime.fromtimestamp(v) + if k == "pipeline": + key_value_text = Text(f"{k}: \n", style="bright_green") + for step in v: + key_value_text.append(f" {step['name']}:") + key_value_text.append(f"{status_symbols[step['status']]}\n") + else: + key_value_text = Text(f"{k}: ", style="bright_green") + key_value_text.append(f"{v}\n", style="white") + text.append_text(key_value_text) + table.add_row(text) console_content = table finally: diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index bc743dc..b1162d9 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -1,6 +1,5 @@ """Manage workflow pipelines schedules.""" -import json from typing import Any, Dict, Optional, Tuple import click @@ -14,6 +13,7 @@ from yaml.loader import SafeLoader from workflow.http.context import HTTPContext +from workflow.utils.variables import status_colors pretty.install() console = Console() @@ -27,11 +27,6 @@ ) BASE_URL = "https://frb.chimenet.ca/schedule" STATUS = ["active", "running", "expired"] -status_colors = { - "active": "bright_blue", - "running": "blue", - "expired": "dark_goldenrod", -} @click.group(name="schedules", help="Manage Workflow Schedules.") @@ -56,37 +51,51 @@ def version(): required=False, help="List only Schedules with provided name.", ) -def ls(name: Optional[str] = None): +@click.option( + "quiet", + "--quiet", + "-q", + is_flag=True, + required=False, + help="Only show IDs.", +) +def ls(name: Optional[str] = None, quiet: Optional[bool] = False): """List schedules. Parameters ---------- name : Optional[str], optional Name of specific schedule, by default None + quiet : Optional[bool], optional + Whether to show only IDs. """ http = HTTPContext() - objects = http.pipelines.list_schedules(name) + objects = http.schedules.list_schedules(name) table.title = "Workflow Scheduled Pipelines" table.add_column("ID", max_width=50, justify="left", style="blue") - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Lives", max_width=50, justify="left") - table.add_column("Has Spawned", max_width=50, justify="left") - table.add_column("Next Time", max_width=50, justify="left") + if not quiet: + table.add_column("Name", max_width=50, justify="left", style="bright_green") + table.add_column("Status", max_width=50, justify="left") + table.add_column("Lives", max_width=50, justify="left") + table.add_column("Has Spawned", max_width=50, justify="left") + table.add_column("Next Time", max_width=50, justify="left") for schedule_obj in objects: - status = Text( - schedule_obj["status"], style=status_colors[schedule_obj["status"]] - ) - lives = schedule_obj["lives"] - lives_text = Text(str(lives) if lives > -1 else "\u221e") - table.add_row( - schedule_obj["id"], - schedule_obj["pipeline_config"]["name"], - status, - lives_text, - str(schedule_obj["has_spawned"]), - str(schedule_obj["next_time"]), - ) + if not quiet: + status = Text( + schedule_obj["status"], style=status_colors[schedule_obj["status"]] + ) + lives = schedule_obj["lives"] + lives_text = Text(str(lives) if lives > -1 else "\u221e") + table.add_row( + schedule_obj["id"], + schedule_obj["pipeline_config"]["name"], + status, + lives_text, + str(schedule_obj["has_spawned"]), + str(schedule_obj["next_time"]), + ) + continue + table.add_row(schedule_obj["id"]) console.print(table) @@ -137,11 +146,26 @@ def deploy(filename: click.Path): @schedules.command("ps", help="Get schedule details.") @click.argument("id", type=str, required=True) -def ps(id: str): +@click.option( + "--detail", + "-d", + is_flag=True, + show_default=True, + help="Returns the Schedule Payload.", +) +def ps(id: str, detail: Optional[bool] = False): """Gets schedules details.""" http = HTTPContext() query: Dict[str, Any] = {"id": id} console_content = None + key_nicknames = { + "id": "ID", + "crontab": "Crontab", + "lives": "To Spawn", + "has_spawned": "Has Spawned", + "status": "Status", + "next_time": "Next Execution", + } try: payload = http.schedules.get_schedule(query) except IndexError: @@ -153,8 +177,19 @@ def ps(id: str): max_width=120, justify="left", ) - text = JSON(json.dumps(payload), indent=2) + text = Text("") + for k, v in payload.items(): + if k == "pipeline_config": + continue + key_value_text = Text(f"{key_nicknames.get(k, k)}: ", style="bright_green") + key_value_text.append( + f"{v}\n", style="white" if k != "status" else status_colors[v] + ) + text.append_text(key_value_text) table.add_row(text) + if detail: + this_payload = JSON.from_data(payload["pipeline_config"], indent=2) + table.add_row(this_payload) console_content = table finally: console.print(console_content) diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index c4008e8..fddba7d 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -90,7 +90,7 @@ def list_pipeline_configs( @try_request def get_pipeline_config( - self, collection: str, query: Dict[str, Any] + self, collection: str, query: Dict[str, Any], projection: Dict[str, Any] ) -> Dict[str, Any]: """Gets details for one pipeline configuration. @@ -100,6 +100,8 @@ def get_pipeline_config( PipelineConfig name. query : Dict[str, Any] Dictionary with search parameters. + projection : Dict[str, Any] + Dictionary with projection parameters. Returns ------- @@ -107,7 +109,11 @@ def get_pipeline_config( Pipeline configuration payload. """ with self.session as session: - params = {"query": dumps(query), "name": collection} + params = { + "query": dumps(query), + "name": collection, + "projection": dumps(projection), + } url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py index f184115..f092eb9 100644 --- a/workflow/http/schedules.py +++ b/workflow/http/schedules.py @@ -9,11 +9,11 @@ from tenacity.stop import stop_after_attempt, stop_after_delay from tenacity.wait import wait_random -from workflow.http.pipelines import Pipelines +from workflow.http.client import Client from workflow.utils.decorators import try_request -class Schedules(Pipelines): +class Schedules(Client): """HTTP Client for interacting with the Schedules endpoints. Args: diff --git a/workflow/utils/variables.py b/workflow/utils/variables.py new file mode 100644 index 0000000..044c570 --- /dev/null +++ b/workflow/utils/variables.py @@ -0,0 +1,21 @@ +"""Variables needed for console printing.""" + +status_colors = { + "active": "bright_blue", + "running": "blue", + "created": "lightblue", + "queued": "yellow", + "success": "green", + "failure": "red", + "cancelled": "dark_goldenrod", + "expired": "dark_goldenrod", +} + +status_symbols = { + "created": "\U000026AA", # white + "active": "\U0001F7E2", # green + "paused": "\U0001F7E1", # yellow + "success": "\U00002705", # Green check + "failure": "\U0000274C", # cross mark + "cancelled": "\U0001F534", # red +} From b79477893c8b6c3628483f43bb1a6bcaf0581baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Wed, 1 May 2024 08:40:20 -0400 Subject: [PATCH 03/41] refactor(schedules.py): adding section when using details option --- workflow/cli/pipelines.py | 8 +++++--- workflow/cli/schedules.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index 7f27452..cf4c338 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -148,13 +148,15 @@ def ps(pipeline: str, id: str): if k in time_fields and v: v = dt.datetime.fromtimestamp(v) if k == "pipeline": - key_value_text = Text(f"{k}: \n", style="bright_green") + key_value_text = Text(f"{k}: \n", style="bright_blue") for step in v: key_value_text.append(f" {step['name']}:") key_value_text.append(f"{status_symbols[step['status']]}\n") else: - key_value_text = Text(f"{k}: ", style="bright_green") - key_value_text.append(f"{v}\n", style="white") + key_value_text = Text(f"{k}: ", style="bright_blue") + key_value_text.append( + f"{v}\n", style="white" if k != "status" else status_colors[v] + ) text.append_text(key_value_text) table.add_row(text) diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index b1162d9..02b0121 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -188,6 +188,9 @@ def ps(id: str, detail: Optional[bool] = False): text.append_text(key_value_text) table.add_row(text) if detail: + table.add_section() + table.add_row(Text("Payload Details", style="magenta")) + table.add_section() this_payload = JSON.from_data(payload["pipeline_config"], indent=2) table.add_row(this_payload) console_content = table From 303bb26018d60783489275cf34cafaad792433ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Wed, 1 May 2024 10:57:39 -0400 Subject: [PATCH 04/41] feat(variables.py): adding missing statuses --- workflow/utils/variables.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/utils/variables.py b/workflow/utils/variables.py index 044c570..f8f8149 100644 --- a/workflow/utils/variables.py +++ b/workflow/utils/variables.py @@ -1,11 +1,12 @@ """Variables needed for console printing.""" status_colors = { - "active": "bright_blue", - "running": "blue", "created": "lightblue", "queued": "yellow", + "active": "bright_blue", + "running": "blue", "success": "green", + "paused": "orange", "failure": "red", "cancelled": "dark_goldenrod", "expired": "dark_goldenrod", @@ -13,9 +14,11 @@ status_symbols = { "created": "\U000026AA", # white + "queued": "\u23F3", # hourglass "active": "\U0001F7E2", # green - "paused": "\U0001F7E1", # yellow + "running": "\u2699", # gear "success": "\U00002705", # Green check + "paused": "\U0001F7E1", # yellow "failure": "\U0000274C", # cross mark "cancelled": "\U0001F534", # red } From f245508adc1dab4033fbcbcc87f5c581780d7b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 09:55:46 -0400 Subject: [PATCH 05/41] feat(cli): adding configs commands confgis commands added so the cli can work with the new version of pipelines(^2.7.0) --- workflow/cli/configs.py | 259 ++++++++++++++++++++++++++++ workflow/cli/main.py | 2 + workflow/http/configs.py | 182 +++++++++++++++++++ workflow/http/context.py | 9 + workflow/utils/renderers.py | 95 ++++++++++ workflow/workspaces/development.yml | 1 + 6 files changed, 548 insertions(+) create mode 100644 workflow/cli/configs.py create mode 100644 workflow/http/configs.py create mode 100644 workflow/utils/renderers.py diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py new file mode 100644 index 0000000..88e42a3 --- /dev/null +++ b/workflow/cli/configs.py @@ -0,0 +1,259 @@ +"""Manage workflow pipelines.""" + +import json +from typing import Any, Dict, Optional + +import click +import requests +import yaml +from rich import pretty +from rich.console import Console +from rich.json import JSON +from rich.table import Table +from rich.text import Text +from yaml import safe_load +from yaml.loader import SafeLoader + +from workflow.http.context import HTTPContext +from workflow.utils.renderers import render_config, render_pipeline +from workflow.utils.variables import status_colors + +pretty.install() +console = Console() + +table = Table( + title="\nWorkflow Pipelines", + show_header=True, + header_style="magenta", + title_style="bold magenta", + min_width=10, +) + +BASE_URL = "https://frb.chimenet.ca/pipelines" +STATUS = ["created", "queued", "running", "success", "failure", "cancelled"] + + +@click.group(name="configs", help="Manage Workflow Configs. Version 2.") +def configs(): + """Manage workflow configs.""" + pass + + +@configs.command("version", help="Backend version.") +def version(): + """Get version of the pipelines service.""" + http = HTTPContext() + console.print(http.configs.info()) + + +@configs.command("count", help="Count objects per collection.") +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +def count(pipelines: bool): + """Count objects in a database. + + Parameters + ---------- + pipelines : bool + Use this command on pipelines database. + """ + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + counts = http.configs.count(database) + table.add_column("Name", max_width=50, justify="left", style="blue") + table.add_column("Count", max_width=50, justify="left") + total = int() + for k, v in counts.items(): + table.add_row(k, str(v)) + total += v + table.add_section() + table.add_row("Total", str(total)) + console.print(table) + + +@configs.command("deploy", help="Deploy a workflow config.") +@click.argument( + "filename", + type=click.Path(exists=True, dir_okay=False, readable=True), + required=True, +) +def deploy(filename: click.Path): + """Deploy a workflow config. + + Parameters + ---------- + filename : click.Path + File path. + """ + http = HTTPContext() + filepath: str = str(filename) + data: Dict[str, Any] = {} + with open(filepath) as reader: + data = yaml.load(reader, Loader=SafeLoader) # type: ignore + try: + deploy_result = http.configs.deploy(data) + except requests.HTTPError as deploy_error: + console.print(deploy_error.response.json()["error_description"][0]["msg"]) + return + table.add_column("IDs", max_width=50, justify="left", style="bright_green") + if isinstance(deploy_result, list): + for _id in deploy_result: + table.add_row(_id) + if isinstance(deploy_result, dict): + for v in deploy_result.values(): + table.add_row(v) + console.print(table) + + +@configs.command("ls", help="List Configs.") +@click.option( + "name", + "--name", + "-n", + type=str, + required=False, + help="List only Configs with provided name.", +) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "quiet", + "--quiet", + "-q", + is_flag=True, + default=False, + help="Only show IDs.", +) +def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): + """List all objects.""" + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + configs_colums = ["name", "version", "children", "user"] + pipelines_columns = ["status", "current_stage", "steps"] + projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + if quiet: + projection = {"id": 1} + http = HTTPContext() + objects = http.configs.get_configs( + database=database, config_name=name, projection=json.dumps(projection) + ) + + # ? Add columns for each key + table.add_column("ID", max_width=40, justify="left", style="blue") + if not quiet: + if database == "configs": + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) + if database == "pipelines": + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + + for obj in objects: + if not quiet: + if database == "configs": + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["children"])), + obj["user"], + ) + if database == "pipelines": + status = Text(obj["status"], style=status_colors[obj["status"]]) + table.add_row( + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) + ) + continue + table.add_row(obj["id"]) + console.print(table) + + +@configs.command("ps", help="Get Configs details.") +@click.argument("name", type=str, required=True) +@click.argument("id", type=str, required=True) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "--details", + "-d", + is_flag=True, + default=False, + show_default=True, + help="Show more details for the object.", +) +def ps(name: str, id: str, pipelines: str, details: bool): + """Show details for an object.""" + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) + console_content = None + column_max_width = 300 + column_min_width = 40 + try: + payload = http.configs.get_configs( + database=database, config_name=name, query=query, projection=projection + )[0] + except IndexError: + error_text = Text(f"No {database.capitalize()} were found", style="red") + console_content = error_text + else: + text = Text("") + if database == "pipelines": + table.add_column( + f"Pipeline: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) + if database == "configs": + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) + if details: + table.add_column("Details", max_width=column_max_width, justify="left") + _details = safe_load(payload["yaml"]) + _details = { + k: v + for k, v in _details.items() + if k not in ["name", "version", "deployments"] + } + table.add_row(text, JSON(json.dumps(_details), indent=2)) + else: + table.add_row(text) + console_content = table + finally: + console.print(console_content) diff --git a/workflow/cli/main.py b/workflow/cli/main.py index ca126bb..db91ef3 100755 --- a/workflow/cli/main.py +++ b/workflow/cli/main.py @@ -3,6 +3,7 @@ import click # from workflow.cli.buckets import buckets +from workflow.cli.configs import configs from workflow.cli.pipelines import pipelines from workflow.cli.results import results from workflow.cli.run import run @@ -19,6 +20,7 @@ def cli(): cli.add_command(run) # cli.add_command(buckets) cli.add_command(results) +cli.add_command(configs) cli.add_command(pipelines) cli.add_command(schedules) cli.add_command(workspace) diff --git a/workflow/http/configs.py b/workflow/http/configs.py new file mode 100644 index 0000000..8cef794 --- /dev/null +++ b/workflow/http/configs.py @@ -0,0 +1,182 @@ +"""Workflow Pipelines API.""" + +from json import dumps +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from requests.models import Response +from tenacity import retry +from tenacity.stop import stop_after_attempt, stop_after_delay +from tenacity.wait import wait_random + +from workflow.http.client import Client +from workflow.utils.decorators import try_request + + +class Configs(Client): + """HTTP Client for interacting with the Configs endpoints on pipelines backend. + + Args: + Client (workflow.http.client): The base class for interacting with the backend. + + Returns: + Configs: A client for interacting with workflow-pipelines. + """ + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def deploy(self, data: Dict[str, Any]): + """Deploys a Config from payload data. + + Parameters + ---------- + data : Dict[str, Any] + YAML data. + + Returns + ------- + List[str] + ID of Config object generated. + """ + with self.session as session: + url = f"{self.baseurl}/v2/configs" + response: Response = session.post(url, json=data) + response.raise_for_status() + return response.json() + + @try_request + def count(self, database: str = "configs") -> Dict[str, Any]: + """Count all documents in a collection. + + Parameters + ---------- + database : str + Database to be used. "configs" or "pipelines". + + Returns + ------- + Dict[str, Any] + Dictionary with count. + """ + with self.session as session: + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?database={database}" + ) + response.raise_for_status() + return response.json() + + @try_request + def get_configs( + self, + database: str, + config_name: str, + query: Optional[str] = "{}", + projection: Optional[str] = "{}", + ) -> List[Dict[str, Any]]: + """View the current configurations on pipelines backend. + + Parameters + ---------- + database : str + Database to query from. + config_name : str + Config name, by default None + query : str, optional + Query payload. + projection : str, optional + Query projection. + + Returns + ------- + List[Dict[str, Any]] + List of Config payloads. + """ + with self.session as session: + # ? When using urlencode, internal dict object get single-quoted + # ? This can trigger error on workflow-pipelines backend + params = {"projection": projection, "query": query} + if config_name: + params.update({"name": config_name}) + if database: + params.update({"database": database}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + print(url) + response: Response = session.get(url=url) + response.raise_for_status() + return response.json() + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def remove(self, pipeline: str, id: str) -> Response: + """Removes a cancelled pipeline configuration. + + Parameters + ---------- + pipeline : str + PipelineConfig name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + Response payload. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + response: Response = session.delete(url=url) + response.raise_for_status() + return response + + @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) + @try_request + def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: + """Stops the manager for a PipelineConfig. + + Parameters + ---------- + pipeline : str + Pipeline name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + List of stopped PipelineConfig objects. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + response: Response = session.put(url) + response.raise_for_status() + if response.status_code == 304: + return [] + return response.json() + + @try_request + def info(self) -> Dict[str, Any]: + """Get the version of the pipelines backend. + + Returns + ------- + Dict[str, Any] + Pipelines backend info. + """ + client_info = self.model_dump() + with self.session as session: + response: Response = session.get(url=f"{self.baseurl}/version") + response.raise_for_status() + server_info = response.json() + return {"client": client_info, "server": server_info} diff --git a/workflow/http/context.py b/workflow/http/context.py index 731447a..ba470c1 100644 --- a/workflow/http/context.py +++ b/workflow/http/context.py @@ -7,6 +7,7 @@ from workflow import DEFAULT_WORKSPACE_PATH from workflow.http.buckets import Buckets +from workflow.http.configs import Configs from workflow.http.pipelines import Pipelines from workflow.http.results import Results from workflow.http.schedules import Schedules @@ -77,6 +78,13 @@ class HTTPContext(BaseSettings): exclude=True, ) + configs: Configs = Field( + default=None, + validate_default=False, + description="Configs API Client.", + exclude=True, + ) + pipelines: Pipelines = Field( default=None, validate_default=False, @@ -103,6 +111,7 @@ def create_clients(self) -> "HTTPContext": "results": Results, "pipelines": Pipelines, "schedules": Schedules, + "configs": Configs, } logger.debug(f"creating http clients for {list(clients.keys())}") config: Dict[str, Any] = read.workspace(self.workspace) diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py new file mode 100644 index 0000000..a39691c --- /dev/null +++ b/workflow/utils/renderers.py @@ -0,0 +1,95 @@ +"""Functions to render objects to rich.console.""" + +import datetime as dt +from json import dumps +from typing import Any, Dict + +from rich.text import Text + +from workflow.http.context import HTTPContext +from workflow.utils.variables import status_colors, status_symbols + + +def render_pipeline(payload: Dict[str, Any]) -> Text: + """Renders a pipeline to rich.Text(). + + Parameters + ---------- + payload : Dict[str, Any] + Pipeline payload. + + Returns + ------- + Text + Rendered text. + """ + steps_field = "steps" + time_fields = ["creation", "start", "stop"] + text = Text() + for k, v in payload.items(): + key_value_text = Text() + if not v: + continue + if k in time_fields and v: + v = dt.datetime.fromtimestamp(v) + if k == steps_field: + key_value_text = Text(f"{k}: \n", style="bright_blue") + for step in v: + key_value_text.append(f" {step['name']}:") + key_value_text.append(f"{status_symbols[step['status']]}\n") + else: + key_value_text = Text(f"{k}: ", style="bright_blue") + key_value_text.append( + f"{v}\n", style="white" if k != "status" else status_colors[v] + ) + text.append_text(key_value_text) + return text + + +def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: + """Renders a config to rich.Text(). + + Parameters + ---------- + http : HTTPContext + Workflow Http context. + payload : Dict[str, Any] + Config payload. + + Returns + ------- + Text + Rendered text. + """ + text = Text() + hidden_keys = ["yaml", "services", "name"] + query = dumps({"id": {"$in": payload["children"]}}) + projection = dumps({"id": 1, "status": 1}) + children_statuses = http.configs.get_configs( + database="pipelines", + config_name=payload["name"], + query=query, + projection=projection, + ) + + for k, v in payload.items(): + if k in hidden_keys: + continue + key_value_text = Text() + if k == "children": + key_value_text.append(f"{k}: \n", style="bright_blue") + for child in children_statuses: + key_value_text.append( + f"\t{child['id']}: ", style=status_colors[child["status"]] + ) + key_value_text.append( + f"{status_symbols[child['status']]}\n", + style=status_colors[child["status"]], + ) + text.append_text(key_value_text) + continue + key_value_text.append(f"{k}: ", style="bright_blue") + key_value_text.append(f"{v}\n", style="white") + text.append_text(key_value_text) + + return text diff --git a/workflow/workspaces/development.yml b/workflow/workspaces/development.yml index f42ca8a..cca26d8 100644 --- a/workflow/workspaces/development.yml +++ b/workflow/workspaces/development.yml @@ -17,6 +17,7 @@ archive: http: baseurls: + configs: http://localhost:8001 pipelines: http://localhost:8001 schedules: http://localhost:8001 buckets: http://localhost:8004 From b7aa758a8219f3057b64178ea891a9ca6387f528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 11:15:30 -0400 Subject: [PATCH 06/41] refactor(http/configs.py): removing print statement --- workflow/http/configs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 8cef794..616e185 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -104,7 +104,6 @@ def get_configs( if database: params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" - print(url) response: Response = session.get(url=url) response.raise_for_status() return response.json() From 391850027adfe944f70d0d200886fc87b00414dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Mon, 6 May 2024 13:59:35 -0400 Subject: [PATCH 07/41] feat(cli/schedules.py): fixing bad formatting --- workflow/cli/schedules.py | 35 ++++++++++++++++++++++++++--------- workflow/http/schedules.py | 18 +++++++++--------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 02b0121..93cee00 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -1,5 +1,6 @@ """Manage workflow pipelines schedules.""" +from datetime import datetime from typing import Any, Dict, Optional, Tuple import click @@ -88,7 +89,7 @@ def ls(name: Optional[str] = None, quiet: Optional[bool] = False): lives_text = Text(str(lives) if lives > -1 else "\u221e") table.add_row( schedule_obj["id"], - schedule_obj["pipeline_config"]["name"], + schedule_obj["config"]["name"], status, lives_text, str(schedule_obj["has_spawned"]), @@ -165,6 +166,7 @@ def ps(id: str, detail: Optional[bool] = False): "has_spawned": "Has Spawned", "status": "Status", "next_time": "Next Execution", + "history": "History", } try: payload = http.schedules.get_schedule(query) @@ -173,26 +175,41 @@ def ps(id: str, detail: Optional[bool] = False): console_content = error_text else: table.add_column( - f"Scheduled Pipeline: {payload['pipeline_config']['name']}", + f"Scheduled Pipeline: {payload['config']['name']}", max_width=120, + min_width=50, justify="left", ) text = Text("") for k, v in payload.items(): - if k == "pipeline_config": + if k == "config": + continue + if k == "history": + key_value_text = Text( + f"{key_nicknames.get(k, k)}: \n", style="bright_green" + ) + for history in v: + history_dt = datetime.fromisoformat(history[0]) + legible_dt = history_dt.strftime("%B %d, %Y at %I:%M:%S %p") + id = history[1] + key_value_text.append(f"-- {legible_dt}:", style="bright_blue") + key_value_text.append(f"\n\t{id}\n\n", style="white") + text.append_text(key_value_text) continue key_value_text = Text(f"{key_nicknames.get(k, k)}: ", style="bright_green") key_value_text.append( f"{v}\n", style="white" if k != "status" else status_colors[v] ) text.append_text(key_value_text) - table.add_row(text) if detail: - table.add_section() - table.add_row(Text("Payload Details", style="magenta")) - table.add_section() - this_payload = JSON.from_data(payload["pipeline_config"], indent=2) - table.add_row(this_payload) + # table.add_section() + table.add_column("Details", style="magenta") + # table.add_row(Text("Payload Details", style="magenta")) + # table.add_section() + this_payload = JSON.from_data(payload["config"], indent=2) + table.add_row(text, this_payload) + else: + table.add_row(text) console_content = table finally: console.print(console_content) diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py index f092eb9..938cac6 100644 --- a/workflow/http/schedules.py +++ b/workflow/http/schedules.py @@ -42,7 +42,7 @@ def deploy(self, data: Dict[str, Any]): IDs of Schedule objects generated. """ with self.session as session: - url = f"{self.baseurl}/v1/schedule" + url = f"{self.baseurl}/v2/schedule" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -63,7 +63,7 @@ def get_schedule(self, query: Dict[str, Any]) -> Dict[str, Any]: """ with self.session as session: params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json()[0] @@ -90,7 +90,7 @@ def remove(self, id: str) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -110,7 +110,7 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: List of schedule payloads. """ with self.session as session: - query = dumps({"pipeline_config.name": schedule_name}) + query = dumps({"config.name": schedule_name}) projection = dumps( { "id": True, @@ -119,13 +119,13 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: "has_spawned": True, "next_time": True, "crontab": True, - "pipeline_config.name": True, + "config.name": True, } ) url = ( - f"{self.baseurl}/v1/schedule?projection={projection}" + f"{self.baseurl}/v2/schedule?projection={projection}" if schedule_name is None - else f"{self.baseurl}/v1/schedule?query={query}&projection={projection}" + else f"{self.baseurl}/v2/schedule?query={query}&projection={projection}" ) response: Response = session.get(url=url) response.raise_for_status() @@ -148,9 +148,9 @@ def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any] with self.session as session: query = dumps({"name": schedule_name}) url = ( - f"{self.baseurl}/v1/schedule/count" + f"{self.baseurl}/v2/schedule/count" if not schedule_name - else f"{self.baseurl}/v1/schedule/count?query={query}" + else f"{self.baseurl}/v2/schedule/count?query={query}" ) response: Response = session.get(url=url) response.raise_for_status() From 253f24d23137bd46bfdedfd983d6681fcac4a722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 10 May 2024 12:17:21 -0400 Subject: [PATCH 08/41] fix(cli): addressing comments on PR --- workflow/cli/configs.py | 215 ++++++++++++++++++------------------ workflow/cli/pipelines.py | 151 ++++++------------------- workflow/cli/schedules.py | 3 +- workflow/http/configs.py | 28 ++--- workflow/http/pipelines.py | 124 ++++----------------- workflow/utils/renderers.py | 11 +- workflow/utils/variables.py | 8 +- 7 files changed, 178 insertions(+), 362 deletions(-) diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py index 88e42a3..cf88142 100644 --- a/workflow/cli/configs.py +++ b/workflow/cli/configs.py @@ -15,18 +15,17 @@ from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.renderers import render_config, render_pipeline -from workflow.utils.variables import status_colors +from workflow.utils.renderers import render_config pretty.install() console = Console() table = Table( - title="\nWorkflow Pipelines", + title="\nWorkflow Configs", show_header=True, header_style="magenta", title_style="bold magenta", - min_width=10, + min_width=50, ) BASE_URL = "https://frb.chimenet.ca/pipelines" @@ -47,26 +46,10 @@ def version(): @configs.command("count", help="Count objects per collection.") -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) -def count(pipelines: bool): - """Count objects in a database. - - Parameters - ---------- - pipelines : bool - Use this command on pipelines database. - """ +def count(): + """Count objects in a database.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - counts = http.configs.count(database) + counts = http.configs.count() table.add_column("Name", max_width=50, justify="left", style="blue") table.add_column("Count", max_width=50, justify="left") total = int() @@ -102,33 +85,29 @@ def deploy(filename: click.Path): except requests.HTTPError as deploy_error: console.print(deploy_error.response.json()["error_description"][0]["msg"]) return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) + table.add_column( + "Deploy Result", + min_width=35, + max_width=50, + justify="left", + style="bright_green", + ) if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) + for k, v in deploy_result.items(): + if k == "config": + row_text = Text(f"{k}: ", style="magenta") + row_text.append(f"{v}", style="white") + table.add_row(row_text) + if k == "pipelines": + row_text = Text(f"{k}:\n", style="bright_blue") + for id in deploy_result[k]: + row_text.append(f"\t{id}\n", style="white") + table.add_row(row_text) console.print(table) @configs.command("ls", help="List Configs.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Configs with provided name.", -) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -137,54 +116,37 @@ def deploy(filename: click.Path): default=False, help="Only show IDs.", ) -def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): +def ls(name: Optional[str] = None, quiet: bool = False): """List all objects.""" - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - configs_colums = ["name", "version", "children", "user"] - pipelines_columns = ["status", "current_stage", "steps"] - projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + configs_colums = ["name", "version", "pipelines", "user"] + projection = {"yaml": 0, "deployments": 0} if quiet: projection = {"id": 1} http = HTTPContext() objects = http.configs.get_configs( - database=database, config_name=name, projection=json.dumps(projection) + config_name=name, projection=json.dumps(projection) ) # ? Add columns for each key table.add_column("ID", max_width=40, justify="left", style="blue") if not quiet: - if database == "configs": - for key in configs_colums: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - style="bright_green" if key == "name" else "white", - ) - if database == "pipelines": - for key in pipelines_columns: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - ) + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) for obj in objects: if not quiet: - if database == "configs": - table.add_row( - obj["id"], - obj["name"], - obj["version"], - str(len(obj["children"])), - obj["user"], - ) - if database == "pipelines": - status = Text(obj["status"], style=status_colors[obj["status"]]) - table.add_row( - obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) - ) + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["pipelines"])), + obj["user"], + ) continue table.add_row(obj["id"]) console.print(table) @@ -193,26 +155,16 @@ def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False) @configs.command("ps", help="Get Configs details.") @click.argument("name", type=str, required=True) @click.argument("id", type=str, required=True) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) @click.option( "--details", - "-d", is_flag=True, default=False, show_default=True, help="Show more details for the object.", ) -def ps(name: str, id: str, pipelines: str, details: bool): +def ps(name: str, id: str, details: bool): """Show details for an object.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" query: str = json.dumps({"id": id}) projection: str = json.dumps({}) console_content = None @@ -220,29 +172,20 @@ def ps(name: str, id: str, pipelines: str, details: bool): column_min_width = 40 try: payload = http.configs.get_configs( - database=database, config_name=name, query=query, projection=projection + config_name=name, query=query, projection=projection )[0] except IndexError: - error_text = Text(f"No {database.capitalize()} were found", style="red") + error_text = Text("No Configs were found", style="red") console_content = error_text else: text = Text("") - if database == "pipelines": - table.add_column( - f"Pipeline: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_pipeline(payload)) - if database == "configs": - table.add_column( - f"Config: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_config(http, payload)) + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) if details: table.add_column("Details", max_width=column_max_width, justify="left") _details = safe_load(payload["yaml"]) @@ -254,6 +197,60 @@ def ps(name: str, id: str, pipelines: str, details: bool): table.add_row(text, JSON(json.dumps(_details), indent=2)) else: table.add_row(text) + table.add_section() + table.add_row( + Text("Explore pipelines in detail: \n", style="magenta i").append( + "workflow pipelines ps ", + style="dark_blue on cyan", + ) + ) console_content = table finally: console.print(console_content) + + +@configs.command("stop", help="Stop managers for a Config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def stop(config: str, id: str): + """Stop managers for a Config.""" + http = HTTPContext() + stop_result = http.configs.stop(config, id) + if not any(stop_result): + text = Text("No configurations were stopped.", style="red") + console.print(text) + return + table.add_column("Stopped IDs", max_width=50, justify="left") + text = Text() + for k in stop_result.keys(): + if k == "stopped_config": + text.append("Config: ", style="bright_blue") + text.append(f"{stop_result[k]}\n") + if k == "stopped_pipelines": + text.append("Pipelines: \n", style="bright_blue") + for id in stop_result["stopped_pipelines"]: + text.append(f"\t{id}\n") + table.add_row(text) + console.print(table) + + +@configs.command("rm", help="Remove a config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def rm(config: str, id: str): + """Remove a config.""" + http = HTTPContext() + content = None + try: + delete_result = http.configs.remove(config, id) + if delete_result.status_code == 204: + text = Text("No pipeline configurations were deleted.", style="red") + content = text + except Exception as e: + text = Text(f"No configurations were deleted.\nError: {e}", style="red") + content = text + else: + table.add_column("Deleted IDs", max_width=50, justify="left", style="red") + table.add_row(id) + content = table + console.print(content) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index cf4c338..83a254c 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,20 +1,18 @@ """Manage workflow pipelines.""" -import datetime as dt import json -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional import click import requests -import yaml from rich import pretty from rich.console import Console from rich.table import Table from rich.text import Text -from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.variables import status_colors, status_symbols +from workflow.utils.renderers import render_pipeline +from workflow.utils.variables import status_colors pretty.install() console = Console() @@ -45,14 +43,7 @@ def version(): @pipelines.command("ls", help="List pipelines.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Pipelines with provided name.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -63,21 +54,24 @@ def version(): ) def ls(name: Optional[str] = None, quiet: Optional[bool] = False): """List all pipelines.""" + pipelines_columns = ["status", "current_stage", "steps"] http = HTTPContext() - objects = http.pipelines.list_pipeline_configs(name) - table.add_column("ID", max_width=50, justify="left", style="blue") - if not quiet: - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Stage", max_width=50, justify="left") - for config in objects: - status = Text(config["status"], style=status_colors[config["status"]]) + objects = http.pipelines.list_pipelines(name) + table.add_column("ID", max_width=100, justify="left", style="blue") + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + for obj in objects: if not quiet: + status = Text(obj["status"], style=status_colors[obj["status"]]) table.add_row( - config["id"], config["name"], status, str(config["current_stage"]) + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) ) continue - table.add_row(config["id"]) + table.add_row(obj["id"]) console.print(table) @@ -97,118 +91,39 @@ def count(): console.print(table) -@pipelines.command("deploy", help="Deploy a workflow pipeline.") -@click.argument( - "filename", - type=click.Path(exists=True, dir_okay=False, readable=True), - required=True, -) -def deploy(filename: click.Path): - """Deploy a workflow pipeline.""" - http = HTTPContext() - filepath: str = str(filename) - data: Dict[str, Any] = {} - with open(filepath) as reader: - data = yaml.load(reader, Loader=SafeLoader) # type: ignore - try: - deploy_result = http.pipelines.deploy(data) - except requests.HTTPError as deploy_error: - console.print(deploy_error.response.json()["message"]) - return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) - if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) - console.print(table) - - @pipelines.command("ps", help="Get pipeline details.") @click.argument("pipeline", type=str, required=True) @click.argument("id", type=str, required=True) def ps(pipeline: str, id: str): """List a pipeline configuration in detail.""" http = HTTPContext() - query: Dict[str, Any] = {"id": id} + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) console_content = None - projection = {"name": False} - time_fields = ["creation", "start", "stop"] + column_max_width = 300 + column_min_width = 40 try: - payload = http.pipelines.get_pipeline_config(pipeline, query, projection) + payload = http.pipelines.get_pipelines( + name=pipeline, query=query, projection=projection + )[0] except IndexError: - error_text = Text("No PipelineConfig were found", style="red") + error_text = Text("No Pipelines were found", style="red") console_content = error_text else: - table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") - text = Text("") - for k, v in payload.items(): - key_value_text = Text() - if k in time_fields and v: - v = dt.datetime.fromtimestamp(v) - if k == "pipeline": - key_value_text = Text(f"{k}: \n", style="bright_blue") - for step in v: - key_value_text.append(f" {step['name']}:") - key_value_text.append(f"{status_symbols[step['status']]}\n") - else: - key_value_text = Text(f"{k}: ", style="bright_blue") - key_value_text.append( - f"{v}\n", style="white" if k != "status" else status_colors[v] - ) - text.append_text(key_value_text) - + text = Text() + table.add_column( + f"Pipeline: {pipeline}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) table.add_row(text) console_content = table finally: console.print(console_content) -@pipelines.command("stop", help="Kill a running pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -def stop(pipeline: str, id: Tuple[str]): - """Kill a running pipeline.""" - http = HTTPContext() - stop_result = http.pipelines.stop(pipeline, id) - if not any(stop_result): - text = Text("No pipeline configurations were stopped.", style="red") - console.print(text) - return - table.add_column("Stopped IDs", max_width=50, justify="left") - for config in stop_result: - table.add_row(config["id"]) - console.print(table) - - -@pipelines.command("rm", help="Remove a pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def rm(pipeline: str, id: Tuple[str], schedule: bool): - """Remove a pipeline.""" - http = HTTPContext() - content = None - try: - delete_result = http.pipelines.remove(pipeline, id, schedule) - if delete_result.status_code == 204: - text = Text("No pipeline configurations were deleted.", style="red") - content = text - except Exception as e: - text = Text( - f"No pipeline configurations were deleted.\nError: {e}", style="red" - ) - content = text - else: - table.add_column("Deleted IDs", max_width=50, justify="left", style="red") - table.add_row(id) - content = table - console.print(content) - - def status( pipeline: Optional[str] = None, query: Optional[Dict[str, Any]] = None, diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 93cee00..853161a 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -148,8 +148,7 @@ def deploy(filename: click.Path): @schedules.command("ps", help="Get schedule details.") @click.argument("id", type=str, required=True) @click.option( - "--detail", - "-d", + "--details", is_flag=True, show_default=True, help="Returns the Schedule Payload.", diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 616e185..43ef565 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -49,14 +49,9 @@ def deploy(self, data: Dict[str, Any]): return response.json() @try_request - def count(self, database: str = "configs") -> Dict[str, Any]: + def count(self) -> Dict[str, Any]: """Count all documents in a collection. - Parameters - ---------- - database : str - Database to be used. "configs" or "pipelines". - Returns ------- Dict[str, Any] @@ -64,7 +59,7 @@ def count(self, database: str = "configs") -> Dict[str, Any]: """ with self.session as session: response: Response = session.get( - url=f"{self.baseurl}/v2/configs/count?database={database}" + url=f"{self.baseurl}/v2/configs/count?database=configs" ) response.raise_for_status() return response.json() @@ -72,7 +67,6 @@ def count(self, database: str = "configs") -> Dict[str, Any]: @try_request def get_configs( self, - database: str, config_name: str, query: Optional[str] = "{}", projection: Optional[str] = "{}", @@ -81,8 +75,6 @@ def get_configs( Parameters ---------- - database : str - Database to query from. config_name : str Config name, by default None query : str, optional @@ -101,8 +93,6 @@ def get_configs( params = {"projection": projection, "query": query} if config_name: params.update({"name": config_name}) - if database: - params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() @@ -114,15 +104,15 @@ def get_configs( stop=(stop_after_delay(5) | stop_after_attempt(1)), ) @try_request - def remove(self, pipeline: str, id: str) -> Response: + def remove(self, config: str, id: str) -> Response: """Removes a cancelled pipeline configuration. Parameters ---------- - pipeline : str - PipelineConfig name. + config : str + Config name. id : str - PipelineConfig ID. + Config ID. Returns ------- @@ -131,8 +121,8 @@ def remove(self, pipeline: str, id: str) -> Response: """ with self.session as session: query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + params = {"query": dumps(query), "name": config} + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -157,7 +147,7 @@ def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() if response.status_code == 304: diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index fddba7d..971051c 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -1,13 +1,9 @@ """Workflow Pipelines API.""" -from json import dumps from typing import Any, Dict, List, Optional from urllib.parse import urlencode from requests.models import Response -from tenacity import retry -from tenacity.stop import stop_after_attempt, stop_after_delay -from tenacity.wait import wait_random from workflow.http.client import Client from workflow.utils.decorators import try_request @@ -23,31 +19,6 @@ class Pipelines(Client): Pipelines: A client for interacting with the Pipelines backend. """ - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def deploy(self, data: Dict[str, Any]): - """Deploys a PipelineConfig from payload data. - - Parameters - ---------- - data : Dict[str, Any] - YAML data. - - Returns - ------- - List[str] - IDs of PipelineConfig objects generated. - """ - with self.session as session: - url = f"{self.baseurl}/v1/pipelines" - response: Response = session.post(url, json=data) - response.raise_for_status() - return response.json() - @try_request def count(self) -> Dict[str, Any]: """Count all documents in a collection. @@ -58,20 +29,21 @@ def count(self) -> Dict[str, Any]: Dictionary with count. """ with self.session as session: - response: Response = session.get(url=f"{self.baseurl}/v1/pipelines/count") + params = {"database": "pipelines"} + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?{urlencode(params)}" + ) response.raise_for_status() return response.json() @try_request - def list_pipeline_configs( - self, config_name: Optional[str] = None - ) -> List[Dict[str, Any]]: + def list_pipelines(self, name: Optional[str] = None) -> List[Dict[str, Any]]: """View the current pipeline configurations in the pipelines backend. Parameters ---------- - config_name : Optional[str], optional - PipelineConfig name, by default None + name : Optional[str], optional + Config name, by default None Returns ------- @@ -79,25 +51,24 @@ def list_pipeline_configs( List of PipelineConfig payloads. """ with self.session as session: - url = ( - f"{self.baseurl}/v1/pipelines" - if config_name is None - else f'{self.baseurl}/v1/pipelines?query={{"name":"{config_name}"}}' - ) + params = {"database": "pipelines"} + if name: + params.update({"name": name}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() @try_request - def get_pipeline_config( - self, collection: str, query: Dict[str, Any], projection: Dict[str, Any] + def get_pipelines( + self, name: str, query: Dict[str, Any], projection: Dict[str, Any] ) -> Dict[str, Any]: """Gets details for one pipeline configuration. Parameters ---------- - collection : str - PipelineConfig name. + name : str + Config name. query : Dict[str, Any] Dictionary with search parameters. projection : Dict[str, Any] @@ -110,69 +81,14 @@ def get_pipeline_config( """ with self.session as session: params = { - "query": dumps(query), - "name": collection, - "projection": dumps(projection), + "query": query, + "name": name, + "projection": projection, + "database": "pipelines", } - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() - return response.json()[0] - - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def remove(self, pipeline: str, id: str) -> Response: - """Removes a cancelled pipeline configuration. - - Parameters - ---------- - pipeline : str - PipelineConfig name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - Response payload. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" - response: Response = session.delete(url=url) - response.raise_for_status() - return response - - @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) - @try_request - def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: - """Stops the manager for a PipelineConfig. - - Parameters - ---------- - pipeline : str - Pipeline name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - List of stopped PipelineConfig objects. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" - response: Response = session.put(url) - response.raise_for_status() - if response.status_code == 304: - return [] return response.json() @try_request diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py index a39691c..c523a04 100644 --- a/workflow/utils/renderers.py +++ b/workflow/utils/renderers.py @@ -63,11 +63,10 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: """ text = Text() hidden_keys = ["yaml", "services", "name"] - query = dumps({"id": {"$in": payload["children"]}}) + query = dumps({"id": {"$in": payload["pipelines"]}}) projection = dumps({"id": 1, "status": 1}) - children_statuses = http.configs.get_configs( - database="pipelines", - config_name=payload["name"], + pipelines_statuses = http.pipelines.get_pipelines( + name=payload["name"], query=query, projection=projection, ) @@ -76,9 +75,9 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: if k in hidden_keys: continue key_value_text = Text() - if k == "children": + if k == "pipelines": key_value_text.append(f"{k}: \n", style="bright_blue") - for child in children_statuses: + for child in pipelines_statuses: key_value_text.append( f"\t{child['id']}: ", style=status_colors[child["status"]] ) diff --git a/workflow/utils/variables.py b/workflow/utils/variables.py index f8f8149..9cfe11b 100644 --- a/workflow/utils/variables.py +++ b/workflow/utils/variables.py @@ -14,11 +14,11 @@ status_symbols = { "created": "\U000026AA", # white - "queued": "\u23F3", # hourglass - "active": "\U0001F7E2", # green - "running": "\u2699", # gear + "queued": "\U0001F4C5", # 📅 + "active": "\U0001F7E2", # green circle + "running": "\U0001F3C3", # 🏃 "success": "\U00002705", # Green check "paused": "\U0001F7E1", # yellow "failure": "\U0000274C", # cross mark - "cancelled": "\U0001F534", # red + "cancelled": "\U0001F6AB", # 🚫 } From 7b17e2a987e09d8a950d9b4202b29c068321971f Mon Sep 17 00:00:00 2001 From: Odarsson <40160237+odarotto@users.noreply.github.com> Date: Fri, 17 May 2024 12:40:37 -0400 Subject: [PATCH 09/41] feat(docker-compose-tutorial.yml): adding docker compose for tutorial (#35) --- docker-compose-tutorial.yml | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 docker-compose-tutorial.yml diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml new file mode 100644 index 0000000..818d701 --- /dev/null +++ b/docker-compose-tutorial.yml @@ -0,0 +1,134 @@ +services: + pipelines_api: + image: chimefrb/pipelines:latest + container_name: pipelines_api + command: python -m pipelines.server + ports: + - "8001:8001" + environment: + - SANIC_HOSTNAME=0.0.0.0 + - SANIC_PORT=8001 + - SANIC_ACCESS_LOG=true + - SANIC_AUTO_RELOAD=true + - SANIC_DEBUG=true + - SANIC_MONGODB_HOSTNAME=mongo + - SANIC_MONGODB_PORT=27017 + - SANIC_START_MANAGER_URL=http://pipelines_managers:8002/v2/start + - SANIC_PAUSE_MANAGER_URL=http://pipelines_managers:8002/v2/pause + - SANIC_STOP_MANAGER_URL=http://pipelines_managers:8002/v2/stop + - SANIC_HEALTH_MANAGERS_URL=http://pipelines_managers:8002/__health__ + - SANIC_HEALTH_MANAGERS_CHECK_TIMES=10 + - SANIC_HEALTH_MANAGERS_CHECK_INTERVAL_SECONDS=30 + - SANIC_LISTENERS_THRESHOLD_SECONDS=120 + - TZ=Etc/UTC + networks: + - workflow-network + + restart: always + + pipelines_managers: + image: chimefrb/pipelines:latest + container_name: pipelines_managers + command: python -m managers.server + ports: + - "8002:8002" + environment: + - SANIC_HOSTNAME=0.0.0.0 + - SANIC_PORT=8002 + - SANIC_ACCESS_LOG=true + - SANIC_AUTO_RELOAD=true + - SANIC_DEBUG=true + - SANIC_MONGODB_HOSTNAME=mongo + - SANIC_MONGODB_PORT=27017 + - SANIC_BUCKETS_URL=http://buckets:8004 + - SANIC_RESULTS_URL=http://results:8005 + - SANIC_UPDATE_INTERVAL_SECONDS=40 + - SANIC_SLEEP_INTERVAL_SECONDS=30 + - SANIC_PURGE_TIME_SECONDS=3600 + - DOCKER_HOST=unix:///var/run/docker.sock # Replace with production address or dind + - TZ=Etc/UTC + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - workflow-network + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:8002/__health__" + ] + interval: 30s + timeout: 10s + retries: 5 + restart: always + + dind: + image: docker:dind + command: [ "--host=tcp://0.0.0.0:2376" ] + deploy: + replicas: 1 + privileged: true + expose: + - 2376 + ports: + - "2375:2375" + - "2376:2376" + environment: + - DOCKER_TLS_CERTDIR= + networks: + - workflow-network + + buckets: + image: chimefrb/buckets:latest + container_name: buckets + command: [ "/bin/bash", "-c", "python -m buckets.server" ] + expose: + - 8004 + ports: + - "8004:8004" + environment: + - SANIC_HOSTNAME=0.0.0.0 + - SANIC_PORT=8004 + - SANIC_ACCESS_LOG=true + - SANIC_AUTO_RELOAD=true + - SANIC_DEBUG=true + - SANIC_MONGODB_HOSTNAME=mongo + - SANIC_MONGODB_PORT=27017 + - SANIC_CORS_ORIGINS=* + networks: + - workflow-network + + results: + image: chimefrb/results:latest + container_name: results + command: [ "/bin/bash", "-c", "python -m results.server" ] + expose: + - 8005 + ports: + - "8005:8005" + environment: + - SANIC_HOSTNAME=0.0.0.0 + - SANIC_PORT=8005 + - SANIC_ACCESS_LOG=true + - SANIC_AUTO_RELOAD=true + - SANIC_DEBUG=true + - SANIC_MONGODB_HOSTNAME=mongo + - SANIC_MONGODB_PORT=27017 + - SANIC_CORS_ORIGINS=* + networks: + - workflow-network + + mongo: + image: mongo:latest + command: mongod --bind_ip_all + container_name: mongo + ports: + - "27017:27017" + networks: + - workflow-network + +networks: + workflow-network: + driver: bridge From 4dfb30ae760f497b22a8ed0144d8b16e84dad2a2 Mon Sep 17 00:00:00 2001 From: Odarsson <40160237+odarotto@users.noreply.github.com> Date: Fri, 17 May 2024 12:41:11 -0400 Subject: [PATCH 10/41] 32 feat create tutorial workspace (#33) * feat(workspaces/tutorial.yml): adding tutorial workspace * feat(workspaces/tutorial.yml): adding baseurl for configs --- workflow/workspaces/tutorial.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 workflow/workspaces/tutorial.yml diff --git a/workflow/workspaces/tutorial.yml b/workflow/workspaces/tutorial.yml new file mode 100644 index 0000000..d1aede9 --- /dev/null +++ b/workflow/workspaces/tutorial.yml @@ -0,0 +1,17 @@ +workspace: tutorial + +# List the valid sites for this workspace +sites: + - local + +http: + baseurls: + configs: http://localhost:8001 + pipelines: http://localhost:8001 + schedules: http://localhost:8001 + buckets: http://localhost:8004 + results: http://localhost:8005 + +config: + archive: + results: true From 3a671365b100d32c37443ed577c4e479a479da0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 09:55:46 -0400 Subject: [PATCH 11/41] feat(cli): adding configs commands confgis commands added so the cli can work with the new version of pipelines(^2.7.0) --- workflow/cli/configs.py | 259 ++++++++++++++++++++++++++++ workflow/cli/main.py | 2 + workflow/http/configs.py | 182 +++++++++++++++++++ workflow/http/context.py | 9 + workflow/utils/renderers.py | 95 ++++++++++ workflow/workspaces/development.yml | 1 + 6 files changed, 548 insertions(+) create mode 100644 workflow/cli/configs.py create mode 100644 workflow/http/configs.py create mode 100644 workflow/utils/renderers.py diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py new file mode 100644 index 0000000..88e42a3 --- /dev/null +++ b/workflow/cli/configs.py @@ -0,0 +1,259 @@ +"""Manage workflow pipelines.""" + +import json +from typing import Any, Dict, Optional + +import click +import requests +import yaml +from rich import pretty +from rich.console import Console +from rich.json import JSON +from rich.table import Table +from rich.text import Text +from yaml import safe_load +from yaml.loader import SafeLoader + +from workflow.http.context import HTTPContext +from workflow.utils.renderers import render_config, render_pipeline +from workflow.utils.variables import status_colors + +pretty.install() +console = Console() + +table = Table( + title="\nWorkflow Pipelines", + show_header=True, + header_style="magenta", + title_style="bold magenta", + min_width=10, +) + +BASE_URL = "https://frb.chimenet.ca/pipelines" +STATUS = ["created", "queued", "running", "success", "failure", "cancelled"] + + +@click.group(name="configs", help="Manage Workflow Configs. Version 2.") +def configs(): + """Manage workflow configs.""" + pass + + +@configs.command("version", help="Backend version.") +def version(): + """Get version of the pipelines service.""" + http = HTTPContext() + console.print(http.configs.info()) + + +@configs.command("count", help="Count objects per collection.") +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +def count(pipelines: bool): + """Count objects in a database. + + Parameters + ---------- + pipelines : bool + Use this command on pipelines database. + """ + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + counts = http.configs.count(database) + table.add_column("Name", max_width=50, justify="left", style="blue") + table.add_column("Count", max_width=50, justify="left") + total = int() + for k, v in counts.items(): + table.add_row(k, str(v)) + total += v + table.add_section() + table.add_row("Total", str(total)) + console.print(table) + + +@configs.command("deploy", help="Deploy a workflow config.") +@click.argument( + "filename", + type=click.Path(exists=True, dir_okay=False, readable=True), + required=True, +) +def deploy(filename: click.Path): + """Deploy a workflow config. + + Parameters + ---------- + filename : click.Path + File path. + """ + http = HTTPContext() + filepath: str = str(filename) + data: Dict[str, Any] = {} + with open(filepath) as reader: + data = yaml.load(reader, Loader=SafeLoader) # type: ignore + try: + deploy_result = http.configs.deploy(data) + except requests.HTTPError as deploy_error: + console.print(deploy_error.response.json()["error_description"][0]["msg"]) + return + table.add_column("IDs", max_width=50, justify="left", style="bright_green") + if isinstance(deploy_result, list): + for _id in deploy_result: + table.add_row(_id) + if isinstance(deploy_result, dict): + for v in deploy_result.values(): + table.add_row(v) + console.print(table) + + +@configs.command("ls", help="List Configs.") +@click.option( + "name", + "--name", + "-n", + type=str, + required=False, + help="List only Configs with provided name.", +) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "quiet", + "--quiet", + "-q", + is_flag=True, + default=False, + help="Only show IDs.", +) +def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): + """List all objects.""" + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + configs_colums = ["name", "version", "children", "user"] + pipelines_columns = ["status", "current_stage", "steps"] + projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + if quiet: + projection = {"id": 1} + http = HTTPContext() + objects = http.configs.get_configs( + database=database, config_name=name, projection=json.dumps(projection) + ) + + # ? Add columns for each key + table.add_column("ID", max_width=40, justify="left", style="blue") + if not quiet: + if database == "configs": + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) + if database == "pipelines": + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + + for obj in objects: + if not quiet: + if database == "configs": + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["children"])), + obj["user"], + ) + if database == "pipelines": + status = Text(obj["status"], style=status_colors[obj["status"]]) + table.add_row( + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) + ) + continue + table.add_row(obj["id"]) + console.print(table) + + +@configs.command("ps", help="Get Configs details.") +@click.argument("name", type=str, required=True) +@click.argument("id", type=str, required=True) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "--details", + "-d", + is_flag=True, + default=False, + show_default=True, + help="Show more details for the object.", +) +def ps(name: str, id: str, pipelines: str, details: bool): + """Show details for an object.""" + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) + console_content = None + column_max_width = 300 + column_min_width = 40 + try: + payload = http.configs.get_configs( + database=database, config_name=name, query=query, projection=projection + )[0] + except IndexError: + error_text = Text(f"No {database.capitalize()} were found", style="red") + console_content = error_text + else: + text = Text("") + if database == "pipelines": + table.add_column( + f"Pipeline: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) + if database == "configs": + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) + if details: + table.add_column("Details", max_width=column_max_width, justify="left") + _details = safe_load(payload["yaml"]) + _details = { + k: v + for k, v in _details.items() + if k not in ["name", "version", "deployments"] + } + table.add_row(text, JSON(json.dumps(_details), indent=2)) + else: + table.add_row(text) + console_content = table + finally: + console.print(console_content) diff --git a/workflow/cli/main.py b/workflow/cli/main.py index ca126bb..db91ef3 100755 --- a/workflow/cli/main.py +++ b/workflow/cli/main.py @@ -3,6 +3,7 @@ import click # from workflow.cli.buckets import buckets +from workflow.cli.configs import configs from workflow.cli.pipelines import pipelines from workflow.cli.results import results from workflow.cli.run import run @@ -19,6 +20,7 @@ def cli(): cli.add_command(run) # cli.add_command(buckets) cli.add_command(results) +cli.add_command(configs) cli.add_command(pipelines) cli.add_command(schedules) cli.add_command(workspace) diff --git a/workflow/http/configs.py b/workflow/http/configs.py new file mode 100644 index 0000000..8cef794 --- /dev/null +++ b/workflow/http/configs.py @@ -0,0 +1,182 @@ +"""Workflow Pipelines API.""" + +from json import dumps +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from requests.models import Response +from tenacity import retry +from tenacity.stop import stop_after_attempt, stop_after_delay +from tenacity.wait import wait_random + +from workflow.http.client import Client +from workflow.utils.decorators import try_request + + +class Configs(Client): + """HTTP Client for interacting with the Configs endpoints on pipelines backend. + + Args: + Client (workflow.http.client): The base class for interacting with the backend. + + Returns: + Configs: A client for interacting with workflow-pipelines. + """ + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def deploy(self, data: Dict[str, Any]): + """Deploys a Config from payload data. + + Parameters + ---------- + data : Dict[str, Any] + YAML data. + + Returns + ------- + List[str] + ID of Config object generated. + """ + with self.session as session: + url = f"{self.baseurl}/v2/configs" + response: Response = session.post(url, json=data) + response.raise_for_status() + return response.json() + + @try_request + def count(self, database: str = "configs") -> Dict[str, Any]: + """Count all documents in a collection. + + Parameters + ---------- + database : str + Database to be used. "configs" or "pipelines". + + Returns + ------- + Dict[str, Any] + Dictionary with count. + """ + with self.session as session: + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?database={database}" + ) + response.raise_for_status() + return response.json() + + @try_request + def get_configs( + self, + database: str, + config_name: str, + query: Optional[str] = "{}", + projection: Optional[str] = "{}", + ) -> List[Dict[str, Any]]: + """View the current configurations on pipelines backend. + + Parameters + ---------- + database : str + Database to query from. + config_name : str + Config name, by default None + query : str, optional + Query payload. + projection : str, optional + Query projection. + + Returns + ------- + List[Dict[str, Any]] + List of Config payloads. + """ + with self.session as session: + # ? When using urlencode, internal dict object get single-quoted + # ? This can trigger error on workflow-pipelines backend + params = {"projection": projection, "query": query} + if config_name: + params.update({"name": config_name}) + if database: + params.update({"database": database}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + print(url) + response: Response = session.get(url=url) + response.raise_for_status() + return response.json() + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def remove(self, pipeline: str, id: str) -> Response: + """Removes a cancelled pipeline configuration. + + Parameters + ---------- + pipeline : str + PipelineConfig name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + Response payload. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + response: Response = session.delete(url=url) + response.raise_for_status() + return response + + @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) + @try_request + def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: + """Stops the manager for a PipelineConfig. + + Parameters + ---------- + pipeline : str + Pipeline name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + List of stopped PipelineConfig objects. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + response: Response = session.put(url) + response.raise_for_status() + if response.status_code == 304: + return [] + return response.json() + + @try_request + def info(self) -> Dict[str, Any]: + """Get the version of the pipelines backend. + + Returns + ------- + Dict[str, Any] + Pipelines backend info. + """ + client_info = self.model_dump() + with self.session as session: + response: Response = session.get(url=f"{self.baseurl}/version") + response.raise_for_status() + server_info = response.json() + return {"client": client_info, "server": server_info} diff --git a/workflow/http/context.py b/workflow/http/context.py index 731447a..ba470c1 100644 --- a/workflow/http/context.py +++ b/workflow/http/context.py @@ -7,6 +7,7 @@ from workflow import DEFAULT_WORKSPACE_PATH from workflow.http.buckets import Buckets +from workflow.http.configs import Configs from workflow.http.pipelines import Pipelines from workflow.http.results import Results from workflow.http.schedules import Schedules @@ -77,6 +78,13 @@ class HTTPContext(BaseSettings): exclude=True, ) + configs: Configs = Field( + default=None, + validate_default=False, + description="Configs API Client.", + exclude=True, + ) + pipelines: Pipelines = Field( default=None, validate_default=False, @@ -103,6 +111,7 @@ def create_clients(self) -> "HTTPContext": "results": Results, "pipelines": Pipelines, "schedules": Schedules, + "configs": Configs, } logger.debug(f"creating http clients for {list(clients.keys())}") config: Dict[str, Any] = read.workspace(self.workspace) diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py new file mode 100644 index 0000000..a39691c --- /dev/null +++ b/workflow/utils/renderers.py @@ -0,0 +1,95 @@ +"""Functions to render objects to rich.console.""" + +import datetime as dt +from json import dumps +from typing import Any, Dict + +from rich.text import Text + +from workflow.http.context import HTTPContext +from workflow.utils.variables import status_colors, status_symbols + + +def render_pipeline(payload: Dict[str, Any]) -> Text: + """Renders a pipeline to rich.Text(). + + Parameters + ---------- + payload : Dict[str, Any] + Pipeline payload. + + Returns + ------- + Text + Rendered text. + """ + steps_field = "steps" + time_fields = ["creation", "start", "stop"] + text = Text() + for k, v in payload.items(): + key_value_text = Text() + if not v: + continue + if k in time_fields and v: + v = dt.datetime.fromtimestamp(v) + if k == steps_field: + key_value_text = Text(f"{k}: \n", style="bright_blue") + for step in v: + key_value_text.append(f" {step['name']}:") + key_value_text.append(f"{status_symbols[step['status']]}\n") + else: + key_value_text = Text(f"{k}: ", style="bright_blue") + key_value_text.append( + f"{v}\n", style="white" if k != "status" else status_colors[v] + ) + text.append_text(key_value_text) + return text + + +def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: + """Renders a config to rich.Text(). + + Parameters + ---------- + http : HTTPContext + Workflow Http context. + payload : Dict[str, Any] + Config payload. + + Returns + ------- + Text + Rendered text. + """ + text = Text() + hidden_keys = ["yaml", "services", "name"] + query = dumps({"id": {"$in": payload["children"]}}) + projection = dumps({"id": 1, "status": 1}) + children_statuses = http.configs.get_configs( + database="pipelines", + config_name=payload["name"], + query=query, + projection=projection, + ) + + for k, v in payload.items(): + if k in hidden_keys: + continue + key_value_text = Text() + if k == "children": + key_value_text.append(f"{k}: \n", style="bright_blue") + for child in children_statuses: + key_value_text.append( + f"\t{child['id']}: ", style=status_colors[child["status"]] + ) + key_value_text.append( + f"{status_symbols[child['status']]}\n", + style=status_colors[child["status"]], + ) + text.append_text(key_value_text) + continue + key_value_text.append(f"{k}: ", style="bright_blue") + key_value_text.append(f"{v}\n", style="white") + text.append_text(key_value_text) + + return text diff --git a/workflow/workspaces/development.yml b/workflow/workspaces/development.yml index f42ca8a..cca26d8 100644 --- a/workflow/workspaces/development.yml +++ b/workflow/workspaces/development.yml @@ -17,6 +17,7 @@ archive: http: baseurls: + configs: http://localhost:8001 pipelines: http://localhost:8001 schedules: http://localhost:8001 buckets: http://localhost:8004 From 32cbff9c24fe60f78a85419e3340ce3c1460a24c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 11:15:30 -0400 Subject: [PATCH 12/41] refactor(http/configs.py): removing print statement --- workflow/http/configs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 8cef794..616e185 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -104,7 +104,6 @@ def get_configs( if database: params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" - print(url) response: Response = session.get(url=url) response.raise_for_status() return response.json() From 22d39b9a12d27e1ab1169123529b0de6df969f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Mon, 6 May 2024 13:59:35 -0400 Subject: [PATCH 13/41] feat(cli/schedules.py): fixing bad formatting --- workflow/cli/schedules.py | 35 ++++++++++++++++++++++++++--------- workflow/http/schedules.py | 18 +++++++++--------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 02b0121..93cee00 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -1,5 +1,6 @@ """Manage workflow pipelines schedules.""" +from datetime import datetime from typing import Any, Dict, Optional, Tuple import click @@ -88,7 +89,7 @@ def ls(name: Optional[str] = None, quiet: Optional[bool] = False): lives_text = Text(str(lives) if lives > -1 else "\u221e") table.add_row( schedule_obj["id"], - schedule_obj["pipeline_config"]["name"], + schedule_obj["config"]["name"], status, lives_text, str(schedule_obj["has_spawned"]), @@ -165,6 +166,7 @@ def ps(id: str, detail: Optional[bool] = False): "has_spawned": "Has Spawned", "status": "Status", "next_time": "Next Execution", + "history": "History", } try: payload = http.schedules.get_schedule(query) @@ -173,26 +175,41 @@ def ps(id: str, detail: Optional[bool] = False): console_content = error_text else: table.add_column( - f"Scheduled Pipeline: {payload['pipeline_config']['name']}", + f"Scheduled Pipeline: {payload['config']['name']}", max_width=120, + min_width=50, justify="left", ) text = Text("") for k, v in payload.items(): - if k == "pipeline_config": + if k == "config": + continue + if k == "history": + key_value_text = Text( + f"{key_nicknames.get(k, k)}: \n", style="bright_green" + ) + for history in v: + history_dt = datetime.fromisoformat(history[0]) + legible_dt = history_dt.strftime("%B %d, %Y at %I:%M:%S %p") + id = history[1] + key_value_text.append(f"-- {legible_dt}:", style="bright_blue") + key_value_text.append(f"\n\t{id}\n\n", style="white") + text.append_text(key_value_text) continue key_value_text = Text(f"{key_nicknames.get(k, k)}: ", style="bright_green") key_value_text.append( f"{v}\n", style="white" if k != "status" else status_colors[v] ) text.append_text(key_value_text) - table.add_row(text) if detail: - table.add_section() - table.add_row(Text("Payload Details", style="magenta")) - table.add_section() - this_payload = JSON.from_data(payload["pipeline_config"], indent=2) - table.add_row(this_payload) + # table.add_section() + table.add_column("Details", style="magenta") + # table.add_row(Text("Payload Details", style="magenta")) + # table.add_section() + this_payload = JSON.from_data(payload["config"], indent=2) + table.add_row(text, this_payload) + else: + table.add_row(text) console_content = table finally: console.print(console_content) diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py index f092eb9..938cac6 100644 --- a/workflow/http/schedules.py +++ b/workflow/http/schedules.py @@ -42,7 +42,7 @@ def deploy(self, data: Dict[str, Any]): IDs of Schedule objects generated. """ with self.session as session: - url = f"{self.baseurl}/v1/schedule" + url = f"{self.baseurl}/v2/schedule" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -63,7 +63,7 @@ def get_schedule(self, query: Dict[str, Any]) -> Dict[str, Any]: """ with self.session as session: params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json()[0] @@ -90,7 +90,7 @@ def remove(self, id: str) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -110,7 +110,7 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: List of schedule payloads. """ with self.session as session: - query = dumps({"pipeline_config.name": schedule_name}) + query = dumps({"config.name": schedule_name}) projection = dumps( { "id": True, @@ -119,13 +119,13 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: "has_spawned": True, "next_time": True, "crontab": True, - "pipeline_config.name": True, + "config.name": True, } ) url = ( - f"{self.baseurl}/v1/schedule?projection={projection}" + f"{self.baseurl}/v2/schedule?projection={projection}" if schedule_name is None - else f"{self.baseurl}/v1/schedule?query={query}&projection={projection}" + else f"{self.baseurl}/v2/schedule?query={query}&projection={projection}" ) response: Response = session.get(url=url) response.raise_for_status() @@ -148,9 +148,9 @@ def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any] with self.session as session: query = dumps({"name": schedule_name}) url = ( - f"{self.baseurl}/v1/schedule/count" + f"{self.baseurl}/v2/schedule/count" if not schedule_name - else f"{self.baseurl}/v1/schedule/count?query={query}" + else f"{self.baseurl}/v2/schedule/count?query={query}" ) response: Response = session.get(url=url) response.raise_for_status() From 46b9330aa70f59025306274b2d25cdc88610e405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 10 May 2024 12:17:21 -0400 Subject: [PATCH 14/41] fix(cli): addressing comments on PR --- workflow/cli/configs.py | 215 ++++++++++++++++++------------------ workflow/cli/pipelines.py | 151 ++++++------------------- workflow/cli/schedules.py | 3 +- workflow/http/configs.py | 28 ++--- workflow/http/pipelines.py | 124 ++++----------------- workflow/utils/renderers.py | 11 +- workflow/utils/variables.py | 8 +- 7 files changed, 178 insertions(+), 362 deletions(-) diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py index 88e42a3..cf88142 100644 --- a/workflow/cli/configs.py +++ b/workflow/cli/configs.py @@ -15,18 +15,17 @@ from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.renderers import render_config, render_pipeline -from workflow.utils.variables import status_colors +from workflow.utils.renderers import render_config pretty.install() console = Console() table = Table( - title="\nWorkflow Pipelines", + title="\nWorkflow Configs", show_header=True, header_style="magenta", title_style="bold magenta", - min_width=10, + min_width=50, ) BASE_URL = "https://frb.chimenet.ca/pipelines" @@ -47,26 +46,10 @@ def version(): @configs.command("count", help="Count objects per collection.") -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) -def count(pipelines: bool): - """Count objects in a database. - - Parameters - ---------- - pipelines : bool - Use this command on pipelines database. - """ +def count(): + """Count objects in a database.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - counts = http.configs.count(database) + counts = http.configs.count() table.add_column("Name", max_width=50, justify="left", style="blue") table.add_column("Count", max_width=50, justify="left") total = int() @@ -102,33 +85,29 @@ def deploy(filename: click.Path): except requests.HTTPError as deploy_error: console.print(deploy_error.response.json()["error_description"][0]["msg"]) return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) + table.add_column( + "Deploy Result", + min_width=35, + max_width=50, + justify="left", + style="bright_green", + ) if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) + for k, v in deploy_result.items(): + if k == "config": + row_text = Text(f"{k}: ", style="magenta") + row_text.append(f"{v}", style="white") + table.add_row(row_text) + if k == "pipelines": + row_text = Text(f"{k}:\n", style="bright_blue") + for id in deploy_result[k]: + row_text.append(f"\t{id}\n", style="white") + table.add_row(row_text) console.print(table) @configs.command("ls", help="List Configs.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Configs with provided name.", -) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -137,54 +116,37 @@ def deploy(filename: click.Path): default=False, help="Only show IDs.", ) -def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): +def ls(name: Optional[str] = None, quiet: bool = False): """List all objects.""" - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - configs_colums = ["name", "version", "children", "user"] - pipelines_columns = ["status", "current_stage", "steps"] - projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + configs_colums = ["name", "version", "pipelines", "user"] + projection = {"yaml": 0, "deployments": 0} if quiet: projection = {"id": 1} http = HTTPContext() objects = http.configs.get_configs( - database=database, config_name=name, projection=json.dumps(projection) + config_name=name, projection=json.dumps(projection) ) # ? Add columns for each key table.add_column("ID", max_width=40, justify="left", style="blue") if not quiet: - if database == "configs": - for key in configs_colums: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - style="bright_green" if key == "name" else "white", - ) - if database == "pipelines": - for key in pipelines_columns: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - ) + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) for obj in objects: if not quiet: - if database == "configs": - table.add_row( - obj["id"], - obj["name"], - obj["version"], - str(len(obj["children"])), - obj["user"], - ) - if database == "pipelines": - status = Text(obj["status"], style=status_colors[obj["status"]]) - table.add_row( - obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) - ) + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["pipelines"])), + obj["user"], + ) continue table.add_row(obj["id"]) console.print(table) @@ -193,26 +155,16 @@ def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False) @configs.command("ps", help="Get Configs details.") @click.argument("name", type=str, required=True) @click.argument("id", type=str, required=True) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) @click.option( "--details", - "-d", is_flag=True, default=False, show_default=True, help="Show more details for the object.", ) -def ps(name: str, id: str, pipelines: str, details: bool): +def ps(name: str, id: str, details: bool): """Show details for an object.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" query: str = json.dumps({"id": id}) projection: str = json.dumps({}) console_content = None @@ -220,29 +172,20 @@ def ps(name: str, id: str, pipelines: str, details: bool): column_min_width = 40 try: payload = http.configs.get_configs( - database=database, config_name=name, query=query, projection=projection + config_name=name, query=query, projection=projection )[0] except IndexError: - error_text = Text(f"No {database.capitalize()} were found", style="red") + error_text = Text("No Configs were found", style="red") console_content = error_text else: text = Text("") - if database == "pipelines": - table.add_column( - f"Pipeline: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_pipeline(payload)) - if database == "configs": - table.add_column( - f"Config: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_config(http, payload)) + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) if details: table.add_column("Details", max_width=column_max_width, justify="left") _details = safe_load(payload["yaml"]) @@ -254,6 +197,60 @@ def ps(name: str, id: str, pipelines: str, details: bool): table.add_row(text, JSON(json.dumps(_details), indent=2)) else: table.add_row(text) + table.add_section() + table.add_row( + Text("Explore pipelines in detail: \n", style="magenta i").append( + "workflow pipelines ps ", + style="dark_blue on cyan", + ) + ) console_content = table finally: console.print(console_content) + + +@configs.command("stop", help="Stop managers for a Config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def stop(config: str, id: str): + """Stop managers for a Config.""" + http = HTTPContext() + stop_result = http.configs.stop(config, id) + if not any(stop_result): + text = Text("No configurations were stopped.", style="red") + console.print(text) + return + table.add_column("Stopped IDs", max_width=50, justify="left") + text = Text() + for k in stop_result.keys(): + if k == "stopped_config": + text.append("Config: ", style="bright_blue") + text.append(f"{stop_result[k]}\n") + if k == "stopped_pipelines": + text.append("Pipelines: \n", style="bright_blue") + for id in stop_result["stopped_pipelines"]: + text.append(f"\t{id}\n") + table.add_row(text) + console.print(table) + + +@configs.command("rm", help="Remove a config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def rm(config: str, id: str): + """Remove a config.""" + http = HTTPContext() + content = None + try: + delete_result = http.configs.remove(config, id) + if delete_result.status_code == 204: + text = Text("No pipeline configurations were deleted.", style="red") + content = text + except Exception as e: + text = Text(f"No configurations were deleted.\nError: {e}", style="red") + content = text + else: + table.add_column("Deleted IDs", max_width=50, justify="left", style="red") + table.add_row(id) + content = table + console.print(content) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index cf4c338..83a254c 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,20 +1,18 @@ """Manage workflow pipelines.""" -import datetime as dt import json -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional import click import requests -import yaml from rich import pretty from rich.console import Console from rich.table import Table from rich.text import Text -from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.variables import status_colors, status_symbols +from workflow.utils.renderers import render_pipeline +from workflow.utils.variables import status_colors pretty.install() console = Console() @@ -45,14 +43,7 @@ def version(): @pipelines.command("ls", help="List pipelines.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Pipelines with provided name.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -63,21 +54,24 @@ def version(): ) def ls(name: Optional[str] = None, quiet: Optional[bool] = False): """List all pipelines.""" + pipelines_columns = ["status", "current_stage", "steps"] http = HTTPContext() - objects = http.pipelines.list_pipeline_configs(name) - table.add_column("ID", max_width=50, justify="left", style="blue") - if not quiet: - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Stage", max_width=50, justify="left") - for config in objects: - status = Text(config["status"], style=status_colors[config["status"]]) + objects = http.pipelines.list_pipelines(name) + table.add_column("ID", max_width=100, justify="left", style="blue") + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + for obj in objects: if not quiet: + status = Text(obj["status"], style=status_colors[obj["status"]]) table.add_row( - config["id"], config["name"], status, str(config["current_stage"]) + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) ) continue - table.add_row(config["id"]) + table.add_row(obj["id"]) console.print(table) @@ -97,118 +91,39 @@ def count(): console.print(table) -@pipelines.command("deploy", help="Deploy a workflow pipeline.") -@click.argument( - "filename", - type=click.Path(exists=True, dir_okay=False, readable=True), - required=True, -) -def deploy(filename: click.Path): - """Deploy a workflow pipeline.""" - http = HTTPContext() - filepath: str = str(filename) - data: Dict[str, Any] = {} - with open(filepath) as reader: - data = yaml.load(reader, Loader=SafeLoader) # type: ignore - try: - deploy_result = http.pipelines.deploy(data) - except requests.HTTPError as deploy_error: - console.print(deploy_error.response.json()["message"]) - return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) - if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) - console.print(table) - - @pipelines.command("ps", help="Get pipeline details.") @click.argument("pipeline", type=str, required=True) @click.argument("id", type=str, required=True) def ps(pipeline: str, id: str): """List a pipeline configuration in detail.""" http = HTTPContext() - query: Dict[str, Any] = {"id": id} + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) console_content = None - projection = {"name": False} - time_fields = ["creation", "start", "stop"] + column_max_width = 300 + column_min_width = 40 try: - payload = http.pipelines.get_pipeline_config(pipeline, query, projection) + payload = http.pipelines.get_pipelines( + name=pipeline, query=query, projection=projection + )[0] except IndexError: - error_text = Text("No PipelineConfig were found", style="red") + error_text = Text("No Pipelines were found", style="red") console_content = error_text else: - table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") - text = Text("") - for k, v in payload.items(): - key_value_text = Text() - if k in time_fields and v: - v = dt.datetime.fromtimestamp(v) - if k == "pipeline": - key_value_text = Text(f"{k}: \n", style="bright_blue") - for step in v: - key_value_text.append(f" {step['name']}:") - key_value_text.append(f"{status_symbols[step['status']]}\n") - else: - key_value_text = Text(f"{k}: ", style="bright_blue") - key_value_text.append( - f"{v}\n", style="white" if k != "status" else status_colors[v] - ) - text.append_text(key_value_text) - + text = Text() + table.add_column( + f"Pipeline: {pipeline}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) table.add_row(text) console_content = table finally: console.print(console_content) -@pipelines.command("stop", help="Kill a running pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -def stop(pipeline: str, id: Tuple[str]): - """Kill a running pipeline.""" - http = HTTPContext() - stop_result = http.pipelines.stop(pipeline, id) - if not any(stop_result): - text = Text("No pipeline configurations were stopped.", style="red") - console.print(text) - return - table.add_column("Stopped IDs", max_width=50, justify="left") - for config in stop_result: - table.add_row(config["id"]) - console.print(table) - - -@pipelines.command("rm", help="Remove a pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def rm(pipeline: str, id: Tuple[str], schedule: bool): - """Remove a pipeline.""" - http = HTTPContext() - content = None - try: - delete_result = http.pipelines.remove(pipeline, id, schedule) - if delete_result.status_code == 204: - text = Text("No pipeline configurations were deleted.", style="red") - content = text - except Exception as e: - text = Text( - f"No pipeline configurations were deleted.\nError: {e}", style="red" - ) - content = text - else: - table.add_column("Deleted IDs", max_width=50, justify="left", style="red") - table.add_row(id) - content = table - console.print(content) - - def status( pipeline: Optional[str] = None, query: Optional[Dict[str, Any]] = None, diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 93cee00..853161a 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -148,8 +148,7 @@ def deploy(filename: click.Path): @schedules.command("ps", help="Get schedule details.") @click.argument("id", type=str, required=True) @click.option( - "--detail", - "-d", + "--details", is_flag=True, show_default=True, help="Returns the Schedule Payload.", diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 616e185..43ef565 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -49,14 +49,9 @@ def deploy(self, data: Dict[str, Any]): return response.json() @try_request - def count(self, database: str = "configs") -> Dict[str, Any]: + def count(self) -> Dict[str, Any]: """Count all documents in a collection. - Parameters - ---------- - database : str - Database to be used. "configs" or "pipelines". - Returns ------- Dict[str, Any] @@ -64,7 +59,7 @@ def count(self, database: str = "configs") -> Dict[str, Any]: """ with self.session as session: response: Response = session.get( - url=f"{self.baseurl}/v2/configs/count?database={database}" + url=f"{self.baseurl}/v2/configs/count?database=configs" ) response.raise_for_status() return response.json() @@ -72,7 +67,6 @@ def count(self, database: str = "configs") -> Dict[str, Any]: @try_request def get_configs( self, - database: str, config_name: str, query: Optional[str] = "{}", projection: Optional[str] = "{}", @@ -81,8 +75,6 @@ def get_configs( Parameters ---------- - database : str - Database to query from. config_name : str Config name, by default None query : str, optional @@ -101,8 +93,6 @@ def get_configs( params = {"projection": projection, "query": query} if config_name: params.update({"name": config_name}) - if database: - params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() @@ -114,15 +104,15 @@ def get_configs( stop=(stop_after_delay(5) | stop_after_attempt(1)), ) @try_request - def remove(self, pipeline: str, id: str) -> Response: + def remove(self, config: str, id: str) -> Response: """Removes a cancelled pipeline configuration. Parameters ---------- - pipeline : str - PipelineConfig name. + config : str + Config name. id : str - PipelineConfig ID. + Config ID. Returns ------- @@ -131,8 +121,8 @@ def remove(self, pipeline: str, id: str) -> Response: """ with self.session as session: query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + params = {"query": dumps(query), "name": config} + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -157,7 +147,7 @@ def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() if response.status_code == 304: diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index fddba7d..971051c 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -1,13 +1,9 @@ """Workflow Pipelines API.""" -from json import dumps from typing import Any, Dict, List, Optional from urllib.parse import urlencode from requests.models import Response -from tenacity import retry -from tenacity.stop import stop_after_attempt, stop_after_delay -from tenacity.wait import wait_random from workflow.http.client import Client from workflow.utils.decorators import try_request @@ -23,31 +19,6 @@ class Pipelines(Client): Pipelines: A client for interacting with the Pipelines backend. """ - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def deploy(self, data: Dict[str, Any]): - """Deploys a PipelineConfig from payload data. - - Parameters - ---------- - data : Dict[str, Any] - YAML data. - - Returns - ------- - List[str] - IDs of PipelineConfig objects generated. - """ - with self.session as session: - url = f"{self.baseurl}/v1/pipelines" - response: Response = session.post(url, json=data) - response.raise_for_status() - return response.json() - @try_request def count(self) -> Dict[str, Any]: """Count all documents in a collection. @@ -58,20 +29,21 @@ def count(self) -> Dict[str, Any]: Dictionary with count. """ with self.session as session: - response: Response = session.get(url=f"{self.baseurl}/v1/pipelines/count") + params = {"database": "pipelines"} + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?{urlencode(params)}" + ) response.raise_for_status() return response.json() @try_request - def list_pipeline_configs( - self, config_name: Optional[str] = None - ) -> List[Dict[str, Any]]: + def list_pipelines(self, name: Optional[str] = None) -> List[Dict[str, Any]]: """View the current pipeline configurations in the pipelines backend. Parameters ---------- - config_name : Optional[str], optional - PipelineConfig name, by default None + name : Optional[str], optional + Config name, by default None Returns ------- @@ -79,25 +51,24 @@ def list_pipeline_configs( List of PipelineConfig payloads. """ with self.session as session: - url = ( - f"{self.baseurl}/v1/pipelines" - if config_name is None - else f'{self.baseurl}/v1/pipelines?query={{"name":"{config_name}"}}' - ) + params = {"database": "pipelines"} + if name: + params.update({"name": name}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() @try_request - def get_pipeline_config( - self, collection: str, query: Dict[str, Any], projection: Dict[str, Any] + def get_pipelines( + self, name: str, query: Dict[str, Any], projection: Dict[str, Any] ) -> Dict[str, Any]: """Gets details for one pipeline configuration. Parameters ---------- - collection : str - PipelineConfig name. + name : str + Config name. query : Dict[str, Any] Dictionary with search parameters. projection : Dict[str, Any] @@ -110,69 +81,14 @@ def get_pipeline_config( """ with self.session as session: params = { - "query": dumps(query), - "name": collection, - "projection": dumps(projection), + "query": query, + "name": name, + "projection": projection, + "database": "pipelines", } - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() - return response.json()[0] - - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def remove(self, pipeline: str, id: str) -> Response: - """Removes a cancelled pipeline configuration. - - Parameters - ---------- - pipeline : str - PipelineConfig name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - Response payload. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" - response: Response = session.delete(url=url) - response.raise_for_status() - return response - - @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) - @try_request - def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: - """Stops the manager for a PipelineConfig. - - Parameters - ---------- - pipeline : str - Pipeline name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - List of stopped PipelineConfig objects. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" - response: Response = session.put(url) - response.raise_for_status() - if response.status_code == 304: - return [] return response.json() @try_request diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py index a39691c..c523a04 100644 --- a/workflow/utils/renderers.py +++ b/workflow/utils/renderers.py @@ -63,11 +63,10 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: """ text = Text() hidden_keys = ["yaml", "services", "name"] - query = dumps({"id": {"$in": payload["children"]}}) + query = dumps({"id": {"$in": payload["pipelines"]}}) projection = dumps({"id": 1, "status": 1}) - children_statuses = http.configs.get_configs( - database="pipelines", - config_name=payload["name"], + pipelines_statuses = http.pipelines.get_pipelines( + name=payload["name"], query=query, projection=projection, ) @@ -76,9 +75,9 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: if k in hidden_keys: continue key_value_text = Text() - if k == "children": + if k == "pipelines": key_value_text.append(f"{k}: \n", style="bright_blue") - for child in children_statuses: + for child in pipelines_statuses: key_value_text.append( f"\t{child['id']}: ", style=status_colors[child["status"]] ) diff --git a/workflow/utils/variables.py b/workflow/utils/variables.py index f8f8149..9cfe11b 100644 --- a/workflow/utils/variables.py +++ b/workflow/utils/variables.py @@ -14,11 +14,11 @@ status_symbols = { "created": "\U000026AA", # white - "queued": "\u23F3", # hourglass - "active": "\U0001F7E2", # green - "running": "\u2699", # gear + "queued": "\U0001F4C5", # 📅 + "active": "\U0001F7E2", # green circle + "running": "\U0001F3C3", # 🏃 "success": "\U00002705", # Green check "paused": "\U0001F7E1", # yellow "failure": "\U0000274C", # cross mark - "cancelled": "\U0001F534", # red + "cancelled": "\U0001F6AB", # 🚫 } From e35e65819486efc9fe806ac47776d966d7c5f443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:30:33 -0400 Subject: [PATCH 15/41] test(test_http_context.py): adding tests Tests added for pipelines client on HTTPContext. --- docker-compose-tutorial.yml | 4 +-- tests/test_http_context.py | 64 ++++++++++++++++++++++++++++++++++--- workflow/http/configs.py | 10 +++--- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml index 818d701..4aad364 100644 --- a/docker-compose-tutorial.yml +++ b/docker-compose-tutorial.yml @@ -1,6 +1,6 @@ services: pipelines_api: - image: chimefrb/pipelines:latest + image: chimefrb/pipelines:pipelines-v2.7.1 container_name: pipelines_api command: python -m pipelines.server ports: @@ -27,7 +27,7 @@ services: restart: always pipelines_managers: - image: chimefrb/pipelines:latest + image: chimefrb/pipelines:pipelines-v2.7.1 container_name: pipelines_managers command: python -m managers.server ports: diff --git a/tests/test_http_context.py b/tests/test_http_context.py index 9ddc6a1..ab6ecb3 100644 --- a/tests/test_http_context.py +++ b/tests/test_http_context.py @@ -1,18 +1,74 @@ """Test the HTTPContext object.""" -import pytest - from workflow.http.context import HTTPContext +config = { + "version": "1", + "name": "demo", + "defaults": {"user": "test"}, + "pipeline": { + "steps": [ + { + "name": "stage-1-a", + "stage": 1, + "matrix": {"event": [123456, 654321], "site": ["chime", "canfar"]}, + "work": { + "site": "${{ matrix.site }}", + "command": ["ls", "${{ matrix.event }}"], + }, + }, + ], + }, +} + class TestHTTPContext: def test_can_be_instantiated(self): """Test that the HTTPContext object can be instantiated.""" HTTPContext() - @pytest.mark.skip def test_clients_connect_to_base_url(self): """Tests HTTPContext.clients have connection to their proper backend.""" http = HTTPContext() assert isinstance(http.buckets.info(), dict) - # TODO test results and pipelines + assert isinstance(http.results.info(), dict) + assert isinstance(http.configs.info(), dict) + assert isinstance(http.pipelines.info(), dict) + + def test_http_configs_deploy(self): + http = HTTPContext() + response = http.configs.deploy(config) + assert isinstance(response, dict) + assert "config" in response.keys() + assert "pipelines" in response.keys() + + def test_http_configs_list(self): + http = HTTPContext() + response = http.configs.get_configs(config_name=None) + assert isinstance(response, list) + + def test_http_configs_count(self): + http = HTTPContext() + response = http.configs.count() + assert isinstance(response, dict) + + def test_http_configs_remove_process(self): + http = HTTPContext() + # ? Post + post_response = http.configs.deploy(config) + stop_response = http.configs.stop(config["name"], post_response["config"]) + assert stop_response["stopped_config"] == post_response["config"] + remove_response = http.configs.remove(config["name"], post_response["config"]) + assert remove_response.status_code == 204 + + def test_http_pipelines_list(self): + http = HTTPContext() + response = http.pipelines.list_pipelines() + assert isinstance(response, list) + + def test_http_pipelines_get(self): + http = HTTPContext() + count_response = http.pipelines.count() + response = http.pipelines.get_pipelines(name="demo", query={}, projection={}) + assert count_response + assert isinstance(response, list) diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 43ef565..b66c93f 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -129,15 +129,15 @@ def remove(self, config: str, id: str) -> Response: @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) @try_request - def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: + def stop(self, config_name: str, id: str) -> List[Dict[str, Any]]: """Stops the manager for a PipelineConfig. Parameters ---------- - pipeline : str - Pipeline name. + config_name : str + Config name. id : str - PipelineConfig ID. + Config ID. Returns ------- @@ -146,7 +146,7 @@ def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: """ with self.session as session: query = {"id": id} - params = {"query": dumps(query), "name": pipeline} + params = {"query": dumps(query), "name": config_name} url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() From d15735b1fcd2c3e26aa67f89adec7918f2b9498e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:36:51 -0400 Subject: [PATCH 16/41] ci(ci.yml): adding steps to tests job Adding steps for: starting docker services needed for tests; setting development workspace and killing services after tests are completed. --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0298b5..5357ae5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,18 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'poetry' + - + name: Run services + run: | + docker-compose -f .\docker-compose-tutorial.yml up -d --build - name: Installing workflow dependencies run: | poetry install + - + name: Set workspace for testing + run: | + poetry run workflow workspace set development - name: Run workflow tests run: | @@ -67,3 +75,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: "coverage.lcov" + - + name: Kill services + if: always() + run: | + docker-compose -f .\docker-compose-tutorial.yml down -v From db3f3ca607c8f4d00f744a0e68cfa73939c9c351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:38:43 -0400 Subject: [PATCH 17/41] ci(ci.yml): changing docker-compose file name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5357ae5..09c0305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Run services run: | - docker-compose -f .\docker-compose-tutorial.yml up -d --build + docker-compose -f docker-compose-tutorial.yml up -d --build - name: Installing workflow dependencies run: | @@ -79,4 +79,4 @@ jobs: name: Kill services if: always() run: | - docker-compose -f .\docker-compose-tutorial.yml down -v + docker-compose -f docker-compose-tutorial.yml down -v From 8ffc08b204ab6b3057a61be602e6c9ffdf18e102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:41:07 -0400 Subject: [PATCH 18/41] ci(ci.yml): adding docker login step --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c0305..9374345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,12 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'poetry' + - + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Run services run: | From 7820c623406f582e7fceef22785cb040d1be04b3 Mon Sep 17 00:00:00 2001 From: Odarsson <40160237+odarotto@users.noreply.github.com> Date: Mon, 27 May 2024 13:40:54 -0400 Subject: [PATCH 19/41] feat(cli/main.py): adding logging on top level function (#37) Logging info about actual workspace being used. --- workflow/cli/main.py | 7 +++++++ workflow/utils/read.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/workflow/cli/main.py b/workflow/cli/main.py index ca126bb..9791721 100755 --- a/workflow/cli/main.py +++ b/workflow/cli/main.py @@ -1,6 +1,7 @@ """Workflow command line interface.""" import click +from rich.console import Console # from workflow.cli.buckets import buckets from workflow.cli.pipelines import pipelines @@ -8,11 +9,17 @@ from workflow.cli.run import run from workflow.cli.schedules import schedules from workflow.cli.workspace import workspace +from workflow.utils.read import get_active_workspace + +console = Console() @click.group() def cli(): """Workflow Command Line Interface.""" + # ? Get workspace + message = get_active_workspace() + console.print(message) pass diff --git a/workflow/utils/read.py b/workflow/utils/read.py index 81b5403..4679f52 100644 --- a/workflow/utils/read.py +++ b/workflow/utils/read.py @@ -6,8 +6,10 @@ from requests import get from requests.exceptions import RequestException +from rich.text import Text from yaml import safe_load +from workflow.cli.workspace import localspaces, localstems, modulespaces, modulestems from workflow.utils.logger import get_logger logger = get_logger("workflow.utils.read") @@ -83,6 +85,33 @@ def filename(source: str) -> Any: raise error +def get_active_workspace() -> Text: + """Returns a Text with info about the active workspace. + + Returns + ------- + Text + Text instance with info. + """ + _workspace = "active" + text = Text() + if _workspace in localstems: + for possibility in localspaces.glob(f"{_workspace}.y*ml"): + config = workspace(possibility) + text.append("Currently using ", style="green") + text.append(f"{config['workspace']} ", style="blue") + text.append("workspace.", style="green") + elif _workspace in modulestems: + for possibility in modulespaces.glob(f"{_workspace}.y*ml"): + config = workspace(possibility) + text.append("Currently using ", style="green") + text.append(f"{config['workspace']} ", style="blue") + text.append("workspace.", style="green") + else: + text.append("There is not active workspace", style="red") + return text + + def is_valid_url(url: str) -> bool: """Return True if the URL is valid. From b0a7c8cede7ae51a116008b1f1d04b3c3bc211df Mon Sep 17 00:00:00 2001 From: Odarsson <40160237+odarotto@users.noreply.github.com> Date: Mon, 27 May 2024 22:26:36 -0400 Subject: [PATCH 20/41] 31 bug fix workspace cli command (#40) * fix(cli/workspace.py): changing filename for active workspace * feat(workflow/http/context.py): adding validator function for workspace field * fix(workspace): fixed workspace cli commands and suppressed UserWarning from pydantic_settings --------- Co-authored-by: Shiny Brar (he/il) --- docs/workspaces.md | 6 +++--- tests/conftest.py | 12 +++++------- tests/test_cli.py | 35 +++++++++++++++++++++++++++++++++++ tests/test_http_context.py | 15 ++++++++++++++- tests/test_token.py | 2 +- tests/test_work.py | 11 +++++++++++ workflow/__init__.py | 9 +++++++-- workflow/cli/workspace.py | 16 +++++----------- workflow/definitions/work.py | 4 ++-- workflow/http/context.py | 34 +++++++++++++++++++++++++++++++--- 10 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 tests/test_cli.py diff --git a/docs/workspaces.md b/docs/workspaces.md index 5f965ae..bf2405b 100644 --- a/docs/workspaces.md +++ b/docs/workspaces.md @@ -6,7 +6,7 @@ A workspace is, - Project-Specific: Each project can have their own workspace, while sharing the same installation. - YAML-Based: Workspaces are defined in YAML files, which can be version controlled. - - Stored in the client/user's home directory under the path `~/.workflow/workspaces/` + - Stored in the client/user's home directory under the path `~/.config/workflow` ## How do I activate a workspace? @@ -32,13 +32,13 @@ To remove an active workspace, workflow workspace rm ``` -This will only remove the active workspace, i.e. `~/.workflow/workspaces/active.yml`. +This will only remove the active workspace, i.e. `~/.config/workflow/workspace.yml`. !!! Important Running workflow without a workspace set will result in an runtime error. -In order to purge all workspaces, from `~/.workflow/workspaces/` run: +In order to purge all workspaces, from `~/.config/workflow/` run: ```bash workflow workspace purge diff --git a/tests/conftest.py b/tests/conftest.py index 7513a76..9a3d4ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,14 @@ """pytest configuration file.""" +import pytest from click.testing import CliRunner from workflow.cli.main import cli as workflow -def pytest_configure(config): - """Initailize pytest configuration. - - Allows plugins and conftest files to perform initial configuration. - This hook is called for every plugin and initial conftest - file after command line options have been parsed. - """ +@pytest.fixture(autouse=True, scope="function") +def set_testing_workspace(): + """Initailize testing workspace.""" runner = CliRunner() runner.invoke(workflow, ["workspace", "set", "development"]) + return True diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6bc74d2 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,35 @@ +"""Test the workspace CLI commands.""" + +import os + +from click.testing import CliRunner + +from workflow import CONFIG_PATH, DEFAULT_WORKSPACE_PATH +from workflow.cli.workspace import ls, set + + +class TestWorkspaceCLI: + def test_workspace_ls(self): + runner = CliRunner() + result = runner.invoke(ls) + assert result.exit_code == 0 + assert "From Workflow Python Module" in result.output + assert "development" in result.output + + def test_workspace_set(self): + runner = CliRunner() + result = runner.invoke(set, ["development"]) + assert result.exit_code == 0 + assert "Locating workspace development" in result.output + assert "Workspace development set to active" in result.output + # ? Check the default folder only contains the active workspace file + entries = os.listdir(CONFIG_PATH) + files = [ + CONFIG_PATH / entry + for entry in entries + if os.path.isfile(os.path.join(CONFIG_PATH, entry)) + ] + files = [f.as_posix() for f in files] + assert files == [DEFAULT_WORKSPACE_PATH.as_posix()] + # ? Re set workspace for other tests + result = runner.invoke(set, ["development"]) diff --git a/tests/test_http_context.py b/tests/test_http_context.py index 9ddc6a1..c173cd2 100644 --- a/tests/test_http_context.py +++ b/tests/test_http_context.py @@ -1,14 +1,27 @@ """Test the HTTPContext object.""" import pytest +from click.testing import CliRunner +from pydantic import ValidationError +from workflow.cli.workspace import set, unset from workflow.http.context import HTTPContext class TestHTTPContext: def test_can_be_instantiated(self): """Test that the HTTPContext object can be instantiated.""" - HTTPContext() + http = HTTPContext() + assert http + + def test_cannot_be_instantiated_without_workspace(self): + """Test that the HTTPContext object cannot be instantiated without workspace.""" + runner = CliRunner() + # ? Set Workspace + runner.invoke(unset) + with pytest.raises(ValidationError): + HTTPContext() + runner.invoke(set, ["development"]) @pytest.mark.skip def test_clients_connect_to_base_url(self): diff --git a/tests/test_token.py b/tests/test_token.py index 293f03c..9beb5fe 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -4,7 +4,7 @@ from workflow.http.context import HTTPContext -def test_work_pass_token_to_client(monkeypatch): +def test_work_pass_token_to_client(monkeypatch, set_testing_workspace): """Test that the Client objects can obtain token from Work object.""" test_token = "ghp_1234567890abcdefg" monkeypatch.setenv("WORKFLOW_HTTP_TOKEN", test_token) diff --git a/tests/test_work.py b/tests/test_work.py index 4ff586f..d5a8459 100644 --- a/tests/test_work.py +++ b/tests/test_work.py @@ -1,8 +1,10 @@ """Test the work object.""" import pytest +from click.testing import CliRunner from pydantic import ValidationError +from workflow.cli.workspace import set, unset from workflow.definitions.work import Work @@ -24,6 +26,15 @@ def test_bad_pipeline(): Work(pipeline="", site="local", user="test") +def test_worskpace_unset(): + """Test that the work object can't be instantiated without a setted workspace.""" + runner = CliRunner() + runner.invoke(unset) + with pytest.raises(ValidationError): + Work(pipeline="", site="local", user="test") + runner.invoke(set, ["development"]) + + def test_pipeline_reformat(): """Test that the work object can't be instantiated with empty pipeline.""" with pytest.raises(ValidationError): diff --git a/workflow/__init__.py b/workflow/__init__.py index 9ab5bd3..32172b5 100644 --- a/workflow/__init__.py +++ b/workflow/__init__.py @@ -1,12 +1,17 @@ """Top-level imports for Tasks API.""" from pathlib import Path +from warnings import filterwarnings + +# Ignore UserWarnings from pydantic_settings module +# These usually come when no docker secrets are found +filterwarnings("ignore", category=UserWarning, module="pydantic_settings") # Root path to the Workflow Module MODULE_PATH: Path = Path(__file__).absolute().parent.parent # Path to local configurations -CONFIG_PATH: Path = Path.home() / ".workflow" +CONFIG_PATH: Path = Path.home() / ".config" / "workflow" # Active Workspace Path -DEFAULT_WORKSPACE_PATH: Path = CONFIG_PATH / "workspaces" / "active.yml" +DEFAULT_WORKSPACE_PATH: Path = CONFIG_PATH / "workspace.yml" # Workflow Client Version __version__ = "0.3.0" # {x-release-please-version} diff --git a/workflow/cli/workspace.py b/workflow/cli/workspace.py index 8deecc1..01df1b7 100644 --- a/workflow/cli/workspace.py +++ b/workflow/cli/workspace.py @@ -86,13 +86,7 @@ def set(workspace: str): name: str = config["workspace"] localspaces.mkdir(parents=True, exist_ok=True) - # Copy the workspace config to ~/.workflow/workspaces/.yml - configpath = localspaces / f"{name}.yml" - activepath = localspaces / "active.yml" - # Write config to configpath, even if it already exists. - with open(configpath, "w") as filename: - dump(config, filename) - console.print(f"Copied {name} to {configpath}.", style="bold green") + activepath = localspaces / "workspace.yml" # Write config to activepath, even if it already exists. with open(activepath, "w") as filename: dump(config, filename) @@ -100,7 +94,7 @@ def set(workspace: str): @workspace.command("read", help="Read workspace config.") -@click.argument("workspace", type=str, required=True, nargs=1, default="active") +@click.argument("workspace", type=str, required=True, nargs=1, default="workspace") def read(workspace: str): """Read the active workspace. @@ -120,7 +114,7 @@ def read(workspace: str): console.print(config, style="green") return else: - console.print(f"Workspace {workspace} not found.", style="bold red") + console.print("No workspace found.", style="italic bold red") return @@ -130,14 +124,14 @@ def unset(): # Set the default console style. console.print("Removing the active workspace.", style="italic red") # If the workspace already exists, warn the user. - (localspaces / "active.yml").unlink(missing_ok=True) + (localspaces / "workspace.yml").unlink(missing_ok=True) console.print("Workspace Removed.", style="bold red") @workspace.command("purge", help="Purge all local workspaces.") def purge(): """Purge all local workspaces.""" - # Remove all files from ~/.workflow/workspaces/ + # Remove all files from ~/.config/workflow/ console.print("Purging all local workspaces", style="italic red") for workspace in localspaces.glob("*.y*ml"): console.print(f"Removing {workspace}", style="italic red") diff --git a/workflow/definitions/work.py b/workflow/definitions/work.py index 7688b4f..22fae88 100644 --- a/workflow/definitions/work.py +++ b/workflow/definitions/work.py @@ -76,7 +76,7 @@ class Work(BaseSettings): notify (Notify): Notification configuration for the work. workspace (FilePath): Path to the active workspace configuration. - Defaults to `~/.workflow/workspaces/active.yml`. (Excluded from payload) + Defaults to `~/.config/workflow/workspace.yml`. (Excluded from payload) token (SecretStr): Workflow Access Token. (Excluded from payload) http (HTTPContext): HTTP Context for backend connections. (Excluded from payload) @@ -130,7 +130,7 @@ class Work(BaseSettings): default=DEFAULT_WORKSPACE_PATH, validate_default=True, description="Default workspace configuration filepath.", - examples=["/home/user/.workflow/active.yml"], + examples=["/home/user/.config/workflow/workspace.yml"], exclude=True, ) token: SecretStr | None = Field( diff --git a/workflow/http/context.py b/workflow/http/context.py index 731447a..26a19b1 100644 --- a/workflow/http/context.py +++ b/workflow/http/context.py @@ -1,8 +1,16 @@ """HTTP client for interacting with the Workflow Servers.""" +import os from typing import Any, Dict, Optional -from pydantic import AliasChoices, Field, FilePath, SecretStr, model_validator +from pydantic import ( + AliasChoices, + Field, + FilePath, + SecretStr, + field_validator, + model_validator, +) from pydantic_settings import BaseSettings, SettingsConfigDict from workflow import DEFAULT_WORKSPACE_PATH @@ -42,8 +50,8 @@ class HTTPContext(BaseSettings): workspace: FilePath = Field( default=DEFAULT_WORKSPACE_PATH, frozen=True, - description="Path to the active workspace configuration.", - examples=["/path/to/workspace/config.yaml"], + description="Path to the workspace configuration.", + examples=["/home/user/.config/workflow/workspace.yaml"], ) timeout: float = Field( default=15.0, @@ -91,6 +99,26 @@ class HTTPContext(BaseSettings): exclude=True, ) + @field_validator("workspace", mode="before") + @classmethod + def check_workspace_is_set(cls, value: str): + """Check that workspace field has a valid filepath. + + Parameters + ---------- + value : str + FilePath str value. + + Raises + ------ + ValueError + If path is not a valid file. + """ + if not os.path.isfile(value): + logger.error("No workspace set.") + raise ValueError("No workspace set.") + return value + @model_validator(mode="after") def create_clients(self) -> "HTTPContext": """Create the HTTP Clients for the Workflow Servers. From 15e0e7ef6802fb9861b5031b356ceeeab821e28c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 22:40:44 -0400 Subject: [PATCH 21/41] chore(main): release 0.4.0 (#26) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ pyproject.toml | 2 +- workflow/__init__.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db256a9..5702720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.4.0](https://github.com/CHIMEFRB/workflow/compare/v0.3.0...v0.4.0) (2024-05-28) + + +### Features + +* add schedule cli ([#25](https://github.com/CHIMEFRB/workflow/issues/25)) ([56dcd6e](https://github.com/CHIMEFRB/workflow/commit/56dcd6e8f53853234de069bac4c77255c448cceb)) +* **cli/main.py:** adding logging on top level function ([#37](https://github.com/CHIMEFRB/workflow/issues/37)) ([7820c62](https://github.com/CHIMEFRB/workflow/commit/7820c623406f582e7fceef22785cb040d1be04b3)) +* **docker-compose-tutorial.yml:** adding docker compose for tutorial ([#35](https://github.com/CHIMEFRB/workflow/issues/35)) ([7b17e2a](https://github.com/CHIMEFRB/workflow/commit/7b17e2a987e09d8a950d9b4202b29c068321971f)) + ## [0.3.0](https://github.com/CHIMEFRB/workflow/compare/v0.2.0...v0.3.0) (2024-03-06) diff --git a/pyproject.toml b/pyproject.toml index a773d16..12729aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "workflow" -version = "0.3.0" +version = "0.4.0" description = "Workflow Core" authors = ["Shiny Brar "] license = "MIT" diff --git a/workflow/__init__.py b/workflow/__init__.py index 32172b5..49a4665 100644 --- a/workflow/__init__.py +++ b/workflow/__init__.py @@ -14,4 +14,4 @@ # Active Workspace Path DEFAULT_WORKSPACE_PATH: Path = CONFIG_PATH / "workspace.yml" # Workflow Client Version -__version__ = "0.3.0" # {x-release-please-version} +__version__ = "0.4.0" # {x-release-please-version} From f5ab3971db3b25fd13e3df57fd012973a4b5ca5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 09:55:46 -0400 Subject: [PATCH 22/41] feat(cli): adding configs commands confgis commands added so the cli can work with the new version of pipelines(^2.7.0) --- workflow/cli/configs.py | 259 ++++++++++++++++++++++++++++ workflow/cli/main.py | 2 + workflow/http/configs.py | 182 +++++++++++++++++++ workflow/http/context.py | 9 + workflow/utils/renderers.py | 95 ++++++++++ workflow/workspaces/development.yml | 1 + 6 files changed, 548 insertions(+) create mode 100644 workflow/cli/configs.py create mode 100644 workflow/http/configs.py create mode 100644 workflow/utils/renderers.py diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py new file mode 100644 index 0000000..88e42a3 --- /dev/null +++ b/workflow/cli/configs.py @@ -0,0 +1,259 @@ +"""Manage workflow pipelines.""" + +import json +from typing import Any, Dict, Optional + +import click +import requests +import yaml +from rich import pretty +from rich.console import Console +from rich.json import JSON +from rich.table import Table +from rich.text import Text +from yaml import safe_load +from yaml.loader import SafeLoader + +from workflow.http.context import HTTPContext +from workflow.utils.renderers import render_config, render_pipeline +from workflow.utils.variables import status_colors + +pretty.install() +console = Console() + +table = Table( + title="\nWorkflow Pipelines", + show_header=True, + header_style="magenta", + title_style="bold magenta", + min_width=10, +) + +BASE_URL = "https://frb.chimenet.ca/pipelines" +STATUS = ["created", "queued", "running", "success", "failure", "cancelled"] + + +@click.group(name="configs", help="Manage Workflow Configs. Version 2.") +def configs(): + """Manage workflow configs.""" + pass + + +@configs.command("version", help="Backend version.") +def version(): + """Get version of the pipelines service.""" + http = HTTPContext() + console.print(http.configs.info()) + + +@configs.command("count", help="Count objects per collection.") +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +def count(pipelines: bool): + """Count objects in a database. + + Parameters + ---------- + pipelines : bool + Use this command on pipelines database. + """ + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + counts = http.configs.count(database) + table.add_column("Name", max_width=50, justify="left", style="blue") + table.add_column("Count", max_width=50, justify="left") + total = int() + for k, v in counts.items(): + table.add_row(k, str(v)) + total += v + table.add_section() + table.add_row("Total", str(total)) + console.print(table) + + +@configs.command("deploy", help="Deploy a workflow config.") +@click.argument( + "filename", + type=click.Path(exists=True, dir_okay=False, readable=True), + required=True, +) +def deploy(filename: click.Path): + """Deploy a workflow config. + + Parameters + ---------- + filename : click.Path + File path. + """ + http = HTTPContext() + filepath: str = str(filename) + data: Dict[str, Any] = {} + with open(filepath) as reader: + data = yaml.load(reader, Loader=SafeLoader) # type: ignore + try: + deploy_result = http.configs.deploy(data) + except requests.HTTPError as deploy_error: + console.print(deploy_error.response.json()["error_description"][0]["msg"]) + return + table.add_column("IDs", max_width=50, justify="left", style="bright_green") + if isinstance(deploy_result, list): + for _id in deploy_result: + table.add_row(_id) + if isinstance(deploy_result, dict): + for v in deploy_result.values(): + table.add_row(v) + console.print(table) + + +@configs.command("ls", help="List Configs.") +@click.option( + "name", + "--name", + "-n", + type=str, + required=False, + help="List only Configs with provided name.", +) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "quiet", + "--quiet", + "-q", + is_flag=True, + default=False, + help="Only show IDs.", +) +def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): + """List all objects.""" + database = "pipelines" if pipelines else "configs" + table.title += f" - {database.capitalize()}" + configs_colums = ["name", "version", "children", "user"] + pipelines_columns = ["status", "current_stage", "steps"] + projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + if quiet: + projection = {"id": 1} + http = HTTPContext() + objects = http.configs.get_configs( + database=database, config_name=name, projection=json.dumps(projection) + ) + + # ? Add columns for each key + table.add_column("ID", max_width=40, justify="left", style="blue") + if not quiet: + if database == "configs": + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) + if database == "pipelines": + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + + for obj in objects: + if not quiet: + if database == "configs": + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["children"])), + obj["user"], + ) + if database == "pipelines": + status = Text(obj["status"], style=status_colors[obj["status"]]) + table.add_row( + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) + ) + continue + table.add_row(obj["id"]) + console.print(table) + + +@configs.command("ps", help="Get Configs details.") +@click.argument("name", type=str, required=True) +@click.argument("id", type=str, required=True) +@click.option( + "--pipelines", + "-p", + is_flag=True, + default=False, + show_default=True, + help="Use this command for pipelines database.", +) +@click.option( + "--details", + "-d", + is_flag=True, + default=False, + show_default=True, + help="Show more details for the object.", +) +def ps(name: str, id: str, pipelines: str, details: bool): + """Show details for an object.""" + http = HTTPContext() + database = "pipelines" if pipelines else "configs" + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) + console_content = None + column_max_width = 300 + column_min_width = 40 + try: + payload = http.configs.get_configs( + database=database, config_name=name, query=query, projection=projection + )[0] + except IndexError: + error_text = Text(f"No {database.capitalize()} were found", style="red") + console_content = error_text + else: + text = Text("") + if database == "pipelines": + table.add_column( + f"Pipeline: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) + if database == "configs": + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) + if details: + table.add_column("Details", max_width=column_max_width, justify="left") + _details = safe_load(payload["yaml"]) + _details = { + k: v + for k, v in _details.items() + if k not in ["name", "version", "deployments"] + } + table.add_row(text, JSON(json.dumps(_details), indent=2)) + else: + table.add_row(text) + console_content = table + finally: + console.print(console_content) diff --git a/workflow/cli/main.py b/workflow/cli/main.py index 9791721..86a734c 100755 --- a/workflow/cli/main.py +++ b/workflow/cli/main.py @@ -4,6 +4,7 @@ from rich.console import Console # from workflow.cli.buckets import buckets +from workflow.cli.configs import configs from workflow.cli.pipelines import pipelines from workflow.cli.results import results from workflow.cli.run import run @@ -26,6 +27,7 @@ def cli(): cli.add_command(run) # cli.add_command(buckets) cli.add_command(results) +cli.add_command(configs) cli.add_command(pipelines) cli.add_command(schedules) cli.add_command(workspace) diff --git a/workflow/http/configs.py b/workflow/http/configs.py new file mode 100644 index 0000000..8cef794 --- /dev/null +++ b/workflow/http/configs.py @@ -0,0 +1,182 @@ +"""Workflow Pipelines API.""" + +from json import dumps +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from requests.models import Response +from tenacity import retry +from tenacity.stop import stop_after_attempt, stop_after_delay +from tenacity.wait import wait_random + +from workflow.http.client import Client +from workflow.utils.decorators import try_request + + +class Configs(Client): + """HTTP Client for interacting with the Configs endpoints on pipelines backend. + + Args: + Client (workflow.http.client): The base class for interacting with the backend. + + Returns: + Configs: A client for interacting with workflow-pipelines. + """ + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def deploy(self, data: Dict[str, Any]): + """Deploys a Config from payload data. + + Parameters + ---------- + data : Dict[str, Any] + YAML data. + + Returns + ------- + List[str] + ID of Config object generated. + """ + with self.session as session: + url = f"{self.baseurl}/v2/configs" + response: Response = session.post(url, json=data) + response.raise_for_status() + return response.json() + + @try_request + def count(self, database: str = "configs") -> Dict[str, Any]: + """Count all documents in a collection. + + Parameters + ---------- + database : str + Database to be used. "configs" or "pipelines". + + Returns + ------- + Dict[str, Any] + Dictionary with count. + """ + with self.session as session: + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?database={database}" + ) + response.raise_for_status() + return response.json() + + @try_request + def get_configs( + self, + database: str, + config_name: str, + query: Optional[str] = "{}", + projection: Optional[str] = "{}", + ) -> List[Dict[str, Any]]: + """View the current configurations on pipelines backend. + + Parameters + ---------- + database : str + Database to query from. + config_name : str + Config name, by default None + query : str, optional + Query payload. + projection : str, optional + Query projection. + + Returns + ------- + List[Dict[str, Any]] + List of Config payloads. + """ + with self.session as session: + # ? When using urlencode, internal dict object get single-quoted + # ? This can trigger error on workflow-pipelines backend + params = {"projection": projection, "query": query} + if config_name: + params.update({"name": config_name}) + if database: + params.update({"database": database}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + print(url) + response: Response = session.get(url=url) + response.raise_for_status() + return response.json() + + @retry( + reraise=True, + wait=wait_random(min=1.5, max=3.5), + stop=(stop_after_delay(5) | stop_after_attempt(1)), + ) + @try_request + def remove(self, pipeline: str, id: str) -> Response: + """Removes a cancelled pipeline configuration. + + Parameters + ---------- + pipeline : str + PipelineConfig name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + Response payload. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + response: Response = session.delete(url=url) + response.raise_for_status() + return response + + @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) + @try_request + def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: + """Stops the manager for a PipelineConfig. + + Parameters + ---------- + pipeline : str + Pipeline name. + id : str + PipelineConfig ID. + + Returns + ------- + List[Dict[str, Any]] + List of stopped PipelineConfig objects. + """ + with self.session as session: + query = {"id": id} + params = {"query": dumps(query), "name": pipeline} + url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + response: Response = session.put(url) + response.raise_for_status() + if response.status_code == 304: + return [] + return response.json() + + @try_request + def info(self) -> Dict[str, Any]: + """Get the version of the pipelines backend. + + Returns + ------- + Dict[str, Any] + Pipelines backend info. + """ + client_info = self.model_dump() + with self.session as session: + response: Response = session.get(url=f"{self.baseurl}/version") + response.raise_for_status() + server_info = response.json() + return {"client": client_info, "server": server_info} diff --git a/workflow/http/context.py b/workflow/http/context.py index 26a19b1..b2aa7b4 100644 --- a/workflow/http/context.py +++ b/workflow/http/context.py @@ -15,6 +15,7 @@ from workflow import DEFAULT_WORKSPACE_PATH from workflow.http.buckets import Buckets +from workflow.http.configs import Configs from workflow.http.pipelines import Pipelines from workflow.http.results import Results from workflow.http.schedules import Schedules @@ -85,6 +86,13 @@ class HTTPContext(BaseSettings): exclude=True, ) + configs: Configs = Field( + default=None, + validate_default=False, + description="Configs API Client.", + exclude=True, + ) + pipelines: Pipelines = Field( default=None, validate_default=False, @@ -131,6 +139,7 @@ def create_clients(self) -> "HTTPContext": "results": Results, "pipelines": Pipelines, "schedules": Schedules, + "configs": Configs, } logger.debug(f"creating http clients for {list(clients.keys())}") config: Dict[str, Any] = read.workspace(self.workspace) diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py new file mode 100644 index 0000000..a39691c --- /dev/null +++ b/workflow/utils/renderers.py @@ -0,0 +1,95 @@ +"""Functions to render objects to rich.console.""" + +import datetime as dt +from json import dumps +from typing import Any, Dict + +from rich.text import Text + +from workflow.http.context import HTTPContext +from workflow.utils.variables import status_colors, status_symbols + + +def render_pipeline(payload: Dict[str, Any]) -> Text: + """Renders a pipeline to rich.Text(). + + Parameters + ---------- + payload : Dict[str, Any] + Pipeline payload. + + Returns + ------- + Text + Rendered text. + """ + steps_field = "steps" + time_fields = ["creation", "start", "stop"] + text = Text() + for k, v in payload.items(): + key_value_text = Text() + if not v: + continue + if k in time_fields and v: + v = dt.datetime.fromtimestamp(v) + if k == steps_field: + key_value_text = Text(f"{k}: \n", style="bright_blue") + for step in v: + key_value_text.append(f" {step['name']}:") + key_value_text.append(f"{status_symbols[step['status']]}\n") + else: + key_value_text = Text(f"{k}: ", style="bright_blue") + key_value_text.append( + f"{v}\n", style="white" if k != "status" else status_colors[v] + ) + text.append_text(key_value_text) + return text + + +def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: + """Renders a config to rich.Text(). + + Parameters + ---------- + http : HTTPContext + Workflow Http context. + payload : Dict[str, Any] + Config payload. + + Returns + ------- + Text + Rendered text. + """ + text = Text() + hidden_keys = ["yaml", "services", "name"] + query = dumps({"id": {"$in": payload["children"]}}) + projection = dumps({"id": 1, "status": 1}) + children_statuses = http.configs.get_configs( + database="pipelines", + config_name=payload["name"], + query=query, + projection=projection, + ) + + for k, v in payload.items(): + if k in hidden_keys: + continue + key_value_text = Text() + if k == "children": + key_value_text.append(f"{k}: \n", style="bright_blue") + for child in children_statuses: + key_value_text.append( + f"\t{child['id']}: ", style=status_colors[child["status"]] + ) + key_value_text.append( + f"{status_symbols[child['status']]}\n", + style=status_colors[child["status"]], + ) + text.append_text(key_value_text) + continue + key_value_text.append(f"{k}: ", style="bright_blue") + key_value_text.append(f"{v}\n", style="white") + text.append_text(key_value_text) + + return text diff --git a/workflow/workspaces/development.yml b/workflow/workspaces/development.yml index f42ca8a..cca26d8 100644 --- a/workflow/workspaces/development.yml +++ b/workflow/workspaces/development.yml @@ -17,6 +17,7 @@ archive: http: baseurls: + configs: http://localhost:8001 pipelines: http://localhost:8001 schedules: http://localhost:8001 buckets: http://localhost:8004 From f8b1547848259ba5b13e3ef2a1635aba9c41b102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 3 May 2024 11:15:30 -0400 Subject: [PATCH 23/41] refactor(http/configs.py): removing print statement --- workflow/http/configs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 8cef794..616e185 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -104,7 +104,6 @@ def get_configs( if database: params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" - print(url) response: Response = session.get(url=url) response.raise_for_status() return response.json() From 25ea64819b45bebaa2404ccb411a8c334e890dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Mon, 6 May 2024 13:59:35 -0400 Subject: [PATCH 24/41] feat(cli/schedules.py): fixing bad formatting --- workflow/cli/schedules.py | 35 ++++++++++++++++++++++++++--------- workflow/http/schedules.py | 18 +++++++++--------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 02b0121..93cee00 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -1,5 +1,6 @@ """Manage workflow pipelines schedules.""" +from datetime import datetime from typing import Any, Dict, Optional, Tuple import click @@ -88,7 +89,7 @@ def ls(name: Optional[str] = None, quiet: Optional[bool] = False): lives_text = Text(str(lives) if lives > -1 else "\u221e") table.add_row( schedule_obj["id"], - schedule_obj["pipeline_config"]["name"], + schedule_obj["config"]["name"], status, lives_text, str(schedule_obj["has_spawned"]), @@ -165,6 +166,7 @@ def ps(id: str, detail: Optional[bool] = False): "has_spawned": "Has Spawned", "status": "Status", "next_time": "Next Execution", + "history": "History", } try: payload = http.schedules.get_schedule(query) @@ -173,26 +175,41 @@ def ps(id: str, detail: Optional[bool] = False): console_content = error_text else: table.add_column( - f"Scheduled Pipeline: {payload['pipeline_config']['name']}", + f"Scheduled Pipeline: {payload['config']['name']}", max_width=120, + min_width=50, justify="left", ) text = Text("") for k, v in payload.items(): - if k == "pipeline_config": + if k == "config": + continue + if k == "history": + key_value_text = Text( + f"{key_nicknames.get(k, k)}: \n", style="bright_green" + ) + for history in v: + history_dt = datetime.fromisoformat(history[0]) + legible_dt = history_dt.strftime("%B %d, %Y at %I:%M:%S %p") + id = history[1] + key_value_text.append(f"-- {legible_dt}:", style="bright_blue") + key_value_text.append(f"\n\t{id}\n\n", style="white") + text.append_text(key_value_text) continue key_value_text = Text(f"{key_nicknames.get(k, k)}: ", style="bright_green") key_value_text.append( f"{v}\n", style="white" if k != "status" else status_colors[v] ) text.append_text(key_value_text) - table.add_row(text) if detail: - table.add_section() - table.add_row(Text("Payload Details", style="magenta")) - table.add_section() - this_payload = JSON.from_data(payload["pipeline_config"], indent=2) - table.add_row(this_payload) + # table.add_section() + table.add_column("Details", style="magenta") + # table.add_row(Text("Payload Details", style="magenta")) + # table.add_section() + this_payload = JSON.from_data(payload["config"], indent=2) + table.add_row(text, this_payload) + else: + table.add_row(text) console_content = table finally: console.print(console_content) diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py index f092eb9..938cac6 100644 --- a/workflow/http/schedules.py +++ b/workflow/http/schedules.py @@ -42,7 +42,7 @@ def deploy(self, data: Dict[str, Any]): IDs of Schedule objects generated. """ with self.session as session: - url = f"{self.baseurl}/v1/schedule" + url = f"{self.baseurl}/v2/schedule" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -63,7 +63,7 @@ def get_schedule(self, query: Dict[str, Any]) -> Dict[str, Any]: """ with self.session as session: params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json()[0] @@ -90,7 +90,7 @@ def remove(self, id: str) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query)} - url = f"{self.baseurl}/v1/schedule?{urlencode(params)}" + url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -110,7 +110,7 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: List of schedule payloads. """ with self.session as session: - query = dumps({"pipeline_config.name": schedule_name}) + query = dumps({"config.name": schedule_name}) projection = dumps( { "id": True, @@ -119,13 +119,13 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: "has_spawned": True, "next_time": True, "crontab": True, - "pipeline_config.name": True, + "config.name": True, } ) url = ( - f"{self.baseurl}/v1/schedule?projection={projection}" + f"{self.baseurl}/v2/schedule?projection={projection}" if schedule_name is None - else f"{self.baseurl}/v1/schedule?query={query}&projection={projection}" + else f"{self.baseurl}/v2/schedule?query={query}&projection={projection}" ) response: Response = session.get(url=url) response.raise_for_status() @@ -148,9 +148,9 @@ def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any] with self.session as session: query = dumps({"name": schedule_name}) url = ( - f"{self.baseurl}/v1/schedule/count" + f"{self.baseurl}/v2/schedule/count" if not schedule_name - else f"{self.baseurl}/v1/schedule/count?query={query}" + else f"{self.baseurl}/v2/schedule/count?query={query}" ) response: Response = session.get(url=url) response.raise_for_status() From bb2eda0e9f86ecfe911b1ccdc558b80fd398535f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 10 May 2024 12:17:21 -0400 Subject: [PATCH 25/41] fix(cli): addressing comments on PR --- workflow/cli/configs.py | 215 ++++++++++++++++++------------------ workflow/cli/pipelines.py | 151 ++++++------------------- workflow/cli/schedules.py | 3 +- workflow/http/configs.py | 28 ++--- workflow/http/pipelines.py | 124 ++++----------------- workflow/utils/renderers.py | 11 +- workflow/utils/variables.py | 8 +- 7 files changed, 178 insertions(+), 362 deletions(-) diff --git a/workflow/cli/configs.py b/workflow/cli/configs.py index 88e42a3..cf88142 100644 --- a/workflow/cli/configs.py +++ b/workflow/cli/configs.py @@ -15,18 +15,17 @@ from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.renderers import render_config, render_pipeline -from workflow.utils.variables import status_colors +from workflow.utils.renderers import render_config pretty.install() console = Console() table = Table( - title="\nWorkflow Pipelines", + title="\nWorkflow Configs", show_header=True, header_style="magenta", title_style="bold magenta", - min_width=10, + min_width=50, ) BASE_URL = "https://frb.chimenet.ca/pipelines" @@ -47,26 +46,10 @@ def version(): @configs.command("count", help="Count objects per collection.") -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) -def count(pipelines: bool): - """Count objects in a database. - - Parameters - ---------- - pipelines : bool - Use this command on pipelines database. - """ +def count(): + """Count objects in a database.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - counts = http.configs.count(database) + counts = http.configs.count() table.add_column("Name", max_width=50, justify="left", style="blue") table.add_column("Count", max_width=50, justify="left") total = int() @@ -102,33 +85,29 @@ def deploy(filename: click.Path): except requests.HTTPError as deploy_error: console.print(deploy_error.response.json()["error_description"][0]["msg"]) return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) + table.add_column( + "Deploy Result", + min_width=35, + max_width=50, + justify="left", + style="bright_green", + ) if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) + for k, v in deploy_result.items(): + if k == "config": + row_text = Text(f"{k}: ", style="magenta") + row_text.append(f"{v}", style="white") + table.add_row(row_text) + if k == "pipelines": + row_text = Text(f"{k}:\n", style="bright_blue") + for id in deploy_result[k]: + row_text.append(f"\t{id}\n", style="white") + table.add_row(row_text) console.print(table) @configs.command("ls", help="List Configs.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Configs with provided name.", -) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -137,54 +116,37 @@ def deploy(filename: click.Path): default=False, help="Only show IDs.", ) -def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False): +def ls(name: Optional[str] = None, quiet: bool = False): """List all objects.""" - database = "pipelines" if pipelines else "configs" - table.title += f" - {database.capitalize()}" - configs_colums = ["name", "version", "children", "user"] - pipelines_columns = ["status", "current_stage", "steps"] - projection = {"yaml": 0, "deployments": 0} if database == "configs" else {} + configs_colums = ["name", "version", "pipelines", "user"] + projection = {"yaml": 0, "deployments": 0} if quiet: projection = {"id": 1} http = HTTPContext() objects = http.configs.get_configs( - database=database, config_name=name, projection=json.dumps(projection) + config_name=name, projection=json.dumps(projection) ) # ? Add columns for each key table.add_column("ID", max_width=40, justify="left", style="blue") if not quiet: - if database == "configs": - for key in configs_colums: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - style="bright_green" if key == "name" else "white", - ) - if database == "pipelines": - for key in pipelines_columns: - table.add_column( - key.capitalize().replace("_", " "), - max_width=50, - justify="left", - ) + for key in configs_colums: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + style="bright_green" if key == "name" else "white", + ) for obj in objects: if not quiet: - if database == "configs": - table.add_row( - obj["id"], - obj["name"], - obj["version"], - str(len(obj["children"])), - obj["user"], - ) - if database == "pipelines": - status = Text(obj["status"], style=status_colors[obj["status"]]) - table.add_row( - obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) - ) + table.add_row( + obj["id"], + obj["name"], + obj["version"], + str(len(obj["pipelines"])), + obj["user"], + ) continue table.add_row(obj["id"]) console.print(table) @@ -193,26 +155,16 @@ def ls(name: Optional[str] = None, pipelines: bool = False, quiet: bool = False) @configs.command("ps", help="Get Configs details.") @click.argument("name", type=str, required=True) @click.argument("id", type=str, required=True) -@click.option( - "--pipelines", - "-p", - is_flag=True, - default=False, - show_default=True, - help="Use this command for pipelines database.", -) @click.option( "--details", - "-d", is_flag=True, default=False, show_default=True, help="Show more details for the object.", ) -def ps(name: str, id: str, pipelines: str, details: bool): +def ps(name: str, id: str, details: bool): """Show details for an object.""" http = HTTPContext() - database = "pipelines" if pipelines else "configs" query: str = json.dumps({"id": id}) projection: str = json.dumps({}) console_content = None @@ -220,29 +172,20 @@ def ps(name: str, id: str, pipelines: str, details: bool): column_min_width = 40 try: payload = http.configs.get_configs( - database=database, config_name=name, query=query, projection=projection + config_name=name, query=query, projection=projection )[0] except IndexError: - error_text = Text(f"No {database.capitalize()} were found", style="red") + error_text = Text("No Configs were found", style="red") console_content = error_text else: text = Text("") - if database == "pipelines": - table.add_column( - f"Pipeline: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_pipeline(payload)) - if database == "configs": - table.add_column( - f"Config: {name}", - min_width=column_min_width, - max_width=column_max_width, - justify="left", - ) - text.append(render_config(http, payload)) + table.add_column( + f"Config: {name}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_config(http, payload)) if details: table.add_column("Details", max_width=column_max_width, justify="left") _details = safe_load(payload["yaml"]) @@ -254,6 +197,60 @@ def ps(name: str, id: str, pipelines: str, details: bool): table.add_row(text, JSON(json.dumps(_details), indent=2)) else: table.add_row(text) + table.add_section() + table.add_row( + Text("Explore pipelines in detail: \n", style="magenta i").append( + "workflow pipelines ps ", + style="dark_blue on cyan", + ) + ) console_content = table finally: console.print(console_content) + + +@configs.command("stop", help="Stop managers for a Config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def stop(config: str, id: str): + """Stop managers for a Config.""" + http = HTTPContext() + stop_result = http.configs.stop(config, id) + if not any(stop_result): + text = Text("No configurations were stopped.", style="red") + console.print(text) + return + table.add_column("Stopped IDs", max_width=50, justify="left") + text = Text() + for k in stop_result.keys(): + if k == "stopped_config": + text.append("Config: ", style="bright_blue") + text.append(f"{stop_result[k]}\n") + if k == "stopped_pipelines": + text.append("Pipelines: \n", style="bright_blue") + for id in stop_result["stopped_pipelines"]: + text.append(f"\t{id}\n") + table.add_row(text) + console.print(table) + + +@configs.command("rm", help="Remove a config.") +@click.argument("config", type=str, required=True) +@click.argument("id", type=str, required=True) +def rm(config: str, id: str): + """Remove a config.""" + http = HTTPContext() + content = None + try: + delete_result = http.configs.remove(config, id) + if delete_result.status_code == 204: + text = Text("No pipeline configurations were deleted.", style="red") + content = text + except Exception as e: + text = Text(f"No configurations were deleted.\nError: {e}", style="red") + content = text + else: + table.add_column("Deleted IDs", max_width=50, justify="left", style="red") + table.add_row(id) + content = table + console.print(content) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index cf4c338..83a254c 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,20 +1,18 @@ """Manage workflow pipelines.""" -import datetime as dt import json -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional import click import requests -import yaml from rich import pretty from rich.console import Console from rich.table import Table from rich.text import Text -from yaml.loader import SafeLoader from workflow.http.context import HTTPContext -from workflow.utils.variables import status_colors, status_symbols +from workflow.utils.renderers import render_pipeline +from workflow.utils.variables import status_colors pretty.install() console = Console() @@ -45,14 +43,7 @@ def version(): @pipelines.command("ls", help="List pipelines.") -@click.option( - "name", - "--name", - "-n", - type=str, - required=False, - help="List only Pipelines with provided name.", -) +@click.argument("name", type=str, required=False) @click.option( "quiet", "--quiet", @@ -63,21 +54,24 @@ def version(): ) def ls(name: Optional[str] = None, quiet: Optional[bool] = False): """List all pipelines.""" + pipelines_columns = ["status", "current_stage", "steps"] http = HTTPContext() - objects = http.pipelines.list_pipeline_configs(name) - table.add_column("ID", max_width=50, justify="left", style="blue") - if not quiet: - table.add_column("Name", max_width=50, justify="left", style="bright_green") - table.add_column("Status", max_width=50, justify="left") - table.add_column("Stage", max_width=50, justify="left") - for config in objects: - status = Text(config["status"], style=status_colors[config["status"]]) + objects = http.pipelines.list_pipelines(name) + table.add_column("ID", max_width=100, justify="left", style="blue") + for key in pipelines_columns: + table.add_column( + key.capitalize().replace("_", " "), + max_width=50, + justify="left", + ) + for obj in objects: if not quiet: + status = Text(obj["status"], style=status_colors[obj["status"]]) table.add_row( - config["id"], config["name"], status, str(config["current_stage"]) + obj["id"], status, str(obj["current_stage"]), str(len(obj["steps"])) ) continue - table.add_row(config["id"]) + table.add_row(obj["id"]) console.print(table) @@ -97,118 +91,39 @@ def count(): console.print(table) -@pipelines.command("deploy", help="Deploy a workflow pipeline.") -@click.argument( - "filename", - type=click.Path(exists=True, dir_okay=False, readable=True), - required=True, -) -def deploy(filename: click.Path): - """Deploy a workflow pipeline.""" - http = HTTPContext() - filepath: str = str(filename) - data: Dict[str, Any] = {} - with open(filepath) as reader: - data = yaml.load(reader, Loader=SafeLoader) # type: ignore - try: - deploy_result = http.pipelines.deploy(data) - except requests.HTTPError as deploy_error: - console.print(deploy_error.response.json()["message"]) - return - table.add_column("IDs", max_width=50, justify="left", style="bright_green") - if isinstance(deploy_result, list): - for _id in deploy_result: - table.add_row(_id) - if isinstance(deploy_result, dict): - for v in deploy_result.values(): - table.add_row(v) - console.print(table) - - @pipelines.command("ps", help="Get pipeline details.") @click.argument("pipeline", type=str, required=True) @click.argument("id", type=str, required=True) def ps(pipeline: str, id: str): """List a pipeline configuration in detail.""" http = HTTPContext() - query: Dict[str, Any] = {"id": id} + query: str = json.dumps({"id": id}) + projection: str = json.dumps({}) console_content = None - projection = {"name": False} - time_fields = ["creation", "start", "stop"] + column_max_width = 300 + column_min_width = 40 try: - payload = http.pipelines.get_pipeline_config(pipeline, query, projection) + payload = http.pipelines.get_pipelines( + name=pipeline, query=query, projection=projection + )[0] except IndexError: - error_text = Text("No PipelineConfig were found", style="red") + error_text = Text("No Pipelines were found", style="red") console_content = error_text else: - table.add_column(f"Pipeline: {pipeline}", max_width=120, justify="left") - text = Text("") - for k, v in payload.items(): - key_value_text = Text() - if k in time_fields and v: - v = dt.datetime.fromtimestamp(v) - if k == "pipeline": - key_value_text = Text(f"{k}: \n", style="bright_blue") - for step in v: - key_value_text.append(f" {step['name']}:") - key_value_text.append(f"{status_symbols[step['status']]}\n") - else: - key_value_text = Text(f"{k}: ", style="bright_blue") - key_value_text.append( - f"{v}\n", style="white" if k != "status" else status_colors[v] - ) - text.append_text(key_value_text) - + text = Text() + table.add_column( + f"Pipeline: {pipeline}", + min_width=column_min_width, + max_width=column_max_width, + justify="left", + ) + text.append(render_pipeline(payload)) table.add_row(text) console_content = table finally: console.print(console_content) -@pipelines.command("stop", help="Kill a running pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -def stop(pipeline: str, id: Tuple[str]): - """Kill a running pipeline.""" - http = HTTPContext() - stop_result = http.pipelines.stop(pipeline, id) - if not any(stop_result): - text = Text("No pipeline configurations were stopped.", style="red") - console.print(text) - return - table.add_column("Stopped IDs", max_width=50, justify="left") - for config in stop_result: - table.add_row(config["id"]) - console.print(table) - - -@pipelines.command("rm", help="Remove a pipeline.") -@click.argument("pipeline", type=str, required=True) -@click.argument("id", type=str, required=True) -@click.option( - "--schedule", "-sch", is_flag=True, help="For interacting with the Schedule API." -) -def rm(pipeline: str, id: Tuple[str], schedule: bool): - """Remove a pipeline.""" - http = HTTPContext() - content = None - try: - delete_result = http.pipelines.remove(pipeline, id, schedule) - if delete_result.status_code == 204: - text = Text("No pipeline configurations were deleted.", style="red") - content = text - except Exception as e: - text = Text( - f"No pipeline configurations were deleted.\nError: {e}", style="red" - ) - content = text - else: - table.add_column("Deleted IDs", max_width=50, justify="left", style="red") - table.add_row(id) - content = table - console.print(content) - - def status( pipeline: Optional[str] = None, query: Optional[Dict[str, Any]] = None, diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 93cee00..853161a 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -148,8 +148,7 @@ def deploy(filename: click.Path): @schedules.command("ps", help="Get schedule details.") @click.argument("id", type=str, required=True) @click.option( - "--detail", - "-d", + "--details", is_flag=True, show_default=True, help="Returns the Schedule Payload.", diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 616e185..43ef565 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -49,14 +49,9 @@ def deploy(self, data: Dict[str, Any]): return response.json() @try_request - def count(self, database: str = "configs") -> Dict[str, Any]: + def count(self) -> Dict[str, Any]: """Count all documents in a collection. - Parameters - ---------- - database : str - Database to be used. "configs" or "pipelines". - Returns ------- Dict[str, Any] @@ -64,7 +59,7 @@ def count(self, database: str = "configs") -> Dict[str, Any]: """ with self.session as session: response: Response = session.get( - url=f"{self.baseurl}/v2/configs/count?database={database}" + url=f"{self.baseurl}/v2/configs/count?database=configs" ) response.raise_for_status() return response.json() @@ -72,7 +67,6 @@ def count(self, database: str = "configs") -> Dict[str, Any]: @try_request def get_configs( self, - database: str, config_name: str, query: Optional[str] = "{}", projection: Optional[str] = "{}", @@ -81,8 +75,6 @@ def get_configs( Parameters ---------- - database : str - Database to query from. config_name : str Config name, by default None query : str, optional @@ -101,8 +93,6 @@ def get_configs( params = {"projection": projection, "query": query} if config_name: params.update({"name": config_name}) - if database: - params.update({"database": database}) url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() @@ -114,15 +104,15 @@ def get_configs( stop=(stop_after_delay(5) | stop_after_attempt(1)), ) @try_request - def remove(self, pipeline: str, id: str) -> Response: + def remove(self, config: str, id: str) -> Response: """Removes a cancelled pipeline configuration. Parameters ---------- - pipeline : str - PipelineConfig name. + config : str + Config name. id : str - PipelineConfig ID. + Config ID. Returns ------- @@ -131,8 +121,8 @@ def remove(self, pipeline: str, id: str) -> Response: """ with self.session as session: query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + params = {"query": dumps(query), "name": config} + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -157,7 +147,7 @@ def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() if response.status_code == 304: diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index fddba7d..971051c 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -1,13 +1,9 @@ """Workflow Pipelines API.""" -from json import dumps from typing import Any, Dict, List, Optional from urllib.parse import urlencode from requests.models import Response -from tenacity import retry -from tenacity.stop import stop_after_attempt, stop_after_delay -from tenacity.wait import wait_random from workflow.http.client import Client from workflow.utils.decorators import try_request @@ -23,31 +19,6 @@ class Pipelines(Client): Pipelines: A client for interacting with the Pipelines backend. """ - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def deploy(self, data: Dict[str, Any]): - """Deploys a PipelineConfig from payload data. - - Parameters - ---------- - data : Dict[str, Any] - YAML data. - - Returns - ------- - List[str] - IDs of PipelineConfig objects generated. - """ - with self.session as session: - url = f"{self.baseurl}/v1/pipelines" - response: Response = session.post(url, json=data) - response.raise_for_status() - return response.json() - @try_request def count(self) -> Dict[str, Any]: """Count all documents in a collection. @@ -58,20 +29,21 @@ def count(self) -> Dict[str, Any]: Dictionary with count. """ with self.session as session: - response: Response = session.get(url=f"{self.baseurl}/v1/pipelines/count") + params = {"database": "pipelines"} + response: Response = session.get( + url=f"{self.baseurl}/v2/configs/count?{urlencode(params)}" + ) response.raise_for_status() return response.json() @try_request - def list_pipeline_configs( - self, config_name: Optional[str] = None - ) -> List[Dict[str, Any]]: + def list_pipelines(self, name: Optional[str] = None) -> List[Dict[str, Any]]: """View the current pipeline configurations in the pipelines backend. Parameters ---------- - config_name : Optional[str], optional - PipelineConfig name, by default None + name : Optional[str], optional + Config name, by default None Returns ------- @@ -79,25 +51,24 @@ def list_pipeline_configs( List of PipelineConfig payloads. """ with self.session as session: - url = ( - f"{self.baseurl}/v1/pipelines" - if config_name is None - else f'{self.baseurl}/v1/pipelines?query={{"name":"{config_name}"}}' - ) + params = {"database": "pipelines"} + if name: + params.update({"name": name}) + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() @try_request - def get_pipeline_config( - self, collection: str, query: Dict[str, Any], projection: Dict[str, Any] + def get_pipelines( + self, name: str, query: Dict[str, Any], projection: Dict[str, Any] ) -> Dict[str, Any]: """Gets details for one pipeline configuration. Parameters ---------- - collection : str - PipelineConfig name. + name : str + Config name. query : Dict[str, Any] Dictionary with search parameters. projection : Dict[str, Any] @@ -110,69 +81,14 @@ def get_pipeline_config( """ with self.session as session: params = { - "query": dumps(query), - "name": collection, - "projection": dumps(projection), + "query": query, + "name": name, + "projection": projection, + "database": "pipelines", } - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" + url = f"{self.baseurl}/v2/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() - return response.json()[0] - - @retry( - reraise=True, - wait=wait_random(min=1.5, max=3.5), - stop=(stop_after_delay(5) | stop_after_attempt(1)), - ) - @try_request - def remove(self, pipeline: str, id: str) -> Response: - """Removes a cancelled pipeline configuration. - - Parameters - ---------- - pipeline : str - PipelineConfig name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - Response payload. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines?{urlencode(params)}" - response: Response = session.delete(url=url) - response.raise_for_status() - return response - - @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) - @try_request - def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: - """Stops the manager for a PipelineConfig. - - Parameters - ---------- - pipeline : str - Pipeline name. - id : str - PipelineConfig ID. - - Returns - ------- - List[Dict[str, Any]] - List of stopped PipelineConfig objects. - """ - with self.session as session: - query = {"id": id} - params = {"query": dumps(query), "name": pipeline} - url = f"{self.baseurl}/v1/pipelines/cancel?{urlencode(params)}" - response: Response = session.put(url) - response.raise_for_status() - if response.status_code == 304: - return [] return response.json() @try_request diff --git a/workflow/utils/renderers.py b/workflow/utils/renderers.py index a39691c..c523a04 100644 --- a/workflow/utils/renderers.py +++ b/workflow/utils/renderers.py @@ -63,11 +63,10 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: """ text = Text() hidden_keys = ["yaml", "services", "name"] - query = dumps({"id": {"$in": payload["children"]}}) + query = dumps({"id": {"$in": payload["pipelines"]}}) projection = dumps({"id": 1, "status": 1}) - children_statuses = http.configs.get_configs( - database="pipelines", - config_name=payload["name"], + pipelines_statuses = http.pipelines.get_pipelines( + name=payload["name"], query=query, projection=projection, ) @@ -76,9 +75,9 @@ def render_config(http: HTTPContext, payload: Dict[str, Any]) -> Text: if k in hidden_keys: continue key_value_text = Text() - if k == "children": + if k == "pipelines": key_value_text.append(f"{k}: \n", style="bright_blue") - for child in children_statuses: + for child in pipelines_statuses: key_value_text.append( f"\t{child['id']}: ", style=status_colors[child["status"]] ) diff --git a/workflow/utils/variables.py b/workflow/utils/variables.py index f8f8149..9cfe11b 100644 --- a/workflow/utils/variables.py +++ b/workflow/utils/variables.py @@ -14,11 +14,11 @@ status_symbols = { "created": "\U000026AA", # white - "queued": "\u23F3", # hourglass - "active": "\U0001F7E2", # green - "running": "\u2699", # gear + "queued": "\U0001F4C5", # 📅 + "active": "\U0001F7E2", # green circle + "running": "\U0001F3C3", # 🏃 "success": "\U00002705", # Green check "paused": "\U0001F7E1", # yellow "failure": "\U0000274C", # cross mark - "cancelled": "\U0001F534", # red + "cancelled": "\U0001F6AB", # 🚫 } From 7ee34b47f78e46e4ecde942be971d150d0308353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 30 Apr 2024 14:08:51 -0400 Subject: [PATCH 26/41] feat(cli): improvements to pipelines and schedules commands --- workflow/cli/pipelines.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index 83a254c..7f5528c 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,5 +1,6 @@ """Manage workflow pipelines.""" +import datetime as dt import json from typing import Any, Dict, Optional From 834ca8c8c9669b54effe693c7e8b1311d80463c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Fri, 10 May 2024 12:17:21 -0400 Subject: [PATCH 27/41] fix(cli): addressing comments on PR --- workflow/cli/pipelines.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/cli/pipelines.py b/workflow/cli/pipelines.py index 7f5528c..83a254c 100644 --- a/workflow/cli/pipelines.py +++ b/workflow/cli/pipelines.py @@ -1,6 +1,5 @@ """Manage workflow pipelines.""" -import datetime as dt import json from typing import Any, Dict, Optional From 371375a78a5ed63acad3f7505686e06d08a863f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:30:33 -0400 Subject: [PATCH 28/41] test(test_http_context.py): adding tests Tests added for pipelines client on HTTPContext. --- docker-compose-tutorial.yml | 4 +-- tests/test_http_context.py | 62 +++++++++++++++++++++++++++++++++++-- workflow/http/configs.py | 10 +++--- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml index 818d701..4aad364 100644 --- a/docker-compose-tutorial.yml +++ b/docker-compose-tutorial.yml @@ -1,6 +1,6 @@ services: pipelines_api: - image: chimefrb/pipelines:latest + image: chimefrb/pipelines:pipelines-v2.7.1 container_name: pipelines_api command: python -m pipelines.server ports: @@ -27,7 +27,7 @@ services: restart: always pipelines_managers: - image: chimefrb/pipelines:latest + image: chimefrb/pipelines:pipelines-v2.7.1 container_name: pipelines_managers command: python -m managers.server ports: diff --git a/tests/test_http_context.py b/tests/test_http_context.py index c173cd2..92fd66b 100644 --- a/tests/test_http_context.py +++ b/tests/test_http_context.py @@ -7,6 +7,25 @@ from workflow.cli.workspace import set, unset from workflow.http.context import HTTPContext +config = { + "version": "1", + "name": "demo", + "defaults": {"user": "test"}, + "pipeline": { + "steps": [ + { + "name": "stage-1-a", + "stage": 1, + "matrix": {"event": [123456, 654321], "site": ["chime", "canfar"]}, + "work": { + "site": "${{ matrix.site }}", + "command": ["ls", "${{ matrix.event }}"], + }, + }, + ], + }, +} + class TestHTTPContext: def test_can_be_instantiated(self): @@ -23,9 +42,48 @@ def test_cannot_be_instantiated_without_workspace(self): HTTPContext() runner.invoke(set, ["development"]) - @pytest.mark.skip def test_clients_connect_to_base_url(self): """Tests HTTPContext.clients have connection to their proper backend.""" http = HTTPContext() assert isinstance(http.buckets.info(), dict) - # TODO test results and pipelines + assert isinstance(http.results.info(), dict) + assert isinstance(http.configs.info(), dict) + assert isinstance(http.pipelines.info(), dict) + + def test_http_configs_deploy(self): + http = HTTPContext() + response = http.configs.deploy(config) + assert isinstance(response, dict) + assert "config" in response.keys() + assert "pipelines" in response.keys() + + def test_http_configs_list(self): + http = HTTPContext() + response = http.configs.get_configs(config_name=None) + assert isinstance(response, list) + + def test_http_configs_count(self): + http = HTTPContext() + response = http.configs.count() + assert isinstance(response, dict) + + def test_http_configs_remove_process(self): + http = HTTPContext() + # ? Post + post_response = http.configs.deploy(config) + stop_response = http.configs.stop(config["name"], post_response["config"]) + assert stop_response["stopped_config"] == post_response["config"] + remove_response = http.configs.remove(config["name"], post_response["config"]) + assert remove_response.status_code == 204 + + def test_http_pipelines_list(self): + http = HTTPContext() + response = http.pipelines.list_pipelines() + assert isinstance(response, list) + + def test_http_pipelines_get(self): + http = HTTPContext() + count_response = http.pipelines.count() + response = http.pipelines.get_pipelines(name="demo", query={}, projection={}) + assert count_response + assert isinstance(response, list) diff --git a/workflow/http/configs.py b/workflow/http/configs.py index 43ef565..b66c93f 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -129,15 +129,15 @@ def remove(self, config: str, id: str) -> Response: @retry(wait=wait_random(min=0.5, max=1.5), stop=(stop_after_delay(30))) @try_request - def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: + def stop(self, config_name: str, id: str) -> List[Dict[str, Any]]: """Stops the manager for a PipelineConfig. Parameters ---------- - pipeline : str - Pipeline name. + config_name : str + Config name. id : str - PipelineConfig ID. + Config ID. Returns ------- @@ -146,7 +146,7 @@ def stop(self, pipeline: str, id: str) -> List[Dict[str, Any]]: """ with self.session as session: query = {"id": id} - params = {"query": dumps(query), "name": pipeline} + params = {"query": dumps(query), "name": config_name} url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() From 719d85e04990f5693d52ce23c06a3509a291c2ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:36:51 -0400 Subject: [PATCH 29/41] ci(ci.yml): adding steps to tests job Adding steps for: starting docker services needed for tests; setting development workspace and killing services after tests are completed. --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0298b5..5357ae5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,18 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'poetry' + - + name: Run services + run: | + docker-compose -f .\docker-compose-tutorial.yml up -d --build - name: Installing workflow dependencies run: | poetry install + - + name: Set workspace for testing + run: | + poetry run workflow workspace set development - name: Run workflow tests run: | @@ -67,3 +75,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: "coverage.lcov" + - + name: Kill services + if: always() + run: | + docker-compose -f .\docker-compose-tutorial.yml down -v From 41b7c920c9989fa49a63e213c1463cb970b35bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:38:43 -0400 Subject: [PATCH 30/41] ci(ci.yml): changing docker-compose file name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5357ae5..09c0305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Run services run: | - docker-compose -f .\docker-compose-tutorial.yml up -d --build + docker-compose -f docker-compose-tutorial.yml up -d --build - name: Installing workflow dependencies run: | @@ -79,4 +79,4 @@ jobs: name: Kill services if: always() run: | - docker-compose -f .\docker-compose-tutorial.yml down -v + docker-compose -f docker-compose-tutorial.yml down -v From 4ed4f67861983c61678582d538e3215642af9d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Thu, 23 May 2024 09:41:07 -0400 Subject: [PATCH 31/41] ci(ci.yml): adding docker login step --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c0305..9374345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,12 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'poetry' + - + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Run services run: | From 1fd950a69303bd7afd4660b9d099e42abbd2420a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 28 May 2024 10:36:52 -0400 Subject: [PATCH 32/41] fix(workflow/utils/read.py): changing active workspace filename --- workflow/utils/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/utils/read.py b/workflow/utils/read.py index 4679f52..f75fb7d 100644 --- a/workflow/utils/read.py +++ b/workflow/utils/read.py @@ -93,7 +93,7 @@ def get_active_workspace() -> Text: Text Text instance with info. """ - _workspace = "active" + _workspace = "workspace" text = Text() if _workspace in localstems: for possibility in localspaces.glob(f"{_workspace}.y*ml"): From 246371139bf760c28948b24000d0d1a0dda09dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Wed, 29 May 2024 13:02:51 -0400 Subject: [PATCH 33/41] fix(cli/schedules.py): fixing variable name --- workflow/cli/schedules.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/workflow/cli/schedules.py b/workflow/cli/schedules.py index 853161a..6310db3 100644 --- a/workflow/cli/schedules.py +++ b/workflow/cli/schedules.py @@ -153,7 +153,7 @@ def deploy(filename: click.Path): show_default=True, help="Returns the Schedule Payload.", ) -def ps(id: str, detail: Optional[bool] = False): +def ps(id: str, details: Optional[bool] = False): """Gets schedules details.""" http = HTTPContext() query: Dict[str, Any] = {"id": id} @@ -200,11 +200,8 @@ def ps(id: str, detail: Optional[bool] = False): f"{v}\n", style="white" if k != "status" else status_colors[v] ) text.append_text(key_value_text) - if detail: - # table.add_section() + if details: table.add_column("Details", style="magenta") - # table.add_row(Text("Payload Details", style="magenta")) - # table.add_section() this_payload = JSON.from_data(payload["config"], indent=2) table.add_row(text, this_payload) else: From b82e4374c7f819968b173eab24175fcbcb1ac9c5 Mon Sep 17 00:00:00 2001 From: "Shiny Brar (he/il)" Date: Wed, 29 May 2024 15:54:14 -0400 Subject: [PATCH 34/41] feat(workflow-dev): added workspace configuration for live system tests --- workflow/workspaces/workflow-dev.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 workflow/workspaces/workflow-dev.yml diff --git a/workflow/workspaces/workflow-dev.yml b/workflow/workspaces/workflow-dev.yml new file mode 100644 index 0000000..ae6eb9c --- /dev/null +++ b/workflow/workspaces/workflow-dev.yml @@ -0,0 +1,17 @@ +workspace: workflow-dev + +# List the valid sites for this workspace +sites: + - local + +http: + baseurls: + configs: http://localhost:37015 + pipelines: http://localhost:37015 + schedules: http://localhost:37015 + buckets: https://frb.chimenet.ca/buckets + results: https://frb.chimenet.ca/results + +config: + archive: + results: true From 7c4034adb39dfefc7ea00fb4bc99a805b8b9fd1b Mon Sep 17 00:00:00 2001 From: "Shiny Brar (he/il)" Date: Fri, 31 May 2024 14:56:57 -0400 Subject: [PATCH 35/41] fix(http): fixed pipelines,schedules and configs to use baseurls with prefixing versioning --- .pre-commit-config.yaml | 12 ++++++------ workflow/http/configs.py | 10 +++++----- workflow/http/pipelines.py | 6 +++--- workflow/http/schedules.py | 14 +++++++------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c50eb2..ca6395c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,11 +23,11 @@ repos: - --py36-plus id: pyupgrade repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.15.2 - hooks: - id: black repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.4.2 - hooks: - additional_dependencies: - types-attrs @@ -40,7 +40,7 @@ repos: - --no-implicit-optional id: mypy repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 - hooks: - args: - --convention=google @@ -70,14 +70,14 @@ repos: args: - --autofix repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 - hooks: - args: - -iii - -lll id: bandit repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.8 - hooks: - additional_dependencies: - radon @@ -96,4 +96,4 @@ repos: stages: - commit-msg repo: https://github.com/commitizen-tools/commitizen - rev: v3.13.0 + rev: v3.27.0 diff --git a/workflow/http/configs.py b/workflow/http/configs.py index b66c93f..a0cab4d 100644 --- a/workflow/http/configs.py +++ b/workflow/http/configs.py @@ -43,7 +43,7 @@ def deploy(self, data: Dict[str, Any]): ID of Config object generated. """ with self.session as session: - url = f"{self.baseurl}/v2/configs" + url = f"{self.baseurl}/configs" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -59,7 +59,7 @@ def count(self) -> Dict[str, Any]: """ with self.session as session: response: Response = session.get( - url=f"{self.baseurl}/v2/configs/count?database=configs" + url=f"{self.baseurl}/configs/count?database=configs" ) response.raise_for_status() return response.json() @@ -93,7 +93,7 @@ def get_configs( params = {"projection": projection, "query": query} if config_name: params.update({"name": config_name}) - url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + url = f"{self.baseurl}/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() @@ -122,7 +122,7 @@ def remove(self, config: str, id: str) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": config} - url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + url = f"{self.baseurl}/configs?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -147,7 +147,7 @@ def stop(self, config_name: str, id: str) -> List[Dict[str, Any]]: with self.session as session: query = {"id": id} params = {"query": dumps(query), "name": config_name} - url = f"{self.baseurl}/v2/configs/cancel?{urlencode(params)}" + url = f"{self.baseurl}/configs/cancel?{urlencode(params)}" response: Response = session.put(url) response.raise_for_status() if response.status_code == 304: diff --git a/workflow/http/pipelines.py b/workflow/http/pipelines.py index 971051c..76f4045 100644 --- a/workflow/http/pipelines.py +++ b/workflow/http/pipelines.py @@ -31,7 +31,7 @@ def count(self) -> Dict[str, Any]: with self.session as session: params = {"database": "pipelines"} response: Response = session.get( - url=f"{self.baseurl}/v2/configs/count?{urlencode(params)}" + url=f"{self.baseurl}/configs/count?{urlencode(params)}" ) response.raise_for_status() return response.json() @@ -54,7 +54,7 @@ def list_pipelines(self, name: Optional[str] = None) -> List[Dict[str, Any]]: params = {"database": "pipelines"} if name: params.update({"name": name}) - url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + url = f"{self.baseurl}/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() @@ -86,7 +86,7 @@ def get_pipelines( "projection": projection, "database": "pipelines", } - url = f"{self.baseurl}/v2/configs?{urlencode(params)}" + url = f"{self.baseurl}/configs?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json() diff --git a/workflow/http/schedules.py b/workflow/http/schedules.py index 938cac6..7805da8 100644 --- a/workflow/http/schedules.py +++ b/workflow/http/schedules.py @@ -42,7 +42,7 @@ def deploy(self, data: Dict[str, Any]): IDs of Schedule objects generated. """ with self.session as session: - url = f"{self.baseurl}/v2/schedule" + url = f"{self.baseurl}/schedule" response: Response = session.post(url, json=data) response.raise_for_status() return response.json() @@ -63,7 +63,7 @@ def get_schedule(self, query: Dict[str, Any]) -> Dict[str, Any]: """ with self.session as session: params = {"query": dumps(query)} - url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" + url = f"{self.baseurl}/schedule?{urlencode(params)}" response: Response = session.get(url=url) response.raise_for_status() return response.json()[0] @@ -90,7 +90,7 @@ def remove(self, id: str) -> Response: with self.session as session: query = {"id": id} params = {"query": dumps(query)} - url = f"{self.baseurl}/v2/schedule?{urlencode(params)}" + url = f"{self.baseurl}/schedule?{urlencode(params)}" response: Response = session.delete(url=url) response.raise_for_status() return response @@ -123,9 +123,9 @@ def list_schedules(self, schedule_name: str) -> List[Dict[str, Any]]: } ) url = ( - f"{self.baseurl}/v2/schedule?projection={projection}" + f"{self.baseurl}/schedule?projection={projection}" if schedule_name is None - else f"{self.baseurl}/v2/schedule?query={query}&projection={projection}" + else f"{self.baseurl}/schedule?query={query}&projection={projection}" ) response: Response = session.get(url=url) response.raise_for_status() @@ -148,9 +148,9 @@ def count_schedules(self, schedule_name: Optional[str] = None) -> Dict[str, Any] with self.session as session: query = dumps({"name": schedule_name}) url = ( - f"{self.baseurl}/v2/schedule/count" + f"{self.baseurl}/schedule/count" if not schedule_name - else f"{self.baseurl}/v2/schedule/count?query={query}" + else f"{self.baseurl}/schedule/count?query={query}" ) response: Response = session.get(url=url) response.raise_for_status() From c34761ab5613be525532a8919facc98853cc63ab Mon Sep 17 00:00:00 2001 From: "Shiny Brar (he/il)" Date: Fri, 31 May 2024 15:04:35 -0400 Subject: [PATCH 36/41] fix(workspace-configs): updated workspace configs to point to v2 --- workflow/workspaces/tutorial.yml | 6 +++--- workflow/workspaces/workflow-dev.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/workspaces/tutorial.yml b/workflow/workspaces/tutorial.yml index d1aede9..e732873 100644 --- a/workflow/workspaces/tutorial.yml +++ b/workflow/workspaces/tutorial.yml @@ -6,9 +6,9 @@ sites: http: baseurls: - configs: http://localhost:8001 - pipelines: http://localhost:8001 - schedules: http://localhost:8001 + configs: http://localhost:8001/v2 + pipelines: http://localhost:8001/v2 + schedules: http://localhost:8001/v2 buckets: http://localhost:8004 results: http://localhost:8005 diff --git a/workflow/workspaces/workflow-dev.yml b/workflow/workspaces/workflow-dev.yml index ae6eb9c..606f0c2 100644 --- a/workflow/workspaces/workflow-dev.yml +++ b/workflow/workspaces/workflow-dev.yml @@ -6,9 +6,9 @@ sites: http: baseurls: - configs: http://localhost:37015 - pipelines: http://localhost:37015 - schedules: http://localhost:37015 + configs: http://localhost:37015/v2 + pipelines: http://localhost:37015/v2 + schedules: http://localhost:37015/v2 buckets: https://frb.chimenet.ca/buckets results: https://frb.chimenet.ca/results From 546b06a7bf05c6b8943718b3c0b38ed4112a635d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Mon, 3 Jun 2024 20:25:55 -0400 Subject: [PATCH 37/41] refactor(test_http_context.py): fixing tests --- docker-compose-tutorial.yml | 2 ++ tests/test_http_context.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml index 4aad364..55130f3 100644 --- a/docker-compose-tutorial.yml +++ b/docker-compose-tutorial.yml @@ -5,6 +5,8 @@ services: command: python -m pipelines.server ports: - "8001:8001" + expose: + - 8001 environment: - SANIC_HOSTNAME=0.0.0.0 - SANIC_PORT=8001 diff --git a/tests/test_http_context.py b/tests/test_http_context.py index 92fd66b..a528e6b 100644 --- a/tests/test_http_context.py +++ b/tests/test_http_context.py @@ -16,7 +16,7 @@ { "name": "stage-1-a", "stage": 1, - "matrix": {"event": [123456, 654321], "site": ["chime", "canfar"]}, + "matrix": {"event": [123456, 654321], "site": ["local"]}, "work": { "site": "${{ matrix.site }}", "command": ["ls", "${{ matrix.event }}"], From fb99ce21b685f4cf1a6928b4d277203663a70f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Mon, 3 Jun 2024 21:19:11 -0400 Subject: [PATCH 38/41] refactor(workspaces/development.yml): adding version suffix to baseurls --- workflow/workspaces/development.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/workspaces/development.yml b/workflow/workspaces/development.yml index cca26d8..61cd629 100644 --- a/workflow/workspaces/development.yml +++ b/workflow/workspaces/development.yml @@ -17,9 +17,9 @@ archive: http: baseurls: - configs: http://localhost:8001 - pipelines: http://localhost:8001 - schedules: http://localhost:8001 + configs: http://localhost:8001/v2 + pipelines: http://localhost:8001/v2 + schedules: http://localhost:8001/v2 buckets: http://localhost:8004 results: http://localhost:8005 # products: http://localhost:8004 From 246abd6d6a8d9e54436ed079755670d2b3228c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 4 Jun 2024 07:21:09 -0400 Subject: [PATCH 39/41] Retrying From 7e066331f8b70ee13f23cc2d089cf81db60f93c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 4 Jun 2024 07:31:35 -0400 Subject: [PATCH 40/41] fix(docker-compose-tutorial.yml): settings services to latests version --- docker-compose-tutorial.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml index 55130f3..2d1b6e5 100644 --- a/docker-compose-tutorial.yml +++ b/docker-compose-tutorial.yml @@ -1,6 +1,6 @@ services: pipelines_api: - image: chimefrb/pipelines:pipelines-v2.7.1 + image: chimefrb/pipelines:latest container_name: pipelines_api command: python -m pipelines.server ports: @@ -29,7 +29,7 @@ services: restart: always pipelines_managers: - image: chimefrb/pipelines:pipelines-v2.7.1 + image: chimefrb/pipelines:latest container_name: pipelines_managers command: python -m managers.server ports: From 081ef217a38402b9b470da7805616d66d048f26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oswaldo=20Alcal=C3=A1?= Date: Tue, 4 Jun 2024 07:43:54 -0400 Subject: [PATCH 41/41] fix(docker-compose-tutorial.yml): adding local service --- docker-compose-tutorial.yml | 49 ++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/docker-compose-tutorial.yml b/docker-compose-tutorial.yml index 2d1b6e5..607c671 100644 --- a/docker-compose-tutorial.yml +++ b/docker-compose-tutorial.yml @@ -24,8 +24,7 @@ services: - SANIC_LISTENERS_THRESHOLD_SECONDS=120 - TZ=Etc/UTC networks: - - workflow-network - + - workflow restart: always pipelines_managers: @@ -52,7 +51,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock networks: - - workflow-network + - workflow healthcheck: test: [ @@ -66,22 +65,6 @@ services: retries: 5 restart: always - dind: - image: docker:dind - command: [ "--host=tcp://0.0.0.0:2376" ] - deploy: - replicas: 1 - privileged: true - expose: - - 2376 - ports: - - "2375:2375" - - "2376:2376" - environment: - - DOCKER_TLS_CERTDIR= - networks: - - workflow-network - buckets: image: chimefrb/buckets:latest container_name: buckets @@ -100,7 +83,7 @@ services: - SANIC_MONGODB_PORT=27017 - SANIC_CORS_ORIGINS=* networks: - - workflow-network + - workflow results: image: chimefrb/results:latest @@ -120,7 +103,7 @@ services: - SANIC_MONGODB_PORT=27017 - SANIC_CORS_ORIGINS=* networks: - - workflow-network + - workflow mongo: image: mongo:latest @@ -129,8 +112,28 @@ services: ports: - "27017:27017" networks: - - workflow-network + - workflow + + local: + image: docker:dind + container_name: local + command: + - "/bin/sh" + - "-c" + - | + dockerd -H tcp://0.0.0.0:4444 --tls=false \ + & while(! docker -H tcp://0.0.0.0:4444 info >/dev/null 2>&1); \ + do sleep 1; \ + done \ + && docker -H tcp://0.0.0.0:4444 swarm init && tail -f /dev/null + ports: + - "4444:4444" + expose: + - 4444 + privileged: true + networks: + - workflow networks: - workflow-network: + workflow: driver: bridge