Add hera CLI and implement generate yaml. (#886)

**Pull Request Checklist**
- [x] Tests added
- [ ] Documentation/examples added
- [x] [Good commit messages](https://cbea.ms/git-commit/) and/or PR
title

**Description of PR**
Per design outlined in #670
and #711

My personal usecase ideally requires generating for a whole folder at a
time, but I only need output to stdout (Argo sidecar cmp plugin).
Anything not in that critical path, I'm happy to remove if it's more
challenging to agree on everything all at once.

* Adds the `hera` cli console script
* optional dependency, informs the user of the required extra if they
use it
* implements `generate yaml`

Notes:
* Notable difference from the design outlined in the above issues:
rather than `--from` as a required "option", I opted for it to be a
positional argument for now. I can certainly change this back to
`--from` if you feel strongly against it
* I will certainly add tests once there's an indication that this looks
roughly like what you'd want/accept
* All of the help text/error messaging is just a first draft, feel free
to suggest different prose.
* I've implemented both `file.py -> stdout`, `folder/ to stdout`,
`file.py -> foo.yaml`, and `folder/ -> to_folder/`
* I think folder to folder is perhaps the most opinionated, in that it
makes the decision that all source files get converted with the same
name but a yaml suffix. It seems logical to me, but I could just as
easily disallow this option.
* I've chosen to use Typer here, because it was what
#674 originally chose to use.
I have a personal (very biased) preference towards
[Cappa](https://cappa.readthedocs.io/), and I think it matches the style
of Hera itself really well. With that said, Typer is the obviously more
established option, so I'm not at all expecting to get buy-in to use
Cappa here (just putting it out there 🤣).

---------

Signed-off-by: Elliot Gunton <[email protected]>
Signed-off-by: DanCardin <[email protected]>
Signed-off-by: DanCardin <[email protected]>
Co-authored-by: Elliot Gunton <[email protected]>

.github/workflows/cicd.yaml

@@ -33,7 +33,7 @@ jobs:
           cache: "poetry"
 
       - name: Install dependencies
-        run: poetry install
+        run: poetry install -E cli
 
       - name: run ci checks
         run: make ci
@@ -80,7 +80,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          poetry install
+          poetry install -E cli
           poetry run pip install "pydantic<2"
 
       - name: run ci checks
@@ -113,7 +113,7 @@ jobs:
           cache: "poetry"
 
       - name: Install dependencies
-        run: poetry install
+        run: poetry install -E cli
 
       - name: setup k3d cluster
         run: make install-k3d

pyproject.toml

@@ -33,14 +33,16 @@ python = ">=3.8,<4"
 pyyaml = { version = ">=6.0", optional = true }
 requests = "*"
 pydantic = { extras = ["email"], version = ">=1.7,<3.0" }
+cappa = {version = "^0.14.3", optional = true}
 
 [tool.poetry.extras]
 yaml = ["PyYAML"]
+cli = ["cappa", "PyYAML"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "*"
 pytest-cov = "*"
-mypy = "*"
+mypy = "1.0.1"
 build = "*"
 ruff = "*"
 types-PyYAML = "*"
@@ -50,6 +52,9 @@ types-requests = "^2.28.11.12"
 pytest-clarity = "^1.0.1"
 pytest-sugar = "^0.9.6"
 
+[tool.poetry.scripts]
+hera = "hera.__main__:main"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
@@ -62,6 +67,7 @@ filterwarnings = [
 ]
 markers = [
     "on_cluster: tests that run on an Argo cluster",
+    "cli: tests that verify CLI functionality",
 ]
 
 # Convert the following to config

src/hera/__main__.py

@@ -0,0 +1,29 @@
+"""Entrypoint for running hera as a CLI."""
+import importlib
+import sys
+
+
+def main(argv=None):
+    """Entrypoint for running hera as a CLI."""
+    try:
+        importlib.import_module("cappa")
+    except ModuleNotFoundError as e:
+        raise ModuleNotFoundError(
+            "Use of the `hera` CLI tool requires installing the 'cli' extra, `pip install hera[cli]`."
+        ) from e
+
+    import cappa
+    import rich
+
+    from hera._cli.base import Hera
+
+    rich.print(
+        "[yellow bold]warning: The `hera` CLI is a work-in-progress, subject to change at any time![/yellow bold]",
+        file=sys.stderr,
+    )
+
+    cappa.invoke(Hera, argv=argv)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

src/hera/_cli/__init__.py

@@ -0,0 +1 @@
+"""Hera's command line interface."""

src/hera/_cli/base.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Union
+
+from cappa import Arg, Subcommands, command
+from typing_extensions import Annotated
+
+
+@dataclass
+class Hera:
+    subcommand: Subcommands[Generate]
+
+
+@command(help="Subcommands for generating yaml, code, and docs from Hera Workflows.")
+@dataclass
+class Generate:
+    subcommand: Subcommands[GenerateYaml]
+
+
+@command(
+    name="yaml",
+    help="Generate yaml from python Workflow definitions.",
+    invoke="hera._cli.generate.yaml.generate_yaml",
+)
+class GenerateYaml:
+    from_: Annotated[
+        Path,
+        Arg(
+            value_name="from",
+            help=(
+                "The path from which the yaml is generated. This can be a file "
+                "or a folder. When a folder is provided, all Python files in the "
+                "folder will be generated."
+            ),
+        ),
+    ]
+    to: Annotated[
+        Union[Path, None],
+        Arg(
+            long=True,
+            help=(
+                "Optional destination for the produced yaml. If 'from' is a "
+                "file this is assumed to be a file. If 'from' is a folder, "
+                "this is assumed to be a folder, and individual file names "
+                "will match the source file."
+            ),
+        ),
+    ] = None
+    recursive: Annotated[bool, Arg(help="Enables recursive traversal of an input folder")] = False

src/hera/_cli/generate/__init__.py

@@ -0,0 +1,5 @@
+from hera._cli.generate import yaml
+
+__all__ = [
+    "yaml",
+]

src/hera/_cli/generate/yaml.py

@@ -0,0 +1,102 @@
+"""The main entrypoint for hera CLI."""
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+from pathlib import Path
+from typing import Generator
+
+from hera._cli.base import GenerateYaml
+from hera.workflows.workflow import Workflow
+
+
+def generate_yaml(options: GenerateYaml):
+    """Generate yaml from Python Workflow definitions.
+
+    If the provided path is a folder, generates yaml for all Python files containing `Workflow`s
+    in that folder
+    """
+    paths = sorted(expand_paths(options.from_, recursive=options.recursive))
+
+    # Generate a collection of source file paths and their resultant yaml.
+    path_to_output: list[tuple[str, str]] = []
+    for path in paths:
+        yaml_outputs = []
+        for workflow in load_workflows_from_module(path):
+            yaml_outputs.append(workflow.to_yaml())
+
+        if not yaml_outputs:
+            continue
+
+        path_to_output.append((path.name, join_workflows(yaml_outputs)))
+
+    # When `to` write file(s) to disk, otherwise output everything to stdout.
+    if options.to:
+        dest_is_file = os.path.exists(options.to) and options.to.is_file()
+
+        if dest_is_file:
+            output = join_workflows(o for _, o in path_to_output)
+            options.to.write_text(output)
+
+        else:
+            os.makedirs(options.to, exist_ok=True)
+
+            for dest_path, content in path_to_output:
+                full_path = (options.to / dest_path).with_suffix(".yaml")
+                full_path.write_text(content)
+
+    else:
+        output = join_workflows(o for _, o in path_to_output)
+        sys.stdout.write(output)
+
+
+def expand_paths(source: Path, recursive: bool = False) -> Generator[Path, None, None]:
+    """Expand a `source` path, return the set of python files matching that path.
+
+    Arguments:
+        source: The source path to expand. In the event `source` references a
+            folder, return all python files in that folder.
+        recursive: If True, recursively traverse the `source` path.
+    """
+    source_is_dir = source.is_dir()
+    if not source_is_dir:
+        yield source
+        return
+
+    iterator = os.walk(source) if recursive else ((next(os.walk(source))),)
+
+    for dir, _, file_names in iterator:
+        for file_name in file_names:
+            path = Path(os.path.join(dir, file_name))
+            if path.suffix == ".py":
+                yield path
+
+
+def load_workflows_from_module(path: Path) -> list[Workflow]:
+    """Load the set of `Workflow` objects defined within a given module.
+
+    Arguments:
+        path: The path to a given python module
+
+    Returns:
+        A list containing all `Workflow` objects defined within that module.
+    """
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    assert spec
+
+    module = importlib.util.module_from_spec(spec)
+
+    assert spec.loader
+    spec.loader.exec_module(module)
+
+    result = []
+    for item in module.__dict__.values():
+        if isinstance(item, Workflow):
+            result.append(item)
+
+    return result
+
+
+def join_workflows(strings):
+    return "\n---\n\n".join(strings)

tests/cli/examples/cluster_workflow_template.py

@@ -0,0 +1,3 @@
+from hera.workflows import ClusterWorkflowTemplate
+
+workflow = ClusterWorkflowTemplate(name="cluster-workflow-template")

tests/cli/examples/multiple_workflow.py

@@ -0,0 +1,4 @@
+from hera.workflows import Workflow
+
+workflow1 = Workflow(name="one")
+workflow2 = Workflow(name="two")

tests/cli/examples/single_workflow.py

@@ -0,0 +1,3 @@
+from hera.workflows import Workflow
+
+workflow = Workflow(name="single")

tests/cli/examples/workflow_template.py

@@ -0,0 +1,3 @@
+from hera.workflows import WorkflowTemplate
+
+workflow = WorkflowTemplate(name="workflow-template")