diff --git a/.config/README.md.template b/.config/README.md.template index a589670..5493d3a 100644 --- a/.config/README.md.template +++ b/.config/README.md.template @@ -31,7 +31,7 @@ runbook run runbook-name.ipynb # Background -### What is a Runbook? +## What is a Runbook? A runbook is an executable document that combines: - Clear markdown documentation - Runnable code blocks @@ -40,13 +40,13 @@ A runbook is an executable document that combines: It's ideal for operations like encoding your Disaster Recovery Operations, spinning up a new cluster, or restoring from snapshots. -### When Should You Use This? +## When Should You Use This? - ✅ When you need **semi-automated tools** with audit trails and safety checks - ✅ When you want **rapid iteration** on operational procedures with built-in rollback steps - ✅ When you need something more powerful than shell scripts but don't want to build a full application - ✅ When you want to make complex operations both **safe and repeatable** -### Runbook Best Practices +## Runbook Best Practices 1. Structure your runbooks with: - Clear purpose and summary - Step-by-step descriptions @@ -72,7 +72,7 @@ It's ideal for operations like encoding your Disaster Recovery Operations, spinn 1. Depending on auditing needs, you can either commit the "runs" folder to your repo or only keep the "binder" folder committed. 1. In case of strict auditing needs, we recommend you add auditing of commands in the local SDK as well as in your cloud provider -## Installation +# Installation We recommend using [uv](https://docs.astral.sh/uv/) for installing runbook as a cli tool. If you already use pipx, you can use that instead. @@ -86,7 +86,7 @@ Or pin to a version uv tool install git+https://github.com/zph/runbook.git@$RUNBOOK_VERSION ``` -## CLI +# CLI ```sh $RUNBOOK_HELP @@ -136,7 +136,7 @@ For development we use the following tools: - [hermit](https://hermit.dev/) to manage developement tool dependencies (see .hermit/bin) - [uv](https://docs.astral.sh/uv/) python package manager and cli runner (see pyproject.toml) -Necessary deps can be seen in [pyproject.toml](pyproject.toml) and .hermit/bin +Necessary deps can be seen in pyproject.toml and .hermit/bin Use .hermit/bin/activate-hermit to activate the environment. diff --git a/.hermit/bin/publish-docs b/.hermit/bin/publish-docs new file mode 100755 index 0000000..b5489c5 --- /dev/null +++ b/.hermit/bin/publish-docs @@ -0,0 +1,40 @@ +#!/bin/bash + +# Stop on errors +set -eou pipefail +set -x + +# Variables +DOCS_DIR="site" +BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) +TEMP_DIR=$(mktemp -d) +TEMP_GIT_DIR=$(mktemp -d) +ORIGINAL_GIT_DIR=$(git rev-parse --show-toplevel) + +# Check if git working directory is clean +# if [ -n "$(git status --porcelain)" ]; then +# echo "Error: Git working directory is not clean. Please commit or stash changes first." +# exit 1 +# fi + +just docs +# Copy the docs to the temporary directory +cp -r "$DOCS_DIR"/* "$TEMP_DIR/" + +( + cd "$TEMP_GIT_DIR" + git clone "$ORIGINAL_GIT_DIR" . + git checkout gh-pages + + rm -rf ./* + cp -r "$TEMP_DIR"/* . + git add -A + git commit -m "Publish documentation [$(date)]" + # Push to local original repo + git push origin gh-pages +) + +# Push to remote +git push origin gh-pages + +echo "Documentation published to gh-pages branch." diff --git a/Justfile b/Justfile index fefad25..571f68b 100644 --- a/Justfile +++ b/Justfile @@ -53,9 +53,10 @@ template-update: readme: .config/templating.sh + cp README.md docs/ docs: - uvx --with mkdocs-click --with . mkdocs serve + uvx --with sphinx-click --with myst_parser --with . --from sphinx sphinx-build -b html docs/ site docs-release: - uvx --with mkdocs-click --with . mkdocs gh-deploy + bash .hermit/bin/publish-docs diff --git a/README.md b/README.md index d679213..6614644 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ runbook run runbook-name.ipynb # Background -### What is a Runbook? +## What is a Runbook? A runbook is an executable document that combines: - Clear markdown documentation - Runnable code blocks @@ -40,13 +40,13 @@ A runbook is an executable document that combines: It's ideal for operations like encoding your Disaster Recovery Operations, spinning up a new cluster, or restoring from snapshots. -### When Should You Use This? +## When Should You Use This? - ✅ When you need **semi-automated tools** with audit trails and safety checks - ✅ When you want **rapid iteration** on operational procedures with built-in rollback steps - ✅ When you need something more powerful than shell scripts but don't want to build a full application - ✅ When you want to make complex operations both **safe and repeatable** -### Runbook Best Practices +## Runbook Best Practices 1. Structure your runbooks with: - Clear purpose and summary - Step-by-step descriptions @@ -72,7 +72,7 @@ It's ideal for operations like encoding your Disaster Recovery Operations, spinn 1. Depending on auditing needs, you can either commit the "runs" folder to your repo or only keep the "binder" folder committed. 1. In case of strict auditing needs, we recommend you add auditing of commands in the local SDK as well as in your cloud provider -## Installation +# Installation We recommend using [uv](https://docs.astral.sh/uv/) for installing runbook as a cli tool. If you already use pipx, you can use that instead. @@ -86,7 +86,7 @@ Or pin to a version uv tool install git+https://github.com/zph/runbook.git@1.0.0-rc2 ``` -## CLI +# CLI ```sh Usage: runbook [OPTIONS] COMMAND [ARGS]... @@ -98,16 +98,16 @@ Options: --help Show this message and exit. Commands: - check Check language validity and formatting of a notebook. - convert Convert an existing runbook to different format - create Create a new runbook from [template] - diff Diff two notebooks + check Check the language validity and formatting of a runbook. + convert Convert a runbook between different formats + create Create a new runbook from a template + diff Compare two runbooks and show their differences edit Edit an existing runbook init Initialize a folder as a runbook repository - list list runbooks + list List runbooks plan Prepares the runbook for execution by injecting parameters. review [Unimplemented] Entrypoint for reviewing runbook - run Run a notebook + run Run a runbook show Show runbook parameters and metadata version Display version information about runbook ``` @@ -156,7 +156,7 @@ For development we use the following tools: - [hermit](https://hermit.dev/) to manage developement tool dependencies (see .hermit/bin) - [uv](https://docs.astral.sh/uv/) python package manager and cli runner (see pyproject.toml) -Necessary deps can be seen in [pyproject.toml](pyproject.toml) and .hermit/bin +Necessary deps can be seen in pyproject.toml and .hermit/bin Use .hermit/bin/activate-hermit to activate the environment. diff --git a/NOTES.md b/docs/NOTES.md similarity index 91% rename from NOTES.md rename to docs/NOTES.md index b495da6..928dfdd 100644 --- a/NOTES.md +++ b/docs/NOTES.md @@ -1,15 +1,14 @@ -# TODO +# Roadmap and Notes ## Triage - [x] Read `runbook plan ...` support reading params from file -- [ ] Re-export confirm(style) and gather from sh lib -- [ ] Make sure we can run directly fully through cli - [ ] If no argument included for RUNBOOK TITLE in edit/plan/run then prompt with options - [x] Include field 'embeds' in the metadata to help with referencing them - [ ] --- Add helper for referencing the embeds? - [ ] Use execute and upload output to S3 for non-interactive: https://github.com/nteract/papermill/tree/main?tab=readme-ov-file#execute-via-cli ## P0 +- [ ] Add linter for runbooks (using `runbook lint`) that checks for presence of title, desc, rollback, cleanup, etc - [ ] Add `runbook init` suggestion or automation to add `export RUNBOOK_WORKING_DIR=...` to shell initializations - [ ] Install pre-commit.yml or git integration during `init` for secure linting and talisman - [x] Setup versioning and bumper (using versioner from npm ecosystem @release-it and @release-it/bumper) @@ -34,10 +33,7 @@ - [x] Should I follow the tf convention of `plan | apply` - [x] Setup decorator to embed dry_run into shell command - [ ] (won't do yet) Allow for executing cell by cell from commandline in a repl? -- [ ] Running cell by cell: https://github.com/odewahn/ipynb-examples/blob/master/Importing%20Notebooks.ipynb -- [ ] figure out how to store and replay individual cells -- [ ] ~~Textualize gui for tui?~~ -- [ ] ~~is euphorie's tui for notebooks helpful?~~ + - [ ] Running cell by cell: https://github.com/odewahn/ipynb-examples/blob/master/Importing%20Notebooks.ipynb - [ ] Build auditability through a custom runner interface -- Or a custom kernel wrapper? -- https://ipython.readthedocs.io/en/stable/config/options/kernel.html diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..6614644 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,165 @@ +# Runbook + +## Summary + +Runbook is a powerful CLI tool that transforms your operational procedures into interactive, executable notebooks. It combines the best of documentation and automation by letting you create dynamic runbooks using Markdown, Deno, or Python. + +Think of it as "infrastructure as code" meets "documentation as code" - perfect for DevOps teams who want both flexibility and reliability. + +**At work, it empowered us to move 300 Mysql Clusters to TiDB with a small team [recording](https://www.youtube.com/watch?app=desktop&v=-_JoqZthrI8) over the course of 18 months.** + +# Quick Start + +```sh +uv tool install git+https://github.com/zph/runbook.git@1.0.0-rc2 + +# Initialize a new runbook project in a repo of your choosing +runbook init + +# Create a new runbook +runbook create -l deno runbook-name.ipynb + +# Edit the runbook +runbook edit runbook-name.ipynb + +# Plan the runbook +runbook plan runbook-name.ipynb --embed file.json --parameters '{"arg": 1, "foo": "baz"}' + +# Run the runbook +runbook run runbook-name.ipynb +``` + +# Background + +## What is a Runbook? +A runbook is an executable document that combines: +- Clear markdown documentation +- Runnable code blocks +- Parameterized inputs for reusability +- Built-in safety checks + +It's ideal for operations like encoding your Disaster Recovery Operations, spinning up a new cluster, or restoring from snapshots. + +## When Should You Use This? +- ✅ When you need **semi-automated tools** with audit trails and safety checks +- ✅ When you want **rapid iteration** on operational procedures with built-in rollback steps +- ✅ When you need something more powerful than shell scripts but don't want to build a full application +- ✅ When you want to make complex operations both **safe and repeatable** + +## Runbook Best Practices +1. Structure your runbooks with: + - Clear purpose and summary + - Step-by-step descriptions + - Warning signs and precautions + - Verification steps + - Execution steps in logical order + - Rollback and cleanup steps +2. Keep read-only operations flexible +3. Require explicit confirmation for destructive actions using the `confirm` flag +4. Include pre-flight checks before any system modifications +5. For critical operations, use pair execution: + - One person to run the procedure + - Another to verify and validate safety checks + +## Workflow + +1. Initialize a new folder project with `runbook init...` +1. Create a new runbook with `runbook create -l deno runbook-name.ipynb` +1. Edit the runbook with `runbook edit runbook-name.ipynb` (or using editor of choice) and add your title, description, steps +1. For complex runbooks, offload the coding details into an SDK that you build beside the runbooks that can be reused across multiple runbooks +1. Plan that runbook for a specific run `runbook plan runbook-name.ipynb --embed file.json --parameters '{"arg": 1, "foo": "baz"}' +1. Run the instance of a runbook with either `runbook run runbook-name.ipynb` or use VSCode to run it `code runbooks/runs/runbook-name.ipynb` +1. Depending on auditing needs, you can either commit the "runs" folder to your repo or only keep the "binder" folder committed. + 1. In case of strict auditing needs, we recommend you add auditing of commands in the local SDK as well as in your cloud provider + +# Installation + +We recommend using [uv](https://docs.astral.sh/uv/) for installing runbook as a cli tool. If you already use pipx, you can use that instead. + +```sh +uv tool install git+https://github.com/zph/runbook.git +``` + +Or pin to a version + +```sh +uv tool install git+https://github.com/zph/runbook.git@1.0.0-rc2 +``` + +# CLI + +```sh +Usage: runbook [OPTIONS] COMMAND [ARGS]... + +Options: + --cwd PATH Directory for operations (normally at root above runbooks, ie + ../.runbook.yaml) and can be set with RUNBOOK_WORKING_DIR or + WORKING_DIR environment variables + --help Show this message and exit. + +Commands: + check Check the language validity and formatting of a runbook. + convert Convert a runbook between different formats + create Create a new runbook from a template + diff Compare two runbooks and show their differences + edit Edit an existing runbook + init Initialize a folder as a runbook repository + list List runbooks + plan Prepares the runbook for execution by injecting parameters. + review [Unimplemented] Entrypoint for reviewing runbook + run Run a runbook + show Show runbook parameters and metadata + version Display version information about runbook +``` + +Shell completion is included via click library and enabled as follows [link](https://click.palletsprojects.com/en/8.1.x/shell-completion/#enabling-completion) + +``` +# Bash +# Add this to ~/.bashrc: +eval "$(_RUNBOOK_COMPLETE=bash_source runbook)" + +# Zsh +# Add this to ~/.zshrc: +eval "$(_RUNBOOK_COMPLETE=zsh_source runbook)" + +# Fish +# Add this to ~/.config/fish/completions/foo-bar.fish: +_RUNBOOK_COMPLETE=fish_source runbook | source +``` + +For advanced completion setup see [docs](https://click.palletsprojects.com/en/8.1.x/shell-completion/#enabling-completion) + +# Principles + +- Prefer deno for better package management and developer ergonomics with typing + - But allow for other kernels (python) as secondary option, via compatible libraries +- Make `runbook` batteries included for interfacing with shell commands and common runbook +operations + +# Caveats + +1. Running notebook in VScode does not set the timings necessary in notebook for being auditable and exported later + 1. Recommendation: if auditable runs are needed, use jupyter via browser `runbook run TITLE` +1. Notebooks have different structured ids per cell depending on run environment + 1. Recommendation: if requiring consistency, write your own pre-processor to standardize on an id format +1. Built-in shell package requires a shell environment and is only expected to run on Linux or Mac not Windows. + 1. Recommendation: Windows support is out of scope for now but we'll review PRs + +## Deno / Typescript +1. Parameter cells must use `let` declarations to allow for param overriding + - `var` or `let` work in Deno notebooks but only `let` works if using `runbook convert a.ipynb a.ts` and running the ts version + +# Developing runbook cli + +For development we use the following tools: +- [hermit](https://hermit.dev/) to manage developement tool dependencies (see .hermit/bin) +- [uv](https://docs.astral.sh/uv/) python package manager and cli runner (see pyproject.toml) + +Necessary deps can be seen in pyproject.toml and .hermit/bin + +Use .hermit/bin/activate-hermit to activate the environment. + +# Readme Changes + +README.md is generated from .config/README.md.template and should be updated there. diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..04de4df --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,15 @@ +.. Runbook documentation master file, created by + sphinx-quickstart on Mon Jan 20 11:01:47 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Runbook CLI +===================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. click:: runbook.cli:cli + :prog: runbook + :show-nested: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..1615cbd --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,37 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +# + +project = "Runbook" +copyright = "2025, Zander Hill" +author = "Zander Hill" +import runbook + +release = runbook.__version__ +# - + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx_click", + "myst_parser", +] + +templates_path = ["_templates"] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 68098c8..0000000 --- a/docs/index.md +++ /dev/null @@ -1,7 +0,0 @@ -# CLI Reference - -This page provides documentation for our command line tools. - -::: mkdocs-click - :module: runbook.cli - :command: cli diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..71a6df1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,19 @@ +.. Runbook documentation master file, created by + sphinx-quickstart on Mon Jan 20 11:01:47 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Runbook documentation +===================== + +CLI for dynamic runbooks: a structured and auditable approach +to creating and executing operational procedures, bridging the +gap between simple shell scripts and more complex tooling. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + README + CLI Documentation + Roadmap and Notes diff --git a/runbook/cli/__init__.py b/runbook/cli/__init__.py index 4a50997..27d413b 100644 --- a/runbook/cli/__init__.py +++ b/runbook/cli/__init__.py @@ -7,7 +7,6 @@ create, diff, edit, - export, init, list, plan, @@ -47,7 +46,6 @@ def cli(ctx, cwd): cli.add_command(version) cli.add_command(list) cli.add_command(show) -# cli.add_command(export) cli.name = "runbook" cli diff --git a/runbook/cli/commands/check.py b/runbook/cli/commands/check.py index 3c73cd0..b01a5fb 100644 --- a/runbook/cli/commands/check.py +++ b/runbook/cli/commands/check.py @@ -25,7 +25,24 @@ ) @click.pass_context def check(ctx, filename, command): - """Check language validity and formatting of a notebook.""" + """Check the language validity and formatting of a runbook. + + This command validates the syntax and formatting of runbook cells based on the kernel + specified in the notebook's metadata. By default: + - For Python kernels: uses 'black' to check code formatting + - For Deno kernels: uses 'deno check' to validate TypeScript/JavaScript + + FILENAME: Path to the runbook file to check. + + Options: + --command, -c: Specify a custom validation command. Use {} as a placeholder for the + filename (e.g., 'mycheck {} --strict'). This overrides the default + checker for the kernel. + + Exit codes: + 0: Check passed successfully + Non-zero: Check failed, see error output for details + """ full_path = path.abspath(filename) content = None with open(full_path, "r") as f: diff --git a/runbook/cli/commands/convert.py b/runbook/cli/commands/convert.py index ee53e46..a6f24c0 100644 --- a/runbook/cli/commands/convert.py +++ b/runbook/cli/commands/convert.py @@ -17,7 +17,27 @@ ) @click.pass_context def convert(ctx, filename, output): - """Convert an existing runbook to different format""" + """Convert a runbook between different formats + + This command converts notebooks between various formats using jupytext. + Supported conversions include: + - .ipynb to .py (Python script) + - .ipynb to .ts (Deno notebook) + - .ipynb to .md (Markdown) + - And other formats supported by jupytext + + FILENAME: Path to the source notebook file to convert. + OUTPUT: Destination path for the converted file. The format is determined + by the file extension. + + Examples: + runbook convert notebook.ipynb script.py # Convert to Python script + runbook convert notebook.ipynb notebook.ts # Convert to Deno notebook + runbook convert notebook.ipynb notebook.md # Convert to Markdown + + The conversion preserves cell metadata, notebook metadata, and execution outputs + where applicable. + """ # Must override argv because it's used in launch instance and there isn't a way # to pass via argument in ExtensionApp.lauch_instance # TODO: diff --git a/runbook/cli/commands/create.py b/runbook/cli/commands/create.py index 76c3f72..e24676e 100644 --- a/runbook/cli/commands/create.py +++ b/runbook/cli/commands/create.py @@ -14,6 +14,7 @@ default="./runbooks/binder/_template-deno.ipynb", type=click.Path(exists=True, file_okay=True), callback=validate_template, + help="Path to the template file to use", ) # TODO: switch to language and template defaulting to Deno @@ -24,10 +25,34 @@ envvar="LANGUAGE", default="deno", callback=validate_create_language, + help="Language to use for the runbook", ) @click.pass_context def create(ctx, filename, template, language): - """Create a new runbook from [template]""" + """Create a new runbook from a template + + This command creates a new runbook using a specified template + or language preset. The new runbook will be created in the runbooks/binder directory. + + FILENAME: Name for the new runbook file (e.g., 'maintenance-task.ipynb'). + Should be a basename only, without directory path. + + Options: + --template, -t: Path to a custom template runbook to use as a base. + Can be set via TEMPLATE environment variable. + Default: ./runbooks/binder/_template-deno.ipynb + + --language, -l: Shortcut to use a predefined language template. + Can be set via LANGUAGE environment variable. + Default: deno + + Examples: + runbook create maintenance-task.ipynb # Creates using default Deno template + runbook create task.ipynb -l python # Creates using Python template + runbook create task.ipynb -t custom-template.ipynb # Creates from custom template + + The command will create the notebook and display the edit command to open it. + """ if language: template = language diff --git a/runbook/cli/commands/diff.py b/runbook/cli/commands/diff.py index 75a504e..aaa2aa3 100644 --- a/runbook/cli/commands/diff.py +++ b/runbook/cli/commands/diff.py @@ -18,7 +18,23 @@ ) @click.pass_context def diff(ctx, notebook_1, notebook_2): - """Diff two notebooks""" + """Compare two runbooks and show their differences + + This command uses nbdime to display a detailed comparison between two runbooks. + + Arguments: + NOTEBOOK_1: Path to the first runbook for comparison + NOTEBOOK_2: Path to the second runbook for comparison + + Examples: + runbook diff notebook1.ipynb notebook2.ipynb # Compare two notebooks + runbook diff original.ipynb modified.ipynb # Show changes between versions + + The diff output will be displayed in a terminal-friendly format, with: + - Added content in green + - Removed content in red + - Modified content showing both versions + """ argv = [path.abspath(notebook_1), path.abspath(notebook_2)] from nbdime import nbdiffapp diff --git a/runbook/cli/commands/export.py b/runbook/cli/commands/export.py deleted file mode 100644 index d850ac1..0000000 --- a/runbook/cli/commands/export.py +++ /dev/null @@ -1,8 +0,0 @@ -import click - - -@click.command() -@click.pass_context -def export(ctx): - """[Unimplemented] Entrypoint for exporting runbook""" - raise RuntimeError("Not Implemented") diff --git a/runbook/cli/commands/init.py b/runbook/cli/commands/init.py index 99ac4ee..83b3762 100644 --- a/runbook/cli/commands/init.py +++ b/runbook/cli/commands/init.py @@ -17,6 +17,7 @@ envvar="DIRECTORY", default="runbooks", type=click.Path(exists=False, dir_okay=True), + help="Path to the runbook directory", ) @click.option( "-s", @@ -24,6 +25,7 @@ envvar="SKIP_CONFIRMATION", default=False, type=click.BOOL, + help="Skip confirmation prompt", ) @click.pass_context def init(ctx, directory, skip_confirmation): diff --git a/runbook/cli/commands/list.py b/runbook/cli/commands/list.py index 6127d2e..f7ce53f 100644 --- a/runbook/cli/commands/list.py +++ b/runbook/cli/commands/list.py @@ -4,7 +4,7 @@ @click.command() @click.pass_context def list(ctx): - """list runbooks""" + """List runbooks""" import glob from rich import print as rprint diff --git a/runbook/cli/commands/plan.py b/runbook/cli/commands/plan.py index 846c6ba..438df42 100644 --- a/runbook/cli/commands/plan.py +++ b/runbook/cli/commands/plan.py @@ -1,5 +1,7 @@ +import ast import json import os +import subprocess from datetime import datetime from os import path from pathlib import Path @@ -38,9 +40,6 @@ def get_notebook_language(notebook_path: str) -> str: return "unknown" -import ast - - def get_parser_by_language(language: str): if language == "typescript": return json.loads @@ -53,16 +52,42 @@ def get_parser_by_language(language: str): @click.command() @click.argument( - "input", type=click.Path(file_okay=True), callback=validate_runbook_file_path + "input", + type=click.Path(file_okay=True), + callback=validate_runbook_file_path, ) # TODO allow for specifying output filename to allow for easier naming -@click.option("-e", "--embed", type=click.Path(exists=True), multiple=True) @click.option( - "-p", "--params", default={}, type=click.UNPROCESSED, callback=validate_plan_params + "-e", + "--embed", + type=click.Path(exists=True), + multiple=True, + help="Path to file(s) to embed in the runbook output directory", +) +@click.option( + "-p", + "--params", + default={}, + type=click.UNPROCESSED, + callback=validate_plan_params, + help="Parameters to inject into the runbook in json object format where the key is the parameter name and the value is the parameter value", +) +@click.option( + "-i", + "--identifier", + default="", + type=click.STRING, + help="Optional identifier to append to the output filename", +) +@click.option( + "-p", + "--prompter", + default="", + type=click.Path(file_okay=True), + help="[Experimental] Path to a prompter script that will be used to gather parameters from the user", ) -@click.option("-i", "--identifier", default="", type=click.STRING) @click.pass_context -def plan(ctx, input, embed, identifier="", params={}): +def plan(ctx, input, embed, identifier="", params={}, prompter=""): """Prepares the runbook for execution by injecting parameters. Doesn't run runbook.""" import shutil @@ -90,23 +115,42 @@ def plan(ctx, input, embed, identifier="", params={}): # TODO: add test cases for auto-planning # As of 2025 Jan it's manual regression testing - if len(params) == 0: + if len(params) == 0 or prompter: inferred_params = pm.inspect_notebook(input) notebook_language = get_notebook_language(input) value_parser = get_parser_by_language(notebook_language) # Inferred_type_name is language specific + formatted_params = {} for key, value in inferred_params.items(): if key != RUNBOOK_METADATA: default = value["default"].rstrip(";") typing = value["inferred_type_name"] or "" - help_hint = value["help"] or "" - + help = value["help"] or "" + formatted_params[key] = { + "default": default, + "typing": typing, + "help": help, + } + + if prompter: + # Input format: params: {'server': {'default': '"main.xargs.io"', 'typing': 'string', 'help': ''}, 'arg': {'default': '1', 'typing': 'number', 'help': ''}, 'anArray': {'default': '["a", "b"]', 'typing': 'string[]', 'help': 'normally a / b'}, '__RUNBOOK_METADATA__': {'default': '{}', 'typing': 'None', 'help': ''}} + # Run prompter with inferred params passed via stdin + result = subprocess.run( + [prompter], + input=json.dumps(formatted_params), + capture_output=True, + text=True, + ) + # Response format: params: {'server': '"main.xargs.io"', 'arg': '1', 'anArray': '["a", "b"]'} + params = json.loads(result.stdout.strip()) + else: + for key, value in formatted_params.items(): parsed_value = click.prompt( - f"""Enter value for {key} {typing} {help_hint}""", - default=default, + f"""Enter value for {key} {value["typing"]} {value["help"]}""", + default=value["default"], value_proc=value_parser, ) - params[key] = parsed_value + params[key] = parsed_value injection_params = {**runbook_param_injection, **params} diff --git a/runbook/cli/commands/run.py b/runbook/cli/commands/run.py index 6813819..6fc60a5 100644 --- a/runbook/cli/commands/run.py +++ b/runbook/cli/commands/run.py @@ -8,7 +8,12 @@ type=click.Path(file_okay=True), callback=validate_planned_runbook_file_path, ) -@click.option("--output", type=click.Path(file_okay=True), default=None) +@click.option( + "--output", + type=click.Path(file_okay=True), + default=None, + help="Path to the output file", +) @click.option( "--interactive/--no-interactive", default=True, @@ -16,7 +21,7 @@ ) @click.pass_context def run(ctx, filename, output, interactive): - """Run a notebook""" + """Run a runbook""" if interactive: argv = [filename] diff --git a/tests/cli_test.py b/tests/cli_test.py index 727a5df..a97a32a 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -46,16 +46,16 @@ def test_cli_help(): --help Show this message and exit. Commands: - check Check language validity and formatting of a notebook. - convert Convert an existing runbook to different format - create Create a new runbook from [template] - diff Diff two notebooks + check Check the language validity and formatting of a runbook. + convert Convert a runbook between different formats + create Create a new runbook from a template + diff Compare two runbooks and show their differences edit Edit an existing runbook init Initialize a folder as a runbook repository - list list runbooks + list List runbooks plan Prepares the runbook for execution by injecting parameters. review [Unimplemented] Entrypoint for reviewing runbook - run Run a notebook + run Run a runbook show Show runbook parameters and metadata version Display version information about runbook """ diff --git a/tests/cli_test.ts b/tests/cli_test.ts index 193a637..9f4c79e 100644 --- a/tests/cli_test.ts +++ b/tests/cli_test.ts @@ -3,7 +3,7 @@ // review // run -import { assertEquals } from "jsr:@std/assert"; +import { assertEquals, assertArrayIncludes } from "jsr:@std/assert"; import { assertSnapshot } from "jsr:@std/testing/snapshot"; import { $ } from "jsr:@david/dax" @@ -81,9 +81,12 @@ Deno.test.ignore("diff", async (t) => { Deno.test("init", async (t) => { const {dir} = await setup(); - const files = await Array.fromAsync(Deno.readDir(dir)); - const filenames = files.map(f => f.name).sort(); - assertSnapshot(t, filenames); + const files = (await getAllFiles(dir)).map(f => f.replace(dir, "")); + assertEquals(files, [ + "/runbooks/binder/_template-deno.ipynb", + "/runbooks/binder/_template-python.ipynb", + "/runbooks/.runbook.json", + ]) }); Deno.test("list", async (t) => { @@ -99,6 +102,27 @@ Deno.test("show", async (t) => { assertSnapshot(t, { stdout: cmd.stdout, stderr: cmd.stderr, exitCode: cmd.code }); }); +// plan +Deno.test("plan: prompter interface", async (t) => { + const cwd = Deno.cwd(); + const {runbook, dir } = await setup(); + const cmd = await runbook(["plan", "runbooks/binder/_template-deno.ipynb", "--prompter", [cwd, "tests/fixtures/prompters/echoer"].join("/")]); + assertEquals(cmd.code, 0); + + const files = await getAllFiles($.path(dir).join("runbooks/runs").toString()); + const planFile = files.find(f => f.endsWith("_template-deno/_template-deno.ipynb")); + if(!planFile) { + throw new Error("Plan file not found"); + } + const json = await Deno.readTextFile(planFile); + const plan = JSON.parse(json); + const maybeParamCells = plan.cells.filter((c: any) => c.cell_type === "code" && c.metadata?.tags?.includes("injected-parameters")); + assertEquals(maybeParamCells.length, 1); + const paramCell = maybeParamCells[0]; + assertArrayIncludes(paramCell.source, [`server = "main.xargs.io";\n`, `arg = 1;\n`, `anArray = ["a", "b"];\n`]); + // assertSnapshot(t, { stdout: cmd.stdout, stderr: cmd.stderr, exitCode: cmd.code }); +}); + // run /* failing on nested dax commands Exception encountered at "In [3]": @@ -122,3 +146,22 @@ Deno.test("version", async (t) => { assertSnapshot(t, { stdout: cmd.stdout.split(":")[0], stderr: cmd.stderr, exitCode: cmd.code }); }); +async function* walkFiles(dir: string): AsyncGenerator { + for await (const entry of Deno.readDir(dir)) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory) { + yield* walkFiles(path); + } else { + yield path; + } + } +} + +async function getAllFiles(dir: string): Promise { + const files = []; + for await (const file of walkFiles(dir)) { + files.push(file); + } + return files; +} + diff --git a/tests/fixtures/prompters/echoer b/tests/fixtures/prompters/echoer new file mode 100755 index 0000000..6e86a8f --- /dev/null +++ b/tests/fixtures/prompters/echoer @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +echo '{"server": "main.xargs.io", "arg": 1, "anArray": ["a", "b"]}' +# cat /dev/stdin diff --git a/tests/fixtures/prompters/interactive-prompter.ts b/tests/fixtures/prompters/interactive-prompter.ts new file mode 100755 index 0000000..4cdfe8f --- /dev/null +++ b/tests/fixtures/prompters/interactive-prompter.ts @@ -0,0 +1,6 @@ +#!/usr/bin/env -S deno run -A + +const response = prompt("What is your name?", "Doe") + +console.error("debug", Deno.env.get("DEBUG")) +console.log(JSON.stringify({server: response}));