Merge pull request #240 from monadicus/docs/in-depth-running-docs

Docs: More running section info

.github/dependabot.yml

@@ -4,12 +4,12 @@ updates:
   - package-ecosystem: cargo
     directory: "/"
     schedule:
-      interval: daily
+      interval: weekly
     open-pull-requests-limit: 10
 
   # Maintain dependencies for GitHub Actions
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
-      interval: daily
-    open-pull-requests-limit: 10
+      interval: weekly
+    open-pull-requests-limit: 10

.github/workflows/mdbook.yml

@@ -19,18 +19,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            book.toml
+            snops_book
+            index.html
+
       - name: Install latest mdbook
         run: |
           tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
           url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
           mkdir mdbook
           curl -sSL $url | tar -xz --directory=./mdbook
           echo `pwd`/mdbook >> $GITHUB_PATH
+
+      # https://github.com/rust-lang/mdBook/issues/984
+      - name: Fix MdBook Issue 984
+        run: |
+          find ./snops_book/ -type f -name "*.md" ! -iname "SUMMARY.md" -exec sed -ri 's/(\[.*\])\((.*\/)?(readme\.md)(#.*)?\)/\1\(\2index\.md\4\)/gI; s/(\[.*\]:[ \t]+)(.*\/)?(readme\.md)/\1\2index\.md/gI' {} \;
+
       - name: Build Book
         run: mdbook build
+
       - name: Setup Pages
         id: pages
         uses: actions/configure-pages@v5
+
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
@@ -45,4 +59,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v4

Cargo.toml

@@ -53,7 +53,7 @@ bincode = "1.3"
 bytes = "1.6"
 checkpoint = { path = "./crates/checkpoint" }
 chrono = { version = "0.4", features = ["now"], default-features = false }
-clap = { version = "4.5", features = ["derive"] }
+clap = { version = "4.5", features = ["derive", "env"] }
 clap_complete = { version = "4.5" }
 clap_mangen = { version = "0.2" }
 clap-markdown = "0.1"

crates/snops-agent/src/cli.rs

@@ -18,19 +18,21 @@ pub const ENV_ENDPOINT_DEFAULT: &str = "127.0.0.1:1234";
 
 #[derive(Debug, Parser)]
 pub struct Cli {
-    #[arg(long)]
+    #[arg(long, env = ENV_ENDPOINT)]
     /// Control plane endpoint address (IP, or wss://host, http://host)
     pub endpoint: Option<String>,
 
+    /// Agent ID, used to identify the agent in the network.
     #[arg(long)]
-    pub id: Option<AgentId>,
+    pub id: AgentId,
 
     /// Locally provided private key file, used for envs where private keys are
     /// locally provided
     #[arg(long)]
     #[clap(long = "private-key-file")]
     pub private_key_file: Option<PathBuf>,
 
+    /// Labels to attach to the agent, used for filtering and grouping.
     #[arg(long, value_delimiter = ',', num_args = 1..)]
     pub labels: Option<Vec<String>>,
 
@@ -103,15 +105,12 @@ impl Cli {
             .endpoint
             .as_ref()
             .cloned()
-            .or_else(|| env::var(ENV_ENDPOINT).ok())
             .unwrap_or(ENV_ENDPOINT_DEFAULT.to_owned());
 
         let mut query = format!("/agent?mode={}", u8::from(self.modes));
 
         // add &id=
-        if let Some(id) = self.id {
-            query.push_str(&format!("&id={}", id));
-        }
+        query.push_str(&format!("&id={}", self.id));
 
         // add local pk flag
         if let Some(file) = self.private_key_file.as_ref() {

crates/snops/src/cli.rs

@@ -26,7 +26,7 @@ pub struct Cli {
 
     // TODO: clarify that this needs to be an IP that agents can reach (handle external/internal?)
     /// Optional URL referencing a Loki server
-    #[arg(long)]
+    #[arg(long, env = "LOKI_URL")]
     pub loki: Option<Url>,
 
     #[arg(long, default_value_t = PrometheusLocation::Docker)]

crates/snops/src/server/mod.rs

@@ -209,7 +209,7 @@ async fn handle_socket(
                 // unwrap safety: this agent was just `mark_connected` with a valid client
                 let client = agent.rpc().cloned().unwrap();
 
-                // drop agent ref to allow for mutable borrow in handhake requests
+                // drop agent ref to allow for mutable borrow in handshake requests
                 drop(agent);
 
                 tokio::spawn(async move {

crates/xtask/src/main.rs

@@ -43,7 +43,11 @@ fn try_main() -> Result<()> {
 fn clipages(sh: &Shell) -> Result<()> {
     cmd!(sh, "cargo run -p snarkos-aot --features=docpages -- md").run()?;
     cmd!(sh, "cargo run -p snops --features=docpages -- md").run()?;
-    cmd!(sh, "cargo run -p snops-agent --features=docpages -- md").run()?;
+    cmd!(
+        sh,
+        "cargo run -p snops-agent --features=docpages -- --id foo md"
+    )
+    .run()?;
     cmd!(sh, "cargo run -p snops-cli --features=docpages -- md").run()?;
     Ok(())
 }

index.html

@@ -88,7 +88,7 @@
 
         <nav id="sidebar" class="sidebar" aria-label="Table of contents">
             <div class="sidebar-scrollbox">
-                <ol class="chapter"><li class="chapter-item expanded affix "><a href="../index.html" class="active">Introduction</a></li><li class="chapter-item expanded affix "><li class="part-title">Architecture</li><li class="chapter-item expanded "><a href="architecture/index.html"><strong aria-hidden="true">1.</strong> Overview</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="architecture/CONTROL_PLANE.html"><strong aria-hidden="true">1.1.</strong> Control Plane</a></li><li class="chapter-item expanded "><a href="architecture/AGENTS.html"><strong aria-hidden="true">1.2.</strong> Agents</a></li><li class="chapter-item expanded "><a href="architecture/RUNNERS.html"><strong aria-hidden="true">1.3.</strong> Runners</a></li></ol></li><li class="chapter-item expanded "><li class="part-title">User Guide</li><li class="chapter-item expanded "><a href="user_guide/index.html"><strong aria-hidden="true">2.</strong> Overview</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/envs/index.html"><strong aria-hidden="true">2.1.</strong> Setting up Environments</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/envs/STORAGE.html"><strong aria-hidden="true">2.1.1.</strong> Storage</a></li><li class="chapter-item expanded "><a href="user_guide/envs/TOPOLOGY.html"><strong aria-hidden="true">2.1.2.</strong> Topology</a></li><li class="chapter-item expanded "><a href="user_guide/envs/CANNONS.html"><strong aria-hidden="true">2.1.3.</strong> Cannons</a></li></ol></li><li class="chapter-item expanded "><a href="user_guide/running/index.html"><strong aria-hidden="true">2.2.</strong> Running</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/running/AGENT.html"><strong aria-hidden="true">2.2.1.</strong> Agent</a></li><li class="chapter-item expanded "><a href="user_guide/running/CONTROL_PLANE.html"><strong aria-hidden="true">2.2.2.</strong> Control Plane</a></li><li class="chapter-item expanded "><a href="user_guide/running/METRICS_AND_LOGGING.html"><strong aria-hidden="true">2.2.3.</strong> Metrics and Logging</a></li></ol></li><li class="chapter-item expanded "><a href="user_guide/clis/index.html"><strong aria-hidden="true">2.3.</strong> CLI Help</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/clis/SNARKOS_AOT.html"><strong aria-hidden="true">2.3.1.</strong> AOT</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS_AGENT.html"><strong aria-hidden="true">2.3.2.</strong> AGENT</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS.html"><strong aria-hidden="true">2.3.3.</strong> SNOPS</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS_CLI.html"><strong aria-hidden="true">2.3.4.</strong> SNOPS_CLI</a></li></ol></li></ol></li><li class="chapter-item expanded "><li class="part-title">Developing</li><li class="chapter-item expanded "><a href="developing/index.html"><strong aria-hidden="true">3.</strong> Developing</a></li><li class="chapter-item expanded affix "><li class="part-title">Glossary</li><li class="chapter-item expanded "><a href="glossary/RETENTION_RULES.html"><strong aria-hidden="true">4.</strong> Retention Rules</a></li><li class="chapter-item expanded "><a href="glossary/NODE_TARGETS.html"><strong aria-hidden="true">5.</strong> Node Targets</a></li><li class="chapter-item expanded "><a href="glossary/FIRE_RATE.html"><strong aria-hidden="true">6.</strong> Fire Rates</a></li></ol>
+                <ol class="chapter"><li class="chapter-item expanded affix "><a href="../index.html" class="active">Introduction</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Architecture</li><li class="chapter-item expanded "><a href="architecture/index.html"><strong aria-hidden="true">1.</strong> Overview</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="architecture/CONTROL_PLANE.html"><strong aria-hidden="true">1.1.</strong> Control Plane</a></li><li class="chapter-item expanded "><a href="architecture/AGENTS.html"><strong aria-hidden="true">1.2.</strong> Agents</a></li><li class="chapter-item expanded "><a href="architecture/RUNNERS.html"><strong aria-hidden="true">1.3.</strong> Runners</a></li></ol></li><li class="chapter-item expanded "><li class="part-title">User Guide</li><li class="chapter-item expanded "><a href="user_guide/index.html"><strong aria-hidden="true">2.</strong> Overview</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/envs/index.html"><strong aria-hidden="true">2.1.</strong> Setting up Environments</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/envs/STORAGE.html"><strong aria-hidden="true">2.1.1.</strong> Storage</a></li><li class="chapter-item expanded "><a href="user_guide/envs/TOPOLOGY.html"><strong aria-hidden="true">2.1.2.</strong> Topology</a></li><li class="chapter-item expanded "><a href="user_guide/envs/CANNONS.html"><strong aria-hidden="true">2.1.3.</strong> Cannons</a></li></ol></li><li class="chapter-item expanded "><a href="user_guide/running/index.html"><strong aria-hidden="true">2.2.</strong> Running</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/running/AGENT.html"><strong aria-hidden="true">2.2.1.</strong> Agent</a></li><li class="chapter-item expanded "><a href="user_guide/running/CONTROL_PLANE.html"><strong aria-hidden="true">2.2.2.</strong> Control Plane</a></li><li class="chapter-item expanded "><a href="user_guide/running/METRICS_AND_LOGGING.html"><strong aria-hidden="true">2.2.3.</strong> Metrics and Logging</a></li></ol></li><li class="chapter-item expanded "><a href="user_guide/clis/index.html"><strong aria-hidden="true">2.3.</strong> CLI Help</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="user_guide/clis/SNARKOS_AOT.html"><strong aria-hidden="true">2.3.1.</strong> AOT</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS_AGENT.html"><strong aria-hidden="true">2.3.2.</strong> AGENT</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS.html"><strong aria-hidden="true">2.3.3.</strong> SNOPS</a></li><li class="chapter-item expanded "><a href="user_guide/clis/SNOPS_CLI.html"><strong aria-hidden="true">2.3.4.</strong> SNOPS_CLI</a></li></ol></li></ol></li><li class="chapter-item expanded "><li class="part-title">Developing</li><li class="chapter-item expanded "><a href="developing/index.html"><strong aria-hidden="true">3.</strong> Developing</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Glossary</li><li class="chapter-item expanded "><a href="glossary/index.html"><strong aria-hidden="true">4.</strong> Overview</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="glossary/ACTIONS.html"><strong aria-hidden="true">4.1.</strong> Actions</a></li><li class="chapter-item expanded "><a href="glossary/IDs.html"><strong aria-hidden="true">4.2.</strong> IDs</a></li><li class="chapter-item expanded "><a href="glossary/NODE_TARGETS.html"><strong aria-hidden="true">4.3.</strong> Node Targets</a></li><li class="chapter-item expanded "><a href="glossary/RETENTION_RULES.html"><strong aria-hidden="true">4.4.</strong> Retention Rules</a></li></ol></li></ol>
             </div>
             <div id="sidebar-resize-handle" class="sidebar-resize-handle">
                 <div class="sidebar-resize-indicator"></div>

snops_book/SUMMARY.md

@@ -2,6 +2,8 @@
 
 [Introduction](../README.md)
 
+---
+
 # Architecture
 
 - [Overview](architecture/README.md)
@@ -30,8 +32,12 @@
 
 - [Developing](developing/README.md)
 
+---
+
 # Glossary
+- [Overview](glossary/README.md)
+  - [Actions](glossary/ACTIONS.md)
+  - [IDs](glossary/IDs.md)
+  - [Node Targets](glossary/NODE_TARGETS.md)
+  - [Retention Rules](glossary/RETENTION_RULES.md)
 
-- [Retention Rules](glossary/RETENTION_RULES.md)
-- [Node Targets](glossary/NODE_TARGETS.md)
-- [Fire Rates](glossary/FIRE_RATE.md)

snops_book/architecture/CONTROL_PLANE.md

@@ -4,12 +4,12 @@ A machine that is running the `snops` crate.
 
 The control plane runs as a daemon. It orchestrates any agents that connect to
 it, and listens for requests from implementors of its HTTP API (such as the
-[snops-cli](TODO)) for further instructions, like preparing an
+[snops-cli](../user_guide/clis/SNOPS_CLI.md)) for further instructions, like preparing an
 environment.
 
-## Responsibilites
+## Responsibilities
 
-The control plane has many responsibilites.
+The control plane has many responsibilities.
 
 ### Binary Distribution
 

snops_book/architecture/README.md

@@ -1,6 +1,6 @@
 # Architecture
 
-A snops "instance" is composed of multiple parts:
+A `snops` "instance" is composed of multiple parts:
 
 - A Control Plane
 - Agents

snops_book/glossary/ACTIONS.md

@@ -0,0 +1,3 @@
+# Actions
+
+TODO

snops_book/glossary/IDs.md

@@ -0,0 +1 @@
+# IDs

snops_book/glossary/NODE_TARGETS.md

@@ -1,3 +1,3 @@
 # Node Targets
 
-TODO decribe the different syntaxes here.
+TODO describe the different syntaxes here.

snops_book/glossary/README.md

@@ -0,0 +1,10 @@
+# Glossary
+
+A glossary of terms we use throughout the documentation.
+
+The format will follow: `link_with_more_details`: `TLDR definition`
+
+- [Actions](./ACTIONS.md): The `action` API offered by the control plane that allows for several conveniences.
+- [IDs](./IDs.md): A unique identifier that aligns with the regex `^[A-Za-z0-9][A-Za-z0-9\-_.]{0,63}$`.
+- [Node Targets](./NODE_TARGETS.md): An identifier to specify a node.
+- [Retention Rules](./RETENTION_RULES.md): Syntax rules to define how long data should be retained for.

snops_book/glossary/RETENTION_RULES.md

@@ -1,3 +1,3 @@
 # Retention Rules
 
-TODO move rentention rules explanation here then link to it.
+TODO move retention rules explanation here then link to it.

snops_book/user_guide/README.md

@@ -4,5 +4,5 @@ This section is broken down into multiple sections to help you learn how to use
 
 - How to set up an Environment.
 - How to deploy the tools.
-- How to run enviroments of various kinds.
-- CLI documenation.
+- How to run environments of various kinds.
+- CLI documentation.

snops_book/user_guide/clis/SNOPS_AGENT.md

@@ -10,7 +10,7 @@ This document contains the help content for the `snops-agent` command-line progr
 
 ## `snops-agent`
 
-**Usage:** `snops-agent [OPTIONS] <COMMAND>`
+**Usage:** `snops-agent [OPTIONS] --id <ID> <COMMAND>`
 
 ###### **Subcommands:**
 
@@ -20,9 +20,9 @@ This document contains the help content for the `snops-agent` command-line progr
 ###### **Options:**
 
 * `--endpoint <ENDPOINT>` — Control plane endpoint address (IP, or wss://host, http://host)
-* `--id <ID>`
+* `--id <ID>` — Agent ID, used to identify the agent in the network
 * `--private-key-file <PRIVATE_KEY_FILE>` — Locally provided private key file, used for envs where private keys are locally provided
-* `--labels <LABELS>`
+* `--labels <LABELS>` — Labels to attach to the agent, used for filtering and grouping
 * `--path <PATH>` — Path to the directory containing the stored data and configuration
 
   Default value: `./snops-data`

snops_book/user_guide/running/AGENT.md

@@ -1,3 +1,120 @@
 # Agent
 
+The `agent` is responsible for:
+
+- Communicating with the running `control plane`.
+- Launching and running the `snarkos-aot` binary.
+- Communicating to the running `snarkos-aot`.
+
+## Running the Agent
+
+The `agent` can be run on the same machine as the control plane or a separate one.
+
+Depending on the [environment](../envs/README.md) you specified, you will need the number agents listed in it, unless they are external.
+
+### Startup Options
+
+The binary has several options you can run mess with at launch and some are required that we will go over here.
+
+However, for a more in depth information you can read about the CLI options [here](../clis/SNOPS_AGENT.md).
+
+#### endpoint
+
+Is an optional argument that can be provided via the CLI or the `SNOPS_ENDPOINT` environment variable.
+
+> NOTE: If both the CLI flag and the ENV variable are present, the ENV variable takes precedence over the CLI flag.
+
+If not provided it will default to `127.0.0.1:1234`.
+
+This is the endpoint of the `Control Plane`.
+
+If you want it to be a secure connection please specify `https://` or `wss://` at the beginning of the endpoint or it will default to `http` and `ws`.
+
+#### _id_
+
+The field where you can give this agent a specific [ID](../../glossary/IDs.md), so it is identifiable within the `snops` ecosystem.
+
+#### labels
+
+Optional comma separated list of labels you can apply to the agent, which are used for filtering and grouping.
+
+#### private-key-file
+
+An optional private key file. If provided is used when starting `snarkOS`.
+
+#### path
+
+Optional path to the directory containing the stored data and configuration for the `agent`.
+
+By default it is `snops-data` local to where the `agent` was run from.
+
+#### external
+
+TODO
+
+#### internal
+
 TODO
+
+#### bind_addr
+
+The optional address for the `agent` to bind to when running.
+
+Defaults to `0.0.0.0`.
+
+#### node
+
+Optional port for the `snarkOS` node server to run on.
+
+Defaults to `4130`.
+
+#### bft
+
+Optional port for the `snarkOS` BFT to run on.
+
+Defaults to `5000`.
+
+#### rest
+
+Optional port for the `snarkOS` REST API to run on.
+
+Defaults to `3030`.
+
+#### metrics
+
+Optional port for the `snarkOS` metrics to run on.
+
+Defaults to `9000`.
+
+#### validator
+
+Enables `validator` mode as an option for the agent's `snarkOS` node.
+
+#### prover
+
+Enables `prover` mode as an option for the agent's `snarkOS` node.
+
+#### client
+
+Enables `client` mode as an option for the agent's `snarkOS` node.
+
+#### compute
+
+Enables `compute` mode as an option for the agent to be able to run transactions fired from within `snops`.
+
+#### quiet
+
+Run the agent in quiet mode which prevents `snarkOS` node output.
+
+## How it Works
+
+The `agent` once running will connect to the control plane. If the control plane goes/offline or isn't online yet that's okay! The `agent` will continuously try to reconnect to the control plane endpoint provided.
+
+For running `snarkOS` the `agent` will download that binary from the `control plane`.
+
+## Updating
+
+You don't have to worry about updating the individual agents.
+
+The control plane will serve the updated `agent` binary to them once there is an update.
+