[LocalNet] Allow to scale actors on LocalNet (#71)

* --wip-- [skip ci]

* --wip-- [skip ci]

* allow to use remote helm charts

* removed unused charts

* LocalNet doc

* link LocalNet

* requested changes

* Update Tiltfile

Co-authored-by: Daniel Olshansky <[email protected]>

---------

Co-authored-by: Daniel Olshansky <[email protected]>

.gitignore

@@ -55,3 +55,6 @@ ts-client/
 
 # Mock
 **/*_mock.go
+
+# Localnet config
+localnet_config.yaml

Makefile

@@ -99,7 +99,7 @@ localnet_down: ## Delete resources created by localnet
 .PHONY: localnet_regenesis
 localnet_regenesis: ## Regenerate the localnet genesis file
 # NOTE: intentionally not using --home <dir> flag to avoid overwriting the test keyring
-	ignite chain init --skip-proto
+	ignite chain init
 	cp -r ${HOME}/.pocket/keyring-test $(POCKETD_HOME)
 	cp ${HOME}/.pocket/config/*_key.json $(POCKETD_HOME)/config/
 	cp ${HOME}/.pocket/config/genesis.json $(POCKETD_HOME)/config/

README.md

@@ -25,10 +25,4 @@ make go_test
 
 ### LocalNet
 
-```bash
-# Create a k8s cluster
-kind create cluster
-
-# Start a LocalNet
-make localnet_up
-```
+Please check out the [LocalNet documentation](./localnet/README.md).

Tiltfile

@@ -1,8 +1,40 @@
 load('ext://restart_process', 'docker_build_with_restart')
+load('ext://helm_resource', "helm_resource", 'helm_repo')
 
 # A list of directories where changes trigger a hot-reload of the sequencer
 hot_reload_dirs = ['app', 'cmd', 'tools', 'x']
 
+# Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values
+localnet_config_path = "localnet_config.yaml"
+localnet_config_defaults = {
+    "relayers": {"count": 1},
+    "gateways": {"count": 1},
+   # By default, we use the `helm_repo` function below to point to the remote repository
+   # but can update it to the locally cloned repo for testing & development
+    "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"},
+}
+localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults)
+localnet_config = {}
+localnet_config.update(localnet_config_defaults)
+localnet_config.update(localnet_config_file)
+if (localnet_config_file != localnet_config) or (
+    not os.path.exists(localnet_config_path)
+):
+    print("Updating " + localnet_config_path + " with defaults")
+    local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config))
+
+# Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
+helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/")
+sequencer_chart = "pokt-network/poktroll-sequencer"
+poktroll_chart = "pokt-network/poktroll"
+if localnet_config["helm_chart_local_repo"]["enabled"]:
+    helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"]
+    hot_reload_dirs.append(helm_chart_local_repo)
+    print("Using local helm chart repo " + helm_chart_local_repo)
+
+    sequencer_chart = helm_chart_local_repo + "/charts/poktroll-sequencer"
+    poktroll_chart = helm_chart_local_repo + "/charts/poktroll"
+
 # Import files into Kubernetes ConfigMap
 def read_files_from_directory(directory):
     files = listdir(directory)
@@ -51,11 +83,15 @@ WORKDIR /
     live_update=[sync("bin/pocketd", "/usr/local/bin/pocketd")],
 )
 
-# Run pocketd, relayer, celestia and anvil nodes
-k8s_yaml(['localnet/kubernetes/celestia-rollkit.yaml', 'localnet/kubernetes/pocketd.yaml', 'localnet/kubernetes/pocketd-relayer.yaml', 'localnet/kubernetes/anvil.yaml'])
+# Run celestia and anvil nodes
+k8s_yaml(['localnet/kubernetes/celestia-rollkit.yaml', 'localnet/kubernetes/anvil.yaml'])
+
+# Run pocket-specific nodes (sequencer, relayers, etc...)
+helm_resource("sequencer", sequencer_chart, flags=['--values=./localnet/kubernetes/values-common.yaml'], image_deps=["pocketd"], image_keys=[('image.repository', 'image.tag')])
+helm_resource("relayers", poktroll_chart, flags=['--values=./localnet/kubernetes/values-common.yaml', '--set=replicaCount=' + str(localnet_config["relayers"]["count"])], image_deps=["pocketd"], image_keys=[('image.repository', 'image.tag')])
 
 # Configure tilt resources (tilt labels and port forawards) for all of the nodes above
 k8s_resource('celestia-rollkit', labels=["blockchains"], port_forwards=['26657', '26658', '26659'])
-k8s_resource('pocketd', labels=["blockchains"], resource_deps=['celestia-rollkit'], port_forwards=['36657', '40004'])
-k8s_resource('pocketd-relayer', labels=["blockchains"], resource_deps=['pocketd'], port_forwards=['8545', '8546', '40005'])
+k8s_resource('sequencer', labels=["blockchains"], resource_deps=['celestia-rollkit'], port_forwards=['36657', '40004'])
+k8s_resource('relayers', labels=["blockchains"], resource_deps=['sequencer'], port_forwards=['8545', '8546', '40005'])
 k8s_resource('anvil', labels=["blockchains"], port_forwards=['8547'])

localnet/README.md

@@ -0,0 +1,80 @@
+# Poktroll LocalNet <!-- omit in toc -->
+
+<!-- TODO(@olshansk, @okdas): Add a video showing how to use & run LocalNet. -->
+
+## Background <!-- omit in toc -->
+
+Poktroll comes with a LocalNet that can be used for development and testing on a local machine. As a rollup, it requires an underlying Data Availability layer, which is provisioned by the locally running celestia node.
+
+## Table of Contents <!-- omit in toc -->
+
+- [Run Poktroll locally](#run-poktroll-locally)
+  - [Report issues](#report-issues)
+  - [TL;DR](#tldr)
+- [Develop on the LocalNet](#develop-on-the-localnet)
+  - [Scaling network actors](#scaling-network-actors)
+  - [Modify Kubernetes workloads](#modify-kubernetes-workloads)
+
+## Run Poktroll locally
+
+### Report issues
+
+If you encounter a problem using this guide, please create a new [GitHub Issue](https://github.com/pokt-network/pocket/issues/new/choose).
+
+### TL;DR
+
+1. Install dependencies:
+   1. [Ignite](https://docs.ignite.com/welcome/install)
+   2. [Docker](https://docs.docker.com/engine/install/)
+   3. [Kind](https://kind.sigs.k8s.io/#installation-and-usage)
+   4. [Helm](https://helm.sh/docs/intro/install/#through-package-managers)
+   5. [Tilt](https://docs.tilt.dev/install.html) (note: we recommend using Kind cluster with Tilt)
+2. Run `make localnet_up` to start the network
+3. When prompted, click `space` to see the web UI with logs and current status of the network. Alternatively, you can go directly to [localhost:10350](http://localhost:10350)
+
+## Develop on the LocalNet
+
+Once LocalNet is started, a new file `localnet_config.yaml` is generated in the root directory of the repository. This file contains the configuration of the network. It looks like this:
+
+```yaml
+helm_chart_local_repo:
+  enabled: false
+  path: ../helm-charts
+relayers:
+  count: 1
+```
+
+### Scaling network actors
+
+To scale the number of actors, edit the `localnet_config.yaml` file and change the `count` of the relayers.
+
+For example:
+
+```diff
+helm_chart_local_repo:
+  enabled: false
+  path: ../helm-charts
+relayers:
+-   count: 1
++   count: 2
+```
+
+_NOTE: You may need to up to 1 minute for the new actors to be registered and deployed locally._
+
+### Modify Kubernetes workloads
+
+If you need to modify Kubernetes resources, follow these steps:
+
+1. Clone the [helm-charts](https://github.com/pokt-network/helm-charts) repository.
+2. In `localnet_config.yaml`, set `helm_chart_local_repo.enabled` to `true` and `path` to the **relative** path of the cloned repository.
+
+The following is an example that has not been tested yet:
+
+```bash
+cd ~/src/pocket
+git clone [email protected]:pokt-network/helm-charts.git
+
+cd ~/src/pocket/poktroll
+sed -i.bak "s/helm_chart_local_repo\.enabled: false.*/helm_chart_local_repo.enabled: true/" localnet_config.yaml
+sed -i.bak "s#path: .*#path: ../helm-charts#" localnet_config.yaml
+```

localnet/kubernetes/values-common.yaml

@@ -0,0 +1,5 @@
+securityContext:
+  # Allows to run delve inside a container.
+  allowPrivilegeEscalation: true
+  capabilities:
+    add: ["SYS_PTRACE"]