diff --git a/.gitignore b/.gitignore index fe560aac0..aa7066b08 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ ts-client/ # Mock **/*_mock.go + +# Localnet config +localnet_config.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index 201b51f48..506962fea 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ localnet_down: ## Delete resources created by localnet .PHONY: localnet_regenesis localnet_regenesis: ## Regenerate the localnet genesis file # NOTE: intentionally not using --home flag to avoid overwriting the test keyring - ignite chain init --skip-proto + ignite chain init cp -r ${HOME}/.pocket/keyring-test $(POCKETD_HOME) cp ${HOME}/.pocket/config/*_key.json $(POCKETD_HOME)/config/ cp ${HOME}/.pocket/config/genesis.json $(POCKETD_HOME)/config/ diff --git a/README.md b/README.md index 0830ee3f8..c73a4b62d 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,4 @@ make go_test ### LocalNet -```bash -# Create a k8s cluster -kind create cluster - -# Start a LocalNet -make localnet_up -``` +Please check out the [LocalNet documentation](./localnet/README.md). diff --git a/Tiltfile b/Tiltfile index 03b807f47..34717c9d2 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,8 +1,40 @@ load('ext://restart_process', 'docker_build_with_restart') +load('ext://helm_resource', "helm_resource", 'helm_repo') # A list of directories where changes trigger a hot-reload of the sequencer hot_reload_dirs = ['app', 'cmd', 'tools', 'x'] +# Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values +localnet_config_path = "localnet_config.yaml" +localnet_config_defaults = { + "relayers": {"count": 1}, + "gateways": {"count": 1}, + # By default, we use the `helm_repo` function below to point to the remote repository + # but can update it to the locally cloned repo for testing & development + "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, +} +localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) +localnet_config = {} +localnet_config.update(localnet_config_defaults) +localnet_config.update(localnet_config_file) +if (localnet_config_file != localnet_config) or ( + not os.path.exists(localnet_config_path) +): + print("Updating " + localnet_config_path + " with defaults") + local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config)) + +# Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo. +helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/") +sequencer_chart = "pokt-network/poktroll-sequencer" +poktroll_chart = "pokt-network/poktroll" +if localnet_config["helm_chart_local_repo"]["enabled"]: + helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"] + hot_reload_dirs.append(helm_chart_local_repo) + print("Using local helm chart repo " + helm_chart_local_repo) + + sequencer_chart = helm_chart_local_repo + "/charts/poktroll-sequencer" + poktroll_chart = helm_chart_local_repo + "/charts/poktroll" + # Import files into Kubernetes ConfigMap def read_files_from_directory(directory): files = listdir(directory) @@ -51,11 +83,15 @@ WORKDIR / live_update=[sync("bin/pocketd", "/usr/local/bin/pocketd")], ) -# Run pocketd, relayer, celestia and anvil nodes -k8s_yaml(['localnet/kubernetes/celestia-rollkit.yaml', 'localnet/kubernetes/pocketd.yaml', 'localnet/kubernetes/pocketd-relayer.yaml', 'localnet/kubernetes/anvil.yaml']) +# Run celestia and anvil nodes +k8s_yaml(['localnet/kubernetes/celestia-rollkit.yaml', 'localnet/kubernetes/anvil.yaml']) + +# Run pocket-specific nodes (sequencer, relayers, etc...) +helm_resource("sequencer", sequencer_chart, flags=['--values=./localnet/kubernetes/values-common.yaml'], image_deps=["pocketd"], image_keys=[('image.repository', 'image.tag')]) +helm_resource("relayers", poktroll_chart, flags=['--values=./localnet/kubernetes/values-common.yaml', '--set=replicaCount=' + str(localnet_config["relayers"]["count"])], image_deps=["pocketd"], image_keys=[('image.repository', 'image.tag')]) # Configure tilt resources (tilt labels and port forawards) for all of the nodes above k8s_resource('celestia-rollkit', labels=["blockchains"], port_forwards=['26657', '26658', '26659']) -k8s_resource('pocketd', labels=["blockchains"], resource_deps=['celestia-rollkit'], port_forwards=['36657', '40004']) -k8s_resource('pocketd-relayer', labels=["blockchains"], resource_deps=['pocketd'], port_forwards=['8545', '8546', '40005']) +k8s_resource('sequencer', labels=["blockchains"], resource_deps=['celestia-rollkit'], port_forwards=['36657', '40004']) +k8s_resource('relayers', labels=["blockchains"], resource_deps=['sequencer'], port_forwards=['8545', '8546', '40005']) k8s_resource('anvil', labels=["blockchains"], port_forwards=['8547']) diff --git a/localnet/README.md b/localnet/README.md new file mode 100644 index 000000000..4a1f3be9d --- /dev/null +++ b/localnet/README.md @@ -0,0 +1,80 @@ +# Poktroll LocalNet + + + +## Background + +Poktroll comes with a LocalNet that can be used for development and testing on a local machine. As a rollup, it requires an underlying Data Availability layer, which is provisioned by the locally running celestia node. + +## Table of Contents + +- [Run Poktroll locally](#run-poktroll-locally) + - [Report issues](#report-issues) + - [TL;DR](#tldr) +- [Develop on the LocalNet](#develop-on-the-localnet) + - [Scaling network actors](#scaling-network-actors) + - [Modify Kubernetes workloads](#modify-kubernetes-workloads) + +## Run Poktroll locally + +### Report issues + +If you encounter a problem using this guide, please create a new [GitHub Issue](https://github.com/pokt-network/pocket/issues/new/choose). + +### TL;DR + +1. Install dependencies: + 1. [Ignite](https://docs.ignite.com/welcome/install) + 2. [Docker](https://docs.docker.com/engine/install/) + 3. [Kind](https://kind.sigs.k8s.io/#installation-and-usage) + 4. [Helm](https://helm.sh/docs/intro/install/#through-package-managers) + 5. [Tilt](https://docs.tilt.dev/install.html) (note: we recommend using Kind cluster with Tilt) +2. Run `make localnet_up` to start the network +3. When prompted, click `space` to see the web UI with logs and current status of the network. Alternatively, you can go directly to [localhost:10350](http://localhost:10350) + +## Develop on the LocalNet + +Once LocalNet is started, a new file `localnet_config.yaml` is generated in the root directory of the repository. This file contains the configuration of the network. It looks like this: + +```yaml +helm_chart_local_repo: + enabled: false + path: ../helm-charts +relayers: + count: 1 +``` + +### Scaling network actors + +To scale the number of actors, edit the `localnet_config.yaml` file and change the `count` of the relayers. + +For example: + +```diff +helm_chart_local_repo: + enabled: false + path: ../helm-charts +relayers: +- count: 1 ++ count: 2 +``` + +_NOTE: You may need to up to 1 minute for the new actors to be registered and deployed locally._ + +### Modify Kubernetes workloads + +If you need to modify Kubernetes resources, follow these steps: + +1. Clone the [helm-charts](https://github.com/pokt-network/helm-charts) repository. +2. In `localnet_config.yaml`, set `helm_chart_local_repo.enabled` to `true` and `path` to the **relative** path of the cloned repository. + +The following is an example that has not been tested yet: + +```bash +cd ~/src/pocket +git clone git@github.com:pokt-network/helm-charts.git + +cd ~/src/pocket/poktroll +sed -i.bak "s/helm_chart_local_repo\.enabled: false.*/helm_chart_local_repo.enabled: true/" localnet_config.yaml +sed -i.bak "s#path: .*#path: ../helm-charts#" localnet_config.yaml +``` diff --git a/localnet/kubernetes/pocketd-relayer.yaml b/localnet/kubernetes/pocketd-relayer.yaml deleted file mode 100644 index fe36d4dff..000000000 --- a/localnet/kubernetes/pocketd-relayer.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pocketd-relayer-scripts -data: - pocket.sh: |- - #!/bin/sh - - ls -la /root/.pocket/config-src/ - mkdir -p /root/.pocket/config/ - cp -r -L /root/.pocket/config-src/* /root/.pocket/config/ - ls -la /root/.pocket/config/ - - # sleep 99999 - - # You can attach to this process with delve (dlv) for debugging purpose with `dlv attach $(pgrep pocketd) --listen :40005 --headless --api-version=2 --accept-multiclient` - run inside the container! - pocketd relayer --node pocketd:36657 --signing-key servicer1 --keyring-backend test - - # OR debug the node (uncomment this line but comment previous line) - # dlv exec --listen :40005 /usr/local/bin/pocketd relayer --pocket-node pocketd:36657 --sequencer-node pocketd:36657 --signing-key servicer1 --keyring-backend test ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pocketd-relayer -spec: - replicas: 1 - selector: - matchLabels: - app: pocketd-relayer - template: - metadata: - labels: - app: pocketd-relayer - spec: - # TODO: Add resource limits - containers: - - name: pocketd - image: pocketd - # To allow Delve to do the thing - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: ["SYS_PTRACE"] - env: - - name: AUTH_TOKEN - valueFrom: - secretKeyRef: - name: celestia-secret - key: auth_token - ports: - - containerPort: 8545 - - containerPort: 8546 - - containerPort: 40005 - command: ["/bin/sh"] - args: ["/scripts/pocket.sh"] - volumeMounts: - - name: scripts-volume - mountPath: /scripts - - name: keys-volume - mountPath: /root/.pocket/keyring-test/ - - name: configs-volume - mountPath: /root/.pocket/config-src/ - volumes: - - name: scripts-volume - configMap: - name: pocketd-relayer-scripts - - name: keys-volume - configMap: - name: pocketd-keys - - name: configs-volume - configMap: - name: pocketd-configs diff --git a/localnet/kubernetes/pocketd.yaml b/localnet/kubernetes/pocketd.yaml deleted file mode 100644 index 907d033db..000000000 --- a/localnet/kubernetes/pocketd.yaml +++ /dev/null @@ -1,128 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pocket-scripts -data: - pocket.sh: |- - #!/bin/sh - - echo "AUTH_TOKEN is: $AUTH_TOKEN" - VALIDATOR_NAME=sequencer1 - CHAIN_ID=pocket - KEY_NAME=pocket-key - KEY_2_NAME=pocket-key-2 - TOKEN_AMOUNT="10000000000000000000000000upokt" - STAKING_AMOUNT="1000000000upokt" - - # reset any existing genesis/chain data - pocketd tendermint unsafe-reset-all - - # create a random Namespace ID for your rollup to post blocks to - openssl rand -hex 10 > /root/.pocket/data/namespace-id - NAMESPACE_ID=$(cat /root/.pocket/data/namespace-id) - - # query the DA Layer start height, in this case we are querying - # our local devnet at port 26657, the RPC. The RPC endpoint is - # to allow users to interact with Celestia's nodes by querying - # the node's state and broadcasting transactions on the Celestia - # network. The default port is 26657. - curl http://celestia-rollkit:26657/block | jq -r '.result.block.header.height' > /root/.pocket/data/da-block-height - DA_BLOCK_HEIGHT=$(cat /root/.pocket/data/da-block-height) - - echo -e "Your NAMESPACE_ID is $NAMESPACE_ID \n\n Your DA_BLOCK_HEIGHT is $DA_BLOCK_HEIGHT" - - ls -la /root/.pocket/config-src/ - mkdir -p /root/.pocket/config/ - cp -r -L /root/.pocket/config-src/* /root/.pocket/config/ - ls -la /root/.pocket/config/ - - # sleep 9999 - - # start the node - # You can attach to this process with delve (dlv) for debugging purpose with `dlv attach $(pgrep pocketd) --listen :40004 --headless --api-version=2 --accept-multiclient` - run inside the container! - pocketd start --rollkit.aggregator true --rollkit.da_layer celestia --rollkit.da_config='{"base_url":"http://celestia-rollkit:26658","timeout":60000000000,"fee":600000,"gas_limit":6000000,"auth_token":"'$AUTH_TOKEN'"}' --rollkit.namespace_id $NAMESPACE_ID --rollkit.da_start_height $DA_BLOCK_HEIGHT --rpc.laddr tcp://0.0.0.0:36657 --p2p.laddr "0.0.0.0:36656" - - # OR debug the node (uncomment this line but comment previous line) - # dlv exec /usr/local/bin/pocketd --listen :40004 --headless --api-version=2 --accept-multiclient -- start --rollkit.aggregator true --rollkit.da_layer celestia --rollkit.da_config='{"base_url":"http://celestia-rollkit:26658","timeout":60000000000,"fee":600000,"gas_limit":6000000,"auth_token":"'$AUTH_TOKEN'"}' --rollkit.namespace_id $NAMESPACE_ID --rollkit.da_start_height $DA_BLOCK_HEIGHT --rpc.laddr tcp://0.0.0.0:36657 --p2p.laddr "0.0.0.0:36656" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pocketd -spec: - replicas: 1 - selector: - matchLabels: - app: pocketd - template: - metadata: - labels: - app: pocketd - spec: - # TODO: Add resource limits - containers: - - name: pocketd - image: pocketd - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: ["SYS_PTRACE"] - env: - - name: AUTH_TOKEN - valueFrom: - secretKeyRef: - name: celestia-secret - key: auth_token - ports: - - containerPort: 36657 - - containerPort: 40004 - command: ["/bin/sh"] - args: ["/scripts/pocket.sh"] - volumeMounts: - - name: scripts-volume - mountPath: /scripts - - name: keys-volume - mountPath: /root/.pocket/keyring-test/ - - name: configs-volume - mountPath: /root/.pocket/config-src/ - - name: data-volume - mountPath: /root/.pocket/data/ - volumes: - - name: scripts-volume - configMap: - name: pocket-scripts - - name: keys-volume - configMap: - name: pocketd-keys - - name: configs-volume - configMap: - name: pocketd-configs - - name: data-volume - persistentVolumeClaim: - claimName: pocketd-data - ---- -apiVersion: v1 -kind: Service -metadata: - name: pocketd -spec: - selector: - app: pocketd - ports: - - protocol: TCP - name: rpc - port: 36657 - targetPort: 36657 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pocketd-data -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/localnet/kubernetes/values-common.yaml b/localnet/kubernetes/values-common.yaml new file mode 100644 index 000000000..b32b36555 --- /dev/null +++ b/localnet/kubernetes/values-common.yaml @@ -0,0 +1,5 @@ +securityContext: + # Allows to run delve inside a container. + allowPrivilegeEscalation: true + capabilities: + add: ["SYS_PTRACE"] \ No newline at end of file