FEAT: mainnet test deploy (#712)

* Rename infra directories, and add documentation on deployment steps * updated docs * add script to generate sign_sk * initial deployment of mainnet nodes * currently deployed mainnet resources * added dev designator and perminant IPs to vars * moved VMs to dev project * misconfig changes * Fixed LB routing issue * added partner code and currently deployed mainnet dev env * added helper script to upload secrets to SM * added an example secrets file * addressing comments * comment suggestions * delete gen cipher keys dir * changed image family to LTS and enabled auto-updates --------- Co-authored-by: Xiangyi Zheng <[email protected]>
near · Jul 24, 2024 · c0abf1d · c0abf1d
1 parent 24c18a4
commit c0abf1d
Show file tree

Hide file tree

Showing 15 changed files with 1,075 additions and 1 deletion.
diff --git a/infra/.gitignore b/infra/.gitignore
@@ -15,10 +15,11 @@ crash.*.log
 # to change depending on the environment.
 *.tfvars
 *.tfvars.json
-!terraform-dev.tfvars
+!terraform-dev*.tfvars
 !backend-config-*.tfvars
 !terraform-testnet-example.tfvars
 !backend.tfvars
+!terraform-mainnet-example.tfvars
 
 # Ignore override files as they are usually used to override resources locally and so
 # are not checked in

diff --git a/infra/modules/instance-from-tpl/outputs.tf b/infra/modules/instance-from-tpl/outputs.tf
@@ -7,3 +7,11 @@ output "available_zones" {
 output "self_links" {
   value = google_compute_instance_from_template.compute_instance[*].self_link
 }
+
+output "hostname" {
+  value = local.hostname
+}
+
+output "ip_address" {
+  value = local.static_ips
+}
diff --git a/infra/multichain-mainnet-dev/main.tf b/infra/multichain-mainnet-dev/main.tf
@@ -0,0 +1,207 @@
+provider "google" {
+  project = var.project_id
+}
+provider "google-beta" {
+  project = var.project_id
+}
+module "gce-container" {
+  count   = length(var.node_configs)
+  source  = "terraform-google-modules/container-vm/google"
+  version = "~> 3.0"
+
+  container = {
+    image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/multichain-public/multichain-dev:mainnet-dev"
+    args  = ["start"]
+    port  = "3000"
+
+    env = concat(var.static_env, [
+      {
+        name  = "MPC_RECOVERY_NODE_ID"
+        value = "${count.index}"
+      },
+      {
+        name  = "MPC_RECOVERY_ACCOUNT_ID"
+        value = var.node_configs["${count.index}"].account
+      },
+      {
+        name  = "MPC_RECOVERY_CIPHER_PK"
+        value = var.node_configs["${count.index}"].cipher_pk
+      },
+      {
+        name  = "MPC_RECOVERY_ACCOUNT_SK"
+        value = data.google_secret_manager_secret_version.account_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_CIPHER_SK"
+        value = data.google_secret_manager_secret_version.cipher_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_SIGN_SK"
+        value =  data.google_secret_manager_secret_version.sign_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "AWS_ACCESS_KEY_ID"
+        value = data.google_secret_manager_secret_version.aws_access_key_secret_id.secret_data
+      },
+      {
+        name  = "AWS_SECRET_ACCESS_KEY"
+        value = data.google_secret_manager_secret_version.aws_secret_key_secret_id.secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_LOCAL_ADDRESS"
+        value = "https://${var.node_configs[count.index].domain}"
+      },
+      {
+        name  = "MPC_RECOVERY_SK_SHARE_SECRET_ID"
+        value = var.node_configs["${count.index}"].sk_share_secret_id
+      },
+      {
+        name  = "MPC_RECOVERY_ENV",
+        value = var.env
+      }
+    ])
+  }
+}
+
+resource "google_compute_global_address" "external_ips" {
+  count        = length(var.node_configs)
+  name         = "multichain-mainnet-dev-${count.index}"
+  address_type = "EXTERNAL"
+  address      = var.node_configs["${count.index}"].ip_address
+}
+
+resource "google_compute_managed_ssl_certificate" "mainnet_dev_ssl" {
+  count = length(var.node_configs)
+  name = "multichain-mainnet-dev-ssl-${count.index}"
+
+  managed {
+    domains = [var.node_configs["${count.index}"].domain]
+  }
+}
+
+module "mig_template" {
+  count      = length(var.node_configs)
+  source     = "../modules/mig_template"
+  network    = "projects/pagoda-shared-infrastructure/global/networks/prod"
+  subnetwork = "projects/pagoda-shared-infrastructure/regions/us-central1/subnetworks/prod-us-central1"
+  region     = var.region
+  service_account = {
+    email  = "mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com",
+    scopes = ["cloud-platform"]
+  }
+  name_prefix          = "multichain-mainnet-dev-${count.index}"
+  source_image_family  = "cos-stable"
+  source_image_project = "cos-cloud"
+  machine_type         = "n2-standard-2"
+
+  startup_script = "docker rm watchtower ; docker run -d --name watchtower -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --debug --interval 30"
+
+  source_image = reverse(split("/", module.gce-container[count.index].source_image))[0]
+  metadata     = merge(var.additional_metadata, { "gce-container-declaration" = module.gce-container["${count.index}"].metadata_value })
+  tags = [
+    "multichain"
+  ]
+  labels = {
+    "container-vm" = module.gce-container[count.index].vm_container_label
+  }
+
+  depends_on = [google_compute_global_address.external_ips]
+}
+
+
+module "instances" {
+  count      = length(var.node_configs)
+  source     = "../modules/instance-from-tpl"
+  region     = var.region
+  project_id = var.project_id
+  hostname   = "multichain-mainnet-dev-${count.index}"
+  network    = "projects/pagoda-shared-infrastructure/global/networks/prod"
+  subnetwork = "projects/pagoda-shared-infrastructure/regions/us-central1/subnetworks/prod-us-central1"
+
+  instance_template = module.mig_template[count.index].self_link_unique
+
+}
+
+resource "google_compute_health_check" "multichain_healthcheck" {
+  name = "multichain-mainnet-dev-healthcheck"
+
+  http_health_check {
+    port         = 3000
+    request_path = "/"
+  }
+
+}
+
+resource "google_compute_global_forwarding_rule" "default" {
+  count                 = length(var.node_configs)
+  name                  = "multichain-mainnet-dev-rule-${count.index}"
+  target                = google_compute_target_http_proxy.default[count.index].id
+  port_range            = "80"
+  load_balancing_scheme = "EXTERNAL"
+  ip_address            = google_compute_global_address.external_ips[count.index].address
+}
+
+resource "google_compute_global_forwarding_rule" "https_fw" {
+  count      = length(var.node_configs)
+  name       = "multichain-mainnet-dev-https-rule-${count.index}"
+  target     = google_compute_target_https_proxy.default_https[count.index].id
+  port_range = "443"
+  ip_protocol = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  ip_address = google_compute_global_address.external_ips[count.index].address
+}
+
+resource "google_compute_target_http_proxy" "default" {
+  count       = length(var.node_configs)
+  name        = "multichain-mainnet-dev-target-http-proxy-${count.index}"
+  description = "a description"
+  url_map     = google_compute_url_map.default_redirect[count.index].id
+}
+
+resource "google_compute_target_https_proxy" "default_https" {
+  count      = length(var.node_configs)
+  name        = "multichain-mainnet-dev-target-https-proxy-${count.index}"
+  description = "a description"
+  ssl_certificates = [ google_compute_managed_ssl_certificate.mainnet_dev_ssl[count.index].self_link ]
+  url_map     = google_compute_url_map.default[count.index].id
+}
+
+resource "google_compute_url_map" "default" {
+  count           = length(var.node_configs)
+  name            = "multichain-mainnet-dev-url-map-${count.index}"
+  default_service = google_compute_backend_service.multichain_backend[count.index].id
+}
+
+resource "google_compute_url_map" "default_redirect" {
+  count           = length(var.node_configs)
+  name            = "multichain-mainnet-dev-redirect-url-map-${count.index}"
+
+  default_url_redirect {
+    strip_query    = false
+    https_redirect = true
+  }
+}
+
+resource "google_compute_backend_service" "multichain_backend" {
+  count                 = length(var.node_configs)
+  name                  = "multichain-service-mainnet-dev-${count.index}"
+  load_balancing_scheme = "EXTERNAL"
+
+  backend {
+    group = google_compute_instance_group.multichain_group[count.index].id
+  }
+
+  health_checks = [google_compute_health_check.multichain_healthcheck.id]
+}
+
+resource "google_compute_instance_group" "multichain_group" {
+  count     = length(var.node_configs)
+  name      = "multichain-instance-group-mainnet-dev-${count.index}"
+  instances = [module.instances[count.index].self_links[0]]
+
+  zone = "us-central1-a"
+  named_port {
+    name = "http"
+    port = 3000
+  }
+}
diff --git a/infra/multichain-mainnet-dev/outputs.tf b/infra/multichain-mainnet-dev/outputs.tf
@@ -0,0 +1,3 @@
+output "instance_ips" {
+  value = local.instance_ip_map
+}
diff --git a/infra/multichain-mainnet-dev/resources.tf b/infra/multichain-mainnet-dev/resources.tf
@@ -0,0 +1,52 @@
+terraform {
+  backend "gcs" {
+    bucket = "multichain-terraform-dev"
+    prefix = "state/multichain-vm-test-mainnet"
+  }
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.73.0"
+    }
+  }
+}
+
+locals {
+  instance_ip_map = {
+    for idx, instance in module.instances : instance["hostname"] => google_compute_global_address.external_ips[idx].address
+  }
+}
+
+# These data blocks grab the values from your GCP secret manager, please adjust secret names as desired
+data "google_secret_manager_secret_version" "account_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[count.index].account_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "cipher_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[count.index].cipher_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "sign_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[count.index].sign_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "sk_share_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[count.index].sk_share_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "aws_access_key_secret_id" {
+  secret = "multichain-indexer-aws-access-key"
+}
+
+data "google_secret_manager_secret_version" "aws_secret_key_secret_id" {
+  secret = "multichain-indexer-aws-secret-key"
+}
diff --git a/infra/multichain-mainnet-dev/terraform-dev.auto.tfvars b/infra/multichain-mainnet-dev/terraform-dev.auto.tfvars
@@ -0,0 +1,83 @@
+env = "mainnet-dev"
+node_configs = [
+  {
+    account              = "multichain-node-dev-0.near"
+    cipher_pk            = "f2b97fe31de2407a6eb83d2d8b75d657ccdf40c77838f4b7a7a0f0f2db947a2b"
+    account_sk_secret_id = "multichain-account-sk-mainnet-0"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-0"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-0"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-0"
+    ip_address           = "34.49.62.49"
+    domain               = "multichain-mainnet-dev-0.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-1.near"
+    cipher_pk            = "11afc3d10766acb5314dfdf8cd832b562277dd428f16c6fe02d88e83db27e508"
+    account_sk_secret_id = "multichain-account-sk-mainnet-1"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-1"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-1"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-1"
+    ip_address           = "34.128.137.203"
+    domain               = "multichain-mainnet-dev-1.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-2.near"
+    cipher_pk            = "1707fd3733c86e51f1bc8eec6c153a25ee33c547bf9873b2b310961ece507c5c"
+    account_sk_secret_id = "multichain-account-sk-mainnet-2"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-2"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-2"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-2"
+    ip_address           = "34.49.228.15"
+    domain               = "multichain-mainnet-dev-2.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-3.near"
+    cipher_pk            = "e80eb77e8af15c3100e5fe65f4adc5ddff7543bbf43c2a4bf1e6c3656f4cec17"
+    account_sk_secret_id = "multichain-account-sk-mainnet-3"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-3"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-3"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-3"
+    ip_address           = "34.111.236.127"
+    domain               = "multichain-mainnet-dev-3.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-4.near"
+    cipher_pk            = "480f247b28a5e057168574d56bf724dbb26f49cb7b018db7c5f41145e2d9db3c"
+    account_sk_secret_id = "multichain-account-sk-mainnet-4"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-4"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-4"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-4"
+    ip_address           = "34.49.151.143"
+    domain               = "multichain-mainnet-dev-4.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-5.near"
+    cipher_pk            = "e74e0463a464b081ac6acc9666836ed7229f1b62b7297bac71c2874e00bcc07c"
+    account_sk_secret_id = "multichain-account-sk-mainnet-5"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-5"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-5"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-5"
+    ip_address           = "34.128.150.24"
+    domain               = "multichain-mainnet-dev-5.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-6.near"
+    cipher_pk            = "f17c9ec7fa3b8cb9bf7bbde10facf91d9737138e3562ad953afeea8c804b9077"
+    account_sk_secret_id = "multichain-account-sk-mainnet-6"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-6"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-6"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-6"
+    ip_address           = "34.120.189.165"
+    domain               = "multichain-mainnet-dev-6.dev.api.pagoda.co"
+  },
+  {
+    account              = "multichain-node-dev-7.near"
+    cipher_pk            = "533c5f6427d6c56b5d720f7f98156a26249138ca0823f0aa57f7c13c5df4d520"
+    account_sk_secret_id = "multichain-account-sk-mainnet-7"
+    cipher_sk_secret_id  = "multichain-cipher-sk-mainnet-7"
+    sign_sk_secret_id    = "multichain-sign-sk-mainnet-7"
+    sk_share_secret_id   = "multichain-sk-share-mainnet-7"
+    ip_address           = "34.111.38.76"
+    domain               = "multichain-mainnet-dev-7.dev.api.pagoda.co"
+  },
+]