diff --git a/automation/terraform/modules/kubernetes/testnet/variables.tf b/automation/terraform/modules/kubernetes/testnet/variables.tf index e89c786549f..4903f91e398 100644 --- a/automation/terraform/modules/kubernetes/testnet/variables.tf +++ b/automation/terraform/modules/kubernetes/testnet/variables.tf @@ -203,15 +203,13 @@ variable "seed_external_port" { variable "seed_configs" { type = list( object({ - name = string, - class = string, - libp2p_secret = string, - libp2p_secret_pw = string - # external_port = number, - external_ip = string, - # private_key_secret = string, - enableArchive = bool, - archiveAddress = string + name = string, + class = string, + libp2p_secret = string, + libp2p_secret_pw = string + external_ip = string, + enableArchive = bool, + archiveAddress = string persist_working_dir = bool, }) ) @@ -225,19 +223,18 @@ variable "log_level" { default = "Trace" } -# variable "block_producer_key_pass" { -# type = string -# } +variable "block_producer_key_pass" { + type = string + default = "naughty blue worm" +} variable "block_producer_configs" { type = list( object({ name = string, class = string, - keypair_name = string, - # private_key = string, - # public_key = string, - privkey_password = string, + keypair_name = string, + privkey_password = string, external_port = number, libp2p_secret = string, enable_gossip_flooding = bool, @@ -259,16 +256,16 @@ variable "plain_node_configs" { # Snark Worker Vars variable "snark_coordinators" { - type = list( + type = list( object({ - snark_coordinator_name = string, - snark_worker_replicas = number - snark_worker_fee = number - snark_worker_public_key = string + snark_coordinator_name = string + snark_worker_replicas = number + snark_worker_fee = number + snark_worker_public_key = string snark_coordinators_host_port = number - persist_working_dir = bool - })) + persist_working_dir = bool + })) default = [] } @@ -354,17 +351,17 @@ variable "make_report_accounts" { variable "archive_configs" { type = list( object({ - name = string - image = string - serverPort = string - externalPort = string - enableLocalDaemon = bool - enablePostgresDB = bool - - postgresHost = string - postgresPort = string - remoteSchemaFile = string - remoteSchemaAuxFiles = list(string) + name = string + image = string + serverPort = string + externalPort = string + enableLocalDaemon = bool + enablePostgresDB = bool + + postgresHost = string + postgresPort = string + remoteSchemaFile = string + remoteSchemaAuxFiles = list(string) persistenceEnabled = bool persistenceSize = string diff --git a/automation/terraform/modules/o1-testnet/inputs.tf b/automation/terraform/modules/o1-testnet/inputs.tf index 6e6227f8530..9841807a9eb 100644 --- a/automation/terraform/modules/o1-testnet/inputs.tf +++ b/automation/terraform/modules/o1-testnet/inputs.tf @@ -126,20 +126,25 @@ variable "seed_discovery_keypairs" { ] } +variable "seed_external_port" { + type = string + default = "10001" +} + # Block Producer Vars variable "whales" { description = "individual whale block producer node deployment configurations" - default = null + default = null } variable "fishes" { description = "individual fish block producer node deployment configurations" - default = null + default = null } variable "nodes_with_user_agent" { - type = list(string) + type = list(string) default = [] } @@ -186,15 +191,15 @@ variable "seed_starting_host_port" { variable "snark_coordinators" { description = "configurations for not just the snark coordinators but also the snark workers they coordinate" - type = list( + type = list( object({ - - snark_coordinator_name = string, - snark_worker_replicas = number - snark_worker_fee = number - snark_worker_public_key = string + snark_coordinator_name = string, + snark_worker_replicas = number + snark_worker_fee = number + snark_worker_public_key = string snark_coordinators_host_port = number - })) + persist_working_dir = bool + })) default = [] } @@ -266,7 +271,7 @@ variable "make_report_accounts" { } variable "log_precomputed_blocks" { - type = bool + type = bool default = false } @@ -299,7 +304,7 @@ variable "archive_node_count" { variable "archive_configs" { description = "individual archive-node deployment configurations" - default = null + default = null } variable "upload_blocks_to_gcloud" { @@ -315,4 +320,4 @@ variable "seed_peers_url" { variable "zkapps_dashboard_key" { type = string default = "" -} \ No newline at end of file +} diff --git a/automation/terraform/modules/o1-testnet/locals.tf b/automation/terraform/modules/o1-testnet/locals.tf index 8113edef4f1..d825c4d4dee 100644 --- a/automation/terraform/modules/o1-testnet/locals.tf +++ b/automation/terraform/modules/o1-testnet/locals.tf @@ -1,19 +1,19 @@ locals { - whale_count_total = length ( flatten( [ + whale_count_total = length(flatten([ for bp in var.whales : - [ - for i in range(bp.duplicates) : "" - - ] ]) ) + [ + for i in range(bp.duplicates) : "" - fish_count_total = length ( flatten( [ + ]])) + + fish_count_total = length(flatten([ for index, bp in var.fishes : - [ - for i in range(bp.duplicates) : "" - - ] ]) ) + [ + for i in range(bp.duplicates) : "" + + ]])) @@ -21,81 +21,82 @@ locals { fish_block_producer_libp2p_names = [for i in range(local.fish_count_total) : "fish-block-producer-${i + 1}"] - whale_configs = flatten( [ + whale_configs = flatten([ for index, bp in var.whales : - [ - for i in range(bp.duplicates) : { - name = "whale-${index+1}-${i+1}" - unique_node_index= index+1 - total_node_index= 1+ i+ length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):0 ] ])) #summation of all duplicates so far - # full_peer = "/dns4/whale-${index+1}-${i+1}.${var.testnet_name}/tcp/${var.block_producer_starting_host_port +i+ length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):0 ] ]))}/p2p/${trimspace(data.local_file.libp2p_peers[element (local.whale_block_producer_libp2p_names,i+ length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):0 ] ])) ) ].content)}", - port = var.block_producer_starting_host_port+i + length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):"" ] ])) - class = "whale" - - } - - ] ]) - - fish_configs = flatten( [ + [ + for i in range(bp.duplicates) : { + name = "whale-${index + 1}-${i + 1}" + unique_node_index = index + 1 + total_node_index = 1 + i + length(flatten([for b in slice(var.whales, 0, index) : [for k in range(b.duplicates) : 0]])) #summation of all duplicates so far + # full_peer = "/dns4/whale-${index+1}-${i+1}.${var.testnet_name}/tcp/${var.block_producer_starting_host_port +i+ length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):0 ] ]))}/p2p/${trimspace(data.local_file.libp2p_peers[element (local.whale_block_producer_libp2p_names,i+ length ( flatten([for b in slice(var.whales,0, index) : [ for k in range(b.duplicates):0 ] ])) ) ].content)}", + port = var.block_producer_starting_host_port + i + length(flatten([for b in slice(var.whales, 0, index) : [for k in range(b.duplicates) : ""]])) + class = "whale" + + } + + ]]) + + fish_configs = flatten([ for index, bp in var.fishes : - [ - for i in range(bp.duplicates) : { - name = "fish-${index+1}-${i+1}" - unique_node_index= index+1 - total_node_index= 1+ i+length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):0 ] ])) - # full_peer = "/dns4/fish-${index+1}-${i+1}.${var.testnet_name}/tcp/${var.block_producer_starting_host_port +i+ length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):0 ] ]))}/p2p/${trimspace(data.local_file.libp2p_peers[element (local.fish_block_producer_libp2p_names,i+ length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):0 ] ])) ) ].content)}", - port = var.block_producer_starting_host_port+i + length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):"" ] ])) - class = "fish" - - } - - ] ]) + [ + for i in range(bp.duplicates) : { + name = "fish-${index + 1}-${i + 1}" + unique_node_index = index + 1 + total_node_index = 1 + i + length(flatten([for b in slice(var.fishes, 0, index) : [for k in range(b.duplicates) : 0]])) + # full_peer = "/dns4/fish-${index+1}-${i+1}.${var.testnet_name}/tcp/${var.block_producer_starting_host_port +i+ length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):0 ] ]))}/p2p/${trimspace(data.local_file.libp2p_peers[element (local.fish_block_producer_libp2p_names,i+ length ( flatten([for b in slice(var.fishes,0, index) : [ for k in range(b.duplicates):0 ] ])) ) ].content)}", + port = var.block_producer_starting_host_port + i + length(flatten([for b in slice(var.fishes, 0, index) : [for k in range(b.duplicates) : ""]])) + class = "fish" + + } + + ]]) block_producer_configs = concat(local.whale_configs, local.fish_configs) whale_count_unique = length(var.whales) - fish_count_unique = length(var.fishes) + fish_count_unique = length(var.fishes) - seed_names = [for i in range(var.seed_count) : "seed-${i + 1}"] + seed_names = [for i in range(var.seed_count) : "seed-${i + 1}"] seed_static_peers = [ for index, name in keys(data.local_file.libp2p_seed_peers) : { # i don't think the seeds need to have different ports full_peer = "/dns4/${name}.${var.testnet_name}.o1test.net/tcp/${var.seed_external_port}/p2p/${trimspace(data.local_file.libp2p_seed_peers[name].content)}", # port = var.seed_starting_host_port + index - name = local.seed_names[index] - unique_node_index= -1 - total_node_index= -1 - class = "seed" + name = local.seed_names[index] + unique_node_index = -1 + total_node_index = -1 + class = "seed" } ] default_archive_node = { - image = var.mina_archive_image - serverPort = "3086" - externalPort = "11010" - enableLocalDaemon = true - enablePostgresDB = true - - postgresHost = "archive-1-postgresql" - postgresPort = 5432 + image = var.mina_archive_image + serverPort = "3086" + externalPort = "11010" + enableLocalDaemon = true + enablePostgresDB = true + + postgresHost = "archive-1-postgresql" + postgresPort = 5432 # remoteSchemaFile needs to be just the script name, not a url. remoteSchemaAuxFiles needs to be a list of urls of scripts, one of these urls needs to be the url of the main sql script that invokes the other ones. sorry it's confusing - remoteSchemaFile = var.mina_archive_schema - remoteSchemaAuxFiles = var.mina_archive_schema_aux_files - + remoteSchemaFile = var.mina_archive_schema + remoteSchemaAuxFiles = var.mina_archive_schema_aux_files + persistenceEnabled = true persistenceSize = "8Gi" persistenceStorageClass = "ssd-delete" persistenceAccessModes = ["ReadWriteOnce"] - spotAllowed = "false" + spotAllowed = "false" + persist_working_dir = true } static_peers = local.seed_static_peers archive_node_configs = var.archive_configs != null ? [for item in var.archive_configs : merge(local.default_archive_node, item)] : [ for i in range(1, var.archive_node_count + 1) : merge(local.default_archive_node, { - name = "archive-${i}" - postgresHost = "archive-${i}-postgresql" + name = "archive-${i}" + postgresHost = "archive-${i}-postgresql" }) ] } diff --git a/automation/terraform/modules/o1-testnet/testnet.tf b/automation/terraform/modules/o1-testnet/testnet.tf index ad92c7afa8c..29c82513025 100644 --- a/automation/terraform/modules/o1-testnet/testnet.tf +++ b/automation/terraform/modules/o1-testnet/testnet.tf @@ -28,20 +28,20 @@ module "kubernetes_testnet" { k8s_context = var.k8s_context testnet_name = var.testnet_name - use_local_charts = true - mina_image = var.mina_image - mina_archive_image = var.mina_archive_image - mina_agent_image = var.mina_agent_image - mina_bots_image = var.mina_bots_image - mina_points_image = var.mina_points_image - watchdog_image = var.watchdog_image + use_local_charts = true + mina_image = var.mina_image + mina_archive_image = var.mina_archive_image + mina_agent_image = var.mina_agent_image + mina_bots_image = var.mina_bots_image + mina_points_image = var.mina_points_image + watchdog_image = var.watchdog_image itn_orchestrator_image = var.itn_orchestrator_image mina_faucet_amount = var.mina_faucet_amount mina_faucet_fee = var.mina_faucet_fee - log_level = var.log_level - log_txn_pool_gossip = var.log_txn_pool_gossip + log_level = var.log_level + log_txn_pool_gossip = var.log_txn_pool_gossip log_precomputed_blocks = var.log_precomputed_blocks agent_min_fee = var.agent_min_fee @@ -52,35 +52,34 @@ module "kubernetes_testnet" { additional_peers = [for peer in local.static_peers : peer.full_peer] runtime_config = var.use_embedded_runtime_config ? "" : data.local_file.genesis_ledger.content - + seed_zone = var.seed_zone seed_region = var.seed_region archive_configs = local.archive_node_configs - mina_archive_schema = var.mina_archive_schema + mina_archive_schema = var.mina_archive_schema mina_archive_schema_aux_files = var.mina_archive_schema_aux_files snark_coordinators = var.snark_coordinators - block_producer_key_pass = var.block_producer_key_pass - block_producer_configs = [for i, bp in local.block_producer_configs: + # block_producer_key_pass = var.block_producer_key_pass + block_producer_configs = [for i, bp in local.block_producer_configs : { name = bp.name class = bp.class - # id = bp.total_node_index - keypair_name = "${bp.class}-${bp.unique_node_index}-key" - privkey_password = "naughty blue worm" + keypair_name = "${bp.class}-${bp.unique_node_index}-key" + privkey_password = "naughty blue worm" external_port = bp.port libp2p_secret = "" enable_gossip_flooding = false - # run_with_user_agent = bp.class =="whale" ? false : ( var.nodes_with_user_agent == [] ? true : contains(var.nodes_with_user_agent, bp.name) ) - run_with_user_agent = bp.class =="whale" ? false : true + run_with_user_agent = bp.class == "whale" ? false : true run_with_bots = false enable_peer_exchange = true isolated = false enableArchive = false - archiveAddress = length(local.archive_node_configs) != 0 ? "${element(local.archive_node_configs, i%(length(local.archive_node_configs)) )["name"]}:${element(local.archive_node_configs, i%(length(local.archive_node_configs)) )["serverPort"]}" : "" + archiveAddress = length(local.archive_node_configs) != 0 ? "${element(local.archive_node_configs, i % (length(local.archive_node_configs)))["name"]}:${element(local.archive_node_configs, i % (length(local.archive_node_configs)))["serverPort"]}" : "" + persist_working_dir = true } ] @@ -88,29 +87,27 @@ module "kubernetes_testnet" { seed_configs = [ for i in range(var.seed_count) : { - name = local.seed_static_peers[i].name - class = "seed" - # id = i + 1 - # external_port = local.seed_static_peers[i].port - libp2p_secret = "seed-${i + 1}-key" - libp2p_secret_pw = "naughty blue worm" - external_ip = google_compute_address.seed_static_ip[i].address - # private_key_secret = "online-seeds-account-${i + 1}-key" - enableArchive = length(local.archive_node_configs) > 0 - archiveAddress = length(local.archive_node_configs) > 0 ? "${element(local.archive_node_configs, i)["name"]}:${element(local.archive_node_configs, i)["serverPort"]}" : "" + name = local.seed_static_peers[i].name + class = "seed" + libp2p_secret = "seed-${i + 1}-key" + libp2p_secret_pw = "naughty blue worm" + external_ip = google_compute_address.seed_static_ip[i].address + enableArchive = length(local.archive_node_configs) > 0 + archiveAddress = length(local.archive_node_configs) > 0 ? "${element(local.archive_node_configs, i)["name"]}:${element(local.archive_node_configs, i)["serverPort"]}" : "" + persist_working_dir = true } ] plain_node_configs = [ for i in range(var.plain_node_count) : { - name = "plain-node-${i+1}" + name = "plain-node-${i + 1}" } ] - cpu_request = var.cpu_request - mem_request= var.mem_request + cpu_request = var.cpu_request + mem_request = var.mem_request worker_cpu_request = var.worker_cpu_request - worker_mem_request= var.worker_mem_request + worker_mem_request = var.worker_mem_request upload_blocks_to_gcloud = var.upload_blocks_to_gcloud restart_nodes = var.restart_nodes @@ -119,7 +116,7 @@ module "kubernetes_testnet" { make_report_every_mins = var.make_report_every_mins make_report_discord_webhook_url = var.make_report_discord_webhook_url make_report_accounts = var.make_report_accounts - seed_peers_url = var.seed_peers_url + # seed_peers_url = var.seed_peers_url zkapps_dashboard_key = var.zkapps_dashboard_key } diff --git a/automation/terraform/testnets/berkeley/main.tf b/automation/terraform/testnets/berkeley/main.tf index 094c14e2557..d6b5c4ebe43 100644 --- a/automation/terraform/testnets/berkeley/main.tf +++ b/automation/terraform/testnets/berkeley/main.tf @@ -76,14 +76,14 @@ locals { } module "berkeley" { - providers = { google.gke = google.google-us-central1 } + providers = { google.gke = google.google-us-east1 } source = "../../modules/o1-testnet" artifact_path = abspath(path.module) - cluster_name = "coda-infra-central1" - cluster_region = "us-central1" - k8s_context = "gke_o1labs-192920_us-central1_coda-infra-central1" + cluster_name = "coda-infra-east" + cluster_region = "us-east1" + k8s_context = "gke_o1labs-192920_us-east1_coda-infra-east" testnet_name = local.testnet_name mina_image = local.mina_image @@ -94,7 +94,7 @@ module "berkeley" { watchdog_image = "gcr.io/o1labs-192920/watchdog:0.4.13" use_embedded_runtime_config = true - archive_node_count = 3 + archive_node_count = 2 mina_archive_schema = "create_schema.sql" mina_archive_schema_aux_files = ["https://raw.githubusercontent.com/MinaProtocol/mina/8468f29424fa5e70cfedf69ee281f4fa24037dfe/src/app/archive/create_schema.sql", "https://raw.githubusercontent.com/MinaProtocol/mina/8468f29424fa5e70cfedf69ee281f4fa24037dfe/src/app/archive/zkapp_tables.sql"] @@ -107,15 +107,9 @@ module "berkeley" { }, { name = "archive-2" - enableLocalDaemon = false - enablePostgresDB = false - postgresHost = "archive-1-postgresql" - }, - { - name = "archive-3" - enableLocalDaemon = false + enableLocalDaemon = true enablePostgresDB = true - postgresHost = "archive-3-postgresql" + postgresHost = "archive-2-postgresql" } ] @@ -145,10 +139,12 @@ module "berkeley" { snark_coordinators = [ { + snark_coordinator_name = "snark-coordinator" snark_worker_replicas = 5 snark_worker_fee = "0.01" snark_worker_public_key = "B62qmQsEHcsPUs5xdtHKjEmWqqhUPRSF2GNmdguqnNvpEZpKftPC69e" snark_coordinators_host_port = 10401 + persist_working_dir = true } ] @@ -167,10 +163,6 @@ module "berkeley" { } ] - # nodes_with_user_agent = ["fish-1-1","fish-2-1"] - - - upload_blocks_to_gcloud = true restart_nodes = false restart_nodes_every_mins = "60" @@ -182,8 +174,7 @@ module "berkeley" { # a pass must be set to connect the zkapps dash to grafana # leave empty to lock down the zkapps dashboard - - # zkapps_dashboard_key = "" + # zkapps_dashboard_key = "" }