Merge pull request #15135 from MinaProtocol/hardfork-package-generati…

…on-automation-develop

Merge "Hardfork package generation automation" into develop

.gitignore

@@ -86,7 +86,7 @@ automation/services/watchdog/check_libp2p/check_libp2p
 _opam
 
 target
-release
+./release
 .vscode
 
 ### IntelliJ IDEA ###

buildkite/scripts/build-artifact.sh

@@ -2,9 +2,9 @@
 
 set -eo pipefail
 
-eval $(opam config env)
-export PATH=/home/opam/.cargo/bin:/usr/lib/go/bin:$PATH
-export GO=/usr/lib/go/bin/go
+([ -z ${DUNE_PROFILE+x} ]) && echo "required env vars were not provided" && exit 1
+
+source ~/.profile
 
 MINA_COMMIT_SHA1=$(git rev-parse HEAD)
 
@@ -19,6 +19,7 @@ echo "--- Build all major tagets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
 echo "Rust Version: $(rustc --version)"
 dune build "--profile=${DUNE_PROFILE}" \
+  ${MAINNET_TARGETS} \
   src/app/logproc/logproc.exe \
   src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \
   src/app/generate_keypair/generate_keypair.exe \
@@ -39,16 +40,3 @@ dune build "--profile=${DUNE_PROFILE}" \
   src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe  \
   src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
-
-echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
-echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
-[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
-
-echo "--- Prepare debian packages"
-./scripts/rebuild-deb.sh $@
-
-echo "--- Upload debs to amazon s3 repo"
-./buildkite/scripts/publish-deb.sh
-
-echo "--- Git diff after build is complete:"
-git diff --exit-code -- .

buildkite/scripts/build-hardfork-package.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -eo pipefail
+
+([ -z ${CONFIG_JSON_GZ_URL+x} ] || [ -z ${NETWORK_NAME+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
+
+# Set the DUNE_PROFILE from the NETWORK_NAME. For now, these are 1-1, but in the future, this may need to be a case statement
+case "${NETWORK_NAME}" in
+  mainnet)
+    DUNE_PROFILE=mainnet
+    ;;
+  devnet|berkeley)
+    DUNE_PROFILE=devnet
+    ;;
+  *)
+    echo "unrecognized network name: ${NETWORK_NAME}"
+    exit 1
+    ;;
+esac
+export DUNE_PROFILE
+
+# Set the base network config for ./scripts/hardfork/create_runtime_config.sh
+export FORKING_FROM_CONFIG_JSON="genesis_ledgers/${NETWORK_NAME}.json"
+[ ! -f "${FORKING_FROM_CONFIG_JSON}" ] && echo "${NETWORK_NAME} is not a known network name; check for existing network configs in 'genesis_ledgers/'" && exit 1
+
+source ~/.profile
+
+MINA_COMMIT_SHA1=$(git rev-parse HEAD)
+
+echo "--- Download and extract previous network config"
+curl -o config.json.gz $CONFIG_JSON_GZ_URL
+gunzip config.json.gz
+
+echo "--- Migrate accounts to new network format"
+# TODO: At this stage, we need to migrate the json accounts into the new network's format.
+#       For now, this is hard-coded to the mainnet -> berkeley migration, but we need to select
+#       a migration to perform in the future.
+# NB: we use sed here instead of jq, because jq is extremely slow at processing this file
+sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "1"}/' config.json
+
+case "${NETWORK_NAME}" in
+  mainnet)
+    MINA_BUILD_MAINNET=1 ./buildkite/scripts/build-artifact.sh
+    ;;
+  *)
+    ./buildkite/scripts/build-artifact.sh
+    ;;
+esac
+
+echo "--- Generate hardfork ledger tarballs"
+mkdir hardfork_ledgers
+_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | _build/default/src/app/logproc/logproc.exe
+
+echo "--- Create hardfork config"
+FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json
+
+echo "--- Build hardfork package for Debian ${MINA_DEB_CODENAME}"
+RUNTIME_CONFIG_JSON=new_config.json LEDGER_TARBALLS="$(echo hardfork_ledgers/*.tar.gz)" ./scripts/create_hardfork_deb.sh
+mkdir -p /tmp/artifacts
+cp _build/mina*.deb /tmp/artifacts/.
+
+echo "--- Upload debs to amazon s3 repo"
+make publish_debs
+
+echo "--- Git diff after build is complete:"
+git diff --exit-code -- .

buildkite/scripts/build-release.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eo pipefail
+
+([ -z ${DUNE_PROFILE+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
+
+source ~/.profile
+
+./buildkite/scripts/build-artifact.sh
+
+echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
+echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
+[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
+
+
+echo "--- Prepare debian packages"
+./scripts/rebuild-deb.sh $@
+
+echo "--- Upload debs to amazon s3 repo"
+./buildkite/scripts/publish-deb.sh
+
+echo "--- Git diff after build is complete:"
+git diff --exit-code -- .

buildkite/scripts/publish-deb.sh

@@ -66,13 +66,15 @@ function verify_size_and_md5 {
 
 }
 
-case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
-  rampup|berkeley|release/2.0.0|develop)
-    TESTNET_NAME="berkeley"
-  ;;
-  *)
-    TESTNET_NAME="mainnet"
-esac
+if [ -z ${TESTNET_NAME+x} ]; then
+  case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
+    rampup|berkeley|release/2.0.0|develop)
+      TESTNET_NAME="berkeley"
+    ;;
+    *)
+      TESTNET_NAME="mainnet"
+  esac
+fi
 
 # In order to prevent anyone to use freshly pushed packages prematurely we need to be sure those packages has correct
 # md5 and sizes before finishing script
@@ -81,4 +83,4 @@ function verify_o1test_repo_is_synced {
     verify_size_and_md5 "mina-archive"
 }
 
-for i in {1..5}; do verify_o1test_repo_is_synced && break || sleep 60; done
+for i in {1..5}; do verify_o1test_repo_is_synced && break || sleep 60; done

buildkite/src/Command/Docker/Type.dhall

@@ -13,14 +13,16 @@
      `mount-buildkite-agent`: Bool,
      `mount-workdir`: Bool,
      privileged: Bool,
-     environment: List Text
+     environment: List Text,
+     user: Optional Text
   },
   default = {
     shell = Some ["/bin/sh", "-e", "-c"],
     `propagate-environment` = True,
     `mount-buildkite-agent` = False,
     `mount-workdir` = False,
     privileged = False,
-    environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ]
+    environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ],
+    user = None Text
   }
 }

buildkite/src/Command/MinaArtifact.dhall

@@ -22,6 +22,140 @@ let Artifacts = ../Constants/Artifacts.dhall
 
 in
 
+--- NB: unlike the regular artifact piopeline, the hardfork pipeline receives many of its parameters as env vars
+let hardforkPipeline : DebianVersions.DebVersion -> Pipeline.Config.Type =
+  \(debVersion : DebianVersions.DebVersion) ->
+      --- TODO: Refactor the dhall interface so that we don't need to keep nesting docker containers.
+      --- I've already refactored some of it such that everything works in the root docker contains,
+      --- EXCEPT that the env secrets injection currently relies on the nested docker configuration.
+
+      --- Once this is done, we can simplify the jobs quite a bit. Below is an example of this job
+      --- after such a refactor has taken place. Note that the commands no longer need to be in a
+      --- separate script, creating less indirection, and that we no longer need to do any weird
+      --- docker nesting and environment passing. But this doesn't have access to any secrets as
+      --- is, and it didn't seem that defining the env vars at the top layer fixed that.
+
+      --- Pipeline.build
+      ---   Pipeline.Config::{
+      ---     spec = JobSpec::{
+      ---       dirtyWhen = [S.everything],
+      ---       path = "Release",
+      ---       name = "MinaArtifactHardfork${DebianVersions.capitalName debVersion}${Profiles.toSuffixUppercase profile}"
+      ---       tags = [ PipelineTag.Type.Release ],
+      ---       mode = PipelineMode.Type.PackageGeneration
+      ---     },
+      ---     steps = [
+      ---       Command.build
+      ---         Command.Config::{
+      ---           commands = [
+      ---             Cmd.run "source ~/.profile"
+      ---             , Cmd.run "curl -o config.json.gz $CONFIG_JSON_GZ_URL"
+      ---             , Cmd.run "gunzip config.json.gz"
+      ---             , Cmd.run "dune build src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe src/app/logproc/logproc.exe"
+      ---             , Cmd.run "mkdir hardfork_ledgers"
+      ---             , Cmd.run "_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | _build/default/src/app/logproc/logproc.exe"
+      ---             -- , Cmd.run ''
+      ---             --     _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json \
+      ---             --     | tee runtime_genesis_ledger.log \
+      ---             --     | _build/default/src/app/logproc/logproc.exe
+      ---             --     ''
+      ---             , Cmd.run "FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json"
+      ---             , Cmd.run "ls hardfork_ledgers"
+      ---             , Cmd.run "cat new_config.json"
+      ---             , Cmd.run "MINA_DEB_CODENAME=bullseye MINA_BUILD_MAINNET=1 RUNTIME_CONFIG_JSON=new_config.json LEDGER_TARBALLS=\"$(echo hardfork_ledgers/*.tar.gz)\" ./buildkite/scripts/build-hardfork-package.sh"
+      ---           ]
+      ---           , docker = Some Docker::{
+      ---             image = ContainerImages.minaToolchain
+      ---             , shell = Some ["/bin/bash", "-e", "-c"]
+      ---             , user = Some "root"
+      ---           }
+      ---           , label = "Ledger tar file generation"
+      ---           , key = "generate-ledger-tars-from-config"
+      ---           , target = Size.XLarge
+      ---         }
+      ---     ]
+      ---   }
+
+      --- TODO: profile is currently hard-coded for standard networks, but should be determined from env vars
+      let profile = Profiles.Type.Standard
+      --- TODO: network is currently hard-coded, but should be determined from env vars
+      let network = "\\\${NETWORK_NAME}-hardfork"
+      let pipelineName = "MinaArtifactHardfork${DebianVersions.capitalName debVersion}${Profiles.toSuffixUppercase profile}"
+      let generateLedgersJobKey = "generate-ledger-tars-from-config"
+
+      in
+
+      Pipeline.Config::{
+        spec = JobSpec::{
+          dirtyWhen = [ S.everything ]
+        , path = "Release"
+        , name = pipelineName
+        , tags = [ PipelineTag.Type.Release ]
+        , mode = PipelineMode.Type.PackageGeneration
+        }
+      , steps =
+        [ Command.build
+            Command.Config::{
+              commands =
+                DebianVersions.toolchainRunner
+                  debVersion
+                  [ "NETWORK_NAME=\$NETWORK_NAME"
+                  , "CONFIG_JSON_GZ_URL=\$CONFIG_JSON_GZ_URL"
+                  , "AWS_ACCESS_KEY_ID"
+                  , "AWS_SECRET_ACCESS_KEY"
+                  , "MINA_BRANCH=\$BUILDKITE_BRANCH"
+                  , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT"
+                  , "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
+                  , "TESTNET_NAME=\$NETWORK_NAME-hardfork"
+                  , "GENESIS_TIMESTAMP=\$GENESIS_TIMESTAMP"
+                  ]
+                  "./buildkite/scripts/build-hardfork-package.sh"
+            , label = "Build Mina Hardfork Package for ${DebianVersions.capitalName debVersion}"
+            , key = generateLedgersJobKey
+            , target = Size.XLarge
+            }
+        , DockerImage.generateStep
+            DockerImage.ReleaseSpec::{
+              deps =
+              [ { name = pipelineName
+                , key = generateLedgersJobKey
+                }
+              ]
+            , service = "mina-daemon"
+            , network = network
+            , deb_codename = "${DebianVersions.lowerName debVersion}"
+            , deb_profile = "${Profiles.lowerName profile}"
+            , step_key = "daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image"
+            }
+        , DockerImage.generateStep
+            DockerImage.ReleaseSpec::{
+              deps =
+              [ { name = pipelineName
+                , key = generateLedgersJobKey
+                }
+              ]
+            , service = "mina-archive"
+            , network = network
+            , deb_codename = "${DebianVersions.lowerName debVersion}"
+            , deb_profile = "${Profiles.lowerName profile}"
+            , step_key = "archive-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image"
+            }
+        , DockerImage.generateStep
+            DockerImage.ReleaseSpec::{
+              deps =
+              [ { name = pipelineName
+                , key = generateLedgersJobKey
+                }
+              ]
+            , service = "mina-rosetta"
+            , network = network
+            , deb_codename = "${DebianVersions.lowerName debVersion}"
+            -- , deb_profile = "${Profiles.lowerName profile}"
+            , step_key = "rosetta-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image"
+            }
+        ]
+      }
+
 let docker_step : Artifacts.Type -> DebianVersions.DebVersion -> Profiles.Type -> DockerImage.ReleaseSpec.Type = 
   \(artifact : Artifacts.Type) ->
   \(debVersion : DebianVersions.DebVersion) ->
@@ -107,7 +241,7 @@ let pipeline : List Artifacts.Type -> DebianVersions.DebVersion  -> Profiles.Typ
               "MINA_BRANCH=$BUILDKITE_BRANCH",
               "MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
               "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
-            ] "./buildkite/scripts/build-artifact.sh ${Artifacts.toDebianNames artifacts}",
+            ] "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames artifacts}",
             label = "Build Mina for ${DebianVersions.capitalName debVersion} ${Profiles.toSuffixUppercase profile}",
             key = "build-deb-pkg",
             target = Size.XLarge,
@@ -138,4 +272,5 @@ let pipeline : List Artifacts.Type -> DebianVersions.DebVersion  -> Profiles.Typ
 in
 {
   pipeline = pipeline
-}
+  , hardforkPipeline = hardforkPipeline
+}

buildkite/src/Jobs/Release/MinaArtifactHardforkBullseye.dhall

@@ -0,0 +1,7 @@
+let MinaArtifact = ../../Command/MinaArtifact.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in  Pipeline.build (MinaArtifact.hardforkPipeline DebianVersions.DebVersion.Bullseye)