Merge branch 'main' into initialize-deposits-from-db

.dockerignore

@@ -0,0 +1,19 @@
+_build/
+.elixir_ls/
+.github/
+.vscode/
+bench/
+deps/
+docs/
+level_db/
+metrics/
+priv/
+test/
+tmp/
+.all-contributorsrc
+.credo.exs
+.formatter.exs
+.spectest_version
+flake.*
+native/**/target/
+Dockerfile

Dockerfile

@@ -0,0 +1,36 @@
+# libp2p port
+FROM golang:1.21.3 AS libp2p_builder
+LABEL stage=builder
+
+RUN mkdir /libp2p_port
+WORKDIR /libp2p_port
+
+COPY native/libp2p_port /libp2p_port
+
+RUN go build -o libp2p_port
+
+
+# Main image
+FROM elixir:1.16.2-otp-26
+
+RUN mkdir /consensus
+WORKDIR /consensus
+
+ENV MIX_ENV=prod
+
+RUN mix local.hex --force
+
+# Install dependencies
+RUN apt-get update && apt-get install -y cmake
+
+# Install rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="${PATH}:/root/.cargo/bin"
+
+COPY . .
+COPY --from=libp2p_builder /libp2p_port/libp2p_port /consensus/priv/native/libp2p_port
+
+RUN mix deps.get
+RUN mix compile
+
+ENTRYPOINT [ "iex", "-S", "mix", "run", "--"]

Makefile

@@ -91,7 +91,7 @@ compile-native: $(OUTPUT_DIR)/libp2p_nif.so $(OUTPUT_DIR)/libp2p_port
 
 #🔨 compile-all: @ Compile the elixir project and its dependencies.
 compile-all: $(CONFIG_FILE) compile-native $(PROTOBUF_EX_FILES) download-beacon-node-oapi
-	mix compile --warnings-as-errors
+	mix compile
 
 #🗑️ clean: @ Remove the build files.
 clean:

README.md

@@ -121,6 +121,20 @@ make dialyzer  # Runs type-checker
 Source code can be formatted using `make fmt`.
 This formats not only the Elixir code, but also the code under [`native/`](./native/).
 
+### Docker
+
+The repo includes a `Dockerfile` for the consensus client. It can be built with:
+
+```bash
+docker build -t consensus .
+```
+
+Then you run it with `docker run`, adding CLI flags as needed:
+
+```bash
+docker run consensus --checkpoint-sync <url> --network <network> ...
+```
+
 ## Consensus spec tests
 
 You can run all of them with:

lib/beacon_api/controllers/v1/beacon_controller.ex

@@ -3,9 +3,9 @@ defmodule BeaconApi.V1.BeaconController do
 
   alias BeaconApi.ApiSpec
   alias BeaconApi.ErrorController
+  alias BeaconApi.Helpers
   alias BeaconApi.Utils
   alias LambdaEthereumConsensus.Beacon.BeaconChain
-  alias LambdaEthereumConsensus.ForkChoice
   alias LambdaEthereumConsensus.Store.BlockDb
   alias LambdaEthereumConsensus.Store.Blocks
 
@@ -40,7 +40,7 @@ defmodule BeaconApi.V1.BeaconController do
 
   @spec get_state_root(Plug.Conn.t(), any) :: Plug.Conn.t()
   def get_state_root(conn, %{state_id: state_id}) do
-    case BeaconApi.Utils.parse_id(state_id) |> ForkChoice.Helpers.state_root_by_state_id() do
+    case BeaconApi.Utils.parse_id(state_id) |> Helpers.state_root_by_state_id() do
       {:ok, {root, execution_optimistic, finalized}} ->
         conn |> root_response(root, execution_optimistic, finalized)
 
@@ -107,7 +107,7 @@ defmodule BeaconApi.V1.BeaconController do
 
   @spec get_finality_checkpoints(Plug.Conn.t(), any) :: Plug.Conn.t()
   def get_finality_checkpoints(conn, %{state_id: state_id}) do
-    case BeaconApi.Utils.parse_id(state_id) |> ForkChoice.Helpers.finality_checkpoint_by_id() do
+    case BeaconApi.Utils.parse_id(state_id) |> Helpers.finality_checkpoint_by_id() do
       {:ok,
        {previous_justified_checkpoint, current_justified_checkpoint, finalized_checkpoint,
         execution_optimistic, finalized}} ->

lib/beacon_api/helpers.ex

@@ -0,0 +1,169 @@
+defmodule BeaconApi.Helpers do
+  @moduledoc """
+  Helper functions for the Beacon API
+  """
+
+  alias LambdaEthereumConsensus.Beacon.BeaconChain
+  alias LambdaEthereumConsensus.Store.BlockDb
+  alias LambdaEthereumConsensus.Store.Blocks
+  alias LambdaEthereumConsensus.Store.StateDb
+
+  alias Types.BeaconState
+  alias Types.SignedBeaconBlock
+
+  @type named_root() :: :genesis | :justified | :finalized | :head
+  @type block_id() :: named_root() | :invalid_id | Types.slot() | Types.root()
+  @type state_id() :: named_root() | :invalid_id | Types.slot() | Types.root()
+  @type block_info() ::
+          {SignedBeaconBlock.t(), execution_optimistic? :: boolean(), finalized? :: boolean()}
+  @type state_info() ::
+          {BeaconState.t(), execution_optimistic? :: boolean(), finalized? :: boolean()}
+  @type root_info() ::
+          {Types.root(), execution_optimistic? :: boolean(), finalized? :: boolean()}
+  @type finality_info() ::
+          {Types.Checkpoint.t(), Types.Checkpoint.t(), Types.Checkpoint.t(),
+           execution_optimistic? :: boolean(), finalized? :: boolean()}
+
+  def root_by_id(:justified) do
+    justified_checkpoint = BeaconChain.get_justified_checkpoint()
+    # TODO compute is_optimistic_or_invalid
+    execution_optimistic = true
+    {:ok, {justified_checkpoint.root, execution_optimistic, false}}
+  end
+
+  def root_by_id(:finalized) do
+    finalized_checkpoint = BeaconChain.get_finalized_checkpoint()
+    # TODO compute is_optimistic_or_invalid
+    execution_optimistic = true
+    {:ok, {finalized_checkpoint.root, execution_optimistic, true}}
+  end
+
+  def root_by_id(hex_root) when is_binary(hex_root) do
+    # TODO compute is_optimistic_or_invalid() and is_finalized()
+    execution_optimistic = true
+    finalized = false
+    {:ok, {hex_root, execution_optimistic, finalized}}
+  end
+
+  @spec block_root_by_block_id(block_id()) ::
+          {:ok, root_info()} | {:error, String.t()} | :not_found | :empty_slot | :invalid_id
+  def block_root_by_block_id(:head) do
+    with current_status <- BeaconChain.get_current_status_message() do
+      # TODO compute is_optimistic_or_invalid
+      execution_optimistic = true
+      {:ok, {current_status.head_root, execution_optimistic, false}}
+    end
+  end
+
+  def block_root_by_block_id(:genesis), do: :not_found
+
+  def block_root_by_block_id(:justified) do
+    with justified_checkpoint <- BeaconChain.get_justified_checkpoint() do
+      # TODO compute is_optimistic_or_invalid
+      execution_optimistic = true
+      {:ok, {justified_checkpoint.root, execution_optimistic, false}}
+    end
+  end
+
+  def block_root_by_block_id(:finalized) do
+    with finalized_checkpoint <- BeaconChain.get_finalized_checkpoint() do
+      # TODO compute is_optimistic_or_invalid
+      execution_optimistic = true
+      {:ok, {finalized_checkpoint.root, execution_optimistic, true}}
+    end
+  end
+
+  def block_root_by_block_id(:invalid_id), do: :invalid_id
+
+  def block_root_by_block_id(slot) when is_integer(slot) do
+    with :ok <- check_valid_slot(slot, BeaconChain.get_current_slot()),
+         {:ok, root} <- BlockDb.get_block_root_by_slot(slot) do
+      # TODO compute is_optimistic_or_invalid() and is_finalized()
+      execution_optimistic = true
+      finalized = false
+      {:ok, {root, execution_optimistic, finalized}}
+    end
+  end
+
+  @spec state_root_by_state_id(state_id()) ::
+          {:ok, root_info()} | {:error, String.t()} | :not_found | :empty_slot | :invalid_id
+  def state_root_by_state_id(hex_root) when is_binary(hex_root) do
+    # TODO compute is_optimistic_or_invalid() and is_finalized()
+    execution_optimistic = true
+    finalized = false
+
+    case StateDb.get_state_by_state_root(hex_root) do
+      {:ok, _state} -> {:ok, {hex_root, execution_optimistic, finalized}}
+      _ -> :not_found
+    end
+  end
+
+  def state_root_by_state_id(id) do
+    with {:ok, {block_root, optimistic, finalized}} <- block_root_by_block_id(id),
+         {:ok, block} <- BlockDb.get_block(block_root) do
+      %{message: %{state_root: state_root}} = block
+      {:ok, {state_root, optimistic, finalized}}
+    end
+  end
+
+  @spec block_by_block_id(block_id()) ::
+          {:ok, block_info()}
+          | {:error, String.t()}
+          | :not_found
+          | :empty_slot
+          | :invalid_id
+  def block_by_block_id(block_id) do
+    with {:ok, {root, optimistic, finalized}} <- block_root_by_block_id(block_id),
+         {:ok, block} <- BlockDb.get_block(root) do
+      {:ok, {block, optimistic, finalized}}
+    end
+  end
+
+  @spec state_by_state_id(state_id()) ::
+          {:ok, state_info()}
+          | {:error, String.t()}
+          | :not_found
+          | :empty_slot
+          | :invalid_id
+  def state_by_state_id(hex_root) when is_binary(hex_root) do
+    # TODO compute is_optimistic_or_invalid() and is_finalized()
+    execution_optimistic = true
+    finalized = false
+
+    case StateDb.get_state_by_state_root(hex_root) do
+      {:ok, state} -> {:ok, {state, execution_optimistic, finalized}}
+      _ -> :not_found
+    end
+  end
+
+  def state_by_state_id(id) do
+    with {:ok, {%{message: %{state_root: state_root}}, optimistic, finalized}} <-
+           block_by_block_id(id),
+         {:ok, state} <-
+           StateDb.get_state_by_state_root(state_root) do
+      {:ok, {state, optimistic, finalized}}
+    end
+  end
+
+  @spec finality_checkpoint_by_id(state_id()) ::
+          {:ok, finality_info()} | {:error, String.t()} | :not_found | :empty_slot | :invalid_id
+  def finality_checkpoint_by_id(id) do
+    with {:ok, {state, optimistic, finalized}} <- state_by_state_id(id) do
+      {:ok,
+       {state.previous_justified_checkpoint, state.current_justified_checkpoint,
+        state.finalized_checkpoint, optimistic, finalized}}
+    end
+  end
+
+  @spec get_state_root(Types.root()) :: Types.root() | nil
+  def get_state_root(root) do
+    with %{} = block <- Blocks.get_block(root) do
+      block.state_root
+    end
+  end
+
+  defp check_valid_slot(slot, current_slot) when slot < current_slot, do: :ok
+
+  defp check_valid_slot(slot, _current_slot),
+    do: {:error, "slot #{slot} cannot be greater than current slot"}
+end

lib/beacon_api/utils.ex

@@ -2,7 +2,7 @@ defmodule BeaconApi.Utils do
   @moduledoc """
    Set of useful utilitary functions in the context of the Beacon API
   """
-  alias LambdaEthereumConsensus.ForkChoice.Helpers
+  alias BeaconApi.Helpers
   alias LambdaEthereumConsensus.Utils.BitList
 
   @spec parse_id(binary) :: Helpers.block_id()

lib/lambda_ethereum_consensus/beacon/beacon_chain.ex

@@ -4,6 +4,7 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconChain do
   use GenServer
 
   alias LambdaEthereumConsensus.ForkChoice
+  alias LambdaEthereumConsensus.P2P.Gossip
   alias LambdaEthereumConsensus.StateTransition.Misc
   alias LambdaEthereumConsensus.Validator
   alias Types.BeaconState
@@ -88,7 +89,7 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconChain do
   @spec get_fork_version() :: Types.version()
   def get_fork_version(), do: GenServer.call(__MODULE__, :get_fork_version)
 
-  @spec get_current_status_message() :: {:ok, Types.StatusMessage.t()} | {:error, any}
+  @spec get_current_status_message() :: Types.StatusMessage.t()
   def get_current_status_message(), do: GenServer.call(__MODULE__, :get_current_status_message)
 
   ##########################
@@ -176,7 +177,7 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconChain do
     new_state = %BeaconChainState{state | time: time}
     new_logical_time = compute_logical_time(new_state)
 
-    if state.synced and old_logical_time != new_logical_time do
+    if old_logical_time != new_logical_time do
       notify_subscribers(new_logical_time)
     end
 
@@ -240,7 +241,10 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconChain do
 
   defp notify_subscribers(logical_time) do
     log_new_slot(logical_time)
-    Validator.notify_tick(logical_time)
+
+    Enum.each([Validator, Gossip.BeaconBlock], fn subscriber ->
+      GenServer.cast(subscriber, {:on_tick, logical_time})
+    end)
   end
 
   defp log_new_slot({slot, :first_third}) do

lib/lambda_ethereum_consensus/beacon/beacon_node.ex

@@ -5,7 +5,7 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
   require Logger
 
   alias LambdaEthereumConsensus.Beacon.StoreSetup
-  alias LambdaEthereumConsensus.ForkChoice.Helpers
+  alias LambdaEthereumConsensus.ForkChoice.Head
   alias LambdaEthereumConsensus.StateTransition.Cache
   alias LambdaEthereumConsensus.Store.Blocks
   alias LambdaEthereumConsensus.Store.BlockStates
@@ -36,7 +36,7 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
 
     time = :os.system_time(:second)
 
-    {:ok, head_root} = Helpers.get_head(store)
+    {:ok, head_root} = Head.get_head(store)
     %{slot: head_slot} = Blocks.get_block!(head_root)
 
     fork_choice_data = %{
@@ -60,7 +60,8 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
         LambdaEthereumConsensus.Beacon.PendingBlocks,
         LambdaEthereumConsensus.Beacon.SyncBlocks,
         LambdaEthereumConsensus.P2P.GossipSub,
-        LambdaEthereumConsensus.P2P.Gossip.Attestation
+        LambdaEthereumConsensus.P2P.Gossip.Attestation,
+        LambdaEthereumConsensus.P2P.Gossip.BeaconBlock
       ] ++ validator_children
 
     Supervisor.init(children, strategy: :one_for_all)

lib/lambda_ethereum_consensus/beacon/sync_blocks.ex

@@ -10,6 +10,7 @@ defmodule LambdaEthereumConsensus.Beacon.SyncBlocks do
   alias LambdaEthereumConsensus.Beacon.BeaconChain
   alias LambdaEthereumConsensus.Beacon.PendingBlocks
   alias LambdaEthereumConsensus.P2P.BlockDownloader
+  alias LambdaEthereumConsensus.P2P.Gossip
   alias LambdaEthereumConsensus.StateTransition.Misc
   alias Types.SignedBeaconBlock
 
@@ -72,6 +73,7 @@ defmodule LambdaEthereumConsensus.Beacon.SyncBlocks do
 
     if Enum.empty?(chunks) do
       Logger.info("[Optimistic Sync] Sync completed")
+      Gossip.BeaconBlock.start()
     else
       Process.sleep(1000)
       perform_sync(remaining_chunks)

lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex

@@ -9,7 +9,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do
   alias LambdaEthereumConsensus.Beacon.BeaconChain
   alias LambdaEthereumConsensus.Execution.ExecutionChain
   alias LambdaEthereumConsensus.ForkChoice.Handlers
-  alias LambdaEthereumConsensus.ForkChoice.Helpers
+  alias LambdaEthereumConsensus.ForkChoice.Head
   alias LambdaEthereumConsensus.P2P.Gossip.OperationsCollector
   alias LambdaEthereumConsensus.Store.Blocks
   alias LambdaEthereumConsensus.Store.StoreDb
@@ -166,7 +166,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do
   @spec recompute_head(Store.t()) :: :ok
   def recompute_head(store) do
     persist_store(store)
-    {:ok, head_root} = Helpers.get_head(store)
+    {:ok, head_root} = Head.get_head(store)
     head_block = Blocks.get_block!(head_root)
 
     Handlers.notify_forkchoice_update(store, head_block)