diff --git a/crates/topos-tce-broadcast/src/double_echo/mod.rs b/crates/topos-tce-broadcast/src/double_echo/mod.rs index d1ff88527..67721444f 100644 --- a/crates/topos-tce-broadcast/src/double_echo/mod.rs +++ b/crates/topos-tce-broadcast/src/double_echo/mod.rs @@ -156,7 +156,8 @@ impl DoubleEcho { DoubleEchoCommand::Echo { certificate_id, validator_id, signature } => { // Check if source is part of known_validators if !self.validators.contains(&validator_id) { - return debug!("ECHO message comes from non-validator: {}", validator_id); + debug!("ECHO message comes from non-validator: {}", validator_id); + continue; } let mut payload = Vec::new(); @@ -164,7 +165,8 @@ impl DoubleEcho { payload.extend_from_slice(validator_id.as_bytes()); if let Err(e) = self.message_signer.verify_signature(signature, &payload, validator_id.address()) { - return debug!("ECHO messag signature cannot be verified from: {}", e); + debug!("ECHO message signature cannot be verified from: {}", e); + continue; } self.handle_echo(certificate_id, validator_id, signature).await @@ -172,7 +174,8 @@ impl DoubleEcho { DoubleEchoCommand::Ready { certificate_id, validator_id, signature } => { // Check if source is part of known_validators if !self.validators.contains(&validator_id) { - return debug!("READY message comes from non-validator: {}", validator_id); + debug!("READY message comes from non-validator: {}", validator_id); + continue; } let mut payload = Vec::new(); @@ -180,7 +183,8 @@ impl DoubleEcho { payload.extend_from_slice(validator_id.as_bytes()); if let Err(e) = self.message_signer.verify_signature(signature, &payload, validator_id.address()) { - return debug!("READY message signature cannot be verified from: {}", e); + debug!("READY message signature cannot be verified from: {}", e); + continue; } self.handle_ready(certificate_id, validator_id, signature).await diff --git a/crates/topos/src/components/node/services/process.rs b/crates/topos/src/components/node/services/process.rs index ce8b4fde4..992111ce2 100644 --- a/crates/topos/src/components/node/services/process.rs +++ b/crates/topos/src/components/node/services/process.rs @@ -83,18 +83,21 @@ pub(crate) fn spawn_tce_process( genesis: Genesis, shutdown: (CancellationToken, mpsc::Sender<()>), ) -> JoinHandle> { + let validators = genesis.validators().expect("Cannot parse validators"); + let tce_params = ReliableBroadcastParams::new(validators.len()); + let tce_config = TceConfiguration { boot_peers: genesis .boot_peers(Some(topos_p2p::constants::TCE_BOOTNODE_PORT)) .into_iter() .chain(config.parse_boot_peers()) .collect::>(), - validators: genesis.validators().expect("Cannot parse validators"), + validators, auth_key: keys.network.map(AuthKey::PrivateKey), signing_key: keys.validator.map(AuthKey::PrivateKey), tce_addr: format!("/ip4/{}", config.libp2p_api_addr.ip()), tce_local_port: config.libp2p_api_addr.port(), - tce_params: ReliableBroadcastParams::new(genesis.validator_count()), + tce_params, api_addr: config.grpc_api_addr, graphql_api_addr: config.graphql_api_addr, metrics_api_addr: config.metrics_api_addr, diff --git a/crates/topos/src/components/regtest/mod.rs b/crates/topos/src/components/regtest/mod.rs index 9cf04db8f..f29e0ed44 100644 --- a/crates/topos/src/components/regtest/mod.rs +++ b/crates/topos/src/components/regtest/mod.rs @@ -22,6 +22,7 @@ pub(crate) async fn handle_command( ) -> Result<(), Box> { match subcommands { Some(RegtestCommands::PushCertificate(cmd)) => { + _ = setup_tracing(verbose, None, None)?; debug!("Start executing PushCertificate command"); match services::push_certificate::check_delivery( cmd.timeout_broadcast, diff --git a/crates/topos/src/config/genesis/mod.rs b/crates/topos/src/config/genesis/mod.rs index 38c0e6934..ab0db46d0 100644 --- a/crates/topos/src/config/genesis/mod.rs +++ b/crates/topos/src/config/genesis/mod.rs @@ -36,14 +36,6 @@ impl Genesis { Ok(Self { path, json }) } - // Considered as being the set of premined addresses for now - // TODO: Parse properly genesis.extraData instead - pub fn validator_count(&self) -> usize { - self.json["genesis"]["alloc"] - .as_object() - .map_or(0, |v| v.len()) - } - // TODO: parse directly with serde pub fn boot_peers(&self, port: Option) -> Vec<(PeerId, Multiaddr)> { match self.json["bootnodes"].as_array() { diff --git a/crates/topos/src/config/genesis/tests.rs b/crates/topos/src/config/genesis/tests.rs index 3dea73b20..a0e769ab3 100644 --- a/crates/topos/src/config/genesis/tests.rs +++ b/crates/topos/src/config/genesis/tests.rs @@ -20,7 +20,8 @@ pub fn genesis() -> Genesis { #[rstest] pub fn test_correct_validator_count(genesis: &Genesis) { - assert_eq!(4, genesis.validator_count()); + let validators = genesis.validators().unwrap(); + assert_eq!(validators.len(), 4); } #[rstest] @@ -39,7 +40,6 @@ pub fn test_extract_validators(genesis: &Genesis) { let third = ValidatorId::from_str("0xb4973cdb10894d1d1547673bd758589034c2bba5").unwrap(); let fourth = ValidatorId::from_str("0xc16d83893cb61872206d4e271b813015d3242d94").unwrap(); - assert_eq!(validators.len(), 4); assert_eq!(validators.get(&first), Some(&first)); assert_eq!(validators.get(&second), Some(&second)); assert_eq!(validators.get(&third), Some(&third)); diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index cc0deae45..bb137de09 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -2,26 +2,20 @@ version: "3.3" services: init: container_name: init - command: sh -c "cp /tmp/libp2p_keys.json /tmp/shared/libp2p_keys.json && \ - cp /tmp/validator_bls_keys.json /tmp/shared/validator_bls_keys.json && \ - cp /tmp/validator_keys.json /tmp/shared/validator_keys.json && \ - chmod a+rwx /tmp/shared/* && ls /tmp && ls /tmp/shared" - image: debian:bullseye-slim + command: init + image: ghcr.io/topos-protocol/topos:main + env_file: + - env/base.env + - env/initializer.env volumes: - shared:/tmp/shared - - ./libp2p_keys.json:/tmp/libp2p_keys.json - - ./validator_bls_keys.json:/tmp/validator_bls_keys.json - - ./validator_keys.json:/tmp/validator_keys.json - healthcheck: - test: ["CMD-SHELL", "test -f /tmp/shared/libp2p_keys.json"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 5s + deploy: + restart_policy: + condition: none boot: container_name: boot - command: node up --name boot --home /tmp/node_config --no-edge-process + command: boot node up --name test --home /tmp/node_config --no-edge-process image: ghcr.io/topos-protocol/topos:main init: true labels: @@ -32,10 +26,7 @@ services: interval: 15s volumes: - shared:/tmp/shared - - ./node_config/node/boot/config.toml:/tmp/node_config/node/boot/config.toml:ro - - ./node_config/node/boot/libp2p/:/tmp/node_config/node/boot/libp2p/:ro - - ./node_config/node/boot/consensus/:/tmp/node_config/node/boot/consensus/:ro - - ./node_config/subnet/topos/genesis.json:/tmp/node_config/subnet/topos/genesis.json:ro + - ./node_config/node/boot/config.toml:/tmp/node_config/node/test/config.toml:ro build: context: ../ args: @@ -55,7 +46,7 @@ services: - env/node.env - env/telemetry.env environment: - - RUST_LOG=topos=info,topos_tce_storage=info,topos_tce_synchronizer=info + - RUST_LOG=topos=debug,topos_tce_storage=info,topos_tce_synchronizer=info peer: image: ghcr.io/topos-protocol/topos:main @@ -69,7 +60,6 @@ services: volumes: - shared:/tmp/shared - ./node_config/node/test/config.toml:/tmp/node_config/node/test/config.toml - - ./node_config/subnet/topos/genesis.json:/tmp/node_config/subnet/topos/genesis.json build: context: ../ args: @@ -93,7 +83,7 @@ services: - env/node.env - env/telemetry.env environment: - - RUST_LOG=topos=info,topos_tce_storage=info,topos_tce_synchronizer=info + - RUST_LOG=topos=debug,topos_tce_storage=info,topos_tce_synchronizer=info sync: image: ghcr.io/topos-protocol/topos:main @@ -104,7 +94,6 @@ services: volumes: - shared:/tmp/shared - ./node_config/node/test/config.toml:/tmp/node_config/node/test/config.toml - - ./node_config/subnet/topos/genesis.json:/tmp/node_config/subnet/topos/genesis.json build: context: ../ args: @@ -124,6 +113,7 @@ services: - env/node.env - env/telemetry.env environment: + - TOPOS_EDGE_VERSION=1.3.0-topos - RUST_LOG=topos=info @@ -151,7 +141,7 @@ services: check: container_name: check image: ghcr.io/topos-protocol/topos:main - command: regtest push-certificate -f json + command: regtest push-certificate -f json --timeout-broadcast=5 profiles: - CI - check @@ -170,7 +160,7 @@ services: condition: service_healthy environment: - LOCAL_TEST_NET=true - - RUST_LOG=info,topos=info + - RUST_LOG=info,topos=debug - TARGET_NODES_PATH=/tmp/peer_nodes.json autoheal: diff --git a/tools/env/base.env b/tools/env/base.env index dff3066d2..264a88536 100644 --- a/tools/env/base.env +++ b/tools/env/base.env @@ -1,3 +1,4 @@ RUST_LOG=warn TOOLCHAIN_VERSION=stable RUST_BACKTRACE=full +LOCAL_TEST_NET=true diff --git a/tools/env/initializer.env b/tools/env/initializer.env new file mode 100644 index 000000000..4ee8e60ed --- /dev/null +++ b/tools/env/initializer.env @@ -0,0 +1,3 @@ +BOOT_NODE=/dns4/boot/tcp/9090/p2p +TOPOS_EDGE_VERSION=1.3.0-topos +VALIDATOR_COUNT=15 diff --git a/tools/env/node.env b/tools/env/node.env index 2539db1a6..235bfaf81 100644 --- a/tools/env/node.env +++ b/tools/env/node.env @@ -1,5 +1,4 @@ RUST_LOG=warn,topos=warn -LOCAL_TEST_NET=true TOPOS_MINIMUM_TCE_CLUSTER_SIZE=7 diff --git a/tools/init.sh b/tools/init.sh index c606db6fd..a84b60622 100755 --- a/tools/init.sh +++ b/tools/init.sh @@ -9,42 +9,123 @@ then fi JQ=$(which jq) -TOPOS_BIN=./topos +TOPOS_BIN=/usr/src/app/topos TOPOS_HOME=/tmp/node_config NODE="http://$HOSTNAME:1340" TCE_EXT_HOST="/dns4/$HOSTNAME" +NODE_LIST_PATH=/tmp/shared/peer_nodes.json FIXED_BOOT_PEER_ID="BOOT_NODE_1" export TOPOS_HOME=$TOPOS_HOME -mkdir -p $TOPOS_HOME/node/test/libp2p -mkdir -p $TOPOS_HOME/node/test/consensus +mkdir -p $TOPOS_HOME/node/test -random_entry () { - local seed=$(hostname -i | tr -d ".") - local result=$(jq -r --arg i $(($seed % $(jq '.keys|length' $1))) '.keys[$i|tonumber]' $1) +install_polygon() { + # Download the polygon-edge binary + [ ! -f "/usr/local/bin/polygon-edge" ] && $TOPOS_BIN setup subnet --path /usr/local/bin --release v${TOPOS_EDGE_VERSION} +} - echo $result +link_genesis() { + mkdir -p $TOPOS_HOME/subnet/topos + [ ! -f "$TOPOS_HOME/subnet/topos/genesis.json" ] && ln -s /tmp/shared/genesis.json $TOPOS_HOME/subnet/topos/genesis.json } case "$1" in + # Try to initialize the shared folder with predefined values + "init") + if [[ ! ${LOCAL_TEST_NET:-"false"} == "true" ]]; then + echo "This command shouldn't be called for other behaviour than test" + exit 1 + fi + + # Create and move to the shared folder + mkdir -p /tmp/shared + cd /tmp/shared + + if [ -f genesis.json ]; then + echo "Configuration already created" + exit 0 + fi + + install_polygon + + # Create nodes folder based on the expected number of validators + polygon-edge secrets init --insecure --data-dir node- --num ${VALIDATOR_COUNT} + + # Create the guard file for pick concurrency + touch /tmp/shared/guard.lock + + # Get the boot node peer id + BOOT_NODE_ID=$(polygon-edge secrets output --data-dir node-1 | grep Node | head -n 1 | awk -F ' ' '{print $4}') + + # Create the bootnode multiaddr + BOOT_NODE=$BOOT_NODE/$BOOT_NODE_ID + + # Generate genesis file + polygon-edge genesis --consensus ibft --ibft-validators-prefix-path node- --bootnode $BOOT_NODE + + mv ./node-1 boot + + # Update permissions on the shared tree + chmod a+rwx /tmp/shared/* + ;; "boot") - exec "$TOPOS_BIN" "${@:2}" + if [[ ${LOCAL_TEST_NET:-"false"} == "true" ]]; then + + echo "Generating node list file..." + $JQ -n --arg NODE $NODE '{"nodes": [$NODE]}' > $NODE_LIST_PATH + echo "Peer nodes list have been successfully generated" + + cp -R /tmp/shared/boot/* $TOPOS_HOME/node/test + + link_genesis + fi + + exec "$TOPOS_BIN" "${@:2}" ;; - "peer") + "peer") if [[ ${LOCAL_TEST_NET:-"false"} == "true" ]]; then - export TOPOS_HOME=$TOPOS_HOME - export TCE_LOCAL_KS=$HOSTNAME - export TCE_EXT_HOST - LIBP2P_KEY=$(random_entry /tmp/shared/libp2p_keys.json) - echo -n $LIBP2P_KEY > $TOPOS_HOME/node/test/libp2p/libp2p.key + until [ -f "$NODE_LIST_PATH" ] + do + echo "Waiting 1s for node_list file $NODE_LIST_PATH to be created by boot container..." + sleep 1 + done + + # Acquire lock and add $NODE to ${NODE_LIST_PATH} only once + ( + flock --exclusive -w 10 200 || exit 1 + cat <<< $($JQ --arg NODE $NODE '.nodes |= (. + [$NODE] | unique)' $NODE_LIST_PATH) > $NODE_LIST_PATH + + ) 200>"${NODE_LIST_PATH}.lock" + + # The init container should have produce a bunch of config folder + # A peer will take one and rename it to its hostname + # If a folder with its hostname already exists we just reuse it + # If not we process to lock the guard and reserve a slot + EXPECTED_FOLDER="/tmp/shared/$HOSTNAME" - VALIDATOR_BLS_KEY=$(random_entry /tmp/shared/validator_bls_keys.json) - echo -n $VALIDATOR_BLS_KEY > $TOPOS_HOME/node/test/consensus/validator-bls.key + if [ ! -d $EXPECTED_FOLDER ]; then - VALIDATOR_KEY=$(random_entry /tmp/shared/validator_keys.json) - echo -n $VALIDATOR_KEY > $TOPOS_HOME/node/test/consensus/validator.key + # Acquire lock for config + ( + flock --exclusive -w 10 200 || exit 1 + + NEXT_NODE_FOLDER=$(ls -d /tmp/shared/node-*/ | head -n1) + if [[ $NEXT_NODE_FOLDER == "" ]]; then + echo "No more configuration available for node, Try to increase VALIDATOR_COUNT" + exit 1; + fi + + echo "PEER has select : $NEXT_NODE_FOLDER" + + mv $NEXT_NODE_FOLDER $EXPECTED_FOLDER + ) 200>"/tmp/shared/guard.lock" + fi + + cp -R $EXPECTED_FOLDER/* $TOPOS_HOME/node/test + + link_genesis fi exec "$TOPOS_BIN" "${@:2}" @@ -53,17 +134,15 @@ case "$1" in "sync") if [[ ${LOCAL_TEST_NET:-"false"} == "true" ]]; then - export TCE_LOCAL_KS=$HOSTNAME - export TCE_EXT_HOST + link_genesis - LIBP2P_KEY=$(random_entry /tmp/shared/libp2p_keys.json) - echo -n $LIBP2P_KEY > $TOPOS_HOME/node/test/libp2p/libp2p.key + cd $TOPOS_HOME/node/test - VALIDATOR_BLS_KEY=$(random_entry /tmp/shared/validator_bls_keys.json) - echo -n $VALIDATOR_BLS_KEY > $TOPOS_HOME/node/test/consensus/validator-bls.key + if [ ! -d "./libp2p" ]; then + install_polygon - VALIDATOR_KEY=$(random_entry /tmp/shared/validator_keys.json) - echo -n $VALIDATOR_KEY > $TOPOS_HOME/node/test/consensus/validator.key + polygon-edge secrets init --insecure --data-dir . + fi fi @@ -75,3 +154,5 @@ case "$1" in exec "$TOPOS_BIN" "$@" ;; esac + +