diff --git a/CHANGELOG.md b/CHANGELOG.md index 091cbf43992..db0e5603224 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ ### Fixed +- [#5111](https://github.com/ChainSafe/forest/issues/5111) Make F3 work when the node Kademlia is disabled. + ## Forest v.0.23.3 "Plumber" Mandatory release for calibnet node operators. It fixes a sync error at epoch 2281645. diff --git a/f3-sidecar/p2p.go b/f3-sidecar/p2p.go index 969861131a7..7818e71d7e0 100644 --- a/f3-sidecar/p2p.go +++ b/f3-sidecar/p2p.go @@ -14,9 +14,10 @@ import ( const ListenAddr = "/ip4/127.0.0.1/tcp/0" type P2PHost struct { - Host host.Host - DHT *dht.IpfsDHT - PubSub *pubsub.PubSub + Host host.Host + DHT *dht.IpfsDHT + BackupDHT *dht.IpfsDHT + PubSub *pubsub.PubSub } func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { @@ -36,6 +37,17 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { return nil, err } + backupDthOpts := []dht.Option{ + dht.Mode(dht.ModeAutoServer), + dht.ProtocolPrefix(protocol.ID(fmt.Sprintf("/fil/kad/f3-sidecar/%s", networkName))), + dht.DisableProviders(), + dht.DisableValues(), + } + backupHostDHT, err := dht.New(ctx, host, backupDthOpts...) + if err != nil { + return nil, err + } + ps, err := pubsub.NewGossipSub(ctx, host, pubsub.WithPeerExchange(true), pubsub.WithFloodPublish(true), @@ -44,5 +56,5 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { return nil, err } - return &P2PHost{host, hostDHT, ps}, nil + return &P2PHost{host, hostDHT, backupHostDHT, ps}, nil } diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 813f2c590ed..879fec318ab 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -5,7 +5,10 @@ set -euxo pipefail source "$(dirname "$0")/harness.sh" -$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --detach --save-token ./admin_token +$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --save-token ./admin_token --exit-after-init +$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" & +FOREST_NODE_PID=$! + FULLNODE_API_INFO="$(cat admin_token):/ip4/127.0.0.1/tcp/2345/http" export FULLNODE_API_INFO @@ -13,3 +16,10 @@ export FULLNODE_API_INFO until $FOREST_CLI_PATH net peers | grep "calib"; do sleep 1s; done + +# Verify F3 is getting certificates from the network +until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do + sleep 1s; +done +kill -KILL $FOREST_NODE_PID +wait $FOREST_NODE_PID || true diff --git a/src/f3/mod.rs b/src/f3/mod.rs index 72afe1c721f..55f373d3a7b 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -100,6 +100,7 @@ pub fn run_f3_sidecar_if_enabled( if is_sidecar_ffi_enabled(chain_config) { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { + tracing::info!("Starting F3 sidecar service ..."); GoF3NodeImpl::run( _rpc_endpoint, _jwt, diff --git a/src/libp2p/discovery.rs b/src/libp2p/discovery.rs index 4a834e8b59f..c08bbccd100 100644 --- a/src/libp2p/discovery.rs +++ b/src/libp2p/discovery.rs @@ -35,6 +35,8 @@ use crate::utils::version::FOREST_VERSION_STRING; pub struct DerivedDiscoveryBehaviour { /// Kademlia discovery. kademlia: Toggle>, + /// Kademlia discovery for bootstrapping F3 sidecar when the main Kademlia is disabled. + kademlia_f3_sidecar: kad::Behaviour, /// Discovers nodes on the local network. mdns: Toggle, /// [`identify::Behaviour`] needs to be manually hooked up with [`kad::Behaviour`] to make discovery work. See @@ -158,6 +160,12 @@ impl<'a> DiscoveryConfig<'a> { } else { None }; + let kademlia_f3_sidecar = new_kademlia( + local_peer_id, + StreamProtocol::try_from_owned(format!( + "/fil/kad/f3-sidecar/{network_name}/kad/1.0.0" + ))?, + ); let mdns_opt = if enable_mdns { Some(Mdns::new(Default::default(), local_peer_id).expect("Could not start mDNS")) @@ -168,6 +176,7 @@ impl<'a> DiscoveryConfig<'a> { Ok(DiscoveryBehaviour { discovery: DerivedDiscoveryBehaviour { kademlia: kademlia_opt.into(), + kademlia_f3_sidecar, mdns: mdns_opt.into(), identify: identify::Behaviour::new( identify::Config::new("ipfs/0.1.0".into(), local_public_key) @@ -441,6 +450,11 @@ impl NetworkBehaviour for DiscoveryBehaviour { kademlia.add_address(peer_id, address.clone()); } } + for address in &info.listen_addrs { + self.discovery + .kademlia_f3_sidecar + .add_address(peer_id, address.clone()); + } } } DerivedDiscoveryBehaviourEvent::Autonat(_) => {} @@ -470,6 +484,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { trace!("Libp2p => Unhandled Kademlia event: {:?}", other) } }, + DerivedDiscoveryBehaviourEvent::KademliaF3Sidecar(_) => {} DerivedDiscoveryBehaviourEvent::Mdns(ev) => match ev { MdnsEvent::Discovered(list) => { if self.n_node_connected >= self.target_peer_count {