From f8c1103ee5ae7439076a3512d19cbc6646b64d2c Mon Sep 17 00:00:00 2001 From: lfg2 Date: Thu, 19 Dec 2024 09:41:35 +0800 Subject: [PATCH 01/94] fix(erigon-lib/kv):readme (#13155) --- erigon-lib/kv/Readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/kv/Readme.md b/erigon-lib/kv/Readme.md index 7c6eadb421f..fbcb7a82648 100644 --- a/erigon-lib/kv/Readme.md +++ b/erigon-lib/kv/Readme.md @@ -29,7 +29,7 @@ open_cursor/seek/write_data_from_current_position/move_to_end/step_back/step_for v v +-----------------------------------+ +-----------------------------------+ | ethdb/kv_mdbx.go | | ethdb/kv_remote.go | -| (tg-specific MDBX implementaion) | | (tg-specific remote DB access) | +| (tg-specific MDBX implementation) | | (tg-specific remote DB access) | +-----------------------------------+ +-----------------------------------+ | | | | @@ -39,7 +39,7 @@ open_cursor/seek/write_data_from_current_position/move_to_end/step_back/step_for | (Common KV interface. DB-friendly, disk-friendly, cpu-cache-friendly. | | Same app code can work with local or remote database. | | Allows experiment with another database implementations. | -| Supports context.Context for cancelation. Any operation can return error) | +| Supports context.Context for cancellation. Any operation can return error) | +----------------------------------------------------------------------------------------------+ Then: @@ -62,7 +62,7 @@ kv_temporal.go - Methods db.Update, db.View - can be used to open and close short transaction. - Methods Begin/Commit/Rollback - for long transaction. -- it's safe to call .Rollback() after .Commit(), multiple rollbacks are also safe. Common transaction patter: +- it's safe to call .Rollback() after .Commit(), multiple rollbacks are also safe. Common transaction pattern: ```golang From 65cbcfd1a0967487804e00804ca388c441e92f40 Mon Sep 17 00:00:00 2001 From: sudeep Date: Thu, 19 Dec 2024 07:12:33 +0530 Subject: [PATCH 02/94] nilness linter (#13146) --- .golangci.yml | 2 ++ cl/antiquary/beacon_states_collector.go | 4 ---- cl/beacon/handler/block_production.go | 2 +- cl/beacon/handler/states.go | 4 ---- cl/beacon/router.go | 4 +--- cl/persistence/beacon_indicies/indicies.go | 2 +- cl/persistence/blob_storage/blob_db.go | 2 +- .../historical_states_reader.go | 9 --------- cl/persistence/state/state_accessors.go | 2 +- cl/phase1/core/state/cache.go | 3 --- cl/phase1/core/state/cache_accessors.go | 3 --- .../execution_client/execution_client_rpc.go | 12 +++++------- cl/phase1/forkchoice/on_attestation.go | 7 ------- cl/phase1/network/gossip_manager.go | 2 +- .../services/sync_committee_messages_service.go | 5 +---- .../services/sync_contribution_service.go | 3 --- .../attestation_producer/attestation_producer.go | 4 ---- cmd/caplin/caplin1/run.go | 2 +- cmd/caplin/main.go | 4 ---- cmd/integration/commands/state_domains.go | 2 +- cmd/snapshots/cmp/cmp.go | 2 +- cmd/snapshots/copy/copy.go | 14 +++++++------- cmd/snapshots/manifest/manifest.go | 8 +++----- cmd/snapshots/torrents/torrents.go | 16 +++++----------- cmd/snapshots/verify/verify.go | 8 +++----- consensus/clique/verifier.go | 2 +- eth/gasprice/gasprice.go | 2 +- ethdb/privateapi/logsfilter.go | 2 +- ethstats/ethstats.go | 2 +- turbo/app/snapshots_cmd.go | 2 +- turbo/execution/eth1/ethereum_execution.go | 5 +---- turbo/execution/eth1/getters.go | 6 ------ turbo/jsonrpc/bor_snapshot.go | 2 +- turbo/jsonrpc/eth_api.go | 2 +- turbo/jsonrpc/tracing.go | 2 +- turbo/stages/headerdownload/header_algos.go | 2 +- turbo/stages/stageloop.go | 14 ++++++-------- 37 files changed, 51 insertions(+), 118 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 168bc592861..774128c38c4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -98,6 +98,8 @@ linters-settings: - fieldalignment - shadow - unsafeptr + enable: + - nilness goconst: min-len: 2 min-occurrences: 2 diff --git a/cl/antiquary/beacon_states_collector.go b/cl/antiquary/beacon_states_collector.go index 9ad867c2ee9..d78f4047ceb 100644 --- a/cl/antiquary/beacon_states_collector.go +++ b/cl/antiquary/beacon_states_collector.go @@ -107,7 +107,6 @@ func (i *beaconStatesCollector) addGenesisState(ctx context.Context, state *stat i.buf.Reset() i.compressor.Reset(i.buf) - var err error slot := state.Slot() epoch := slot / i.beaconCfg.SlotsPerEpoch // Setup state events handlers @@ -121,9 +120,6 @@ func (i *beaconStatesCollector) addGenesisState(ctx context.Context, state *stat events.AddValidator(uint64(index), v) return true }) - if err != nil { - return err - } roundedSlotToDump := slot - (slot % clparams.SlotsPerDump) if err := antiquateField(ctx, roundedSlotToDump, state.RawBalances(), i.buf, i.compressor, i.balancesDumpsCollector); err != nil { diff --git a/cl/beacon/handler/block_production.go b/cl/beacon/handler/block_production.go index bfc24b096e7..710fd009c4c 100644 --- a/cl/beacon/handler/block_production.go +++ b/cl/beacon/handler/block_production.go @@ -546,7 +546,7 @@ func (a *ApiHandler) getBuilderPayload( ethHeader.SetVersion(baseState.Version()) } // check kzg commitments - if header != nil && baseState.Version() >= clparams.DenebVersion { + if baseState.Version() >= clparams.DenebVersion { if header.Data.Message.BlobKzgCommitments.Len() >= cltypes.MaxBlobsCommittmentsPerBlock { return nil, fmt.Errorf("too many blob kzg commitments: %d", header.Data.Message.BlobKzgCommitments.Len()) } diff --git a/cl/beacon/handler/states.go b/cl/beacon/handler/states.go index b63deedd626..ac1779a6ff9 100644 --- a/cl/beacon/handler/states.go +++ b/cl/beacon/handler/states.go @@ -256,10 +256,6 @@ func (a *ApiHandler) getFinalityCheckpoints(w http.ResponseWriter, r *http.Reque } finalizedCheckpoint, currentJustifiedCheckpoint, previousJustifiedCheckpoint, ok := a.forkchoiceStore.GetFinalityCheckpoints(blockRoot) - if err != nil { - return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err) - } - snRoTx := a.caplinStateSnapshots.View() defer snRoTx.Close() diff --git a/cl/beacon/router.go b/cl/beacon/router.go index da43116c618..a6c0e72d723 100644 --- a/cl/beacon/router.go +++ b/cl/beacon/router.go @@ -37,6 +37,7 @@ type LayeredBeaconHandler struct { func ListenAndServe(beaconHandler *LayeredBeaconHandler, routerCfg beacon_router_configuration.RouterConfiguration) error { listener, err := net.Listen(routerCfg.Protocol, routerCfg.Address) if err != nil { + log.Warn("[Beacon API] Failed to start listening", "addr", routerCfg.Address, "err", err) return err } defer listener.Close() @@ -72,9 +73,6 @@ func ListenAndServe(beaconHandler *LayeredBeaconHandler, routerCfg beacon_router IdleTimeout: routerCfg.IdleTimeout, WriteTimeout: routerCfg.WriteTimeout, } - if err != nil { - log.Warn("[Beacon API] Failed to start listening", "addr", routerCfg.Address, "err", err) - } if err := server.Serve(listener); err != nil { log.Warn("[Beacon API] failed to start serving", "addr", routerCfg.Address, "err", err) diff --git a/cl/persistence/beacon_indicies/indicies.go b/cl/persistence/beacon_indicies/indicies.go index ac25f94e6b1..c83fa08b211 100644 --- a/cl/persistence/beacon_indicies/indicies.go +++ b/cl/persistence/beacon_indicies/indicies.go @@ -253,7 +253,7 @@ func PruneSignedHeaders(tx kv.RwTx, from uint64) error { return err } for k, _, err := cursor.Seek(base_encoding.Encode64ToBytes4(from)); err == nil && k != nil; k, _, err = cursor.Prev() { - if err != nil { + if err != nil { //nolint:govet return err } if err := cursor.DeleteCurrent(); err != nil { diff --git a/cl/persistence/blob_storage/blob_db.go b/cl/persistence/blob_storage/blob_db.go index 51c8244da45..0377bde4d7a 100644 --- a/cl/persistence/blob_storage/blob_db.go +++ b/cl/persistence/blob_storage/blob_db.go @@ -265,7 +265,7 @@ func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, stor } if verifySignatureFn != nil { // verify the signature of the sidecar head, we leave this step up to the caller to define - if verifySignatureFn(sidecar.SignedBlockHeader); err != nil { + if err := verifySignatureFn(sidecar.SignedBlockHeader); err != nil { return 0, 0, err } } diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go index a2536572f2b..f193ed2cda9 100644 --- a/cl/persistence/state/historical_states_reader/historical_states_reader.go +++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go @@ -749,9 +749,6 @@ func (r *HistoricalStatesReader) readPendingEpochs(tx kv.Tx, slot uint64) (*soli } return true }) - if err != nil { - return nil, nil, err - } } return currentEpochAttestations, previousEpochAttestations, nil } @@ -789,9 +786,6 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, kvGetter state_acc currentIdxs := solid.NewParticipationBitList(int(validatorLength), int(r.cfg.ValidatorRegistryLimit)) previousIdxs := solid.NewParticipationBitList(int(validatorLength), int(r.cfg.ValidatorRegistryLimit)) - if err != nil { - return nil, nil, err - } // Read the previous idxs for i := beginSlot; i <= slot; i++ { @@ -864,9 +858,6 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, kvGetter state_acc } return true }) - if err != nil { - return nil, nil, err - } } return currentIdxs, previousIdxs, nil } diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go index 143cf1189b4..3340f9425c7 100644 --- a/cl/persistence/state/state_accessors.go +++ b/cl/persistence/state/state_accessors.go @@ -151,7 +151,7 @@ func ReadValidatorsTable(tx kv.Tx, out *StaticValidatorTable) error { } out.validatorTable = append(out.validatorTable, staticValidator) } - if err != nil { + if err != nil { //nolint:govet return err } slot, err := GetStateProcessingProgress(tx) diff --git a/cl/phase1/core/state/cache.go b/cl/phase1/core/state/cache.go index 169c7ec18f3..a04090888c7 100644 --- a/cl/phase1/core/state/cache.go +++ b/cl/phase1/core/state/cache.go @@ -178,9 +178,6 @@ func (b *CachingBeaconState) _initializeValidatorsPhase0() error { if err != nil { return err } - if err != nil { - return err - } attestation := &solid.Attestation{ AggregationBits: pa.AggregationBits, Data: attestationData, diff --git a/cl/phase1/core/state/cache_accessors.go b/cl/phase1/core/state/cache_accessors.go index aa7880a8784..b9cb03bb42b 100644 --- a/cl/phase1/core/state/cache_accessors.go +++ b/cl/phase1/core/state/cache_accessors.go @@ -188,9 +188,6 @@ func (b *CachingBeaconState) BaseReward(index uint64) (uint64, error) { // It grabs values from cache as needed func (b *CachingBeaconState) SyncRewards() (proposerReward, participantReward uint64, err error) { activeBalance := b.GetTotalActiveBalance() - if err != nil { - return 0, 0, err - } totalActiveIncrements := activeBalance / b.BeaconConfig().EffectiveBalanceIncrement baseRewardPerInc := b.BaseRewardPerIncrement() totalBaseRewards := baseRewardPerInc * totalActiveIncrements diff --git a/cl/phase1/execution_client/execution_client_rpc.go b/cl/phase1/execution_client/execution_client_rpc.go index 486eacf04d9..87574f52acb 100644 --- a/cl/phase1/execution_client/execution_client_rpc.go +++ b/cl/phase1/execution_client/execution_client_rpc.go @@ -169,15 +169,13 @@ func (cc *ExecutionClientRpc) ForkChoiceUpdate(ctx context.Context, finalized li err := cc.client.CallContext(ctx, forkChoiceResp, rpc_helper.ForkChoiceUpdatedV1, args...) if err != nil { + if err.Error() == errContextExceeded { + // ignore timeouts + return nil, nil + } return nil, fmt.Errorf("execution Client RPC failed to retrieve ForkChoiceUpdate response, err: %w", err) } - // Ignore timeouts - if err != nil && err.Error() == errContextExceeded { - return nil, nil - } - if err != nil { - return nil, err - } + if forkChoiceResp.PayloadId == nil { return []byte{}, checkPayloadStatus(forkChoiceResp.PayloadStatus) } diff --git a/cl/phase1/forkchoice/on_attestation.go b/cl/phase1/forkchoice/on_attestation.go index deeb757442a..65ede2792a5 100644 --- a/cl/phase1/forkchoice/on_attestation.go +++ b/cl/phase1/forkchoice/on_attestation.go @@ -116,10 +116,6 @@ func (f *ForkChoiceStore) verifyAttestationWithCheckpointState( } if !fromBlock { indexedAttestation := state.GetIndexedAttestation(attestation, attestationIndicies) - if err != nil { - return nil, err - } - valid, err := targetState.isValidIndexedAttestation(indexedAttestation) if err != nil { return nil, err @@ -142,9 +138,6 @@ func (f *ForkChoiceStore) verifyAttestationWithState( } if !fromBlock { indexedAttestation := state.GetIndexedAttestation(attestation, attestationIndicies) - if err != nil { - return nil, err - } valid, err := state.IsValidIndexedAttestation(s, indexedAttestation) if err != nil { return nil, err diff --git a/cl/phase1/network/gossip_manager.go b/cl/phase1/network/gossip_manager.go index b370de6ff78..1955f27e89c 100644 --- a/cl/phase1/network/gossip_manager.go +++ b/cl/phase1/network/gossip_manager.go @@ -131,7 +131,7 @@ func (g *GossipManager) onRecv(ctx context.Context, data *sentinel.GossipData, l if errors.Is(err, services.ErrIgnore) || errors.Is(err, synced_data.ErrNotSynced) { return nil } - if err != nil { + if err != nil { //nolint:govet g.sentinel.BanPeer(ctx, data.Peer) return err } diff --git a/cl/phase1/network/services/sync_committee_messages_service.go b/cl/phase1/network/services/sync_committee_messages_service.go index 9af0269cc93..83c7c224c57 100644 --- a/cl/phase1/network/services/sync_committee_messages_service.go +++ b/cl/phase1/network/services/sync_committee_messages_service.go @@ -160,9 +160,6 @@ func verifySyncCommitteeMessageSignature(s *state.CachingBeaconState, msg *cltyp if err != nil { return nil, nil, nil, err } - signingRoot, err := utils.Sha256(msg.BeaconBlockRoot[:], domain), nil - if err != nil { - return nil, nil, nil, err - } + signingRoot := utils.Sha256(msg.BeaconBlockRoot[:], domain) return msg.Signature[:], signingRoot[:], publicKey[:], nil } diff --git a/cl/phase1/network/services/sync_contribution_service.go b/cl/phase1/network/services/sync_contribution_service.go index a0799c4f568..27da98964e1 100644 --- a/cl/phase1/network/services/sync_contribution_service.go +++ b/cl/phase1/network/services/sync_contribution_service.go @@ -285,9 +285,6 @@ func verifySyncContributionProofAggregatedSignature(s *state.CachingBeaconState, } msg := utils.Sha256(contribution.BeaconBlockRoot[:], domain) - if err != nil { - return nil, nil, nil, err - } // only use the ones pertaining to the aggregation bits subCommitteePubsKeys := make([][]byte, 0, len(subCommitteeKeys)) for i, key := range subCommitteeKeys { diff --git a/cl/validator/attestation_producer/attestation_producer.go b/cl/validator/attestation_producer/attestation_producer.go index 4f16a6db3fa..0e277f06e72 100644 --- a/cl/validator/attestation_producer/attestation_producer.go +++ b/cl/validator/attestation_producer/attestation_producer.go @@ -183,10 +183,6 @@ func (ap *attestationProducer) ProduceAndCacheAttestationData(tx kv.Tx, baseStat log.Warn("Failed to process slots", "slot", slot, "err", err) return solid.AttestationData{}, err } - if err != nil { - return solid.AttestationData{}, err - } - } targetCheckpoint, err := ap.computeTargetCheckpoint(tx, baseState, baseStateBlockRoot, slot) diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index 99877a288a2..81d4cbd331a 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -165,7 +165,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi } genesisState = state.New(beaconConfig) - if genesisState.DecodeSSZ(stateBytes, int(beaconConfig.GetCurrentStateVersion(beaconConfig.GenesisEpoch))); err != nil { + if err := genesisState.DecodeSSZ(stateBytes, int(beaconConfig.GetCurrentStateVersion(beaconConfig.GenesisEpoch))); err != nil { return fmt.Errorf("could not decode genesis state: %s", err) } } else { diff --git a/cmd/caplin/main.go b/cmd/caplin/main.go index 7c6fe7d2ce2..2f1a10c0c23 100644 --- a/cmd/caplin/main.go +++ b/cmd/caplin/main.go @@ -87,10 +87,6 @@ func runCaplinNode(cliCtx *cli.Context) error { ctx, cn := context.WithCancel(cliCtx.Context) defer cn() - if err != nil { - log.Error("[Checkpoint Sync] Failed", "reason", err) - return err - } var executionEngine execution_client2.ExecutionEngine if cfg.RunEngineAPI { cc, err := execution_client2.NewExecutionClientRPC(cfg.JwtSecret, cfg.EngineAPIAddr, cfg.EngineAPIPort) diff --git a/cmd/integration/commands/state_domains.go b/cmd/integration/commands/state_domains.go index 6ca2ecfbea1..e754165af75 100644 --- a/cmd/integration/commands/state_domains.go +++ b/cmd/integration/commands/state_domains.go @@ -139,7 +139,7 @@ func requestDomains(chainDb, stateDb kv.RwDB, ctx context.Context, readDomain st defer agg.Close() r := state.NewReaderV3(domains) - if err != nil && startTxNum != 0 { + if startTxNum != 0 { return fmt.Errorf("failed to seek commitment to txn %d: %w", startTxNum, err) } latestTx := domains.TxNum() diff --git a/cmd/snapshots/cmp/cmp.go b/cmd/snapshots/cmp/cmp.go index 4ccbc6057f1..31918ea56ef 100644 --- a/cmd/snapshots/cmp/cmp.go +++ b/cmd/snapshots/cmp/cmp.go @@ -225,7 +225,7 @@ func cmp(cliCtx *cli.Context) error { return errors.New("no first session established") } - if session1 == nil { + if session2 == nil { return errors.New("no second session established") } diff --git a/cmd/snapshots/copy/copy.go b/cmd/snapshots/copy/copy.go index 2ddedb6ec38..4c632ada244 100644 --- a/cmd/snapshots/copy/copy.go +++ b/cmd/snapshots/copy/copy.go @@ -124,12 +124,12 @@ func copy(cliCtx *cli.Context) error { pos++ } - switch dst.LType { + switch dst.LType { //nolint:govet case sync.TorrentFs: return errors.New("can't copy to torrent - need intermediate local fs") case sync.RemoteFs: - if rcCli == nil { + if rcCli == nil { //nolint:govet rcCli, err = downloader.NewRCloneClient(logger) if err != nil { @@ -144,7 +144,7 @@ func copy(cliCtx *cli.Context) error { switch src.LType { case sync.TorrentFs: - config := sync.NewTorrentClientConfigFromCobra(cliCtx, dst.Chain) + config := sync.NewTorrentClientConfigFromCobra(cliCtx, dst.Chain) //nolint:govet torrentCli, err = sync.NewTorrentClient(cliCtx.Context, config) if err != nil { return fmt.Errorf("can't create torrent: %w", err) @@ -186,7 +186,7 @@ func copy(cliCtx *cli.Context) error { version := cliCtx.Int(VersionFlag.Name) if version != 0 { - dst.Version = snaptype.Version(version) + dst.Version = snaptype.Version(version) //nolint:govet } if cliCtx.Args().Len() > pos { @@ -205,7 +205,7 @@ func copy(cliCtx *cli.Context) error { switch src.LType { case sync.LocalFs: - switch dst.LType { + switch dst.LType { //nolint:govet case sync.LocalFs: return localToLocal(src, dst, firstBlock, lastBlock, snapTypes, torrents, hashes, manifest) case sync.RemoteFs: @@ -215,7 +215,7 @@ func copy(cliCtx *cli.Context) error { } case sync.RemoteFs: - switch dst.LType { + switch dst.LType { //nolint:govet case sync.LocalFs: return remoteToLocal(cliCtx.Context, rcCli, src, dst, firstBlock, lastBlock, snapTypes, torrents, hashes, manifest) case sync.RemoteFs: @@ -225,7 +225,7 @@ func copy(cliCtx *cli.Context) error { } case sync.TorrentFs: - switch dst.LType { + switch dst.LType { //nolint:govet case sync.LocalFs: return torrentToLocal(torrentCli, src, dst, firstBlock, lastBlock, snapTypes, torrents, hashes, manifest) case sync.RemoteFs: diff --git a/cmd/snapshots/manifest/manifest.go b/cmd/snapshots/manifest/manifest.go index 617b442a19b..540c8c3a52d 100644 --- a/cmd/snapshots/manifest/manifest.go +++ b/cmd/snapshots/manifest/manifest.go @@ -111,12 +111,10 @@ func manifest(cliCtx *cli.Context, command string) error { switch src.LType { case sync.RemoteFs: - if rcCli == nil { - rcCli, err = downloader.NewRCloneClient(logger) + rcCli, err = downloader.NewRCloneClient(logger) - if err != nil { - return err - } + if err != nil { + return err } if err = sync.CheckRemote(rcCli, src.Src); err != nil { diff --git a/cmd/snapshots/torrents/torrents.go b/cmd/snapshots/torrents/torrents.go index 49e20614fa8..f1b93ace28b 100644 --- a/cmd/snapshots/torrents/torrents.go +++ b/cmd/snapshots/torrents/torrents.go @@ -111,10 +111,6 @@ func torrents(cliCtx *cli.Context, command string) error { if src, err = sync.ParseLocator(cliCtx.Args().Get(pos)); err != nil { return err } - - if err != nil { - return err - } } pos++ @@ -144,12 +140,10 @@ func torrents(cliCtx *cli.Context, command string) error { switch src.LType { case sync.RemoteFs: - if rcCli == nil { - rcCli, err = downloader.NewRCloneClient(logger) + rcCli, err = downloader.NewRCloneClient(logger) - if err != nil { - return err - } + if err != nil { + return err } if err = sync.CheckRemote(rcCli, src.Src); err != nil { @@ -178,7 +172,7 @@ func torrents(cliCtx *cli.Context, command string) error { } if rcCli != nil { - if src != nil && src.LType == sync.RemoteFs { + if src.LType == sync.RemoteFs { ctx := cliCtx.Context // avoiding sonar dup complaint srcSession, err = rcCli.NewSession(ctx, filepath.Join(tempDir, "src"), src.Src+":"+src.Root, nil) @@ -188,7 +182,7 @@ func torrents(cliCtx *cli.Context, command string) error { } } - if src != nil && srcSession == nil { + if srcSession == nil { return errors.New("no src session established") } diff --git a/cmd/snapshots/verify/verify.go b/cmd/snapshots/verify/verify.go index 15ef7ebc735..95ef16ce9d2 100644 --- a/cmd/snapshots/verify/verify.go +++ b/cmd/snapshots/verify/verify.go @@ -124,12 +124,10 @@ func verify(cliCtx *cli.Context) error { } case sync.RemoteFs: - if rcCli == nil { - rcCli, err = downloader.NewRCloneClient(logger) + rcCli, err = downloader.NewRCloneClient(logger) - if err != nil { - return err - } + if err != nil { + return err } if err = sync.CheckRemote(rcCli, src.Src); err != nil { diff --git a/consensus/clique/verifier.go b/consensus/clique/verifier.go index fd2c300c3f1..59bb88af6ef 100644 --- a/consensus/clique/verifier.go +++ b/consensus/clique/verifier.go @@ -176,7 +176,7 @@ func (c *Clique) Snapshot(chain consensus.ChainHeaderReader, number uint64, hash headers []*types.Header snap *Snapshot ) - for snap == nil { + for snap == nil { //nolint:govet // If an in-memory snapshot was found, use that if s, ok := c.recents.Get(hash); ok { snap = s diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index b091b2a9dc4..47133618cae 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -261,7 +261,7 @@ func (oracle *Oracle) getBlockPrices(ctx context.Context, blockNum uint64, limit continue } sender, _ := tx.GetSender() - if err == nil && sender != block.Coinbase() { + if sender != block.Coinbase() { heap.Push(s, tip) count = count + 1 } diff --git a/ethdb/privateapi/logsfilter.go b/ethdb/privateapi/logsfilter.go index a791ac60439..2cdac94385e 100644 --- a/ethdb/privateapi/logsfilter.go +++ b/ethdb/privateapi/logsfilter.go @@ -149,7 +149,7 @@ func (a *LogsFilterAggregator) subscribeLogs(server remote.ETHBACKEND_SubscribeL for filterReq, recvErr = server.Recv(); recvErr == nil; filterReq, recvErr = server.Recv() { a.updateLogsFilter(filter, filterReq) } - if recvErr != nil && recvErr != io.EOF { // termination + if recvErr != io.EOF { // termination return fmt.Errorf("receiving log filter request: %w", recvErr) } return nil diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 8f8cd85f049..eb5ebb51ab4 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -207,7 +207,7 @@ func (s *Service) loop() { break } } - if err != nil || conn == nil { + if conn == nil { log.Warn("Stats server unreachable") errTimer.Reset(10 * time.Second) continue diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 68100116297..72269b57514 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -1254,7 +1254,7 @@ func doCompress(cliCtx *cli.Context) error { default: } } - if err != nil && !errors.Is(err, io.EOF) { + if !errors.Is(err, io.EOF) { return err } if err := c.Compress(); err != nil { diff --git a/turbo/execution/eth1/ethereum_execution.go b/turbo/execution/eth1/ethereum_execution.go index 6e6c56745ca..f2019dfbefb 100644 --- a/turbo/execution/eth1/ethereum_execution.go +++ b/turbo/execution/eth1/ethereum_execution.go @@ -167,16 +167,13 @@ func (e *EthereumExecutionModule) unwindToCommonCanonical(tx kv.RwTx, header *ty currentHeader := header for isCanonical, err := e.isCanonicalHash(e.bacgroundCtx, tx, currentHeader.Hash()); !isCanonical && err == nil; isCanonical, err = e.isCanonicalHash(e.bacgroundCtx, tx, currentHeader.Hash()) { + currentHeader, err = e.getHeader(e.bacgroundCtx, tx, currentHeader.ParentHash, currentHeader.Number.Uint64()-1) if err != nil { return err } if currentHeader == nil { return fmt.Errorf("header %v not found", currentHeader.Hash()) } - currentHeader, err = e.getHeader(e.bacgroundCtx, tx, currentHeader.ParentHash, currentHeader.Number.Uint64()-1) - if err != nil { - return err - } } if err := e.hook.BeforeRun(tx, true); err != nil { return err diff --git a/turbo/execution/eth1/getters.go b/turbo/execution/eth1/getters.go index ffd9b008a78..d5758009723 100644 --- a/turbo/execution/eth1/getters.go +++ b/turbo/execution/eth1/getters.go @@ -155,9 +155,6 @@ func (e *EthereumExecutionModule) GetBodiesByHashes(ctx context.Context, req *ex return nil, fmt.Errorf("ethereumExecutionModule.GetBodiesByHashes: MarshalTransactionsBinary error %w", err) } - if err != nil { - return nil, fmt.Errorf("ethereumExecutionModule.GetBodiesByHashes: MarshalRequestsBinary error %w", err) - } bodies = append(bodies, &execution.BlockBody{ Transactions: txs, Withdrawals: eth1_utils.ConvertWithdrawalsToRpc(body.Withdrawals), @@ -201,9 +198,6 @@ func (e *EthereumExecutionModule) GetBodiesByRange(ctx context.Context, req *exe return nil, fmt.Errorf("ethereumExecutionModule.GetBodiesByRange: MarshalTransactionsBinary error %w", err) } - if err != nil { - return nil, fmt.Errorf("ethereumExecutionModule.GetBodiesByHashes: MarshalRequestsBinary error %w", err) - } bodies = append(bodies, &execution.BlockBody{ Transactions: txs, Withdrawals: eth1_utils.ConvertWithdrawalsToRpc(body.Withdrawals), diff --git a/turbo/jsonrpc/bor_snapshot.go b/turbo/jsonrpc/bor_snapshot.go index 3c71a2344ab..7183356259d 100644 --- a/turbo/jsonrpc/bor_snapshot.go +++ b/turbo/jsonrpc/bor_snapshot.go @@ -656,7 +656,7 @@ func snapshot(ctx context.Context, api *BorImpl, db kv.Tx, borDb kv.Tx, header * number := header.Number.Uint64() hash := header.Hash() - for snap == nil { + for snap == nil { //nolint:govet // If an on-disk checkpoint snapshot can be found, use that if number%checkpointInterval == 0 { if s, err := loadSnapshot(api, db, borDb, hash); err == nil { diff --git a/turbo/jsonrpc/eth_api.go b/turbo/jsonrpc/eth_api.go index 8190588c2aa..5b27ad313c1 100644 --- a/turbo/jsonrpc/eth_api.go +++ b/turbo/jsonrpc/eth_api.go @@ -270,7 +270,7 @@ func (api *BaseAPI) chainConfigWithGenesis(ctx context.Context, tx kv.Tx) (*chai if err != nil { return nil, nil, err } - if cc != nil && genesisBlock != nil { + if cc != nil { api._genesis.Store(genesisBlock) api._chainConfig.Store(cc) } diff --git a/turbo/jsonrpc/tracing.go b/turbo/jsonrpc/tracing.go index b5735979e09..c368dbcecf3 100644 --- a/turbo/jsonrpc/tracing.go +++ b/turbo/jsonrpc/tracing.go @@ -412,7 +412,7 @@ func (api *PrivateDebugAPIImpl) TraceCall(ctx context.Context, args ethapi.CallA } var baseFee *uint256.Int - if header != nil && header.BaseFee != nil { + if header.BaseFee != nil { var overflow bool baseFee, overflow = uint256.FromBig(header.BaseFee) if overflow { diff --git a/turbo/stages/headerdownload/header_algos.go b/turbo/stages/headerdownload/header_algos.go index 219f73de9d7..7bc99c39788 100644 --- a/turbo/stages/headerdownload/header_algos.go +++ b/turbo/stages/headerdownload/header_algos.go @@ -432,7 +432,7 @@ func (hd *HeaderDownload) RequestMoreHeaders(currentTime time.Time) (*HeaderRequ func (hd *HeaderDownload) requestMoreHeadersForPOS(currentTime time.Time) (timeout bool, request *HeaderRequest, penalties []PenaltyItem) { anchor := hd.posAnchor if anchor == nil { - dataflow.HeaderDownloadStates.AddChange(anchor.blockHeight-1, dataflow.HeaderEmpty) + //dataflow.HeaderDownloadStates.AddChange(anchor.blockHeight-1, dataflow.HeaderEmpty) hd.logger.Debug("[downloader] No PoS anchor") return } diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 38bc7de55d6..acbb47371f3 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -75,15 +75,13 @@ func StageLoop( defer close(waitForDone) if err := ProcessFrozenBlocks(ctx, db, blockReader, sync, hook); err != nil { - if err != nil { - if errors.Is(err, libcommon.ErrStopped) || errors.Is(err, context.Canceled) { - return - } + if errors.Is(err, libcommon.ErrStopped) || errors.Is(err, context.Canceled) { + return + } - logger.Error("Staged Sync", "err", err) - if recoveryErr := hd.RecoverFromDb(db); recoveryErr != nil { - logger.Error("Failed to recover header sentriesClient", "err", recoveryErr) - } + logger.Error("Staged Sync", "err", err) + if recoveryErr := hd.RecoverFromDb(db); recoveryErr != nil { + logger.Error("Failed to recover header sentriesClient", "err", recoveryErr) } } From b408f78d9472ebc4e01fdf98560ff6dc4389d0f1 Mon Sep 17 00:00:00 2001 From: Ostroukhov Nikita Date: Thu, 19 Dec 2024 04:28:56 +0000 Subject: [PATCH 03/94] Fixed snapshot read from disk (#13144) --- turbo/snapshotsync/snapshots.go | 40 +++++++++------------------------ 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/turbo/snapshotsync/snapshots.go b/turbo/snapshotsync/snapshots.go index c46d4a5ff02..6ecdc947dcf 100644 --- a/turbo/snapshotsync/snapshots.go +++ b/turbo/snapshotsync/snapshots.go @@ -35,7 +35,6 @@ import ( "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/dbg" - dir2 "github.com/erigontech/erigon-lib/common/dir" "github.com/erigontech/erigon-lib/diagnostics" "github.com/erigontech/erigon-lib/downloader/snaptype" "github.com/erigontech/erigon-lib/log/v3" @@ -56,6 +55,10 @@ type SortedRange interface { // NoOverlaps - keep largest ranges and avoid overlap func NoOverlaps[T SortedRange](in []T) (res []T) { + if len(in) == 1 { + return in + } + for i := 0; i < len(in); i++ { r := in[i] iFrom, iTo := r.GetRange() @@ -83,6 +86,10 @@ func NoGaps[T SortedRange](in []T) (out []T, missingRanges []Range) { if len(in) == 0 { return nil, nil } + if len(in) == 1 { + return in, nil + } + prevTo, _ := in[0].GetRange() for _, f := range in { from, to := f.GetRange() @@ -980,10 +987,6 @@ func (s *RoSnapshots) InitSegments(fileNames []string) error { } func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps bool) (res []snaptype.FileInfo, missingSnapshots []Range, err error) { - segmentsTypeCheck := func(dir string, in []snaptype.FileInfo) (res []snaptype.FileInfo) { - return typeOfSegmentsMustExist(dir, in, types) - } - list, err := snaptype.Segments(dir) if err != nil { @@ -1002,10 +1005,11 @@ func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps } if allowGaps { - l = NoOverlaps(segmentsTypeCheck(dir, l)) + l = NoOverlaps(l) } else { - l, m = NoGaps(NoOverlaps(segmentsTypeCheck(dir, l))) + l, m = NoGaps(NoOverlaps(l)) } + if len(m) > 0 { lst := m[len(m)-1] log.Debug("[snapshots] see gap", "type", segType, "from", lst.from) @@ -1619,28 +1623,6 @@ func sendDiagnostics(startIndexingTime time.Time, indexPercent map[string]int, a }) } -func typeOfSegmentsMustExist(dir string, in []snaptype.FileInfo, types []snaptype.Type) (res []snaptype.FileInfo) { -MainLoop: - for _, f := range in { - if f.From == f.To { - continue - } - for _, t := range types { - p := filepath.Join(dir, snaptype.SegmentFileName(f.Version, f.From, f.To, t.Enum())) - exists, err := dir2.FileExist(p) - if err != nil { - log.Debug("[snapshots] FileExist error", "err", err, "path", p) - continue MainLoop - } - if !exists { - continue MainLoop - } - res = append(res, f) - } - } - return res -} - func removeOldFiles(toDel []string, snapDir string) { for _, f := range toDel { _ = os.Remove(f) From 94a1581a5d5fce01fa1fedc1f4e8a33c2c4f1773 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:14:57 +0100 Subject: [PATCH 04/94] deps: upgrade golang.org/x/net to v0.33.0 (#13177) --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index d337d90d9f7..b46410c6a3a 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -153,7 +153,7 @@ require ( go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect go.uber.org/goleak v1.3.0 // indirect - golang.org/x/net v0.30.0 + golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 150d86f95c5..65818353ba7 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -554,8 +554,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/go.mod b/go.mod index 07949c43314..b1d51ea37a2 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/time v0.8.0 diff --git a/go.sum b/go.sum index e58d9b5bbe3..ff4b776ceda 100644 --- a/go.sum +++ b/go.sum @@ -1058,8 +1058,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= From a93dd01b7550a0289b9d5db4c168ca88ceaf7f93 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:30:44 +0100 Subject: [PATCH 05/94] Selective check for manifest (#13179) This restricts the manifest check to those sources which are actually contributing to snapshots distribution. --- .github/workflows/manifest.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index ee14b9f153e..13c054ed578 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -31,9 +31,9 @@ jobs: go-version: '1.23' - run: make downloader - run: echo $ModModified - - run: ./build/bin/downloader manifest-verify --chain mainnet - - run: ./build/bin/downloader manifest-verify --chain bor-mainnet - - run: ./build/bin/downloader manifest-verify --chain gnosis - - run: ./build/bin/downloader manifest-verify --chain chiado - - run: ./build/bin/downloader manifest-verify --chain sepolia - - run: ./build/bin/downloader manifest-verify --chain amoy + - run: ./build/bin/downloader manifest-verify --chain mainnet --webseed 'https://erigon3-v1-snapshots-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain bor-mainnet --webseed 'https://erigon3-v1-snapshots-bor-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain gnosis --webseed 'https://erigon3-v1-snapshots-gnosis.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain chiado --webseed 'https://erigon3-v1-snapshots-chiado.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain sepolia --webseed 'https://erigon3-v1-snapshots-sepolia.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain amoy --webseed 'https://erigon3-v1-snapshots-amoy.erigon.network' From c6bf543d5db5c65bf23b091b29c1e1e3c19b59e4 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:47:25 +0100 Subject: [PATCH 06/94] Automation for dashboards backup (#13182) This introduces a new workflow for automating the dashboards backup. --------- Co-authored-by: Michele Modolo --- .github/workflows/backups-dashboards.yml | 130 +++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 .github/workflows/backups-dashboards.yml diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml new file mode 100644 index 00000000000..1533b3d2664 --- /dev/null +++ b/.github/workflows/backups-dashboards.yml @@ -0,0 +1,130 @@ +# Author: Michele@DevOpsTeam.Erigon +# Maintainers: DevOps@DevOpsTeam.Erigon +# Description: This workflow is responsible for backing up the dashboards of the Erigon project. +# Status: Mergeable for testing + +name: Dashboards backup + +env: + GREEN: '\033[0;32m' + RED: '\033[0;31m' + YELLOW: '\033[0;33m' + NOCOLOUR: '\033[0m' + +on: + workflow_dispatch: + inputs: + TEMPLATE_BRANCH: + required: false + type: string + default: main + description: 'The branch to pull the backup script from (default: main)' + ERIGON_BRANCH: + required: false + type: string + default: main + description: 'The branch to pull the erigon repo from (default: main)' + +jobs: + # + # NOTE: This workflow splits the backup process in 3 jobs even though 2 (or even 1 if you're particularly naive) would have been enough. I decided to do that just as a reference for passing data between jobs. The overhead is minimal. + # + preparation: + runs-on: ubuntu-latest + environment: dashboards_backups + steps: + - name: CLONE THE ERIGON REPO from ${{ inputs.ERIGON_BRANCH }} branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ERIGON_BRANCH }} + fetch-depth: 1 + path: ${{ github.workspace }}/erigon + + - name: Upload Erigon repository + uses: actions/upload-artifact@v3 + with: + name: erigon-repo + path: ${{ github.workspace }}/erigon + retention-days: 1 + + - name: PULL THE LATEST VERSION OF BACKUP SCRIPT from ${{ inputs.TEMPLATE_BRANCH}} branch + run: | + set +x + curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup + + - name: Upload dashboard-backup + uses: actions/upload-artifact@v3 + with: + name: dashboard-backup + path: /tmp/dashboard-backup + retention-days: 1 + + + backup_dashboard: + needs: preparation + runs-on: ubuntu-latest + environment: dashboards_backups + strategy: + matrix: + # For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!) + dashboard: [erigon_custom_metrics] + env: + DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }} + steps: + - name: SET-UP GIT CONFIG + run: | + echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig + + - name: Download Erigon repository + uses: actions/download-artifact@v3 + with: + name: erigon-repo + path: ${{ github.workspace }}/erigon + + - name: Move Erigon repository to $HOME # as the backup script expects it to be there + run: mv ${{ github.workspace }}/erigon $HOME/ + + - name: Download dashboard-backup + uses: actions/download-artifact@v3 + with: + name: dashboard-backup + path: /tmp + + - name: Set dashboard-backup permissions + run: chmod +x /tmp/dashboard-backup + + - name: BACKUP OF ${{ matrix.dashboard }} DASHBOARD + run: | + set +x + echo -e "${{ env.GREEN }} I'm processing the ${{ matrix.dashboard }} dashboard... ${{ env.NOCOLOUR }}" + /tmp/dashboard-backup ${{ matrix.dashboard }} + + + pulizie_di_primavera: + needs: [preparation, backup_dashboard] + runs-on: ubuntu-latest + if: always() + steps: + - name: housekeeping + uses: geekyeggo/delete-artifact@v2 + with: + name: | + dashboard-backup + erigon-repo + + - name: Bye ${{ github.actor }} + if: always() + run: | + set +x + if [ "${{ needs.preparation.result }}" == "success" ] && [ "${{ needs.backup_dashboard.result }}" == "success" ]; then + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} *** It was a true pleasure to serve you ${{ github.actor }}, SEE YOU NEXT TIME! ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + else + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + echo -e "${{ env.RED }} *** Sorry ${{ github.actor }}, it seems there was an error this time: just fix it and COME BACK! ${{ env.RED }}" + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + fi + \ No newline at end of file From ee9d3fccb6083af73209e5a1f5afdac70a82cb5d Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:45:25 +0100 Subject: [PATCH 07/94] dashboards backup (#13186) I streamlined the workflow for backing up dashboards and avoided unnecessary fiddling of users with erigon repo branch. --- .github/workflows/backups-dashboards.yml | 46 ++++++++---------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml index 1533b3d2664..a30f72c94e8 100644 --- a/.github/workflows/backups-dashboards.yml +++ b/.github/workflows/backups-dashboards.yml @@ -10,6 +10,7 @@ env: RED: '\033[0;31m' YELLOW: '\033[0;33m' NOCOLOUR: '\033[0m' + ERIGON_BRANCH: main on: workflow_dispatch: @@ -19,11 +20,6 @@ on: type: string default: main description: 'The branch to pull the backup script from (default: main)' - ERIGON_BRANCH: - required: false - type: string - default: main - description: 'The branch to pull the erigon repo from (default: main)' jobs: # @@ -32,22 +28,8 @@ jobs: preparation: runs-on: ubuntu-latest environment: dashboards_backups - steps: - - name: CLONE THE ERIGON REPO from ${{ inputs.ERIGON_BRANCH }} branch - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ERIGON_BRANCH }} - fetch-depth: 1 - path: ${{ github.workspace }}/erigon - - - name: Upload Erigon repository - uses: actions/upload-artifact@v3 - with: - name: erigon-repo - path: ${{ github.workspace }}/erigon - retention-days: 1 - - - name: PULL THE LATEST VERSION OF BACKUP SCRIPT from ${{ inputs.TEMPLATE_BRANCH}} branch + steps: + - name: Pull backup script from ${{ inputs.TEMPLATE_BRANCH}} branch run: | set +x curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup @@ -72,19 +54,20 @@ jobs: DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }} GH_TOKEN: ${{ secrets.GH_TOKEN }} DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }} - steps: - - name: SET-UP GIT CONFIG - run: | - echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig - - - name: Download Erigon repository - uses: actions/download-artifact@v3 + steps: + - name: Clone erigon from ${{ env.ERIGON_BRANCH }} branch + uses: actions/checkout@v4 with: - name: erigon-repo - path: ${{ github.workspace }}/erigon + ref: ${{ env.ERIGON_BRANCH }} + fetch-depth: 1 + path: ${{ github.workspace }}/erigon - - name: Move Erigon repository to $HOME # as the backup script expects it to be there + - name: Move repository to $HOME # as the backup script expects it to be there run: mv ${{ github.workspace }}/erigon $HOME/ + + - name: Set-up git + run: | + echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig - name: Download dashboard-backup uses: actions/download-artifact@v3 @@ -112,7 +95,6 @@ jobs: with: name: | dashboard-backup - erigon-repo - name: Bye ${{ github.actor }} if: always() From c095292cbaa6590cd122e232fab7002bfbfbd038 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:03:21 +0100 Subject: [PATCH 08/94] dashboards backup (#13192) Update comments and rename the final step. --- .github/workflows/backups-dashboards.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml index a30f72c94e8..f1af4e03cfc 100644 --- a/.github/workflows/backups-dashboards.yml +++ b/.github/workflows/backups-dashboards.yml @@ -1,7 +1,7 @@ # Author: Michele@DevOpsTeam.Erigon # Maintainers: DevOps@DevOpsTeam.Erigon # Description: This workflow is responsible for backing up the dashboards of the Erigon project. -# Status: Mergeable for testing +# Status: Production (further improvements are planned though) name: Dashboards backup @@ -23,7 +23,7 @@ on: jobs: # - # NOTE: This workflow splits the backup process in 3 jobs even though 2 (or even 1 if you're particularly naive) would have been enough. I decided to do that just as a reference for passing data between jobs. The overhead is minimal. + # NOTE: This workflow splits the backup process in 2 jobs to spot any pulling issue early on # preparation: runs-on: ubuntu-latest @@ -85,12 +85,12 @@ jobs: /tmp/dashboard-backup ${{ matrix.dashboard }} - pulizie_di_primavera: + housekeeping: needs: [preparation, backup_dashboard] runs-on: ubuntu-latest if: always() steps: - - name: housekeeping + - name: cleaning up uses: geekyeggo/delete-artifact@v2 with: name: | From 3b7ac415c1383fb86923dad0c5080cac39e28485 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Fri, 20 Dec 2024 18:47:14 +0100 Subject: [PATCH 09/94] [automation] Backup of erigon_custom_metrics dashboard - 20 Dec 2024 (#13197) The erigon_custom_metrics dashboard got changed so the new version should overtake the currently stored one, which should in turn get archived in a compressed form --- .../erigon_custom_metrics.internal.json | 8513 +++++++++++++++++ .../history/20241220.tar.gz | Bin 0 -> 180 bytes 2 files changed, 8513 insertions(+) create mode 100644 dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json create mode 100644 dashboards/erigon_custom_metrics/history/20241220.tar.gz diff --git a/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json b/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json new file mode 100644 index 00000000000..085644e9e49 --- /dev/null +++ b/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json @@ -0,0 +1,8513 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 241, + "panels": [], + "title": "Agg CL", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + " dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 240, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "((aggregate_and_proof_signatures{instance=\"validator-bm-ethgiuweak-e3caplin-c1-n1\"}/577)*3)*0.4", + "instant": true, + "legendFormat": "Aggregate verification", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "3000", + "hide": false, + "instant": false, + "legendFormat": "Spare time", + "range": true, + "refId": "B" + } + ], + "title": "Aggregate verificiation per slot", + "type": "piechart" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 218, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 200 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 219, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "committee_size{instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Attestating Committee Size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 220, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "aggregate_quality_50{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", + "legendFormat": "{{label_name}} {{instance}} 50th", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "aggregate_quality_25{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} 25th", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "aggregate_quality_75{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} 75th", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "aggregate_quality_max{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} max", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "aggregate_quality_min{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} min", + "range": true, + "refId": "E" + } + ], + "title": "Network aggregate quality", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 141 + }, + "id": 222, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "current_slot{instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Current slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 145 + }, + "id": 223, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "current_epoch{instance=~\"$instance\"}", + "legendFormat": "Epoch {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Current epoch", + "type": "stat" + } + ], + "title": "Beacon chain", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 212, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 232, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "execution_time{instance=\"dev-bm-e3-ethmainnet-n1\"}", + "legendFormat": "Block Execution", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "blob_verification_time{instance=\"dev-bm-e3-ethmainnet-n1\"}", + "hide": false, + "instant": false, + "legendFormat": "Blob verificaton time", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_importing_latency{instance=\"dev-bm-e3-ethmainnet-n1\"}", + "hide": false, + "instant": false, + "legendFormat": "Network Latency", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "(4000-block_importing_latency{instance=\"dev-bm-e3-ethmainnet-n1\"}-blob_verification_time{instance=\"dev-bm-e3-ethmainnet-n1\"}-execution_time{instance=\"dev-bm-e3-ethmainnet-n1\"}-full_block_processing_time{instance=\"dev-bm-e3-ethmainnet-n1\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Spare time", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "full_block_processing_time{instance=\"dev-bm-e3-ethmainnet-n1\"}", + "hide": false, + "instant": false, + "legendFormat": "Consensus Processing", + "range": true, + "refId": "E" + } + ], + "title": "Block time distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 22, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 230, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "blob_verification_time{instance=~\"$instance\"}", + "legendFormat": "{{label_name}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Blob sidecar computation time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 229, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "block_importing_latency{instance=~\"$instance\"}", + "legendFormat": "{{label_name}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Block importing latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 215, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(gossip_topics_seen_sync_committee_contribution_and_proof{instance=~\"$instance\"}[$rate_interval]))/125 > 0\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total SyncCommitteeAggregate bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 236, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(total_out_bytes{instance=~\"$instance\"}[$rate_interval])+rate(total_in_bytes{instance=~\"$instance\"}[$rate_interval])+(rate(gossip_topics_seen_beacon_aggregate_and_proof{instance=~\"$instance\"}[$rate_interval])*15)) > 0\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Caplin: Total bandwidth", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 214, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(gossip_topics_seen_beacon_aggregate_and_proof{instance=~\"$instance\"}[$rate_interval]))/125 > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total AggregateAndProof bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 235, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(total_out_bytes{instance=~\"$instance\"}[$rate_interval])) > 0\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Caplin: Outgoing bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 210, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "attestation_block_processing_time{instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": " {{ method }} {{ instance }}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Caplin: Attestation in block latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 234, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(total_in_bytes{instance=~\"$instance\"}[$rate_interval])) > 0\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Caplin: Ingoing bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 37, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + " dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 211, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "full_block_processing_time{instance=~\"$instance\"}", + "legendFormat": " {{ method }} {{ instance }}", + "range": true, + "refId": "A" + } + ], + "title": "Caplin: Block processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 27, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 231, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "execution_time{instance=~\"$instance\"}", + "legendFormat": "{{label_name}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "ValidateChain: time spent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 221, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "process_justification_bits_and_finality_time{instance=~\"$instance\"}", + "legendFormat": "{{label_name}} {{instance}} process_justification_bits_and_finality", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_inactivity_ccores_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_inactivity_scores", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_rewards_and_penalties_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_rewards_and_penalties_time", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_registry_updates_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_registry_updates_time", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_slashings_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_slashings_time", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_historical_roots_update_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_historical_roots_update_time", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "process_sync_committee_update_time{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}} {{instance}} process_sync_committee_update_time", + "range": true, + "refId": "G" + } + ], + "title": "Intra-epoch processing time", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 27, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 216, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(gossip_topics_seen_beacon_attestation_0{instance=~\"$instance\"}[$rate_interval]))/125 > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Single attestation subnet bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + " snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 217, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "epoch_processing_time{instance=~\"$instance\"}", + "legendFormat": " {{ method }} {{ instance }}", + "range": true, + "refId": "A" + } + ], + "title": "Epoch processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 17, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 101 + }, + "id": 228, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "compute_shuffled_indices{instance=~\"$instance\"} > 0", + "legendFormat": "{{label_name}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Shuffling validator set", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "id": 213, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(gossip_topics_seen_beacon_block{instance=~\"$instance\"}[$rate_interval])/125 > 0 ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Beacon Block Gossip bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 109 + }, + "id": 226, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(rate(gossip_topics_seen_blob_sidecar_0{instance=~\"$instance\"}[$rate_interval])+rate(gossip_topics_seen_blob_sidecar_1{instance=~\"$instance\"}[$rate_interval])+rate(gossip_topics_seen_blob_sidecar_2{instance=~\"$instance\"}[$rate_interval])+rate(gossip_topics_seen_blob_sidecar_3{instance=~\"$instance\"}[$rate_interval])+rate(gossip_topics_seen_blob_sidecar_4{instance=~\"$instance\"}[$rate_interval])+rate(gossip_topics_seen_blob_sidecar_5{instance=~\"$instance\"}[$rate_interval]))/125 > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total blobs bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 109 + }, + "id": 227, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(\n rate(gossip_topics_seen_beacon_attestation_0{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_1{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_2{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_3{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_4{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_5{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_6{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_7{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_8{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_9{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_10{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_11{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_12{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_13{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_14{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_15{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_16{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_17{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_18{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_19{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_20{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_21{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_22{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_23{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_24{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_25{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_26{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_27{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_28{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_29{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_30{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_31{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_32{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_33{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_34{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_35{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_36{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_37{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_38{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_39{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_40{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_41{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_42{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_43{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_44{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_45{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_46{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_47{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_48{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_49{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_50{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_51{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_52{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_53{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_54{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_55{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_56{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_57{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_58{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_59{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_60{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_61{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_62{instance=~\"$instance\"}[$rate_interval]) +\n rate(gossip_topics_seen_beacon_attestation_63{instance=~\"$instance\"}[$rate_interval])\n) > 0\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "kb/s {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total attestation subnets bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 4, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + " snapshotter-bm-e3-ethmainnet-n1 blobs" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 117 + }, + "id": 224, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "frozen_blobs{instance=~\"$instance\"} > 0", + "legendFormat": "{{label_name}} {{instance}} blobs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "frozen_blocks{instance=~\"$instance\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} blocks", + "range": true, + "refId": "B" + } + ], + "title": "Caplin snapshot tracker", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"aggregation_per_signature\", instance=\"snapshotter-bm-e3-ethmainnet-n1\", job=\"erigon\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 126 + }, + "id": 209, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "aggregation_per_signature{instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "", + "hide": false, + "instant": false, + "range": true, + "refId": "B" + } + ], + "title": "Average BLS verification time per signature (MS) ", + "type": "timeseries" + } + ], + "title": "Caplin", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 205, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "rate(kv_get_count{instance=~\"$instance\"}[$rate_interval])", + "legendFormat": "{{domain}} {{level}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Files Level", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "kv_get{quantile=\"$quantile\",instance=~\"$instance\"}", + "legendFormat": "{{domain}} {{level}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Files Level", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 238, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "sum by(instance, level) (rate(trie_state_levelled_load_rate{instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "loaded {{level}} {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(trie_state_levelled_skip_rate{instance=~\"$instance\"}[$__rate_interval])", + "hide": true, + "instant": false, + "legendFormat": "skipped {{level}} {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum by(instance, level) (rate(trie_state_levelled_skip_rate{instance=~\"$instance\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "skipped {{level}} {{instance}}", + "range": true, + "refId": "C" + } + ], + "title": "Load/Skip state per levels", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 239, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "editorMode": "code", + "expr": "100*(trie_state_skip_rate{instance=~\"$instance\"}/(trie_state_skip_rate{instance=~\"$instance\"}+trie_state_load_rate{instance=~\"$instance\"})) ", + "legendFormat": "skip {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "state load Skip Ratio", + "type": "stat" + } + ], + "title": "E3 Files Timings", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 171, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "execution: validator-bm-holesky-e3caplin-cluster1-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 196, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sync{instance=~\"$instance\",stage=\"execution\"} > 0 ", + "instant": false, + "legendFormat": "{{ stage }}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync Stages progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 206, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_gas{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "gas/s {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Exec throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "dev-bm-e3-ethmainnet-n1 blocks " + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 200, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "prune_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}} {{type}} ", + "range": true, + "refId": "A" + } + ], + "title": "Prune, seconds", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 53 + }, + "id": 197, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "irate(domain_collation_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "collated [domain]: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "irate(domain_collation_hist_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "collated [history]: {{instance}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "sum(rate(domain_commitment_keys{instance=~\"$instance\"}[$rate_interval])) by (instance) > 0", + "hide": false, + "legendFormat": "keys committed: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "irate(domain_commitment_updates{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment node updates: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "irate(domain_commitment_updates_applied{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment trie node updates: {{instance}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "irate(domain_prune_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "pruned keys [{{type}}]: {{instance}}", + "range": true, + "refId": "G" + } + ], + "title": "State: Collate/Prune/Merge/Commitment", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 53 + }, + "id": 207, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "txs apply: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(exec_txns{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "txn/s {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Exec v3: txs/s ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 53 + }, + "id": 202, + "options": { + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 81, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_prunable{instance=~\"$instance\",type=\"domain\"}", + "hide": false, + "legendFormat": "{{instance}} {{type}}-{{table}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_prunable{instance=~\"$instance\",type=\"history\",table!=\"commitment\"}/1562500", + "hide": false, + "legendFormat": "{{instance}} {{type}}-{{table}}", + "range": true, + "refId": "C" + } + ], + "title": "pruning availability, steps", + "transparent": true, + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 158, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(sync{instance=~\"$instance\",stage=\"execution\"}[$rate_interval]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ stage }}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync Stages progress rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 59 + }, + "id": 199, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "chain_execution_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "execution: {{instance}}", + "refId": "A" + } + ], + "title": "Block Execution speed ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "commitment took: dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 61 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_collate_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "collation took: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_step_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "step took: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_prune_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "prune took [{{type}}]: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_commitment_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment took: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(domain_commitment_write_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "commitment update write took: {{instance}}", + "range": true, + "refId": "F" + } + ], + "title": "State: timins", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 64 + }, + "id": 198, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_running_merges{instance=~\"$instance\"}", + "legendFormat": "running merges: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_running_collations{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "running collations: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_pruning_progress{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "running prunes: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_running_commitment{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "running commitment: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_running_files_building{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "running files building: {{instance}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "domain_wal_flushes{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "WAL flushes {{instance}}", + "range": true, + "refId": "F" + } + ], + "title": "State: running collate/merge/prune", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 64 + }, + "id": 201, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"header_download\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "header: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"body_download\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "body: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"pre_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "execution_start: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"post_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "execution_end: {{instance}}", + "range": true, + "refId": "D" + } + ], + "title": "Block execution delays", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mgas/sec" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "mgas: snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 208, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_mgas{instance=~\"$instance\"} < 5000", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mgas: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "mgas/s ", + "type": "stat" + } + ], + "title": "Blocks execution", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 17, + "panels": [], + "title": "Database", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0.001, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_commit_seconds_count{phase=\"total\",instance=~\"$instance\"}[$rate_interval]) > 0 ", + "interval": "", + "legendFormat": "commit: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Commit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 14 + }, + "id": 166, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"total\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "total: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"gc_wall_clock\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_wall_clock: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"write\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "write: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"sync\",quantile=\"$quantile\",instance=~\"$instance\"} > 0.002", + "hide": false, + "interval": "", + "legendFormat": "sync: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"gc_cpu_time\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_cpu_time: {{instance}}", + "range": true, + "refId": "I" + } + ], + "title": "Commit speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 159, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "db_size{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "size: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "db_mi_last_pgno{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "db_mi_last_pgno: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "DB Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 23 + }, + "id": 168, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"newly\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "newly: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"cow\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "cow: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"clone\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "clone: {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"split\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "split: {{instance}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"merge\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "merge: {{instance}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"spill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "spill: {{instance}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"wops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "wops: {{instance}}", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"unspill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "unspill: {{instance}}", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcrloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcrloops: {{instance}}", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcwloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcwloops: {{instance}}", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcxpages\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcxpages: {{instance}}", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"msync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "msync: {{instance}}", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"fsync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "fsync: {{instance}}", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"minicore\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "minicore: {{instance}}", + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"prefault\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "prefault: {{instance}}", + "refId": "O" + } + ], + "title": "DB Pages Ops/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 167, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "tx_limit{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "limit: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "tx_dirty{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "dirty: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Tx Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 30 + }, + "id": 169, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_gc_leaf{instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "gc_leaf: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_gc_overflow{instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_overflow: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_steps_in_db{instance=~\"$instance\"}/100 > 0", + "hide": false, + "interval": "", + "legendFormat": "exec_steps_in_db: {{instance}}", + "range": true, + "refId": "E" + } + ], + "title": "GC and State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 30 + }, + "id": 150, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_minor_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "interval": "", + "legendFormat": "soft: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_major_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "interval": "", + "legendFormat": "hard: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "getrusage: minflt - soft page faults (reclaims), majflt - hard faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 194, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-79146", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_repeats{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "repeats: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_triggers{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "triggers: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Exec v3", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 134, + "panels": [], + "title": "Process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 155, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_io_write_syscalls_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "in: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_io_read_syscalls_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "out: {{instance}}", + "range": true, + "refId": "D" + } + ], + "title": "Read/Write syscall/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 300000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "process_virtual_memory_bytes{instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "resident virtual mem: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "process_resident_memory_anon_bytes{instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "resident anon mem: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "resident mem: {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "mem_data{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "data: {{instance}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "mem_stack{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "stack: {{instance}}", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "mem_locked{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "locked: {{instance}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "mem_swap{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "swap: {{instance}}", + "refId": "G" + } + ], + "title": "mem: resident set size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(process_cpu_seconds_total{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "system: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "write: dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_io_storage_read_bytes_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "read: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_io_storage_written_bytes_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "write: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk bytes/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 153, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(go_cgo_calls_count{instance=~\"$instance\"}[$rate_interval]) > 0", + "interval": "", + "legendFormat": "cgo_calls_count: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "cgo calls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 300000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 54 + }, + "id": 154, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_stack_sys_bytes{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "stack_sys: {{ instance }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_sys_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "sys: {{ instance }}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_stack_inuse_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "stack_inuse: {{ instance }}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_mspan_sys_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "mspan_sys: {{ instance }}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_mcache_sys_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "mcache_sys: {{ instance }}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "go_memstats_heap_alloc_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "heap_alloc: {{ instance }}", + "range": true, + "refId": "F" + } + ], + "title": "go memstat", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 300000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 58 + }, + "id": 128, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "go_goroutines{instance=~\"$instance\"}", + "instant": false, + "interval": "", + "legendFormat": "goroutines: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "go_threads{instance=~\"$instance\"}", + "instant": false, + "interval": "", + "legendFormat": "threads: {{instance}}", + "refId": "B" + } + ], + "title": "GO Goroutines and Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 300000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 60 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(go_memstats_mallocs_total{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "memstats_mallocs_total: {{ instance }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(go_memstats_frees_total{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "memstats_frees_total: {{ instance }}", + "range": true, + "refId": "B" + } + ], + "title": "Process Mem: allocate objects/sec, free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 64 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(go_gc_duration_seconds{quantile=\"0.75\",instance=~\"$instance\"}[$rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GC Stop the World per sec", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 183, + "panels": [], + "title": "RPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 185, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(rpc_duration_seconds_count{instance=~\"$instance\",success=\"success\"}[1m])", + "interval": "", + "legendFormat": "success {{ method }} {{ instance }} ", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(rpc_duration_seconds_count{instance=~\"$instance\",success=\"failure\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "failure {{ method }} {{ instance }} ", + "refId": "B" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 186, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "db_begin_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "db_begin_seconds: {{ method }} {{ instance }}", + "refId": "A" + } + ], + "title": "DB begin", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 187, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rpc_duration_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": " {{ method }} {{ instance }} {{ success }}", + "refId": "A" + } + ], + "title": "Timings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 188, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "go_goroutines{instance=~\"$instance\"}", + "instant": false, + "interval": "", + "legendFormat": "go/goroutines: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "go_threads{instance=~\"$instance\"}", + "instant": false, + "interval": "", + "legendFormat": "go/threads: {{instance}}", + "refId": "B" + } + ], + "title": "GO Goroutines and Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 86 + }, + "id": 189, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_keys_total{name=\"rpc\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "keys: {{ instance }} ", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_list_total{name=\"rpc\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "list: {{ instance }} ", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_code_keys_total{name=\"rpc\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "code_keys: {{ instance }} ", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_code_list_total{name=\"rpc\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "code_list: {{ instance }} ", + "refId": "D" + } + ], + "title": "Cache keys", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 86 + }, + "id": 184, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(delta(cache_total{result=\"hit\",name=\"rpc\",instance=~\"$instance\"}[1m]))/sum(delta(cache_total{name=\"rpc\",instance=~\"$instance\"}[1m])) ", + "hide": false, + "interval": "", + "legendFormat": "hit rate: {{ instance }} ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(delta(cache_code_total{result=\"hit\",name=\"rpc\",instance=~\"$instance\"}[1m]))/sum(delta(cache_code_total{name=\"rpc\",instance=~\"$instance\"}[1m])) ", + "hide": false, + "interval": "", + "legendFormat": "code hit rate: {{ instance }} ", + "refId": "B" + } + ], + "title": "Cache hit-rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 75, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "ingress: snapshotter-bm-e3-bormainnet-n1", + "ingress: snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(p2p_ingress{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ingress: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(p2p_egress{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "egress: {{instance}}", + "range": true, + "refId": "C" + } + ], + "title": "Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "p2p_peers{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "peers: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "rate(p2p_dials{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "dials: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "rate(p2p_serves{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "serves: {{instance}}", + "refId": "C" + } + ], + "title": "Peers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 173, + "panels": [], + "title": "TxPool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 175, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_process_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_remote_txs: {{ instance }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_add_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "add_remote_txs: {{ instance }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_new_block{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "new_block: {{ instance }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_write_to_db{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "write_to_db: {{ instance }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_propagate_to_new_peer{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "propagate_to_new_peer: {{ instance }}", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "pool_propagate_new_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "propagate_new_txs: {{ instance }}", + "refId": "F" + } + ], + "title": "Timings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 177, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(pool_process_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_process_remote_txs_count: {{ instance }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(pool_add_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_add_remote_txs_count: {{ instance }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(pool_new_block_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_new_block_count: {{ instance }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(pool_write_to_db_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_write_to_db_count: {{ instance }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(pool_p2p_out{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "E" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 111 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(delta(cache_total{result=\"hit\",name=\"txpool\",instance=~\"$instance\"}[1m]))/sum(delta(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])) ", + "hide": false, + "interval": "", + "legendFormat": "hit rate: {{ instance }} ", + "refId": "A" + } + ], + "title": "Cache hit-rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 111 + }, + "id": 180, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{ result }}: {{ instance }} ", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(cache_timeout_total{name=\"txpool\",instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "timeout: {{ instance }} ", + "refId": "B" + } + ], + "title": "Cache rps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 111 + }, + "id": 181, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_keys_total{name=\"txpool\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "keys: {{ instance }} ", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "cache_list_total{name=\"txpool\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "list: {{ instance }} ", + "refId": "B" + } + ], + "title": "Cache keys", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 117 + }, + "id": 178, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(pool_write_to_db_bytes{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_write_to_db_bytes: {{ instance }}", + "refId": "A" + } + ], + "title": "DB", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "10s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "0.97", + "value": "0.97" + }, + "includeAll": false, + "name": "quantile", + "options": [ + { + "selected": false, + "text": "0.0", + "value": "0.0" + }, + { + "selected": false, + "text": "0.25", + "value": "0.25" + }, + { + "selected": false, + "text": "0.5", + "value": "0.5" + }, + { + "selected": false, + "text": "0.9", + "value": "0.9" + }, + { + "selected": true, + "text": "0.97", + "value": "0.97" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "1", + "value": "1" + } + ], + "query": "0.0,0.25,0.5, 0.9, 0.97, 0.99, 1", + "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "definition": "go_goroutines", + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 4, + "query": "go_goroutines", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*instance=\"([^\"]*).*/", + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "1m", + "value": "1m" + }, + "label": "Rate Interval", + "name": "rate_interval", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,3h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "1. Erigon CUSTOM METRICS", + "uid": "b42a61d7-02b1-416c-8ab4-b9c864356174", + "version": 276, + "weekStart": "" +} diff --git a/dashboards/erigon_custom_metrics/history/20241220.tar.gz b/dashboards/erigon_custom_metrics/history/20241220.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..7f00d449962ce4abeaa785c9b3b9c94fd586a726 GIT binary patch literal 180 zcmb2|=3oE==C>Ce`I-y_S|7?SRliWC@x`J=!EnVE&&J=}E_V}qCbE9~Z#H?E?=oZm z9t-WHzAWpsN1tofU3*?;d3#aBtL9DT(!=(A^qaELOn7<8zK8SUj@7N0bWN*Q(IoTl zUCnFP%fDNzp8qrZ%DiNa*O~GgwRfj^ Date: Sat, 21 Dec 2024 00:49:28 +0100 Subject: [PATCH 10/94] E3: make state snapshots download optional (#13107) --- cmd/utils/flags.go | 7 +++++++ eth/ethconfig/config.go | 15 ++++++++------- turbo/snapshotsync/snapshotsync.go | 8 ++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d1a9b1a882b..722ec2aad00 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -675,6 +675,11 @@ var ( Name: ethconfig.FlagSnapStateStop, Usage: "Workaround to stop producing new state files, if you meet some state-related critical bug. It will stop aggregate DB history in a state files. DB will grow and may slightly slow-down - and removing this flag in future will not fix this effect (db size will not greatly reduce).", } + SnapSkipStateSnapshotDownloadFlag = cli.BoolFlag{ + Name: "snap.skip-state-snapshot-download", + Usage: "Skip state download and start from genesis block", + Value: false, + } TorrentVerbosityFlag = cli.IntFlag{ Name: "torrent.verbosity", Value: 2, @@ -749,6 +754,7 @@ var ( Usage: "Enable WRITE_MAP feature for fast database writes and fast commit times", Value: true, } + HealthCheckFlag = cli.BoolFlag{ Name: "healthcheck", Usage: "Enabling grpc health check", @@ -1860,6 +1866,7 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C cfg.Snapshot.KeepBlocks = ctx.Bool(SnapKeepBlocksFlag.Name) cfg.Snapshot.ProduceE2 = !ctx.Bool(SnapStopFlag.Name) cfg.Snapshot.ProduceE3 = !ctx.Bool(SnapStateStopFlag.Name) + cfg.Snapshot.DisableDownloadE3 = ctx.Bool(SnapSkipStateSnapshotDownloadFlag.Name) cfg.Snapshot.NoDownloader = ctx.Bool(NoDownloaderFlag.Name) cfg.Snapshot.Verify = ctx.Bool(DownloaderVerifyFlag.Name) cfg.Snapshot.DownloaderAddr = strings.TrimSpace(ctx.String(DownloaderAddrFlag.Name)) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8ea5f2bbab1..d8f9c9e9cc5 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -135,13 +135,14 @@ func init() { //go:generate gencodec -dir . -type Config -formats toml -out gen_config.go type BlocksFreezing struct { - KeepBlocks bool // produce new snapshots of blocks but don't remove blocks from DB - ProduceE2 bool // produce new block files - ProduceE3 bool // produce new state files - NoDownloader bool // possible to use snapshots without calling Downloader - Verify bool // verify snapshots on startup - DownloaderAddr string - ChainName string + KeepBlocks bool // produce new snapshots of blocks but don't remove blocks from DB + ProduceE2 bool // produce new block files + ProduceE3 bool // produce new state files + NoDownloader bool // possible to use snapshots without calling Downloader + Verify bool // verify snapshots on startup + DisableDownloadE3 bool // disable download state snapshots + DownloaderAddr string + ChainName string } func (s BlocksFreezing) String() string { diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 7bbfee50d17..9182cfb319a 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -158,7 +158,7 @@ func adjustBlockPrune(blocks, minBlocksToDownload uint64) uint64 { return blocks - blocks%snaptype.Erigon2MergeLimit } -func shouldUseStepsForPruning(name string) bool { +func isStateSnapshot(name string) bool { return strings.HasPrefix(name, "idx") || strings.HasPrefix(name, "history") || strings.HasPrefix(name, "accessor") } @@ -182,7 +182,7 @@ func buildBlackListForPruning(pruneMode bool, stepPrune, minBlockToDownload, blo } var _, to uint64 var err error - if shouldUseStepsForPruning(name) { + if isStateSnapshot(name) { // parse "from" (0) and "to" (64) from the name // parse the snapshot "kind". e.g kind of 'idx/v1-accounts.0-64.ef' is "idx/v1-accounts" rangeString := strings.Split(name, ".")[1] @@ -337,6 +337,10 @@ func WaitForDownloader(ctx context.Context, logPrefix string, dirs datadir.Dirs, if caplin == OnlyCaplin && !strings.Contains(p.Name, "beaconblocks") && !strings.Contains(p.Name, "blobsidecars") && !strings.Contains(p.Name, "caplin") { continue } + + if isStateSnapshot(p.Name) && blockReader.FreezingCfg().DisableDownloadE3 { + continue + } if !blobs && strings.Contains(p.Name, "blobsidecars") { continue } From 78a3f430e6667f8953fa4707878f1bd3b23e14dc Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Sat, 21 Dec 2024 06:35:21 +0200 Subject: [PATCH 11/94] txpool: tidy NoGossip config (#13203) pretty straightforward tidy up PR 1. moves DisableTxPoolGossip from ethconfig.Config to txpoolcfg.Config 2. adds some license/doc comments to tidy up previous PR --- cmd/utils/flags.go | 7 +++---- eth/backend.go | 1 - eth/ethconfig/config.go | 2 -- eth/ethconfig/gen_config.go | 6 ------ txnprovider/provider.go | 6 ++++++ txnprovider/shutter/config.go | 16 ++++++++++++++++ txnprovider/shutter/pool.go | 16 ++++++++++++++++ 7 files changed, 41 insertions(+), 13 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 722ec2aad00..7e63d1cb44d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1568,6 +1568,9 @@ func setTxPool(ctx *cli.Context, dbDir string, fullCfg *ethconfig.Config) { if ctx.IsSet(DbWriteMapFlag.Name) { cfg.MdbxWriteMap = ctx.Bool(DbWriteMapFlag.Name) } + if ctx.IsSet(TxPoolGossipDisableFlag.Name) { + cfg.NoGossip = ctx.Bool(TxPoolGossipDisableFlag.Name) + } cfg.LogEvery = 3 * time.Minute cfg.CommitEvery = libcommon.RandomizeDuration(ctx.Duration(TxPoolCommitEveryFlag.Name)) cfg.DBDir = dbDir @@ -1992,10 +1995,6 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C if ctx.IsSet(TrustedSetupFile.Name) { libkzg.SetTrustedSetupFilePath(ctx.String(TrustedSetupFile.Name)) } - - if ctx.IsSet(TxPoolGossipDisableFlag.Name) { - cfg.DisableTxPoolGossip = ctx.Bool(TxPoolGossipDisableFlag.Name) - } } // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if diff --git a/eth/backend.go b/eth/backend.go index 587eca7d4fd..5b7a8584928 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -645,7 +645,6 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger } var txnProvider txnprovider.TxnProvider - config.TxPool.NoGossip = config.DisableTxPoolGossip var miningRPC txpoolproto.MiningServer stateDiffClient := direct.NewStateDiffClientDirect(kvRPC) if config.TxPool.Disable { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d8f9c9e9cc5..af4871915bc 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -258,8 +258,6 @@ type Config struct { SilkwormRpcLogDumpResponse bool SilkwormRpcNumWorkers uint32 SilkwormRpcJsonCompatibility bool - - DisableTxPoolGossip bool } type Sync struct { diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 94abe593f55..7a5d015e379 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -67,7 +67,6 @@ func (c Config) MarshalTOML() (interface{}, error) { SilkwormRpcLogDumpResponse bool SilkwormRpcNumWorkers uint32 SilkwormRpcJsonCompatibility bool - DisableTxPoolGossip bool } var enc Config enc.Genesis = c.Genesis @@ -114,7 +113,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SilkwormRpcLogDumpResponse = c.SilkwormRpcLogDumpResponse enc.SilkwormRpcNumWorkers = c.SilkwormRpcNumWorkers enc.SilkwormRpcJsonCompatibility = c.SilkwormRpcJsonCompatibility - enc.DisableTxPoolGossip = c.DisableTxPoolGossip return &enc, nil } @@ -165,7 +163,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SilkwormRpcLogDumpResponse *bool SilkwormRpcNumWorkers *uint32 SilkwormRpcJsonCompatibility *bool - DisableTxPoolGossip *bool } var dec Config if err := unmarshal(&dec); err != nil { @@ -303,8 +300,5 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SilkwormRpcJsonCompatibility != nil { c.SilkwormRpcJsonCompatibility = *dec.SilkwormRpcJsonCompatibility } - if dec.DisableTxPoolGossip != nil { - c.DisableTxPoolGossip = *dec.DisableTxPoolGossip - } return nil } diff --git a/txnprovider/provider.go b/txnprovider/provider.go index eff293a7996..036239e9ac3 100644 --- a/txnprovider/provider.go +++ b/txnprovider/provider.go @@ -26,6 +26,12 @@ import ( ) type TxnProvider interface { + // ProvideTxns provides transactions ready to be included in a block for block building. Available request options: + // - WithParentBlockNum + // - WithAmount + // - WithGasTarget + // - WithBlobGasTarget + // - WithTxnIdsFilter ProvideTxns(ctx context.Context, opts ...ProvideOption) ([]types.Transaction, error) } diff --git a/txnprovider/shutter/config.go b/txnprovider/shutter/config.go index 2464404f6b4..bd5a3e6c873 100644 --- a/txnprovider/shutter/config.go +++ b/txnprovider/shutter/config.go @@ -1,3 +1,19 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + package shutter import ( diff --git a/txnprovider/shutter/pool.go b/txnprovider/shutter/pool.go index 3b28c03a9ba..cce91c1d309 100644 --- a/txnprovider/shutter/pool.go +++ b/txnprovider/shutter/pool.go @@ -1,3 +1,19 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + package shutter import ( From 31789923a135d92e80a5a44d4dc229091e4b7034 Mon Sep 17 00:00:00 2001 From: lupin012 <58134934+lupin012@users.noreply.github.com> Date: Sun, 22 Dec 2024 15:28:49 +0100 Subject: [PATCH 12/94] qa-integration-test: update integration test version and tests list (#13208) --- .github/workflows/qa-rpc-integration-tests.yml | 2 +- .github/workflows/scripts/run_rpc_tests.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index 7a8ead873ab..bf9f0d560d6 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -32,7 +32,7 @@ jobs: - name: Checkout RPC Tests Repository & Install Requirements run: | rm -rf ${{ runner.workspace }}/rpc-tests - git -c advice.detachedHead=false clone --depth 1 --branch v1.26.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests + git -c advice.detachedHead=false clone --depth 1 --branch v1.27.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests cd ${{ runner.workspace }}/rpc-tests pip3 install -r requirements.txt diff --git a/.github/workflows/scripts/run_rpc_tests.sh b/.github/workflows/scripts/run_rpc_tests.sh index 74e68533e7b..d4551843eeb 100755 --- a/.github/workflows/scripts/run_rpc_tests.sh +++ b/.github/workflows/scripts/run_rpc_tests.sh @@ -6,8 +6,6 @@ set +e # Disable exit on error disabled_tests=( # Erigon2 and Erigon3 never supported this api methods trace_rawTransaction - # Erigon bug https://github.com/erigontech/erigon/issues/12603 - erigon_getLatestLogs # to investigate engine_exchangeCapabilities/test_1.json engine_exchangeTransitionConfigurationV1/test_01.json From 1cee6584013282b9d5c5cb2b2c8506a6821c9a0a Mon Sep 17 00:00:00 2001 From: Somnath Date: Mon, 23 Dec 2024 06:22:19 +0400 Subject: [PATCH 13/94] accounts/abi: substitude arg%d to the range keyword (#25307) (#13190) Cherrypick https://github.com/ethereum/go-ethereum/pull/25307 * accounts/abi: substitude arg%d to the range keyword * support more keywords * review feedback Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- accounts/abi/bind/bind.go | 41 +++++- accounts/abi/bind/bind_test.go | 255 +++++++++++++++++++++++++++++++++ 2 files changed, 294 insertions(+), 2 deletions(-) diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 06e2730f328..0087826d1bb 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -56,6 +56,43 @@ const ( typeString string = "String" ) +func isKeyWord(arg string) bool { + switch arg { + case "break": + case "case": + case "chan": + case "const": + case "continue": + case "default": + case "defer": + case "else": + case "fallthrough": + case "for": + case "func": + case "go": + case "goto": + case "if": + case "import": + case "interface": + case "iota": + case "map": + case "make": + case "new": + case "package": + case "range": + case "return": + case "select": + case "struct": + case "switch": + case "type": + case "var": + default: + return false + } + + return true +} + // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant // to be used as is in client code, but rather as an intermediate struct which // enforces compile time type safety and naming convention opposed to having to @@ -118,7 +155,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } if hasStruct(input.Type) { @@ -161,7 +198,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } if hasStruct(input.Type) { diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index d91c5426134..a64a6799287 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1788,6 +1788,261 @@ var bindTests = []struct { nil, nil, }, + // Test resolving single struct argument + { + `NewSingleStructArgument`, + ` + pragma solidity ^0.8.0; + + contract NewSingleStructArgument { + struct MyStruct{ + uint256 a; + uint256 b; + } + event StructEvent(MyStruct s); + function TestEvent() public { + emit StructEvent(MyStruct({a: 1, b: 2})); + } + } + `, + []string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"}, + []string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`}, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, _, d, err := DeployNewSingleStructArgument(user, sim) + if err != nil { + t.Fatalf("Failed to deploy contract %v", err) + } + sim.Commit() + + _, err = d.TestEvent(user) + if err != nil { + t.Fatalf("Failed to call contract %v", err) + } + sim.Commit() + + it, err := d.FilterStructEvent(nil) + if err != nil { + t.Fatalf("Failed to filter contract event %v", err) + } + var count int + for it.Next() { + if it.Event.S.A.Cmp(big.NewInt(1)) != 0 { + t.Fatal("Unexpected contract event") + } + if it.Event.S.B.Cmp(big.NewInt(2)) != 0 { + t.Fatal("Unexpected contract event") + } + count += 1 + } + if count != 1 { + t.Fatal("Unexpected contract event number") + } + `, + nil, + nil, + nil, + nil, + }, + // Test errors introduced in v0.8.4 + { + `NewErrors`, + ` + pragma solidity >0.8.4; + + contract NewErrors { + error MyError(uint256); + error MyError1(uint256); + error MyError2(uint256, uint256); + error MyError3(uint256 a, uint256 b, uint256 c); + function Error() public pure { + revert MyError3(1,2,3); + } + } + `, + []string{"0x6080604052348015600f57600080fd5b5060998061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063726c638214602d575b600080fd5b60336035565b005b60405163024876cd60e61b815260016004820152600260248201526003604482015260640160405180910390fdfea264697066735822122093f786a1bc60216540cd999fbb4a6109e0fef20abcff6e9107fb2817ca968f3c64736f6c63430008070033"}, + []string{`[{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError","type":"error"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError1","type":"error"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"},{"internalType":"uint256","name":"","type":"uint256"}],"name":"MyError2","type":"error"},{"inputs":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"},{"internalType":"uint256","name":"c","type":"uint256"}],"name":"MyError3","type":"error"},{"inputs":[],"name":"Error","outputs":[],"stateMutability":"pure","type":"function"}]`}, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, contract, err := DeployNewErrors(user, sim) + if err != nil { + t.Fatal(err) + } + sim.Commit() + _, err = bind.WaitDeployed(nil, sim, tx) + if err != nil { + t.Error(err) + } + if err := contract.Error(new(bind.CallOpts)); err == nil { + t.Fatalf("expected contract to throw error") + } + // TODO (MariusVanDerWijden unpack error using abigen + // once that is implemented + `, + nil, + nil, + nil, + nil, + }, + { + name: `ConstructorWithStructParam`, + contract: ` + pragma solidity >=0.8.0 <0.9.0; + + contract ConstructorWithStructParam { + struct StructType { + uint256 field; + } + + constructor(StructType memory st) {} + } + `, + bytecode: []string{`0x608060405234801561001057600080fd5b506040516101c43803806101c48339818101604052810190610032919061014a565b50610177565b6000604051905090565b600080fd5b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6100958261004c565b810181811067ffffffffffffffff821117156100b4576100b361005d565b5b80604052505050565b60006100c7610038565b90506100d3828261008c565b919050565b6000819050919050565b6100eb816100d8565b81146100f657600080fd5b50565b600081519050610108816100e2565b92915050565b60006020828403121561012457610123610047565b5b61012e60206100bd565b9050600061013e848285016100f9565b60008301525092915050565b6000602082840312156101605761015f610042565b5b600061016e8482850161010e565b91505092915050565b603f806101856000396000f3fe6080604052600080fdfea2646970667358221220cdffa667affecefac5561f65f4a4ba914204a8d4eb859d8cd426fb306e5c12a364736f6c634300080a0033`}, + abi: []string{`[{"inputs":[{"components":[{"internalType":"uint256","name":"field","type":"uint256"}],"internalType":"struct ConstructorWithStructParam.StructType","name":"st","type":"tuple"}],"stateMutability":"nonpayable","type":"constructor"}]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, _, err := DeployConstructorWithStructParam(user, sim, ConstructorWithStructParamStructType{Field: big.NewInt(42)}) + if err != nil { + t.Fatalf("DeployConstructorWithStructParam() got err %v; want nil err", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Logf("Deployment tx: %+v", tx) + t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) + } + `, + }, + { + name: `NameConflict`, + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract oracle { + struct request { + bytes data; + bytes _data; + } + event log (int msg, int _msg); + function addRequest(request memory req) public pure {} + function getRequest() pure public returns (request memory) { + return request("", ""); + } + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033"}, + abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, _, err := DeployNameConflict(user, sim) + if err != nil { + t.Fatalf("DeployNameConflict() got err %v; want nil err", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Logf("Deployment tx: %+v", tx) + t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) + } + `, + }, + { + name: "RangeKeyword", + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract keywordcontract { + function functionWithKeywordParameter(range uint256) public pure {} + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5060dc8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063527a119f14602d575b600080fd5b60436004803603810190603f9190605b565b6045565b005b50565b6000813590506055816092565b92915050565b600060208284031215606e57606d608d565b5b6000607a848285016048565b91505092915050565b6000819050919050565b600080fd5b6099816083565b811460a357600080fd5b5056fea2646970667358221220d4f4525e2615516394055d369fb17df41c359e5e962734f27fd683ea81fd9db164736f6c63430008070033"}, + abi: []string{`[{"inputs":[{"internalType":"uint256","name":"range","type":"uint256"}],"name":"functionWithKeywordParameter","outputs":[],"stateMutability":"pure","type":"function"}]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + _, tx, _, err := DeployRangeKeyword(user, sim) + if err != nil { + t.Fatalf("error deploying contract: %v", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Errorf("error deploying the contract: %v", err) + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and From d867a48ed13b222942b71d0da6d77102d2ccacf4 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 23 Dec 2024 18:02:07 +0700 Subject: [PATCH 14/94] link to erigon3 docs (#13200) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c94afefb143..f7355131e8a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Erigon -Documentation: **[erigon.gitbook.io](https://erigon.gitbook.io)** +Documentation: **[docs.erigon.tech](https://docs.erigon.tech)** Blog: **[erigon.substack.com](https://erigon.substack.com/)** Twitter: [x.com/ErigonEth](https://x.com/ErigonEth) From 5900d21c0a001edf65b79e6fe6a15bc967b0e375 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 23 Dec 2024 18:02:34 +0700 Subject: [PATCH 15/94] agg: move integrity methods to `integrity.go` (#13188) --- erigon-lib/state/aggregator.go | 73 ---------- erigon-lib/state/domain.go | 69 +-------- erigon-lib/state/integrity.go | 215 +++++++++++++++++++++++++++++ erigon-lib/state/inverted_index.go | 60 -------- eth/integrity/e3_ef_files.go | 2 +- turbo/app/snapshots_cmd.go | 4 +- 6 files changed, 219 insertions(+), 204 deletions(-) create mode 100644 erigon-lib/state/integrity.go diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 21237df88ab..7c879fb576e 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -1863,79 +1863,6 @@ func (ac *AggregatorRoTx) GetLatest(domain kv.Domain, k []byte, tx kv.Tx) (v []b return ac.d[domain].GetLatest(k, tx) } -// search key in all files of all domains and print file names -func (ac *AggregatorRoTx) DebugKey(domain kv.Domain, k []byte) error { - l, err := ac.d[domain].DebugKVFilesWithKey(k) - if err != nil { - return err - } - if len(l) > 0 { - ac.a.logger.Info("[dbg] found in", "files", l) - } - return nil -} -func (ac *AggregatorRoTx) DebugEFKey(domain kv.Domain, k []byte) error { - return ac.d[domain].DebugEFKey(k) -} - -func (ac *AggregatorRoTx) DebugEFAllValuesAreInRange(ctx context.Context, name kv.InvertedIdx, failFast bool, fromStep uint64) error { - switch name { - case kv.AccountsHistoryIdx: - err := ac.d[kv.AccountsDomain].ht.iit.DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.StorageHistoryIdx: - err := ac.d[kv.CodeDomain].ht.iit.DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.CodeHistoryIdx: - err := ac.d[kv.StorageDomain].ht.iit.DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.CommitmentHistoryIdx: - err := ac.d[kv.CommitmentDomain].ht.iit.DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.ReceiptHistoryIdx: - err := ac.d[kv.ReceiptDomain].ht.iit.DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - //case kv.GasUsedHistoryIdx: - // err := ac.d[kv.GasUsedDomain].ht.iit.DebugEFAllValuesAreInRange(ctx) - // if err != nil { - // return err - // } - case kv.TracesFromIdx: - err := ac.iis[kv.TracesFromIdxPos].DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.TracesToIdx: - err := ac.iis[kv.TracesToIdxPos].DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.LogAddrIdx: - err := ac.iis[kv.LogAddrIdxPos].DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - case kv.LogTopicIdx: - err := ac.iis[kv.LogTopicIdxPos].DebugEFAllValuesAreInRange(ctx, failFast, fromStep) - if err != nil { - return err - } - default: - panic(fmt.Sprintf("unexpected: %s", name)) - } - return nil -} - // --- Domain part END --- func (ac *AggregatorRoTx) Close() { diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 4cc9ec20e77..d34e39be8c9 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -32,9 +32,6 @@ import ( btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon-lib/recsplit/eliasfano32" - "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/dbg" @@ -43,6 +40,7 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/stream" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon-lib/seg" ) @@ -703,71 +701,6 @@ func (dt *DomainRoTx) getLatestFromFile(i int, filekey []byte) (v []byte, ok boo } -func (dt *DomainRoTx) DebugKVFilesWithKey(k []byte) (res []string, err error) { - for i := len(dt.files) - 1; i >= 0; i-- { - _, ok, _, err := dt.getLatestFromFile(i, k) - if err != nil { - return res, err - } - if ok { - res = append(res, dt.files[i].src.decompressor.FileName()) - } - } - return res, nil -} -func (dt *DomainRoTx) DebugEFKey(k []byte) error { - dt.ht.iit.ii.dirtyFiles.Walk(func(items []*filesItem) bool { - for _, item := range items { - if item.decompressor == nil { - continue - } - accessor := item.index - if accessor == nil { - fPath := dt.d.efAccessorFilePath(item.startTxNum/dt.d.aggregationStep, item.endTxNum/dt.d.aggregationStep) - exists, err := dir.FileExist(fPath) - if err != nil { - _, fName := filepath.Split(fPath) - dt.d.logger.Warn("[agg] InvertedIndex.openDirtyFiles", "err", err, "f", fName) - continue - } - if exists { - var err error - accessor, err = recsplit.OpenIndex(fPath) - if err != nil { - _, fName := filepath.Split(fPath) - dt.d.logger.Warn("[agg] InvertedIndex.openDirtyFiles", "err", err, "f", fName) - continue - } - defer accessor.Close() - } else { - continue - } - } - - offset, ok := accessor.GetReaderFromPool().Lookup(k) - if !ok { - continue - } - g := item.decompressor.MakeGetter() - g.Reset(offset) - key, _ := g.NextUncompressed() - if !bytes.Equal(k, key) { - continue - } - eliasVal, _ := g.NextUncompressed() - ef, _ := eliasfano32.ReadEliasFano(eliasVal) - - last2 := uint64(0) - if ef.Count() > 2 { - last2 = ef.Get(ef.Count() - 2) - } - log.Warn(fmt.Sprintf("[dbg] see1: %s, min=%d,max=%d, before_max=%d, all: %d\n", item.decompressor.FileName(), ef.Min(), ef.Max(), last2, stream.ToArrU64Must(ef.Iterator()))) - } - return true - }) - return nil -} - func (d *Domain) BeginFilesRo() *DomainRoTx { for i := 0; i < len(d._visible.files); i++ { if !d._visible.files[i].src.frozen { diff --git a/erigon-lib/state/integrity.go b/erigon-lib/state/integrity.go new file mode 100644 index 00000000000..6ac91d24943 --- /dev/null +++ b/erigon-lib/state/integrity.go @@ -0,0 +1,215 @@ +package state + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "time" + + "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/dir" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/stream" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/recsplit" + "github.com/erigontech/erigon-lib/recsplit/eliasfano32" +) + +// search key in all files of all domains and print file names +func (ac *AggregatorRoTx) IntegrityKey(domain kv.Domain, k []byte) error { + l, err := ac.d[domain].IntegrityDomainFilesWithKey(k) + if err != nil { + return err + } + if len(l) > 0 { + ac.a.logger.Info("[dbg] found in", "files", l) + } + return nil +} +func (ac *AggregatorRoTx) IntegirtyInvertedIndexKey(domain kv.Domain, k []byte) error { + return ac.d[domain].IntegrityKey(k) +} + +func (ac *AggregatorRoTx) IntegrityInvertedIndexAllValuesAreInRange(ctx context.Context, name kv.InvertedIdx, failFast bool, fromStep uint64) error { + switch name { + case kv.AccountsHistoryIdx: + err := ac.d[kv.AccountsDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.StorageHistoryIdx: + err := ac.d[kv.CodeDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.CodeHistoryIdx: + err := ac.d[kv.StorageDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.CommitmentHistoryIdx: + err := ac.d[kv.CommitmentDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.ReceiptHistoryIdx: + err := ac.d[kv.ReceiptDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + //case kv.GasUsedHistoryIdx: + // err := ac.d[kv.GasUsedDomain].ht.iit.IntegrityInvertedIndexAllValuesAreInRange(ctx) + // if err != nil { + // return err + // } + case kv.TracesFromIdx: + err := ac.iis[kv.TracesFromIdxPos].IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.TracesToIdx: + err := ac.iis[kv.TracesToIdxPos].IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.LogAddrIdx: + err := ac.iis[kv.LogAddrIdxPos].IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + case kv.LogTopicIdx: + err := ac.iis[kv.LogTopicIdxPos].IntegrityInvertedIndexAllValuesAreInRange(ctx, failFast, fromStep) + if err != nil { + return err + } + default: + panic(fmt.Sprintf("unexpected: %s", name)) + } + return nil +} + +func (dt *DomainRoTx) IntegrityDomainFilesWithKey(k []byte) (res []string, err error) { + for i := len(dt.files) - 1; i >= 0; i-- { + _, ok, _, err := dt.getLatestFromFile(i, k) + if err != nil { + return res, err + } + if ok { + res = append(res, dt.files[i].src.decompressor.FileName()) + } + } + return res, nil +} +func (dt *DomainRoTx) IntegrityKey(k []byte) error { + dt.ht.iit.ii.dirtyFiles.Walk(func(items []*filesItem) bool { + for _, item := range items { + if item.decompressor == nil { + continue + } + accessor := item.index + if accessor == nil { + fPath := dt.d.efAccessorFilePath(item.startTxNum/dt.d.aggregationStep, item.endTxNum/dt.d.aggregationStep) + exists, err := dir.FileExist(fPath) + if err != nil { + _, fName := filepath.Split(fPath) + dt.d.logger.Warn("[agg] InvertedIndex.openDirtyFiles", "err", err, "f", fName) + continue + } + if exists { + var err error + accessor, err = recsplit.OpenIndex(fPath) + if err != nil { + _, fName := filepath.Split(fPath) + dt.d.logger.Warn("[agg] InvertedIndex.openDirtyFiles", "err", err, "f", fName) + continue + } + defer accessor.Close() + } else { + continue + } + } + + offset, ok := accessor.GetReaderFromPool().Lookup(k) + if !ok { + continue + } + g := item.decompressor.MakeGetter() + g.Reset(offset) + key, _ := g.NextUncompressed() + if !bytes.Equal(k, key) { + continue + } + eliasVal, _ := g.NextUncompressed() + ef, _ := eliasfano32.ReadEliasFano(eliasVal) + + last2 := uint64(0) + if ef.Count() > 2 { + last2 = ef.Get(ef.Count() - 2) + } + log.Warn(fmt.Sprintf("[dbg] see1: %s, min=%d,max=%d, before_max=%d, all: %d\n", item.decompressor.FileName(), ef.Min(), ef.Max(), last2, stream.ToArrU64Must(ef.Iterator()))) + } + return true + }) + return nil +} + +func (iit *InvertedIndexRoTx) IntegrityInvertedIndexAllValuesAreInRange(ctx context.Context, failFast bool, fromStep uint64) error { + logEvery := time.NewTicker(30 * time.Second) + defer logEvery.Stop() + fromTxNum := fromStep * iit.ii.aggregationStep + iterStep := func(item visibleFile) error { + g := item.src.decompressor.MakeGetter() + g.Reset(0) + defer item.src.decompressor.EnableReadAhead().DisableReadAhead() + + for g.HasNext() { + k, _ := g.NextUncompressed() + _ = k + eliasVal, _ := g.NextUncompressed() + ef, _ := eliasfano32.ReadEliasFano(eliasVal) + if ef.Count() == 0 { + continue + } + if item.startTxNum > ef.Min() { + err := fmt.Errorf("[integrity] .ef file has foreign txNum: %d > %d, %s, %x", item.startTxNum, ef.Min(), g.FileName(), common.Shorten(k, 8)) + if failFast { + return err + } else { + log.Warn(err.Error()) + } + } + if item.endTxNum < ef.Max() { + err := fmt.Errorf("[integrity] .ef file has foreign txNum: %d < %d, %s, %x", item.endTxNum, ef.Max(), g.FileName(), common.Shorten(k, 8)) + if failFast { + return err + } else { + log.Warn(err.Error()) + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-logEvery.C: + log.Info(fmt.Sprintf("[integrity] InvertedIndex: %s, prefix=%x", g.FileName(), common.Shorten(k, 8))) + default: + } + } + return nil + } + + for _, item := range iit.files { + if item.src.decompressor == nil { + continue + } + if item.endTxNum <= fromTxNum { + continue + } + if err := iterStep(item); err != nil { + return err + } + //log.Warn(fmt.Sprintf("[dbg] see1: %s, min=%d,max=%d, before_max=%d, all: %d\n", item.src.decompressor.FileName(), ef.Min(), ef.Max(), last2, stream.ToArrU64Must(ef.Iterator()))) + } + return nil +} diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 7c351294536..0c805f06e9b 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -928,66 +928,6 @@ func (iit *InvertedIndexRoTx) Prune(ctx context.Context, rwTx kv.RwTx, txFrom, t return stat, err } -func (iit *InvertedIndexRoTx) DebugEFAllValuesAreInRange(ctx context.Context, failFast bool, fromStep uint64) error { - logEvery := time.NewTicker(30 * time.Second) - defer logEvery.Stop() - fromTxNum := fromStep * iit.ii.aggregationStep - iterStep := func(item visibleFile) error { - g := item.src.decompressor.MakeGetter() - g.Reset(0) - defer item.src.decompressor.EnableReadAhead().DisableReadAhead() - - for g.HasNext() { - k, _ := g.NextUncompressed() - _ = k - eliasVal, _ := g.NextUncompressed() - ef, _ := eliasfano32.ReadEliasFano(eliasVal) - if ef.Count() == 0 { - continue - } - if item.startTxNum > ef.Min() { - err := fmt.Errorf("[integrity] .ef file has foreign txNum: %d > %d, %s, %x", item.startTxNum, ef.Min(), g.FileName(), common.Shorten(k, 8)) - if failFast { - return err - } else { - log.Warn(err.Error()) - } - } - if item.endTxNum < ef.Max() { - err := fmt.Errorf("[integrity] .ef file has foreign txNum: %d < %d, %s, %x", item.endTxNum, ef.Max(), g.FileName(), common.Shorten(k, 8)) - if failFast { - return err - } else { - log.Warn(err.Error()) - } - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-logEvery.C: - log.Info(fmt.Sprintf("[integrity] InvertedIndex: %s, prefix=%x", g.FileName(), common.Shorten(k, 8))) - default: - } - } - return nil - } - - for _, item := range iit.files { - if item.src.decompressor == nil { - continue - } - if item.endTxNum <= fromTxNum { - continue - } - if err := iterStep(item); err != nil { - return err - } - //log.Warn(fmt.Sprintf("[dbg] see1: %s, min=%d,max=%d, before_max=%d, all: %d\n", item.src.decompressor.FileName(), ef.Min(), ef.Max(), last2, stream.ToArrU64Must(ef.Iterator()))) - } - return nil -} - func (iit *InvertedIndexRoTx) IterateChangedKeys(startTxNum, endTxNum uint64, roTx kv.Tx) InvertedIterator1 { var ii1 InvertedIterator1 ii1.hasNextInDb = true diff --git a/eth/integrity/e3_ef_files.go b/eth/integrity/e3_ef_files.go index 929413725c4..6103acfa9d7 100644 --- a/eth/integrity/e3_ef_files.go +++ b/eth/integrity/e3_ef_files.go @@ -43,7 +43,7 @@ func E3EfFiles(ctx context.Context, chainDB kv.RwDB, agg *state.Aggregator, fail } defer tx.Rollback() - err = tx.(state.HasAggTx).AggTx().(*state.AggregatorRoTx).DebugEFAllValuesAreInRange(ctx, idx, failFast, fromStep) + err = tx.(state.HasAggTx).AggTx().(*state.AggregatorRoTx).IntegrityInvertedIndexAllValuesAreInRange(ctx, idx, failFast, fromStep) if err != nil { return err } diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 72269b57514..e85985e0f53 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -508,10 +508,10 @@ func doDebugKey(cliCtx *cli.Context) error { view := agg.BeginFilesRo() defer view.Close() - if err := view.DebugKey(domain, key); err != nil { + if err := view.IntegrityKey(domain, key); err != nil { return err } - if err := view.DebugEFKey(domain, key); err != nil { + if err := view.IntegirtyInvertedIndexKey(domain, key); err != nil { return err } return nil From 83faf86e592a0bad7f937ddd634408a5f3589305 Mon Sep 17 00:00:00 2001 From: Shota Date: Mon, 23 Dec 2024 20:10:03 +0400 Subject: [PATCH 16/94] not synced not logging (#13210) [#closes 13154](https://github.com/erigontech/erigon/issues/13154) Co-authored-by: shota.silagadze --- cl/phase1/network/gossip_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cl/phase1/network/gossip_manager.go b/cl/phase1/network/gossip_manager.go index 1955f27e89c..847d93c5a4b 100644 --- a/cl/phase1/network/gossip_manager.go +++ b/cl/phase1/network/gossip_manager.go @@ -311,7 +311,7 @@ func (g *GossipManager) Start(ctx context.Context) { return case data := <-ch: l := log.Ctx{} - if err := g.onRecv(ctx, data, l); err != nil && !errors.Is(err, services.ErrIgnore) { + if err := g.onRecv(ctx, data, l); err != nil && !errors.Is(err, services.ErrIgnore) && !errors.Is(err, synced_data.ErrNotSynced) { log.Debug("[Beacon Gossip] Recoverable Error", "err", err) } } From 13476765ec7a386281cead4a422fc8bbc591bab9 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 24 Dec 2024 09:12:26 +0700 Subject: [PATCH 17/94] mdbx: v13 (#13205) --- Makefile | 2 +- core/rawdb/accessors_indexes.go | 2 +- core/state/intra_block_state_logger_test.go | 9 ++-- erigon-lib/config3/config3.go | 1 + erigon-lib/go.mod | 4 +- erigon-lib/go.sum | 8 +-- erigon-lib/kv/mdbx/kv_mdbx.go | 2 +- erigon-lib/kv/mdbx/kv_mdbx_test.go | 58 --------------------- erigon-lib/state/history_stream.go | 44 +++++++++++----- erigon-lib/state/history_test.go | 14 +++-- erigon-lib/state/inverted_index_stream.go | 18 +++++-- erigon-lib/state/merge_test.go | 3 +- go.mod | 4 +- go.sum | 8 +-- 14 files changed, 72 insertions(+), 105 deletions(-) diff --git a/Makefile b/Makefile index c78ffa8530d..6a7fbe8a17e 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ GOPRIVATE = github.com/erigontech/silkworm-go PACKAGE = github.com/erigontech/erigon -GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false +GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}" GOBUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build $(GO_FLAGS) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index c1b5175cb46..6a4d5bdb537 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -21,10 +21,10 @@ package rawdb import ( "encoding/binary" + libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/core/types" ) diff --git a/core/state/intra_block_state_logger_test.go b/core/state/intra_block_state_logger_test.go index cc5fed21804..64cc73d59f6 100644 --- a/core/state/intra_block_state_logger_test.go +++ b/core/state/intra_block_state_logger_test.go @@ -20,17 +20,16 @@ import ( "reflect" "testing" + "github.com/holiman/uint256" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv/rawdbv3" "github.com/erigontech/erigon-lib/log/v3" stateLib "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/tracing/mocks" - - "github.com/holiman/uint256" - "github.com/stretchr/testify/require" - gomock "go.uber.org/mock/gomock" ) func TestStateLogger(t *testing.T) { diff --git a/erigon-lib/config3/config3.go b/erigon-lib/config3/config3.go index be0b43b3d98..8c2c07bdb92 100644 --- a/erigon-lib/config3/config3.go +++ b/erigon-lib/config3/config3.go @@ -18,6 +18,7 @@ package config3 // Default number of transactions (txNums) in one "step". One static file can have [1, 64] steps. const DefaultStepSize = 1_562_500 // = 100M / 64. Dividers: 2, 5, 10, 20, 50, 100, 500 +//const DefaultStepSize = 1_562_500 / 10 // StepsInFrozenFile - files of this size are completely frozen/immutable. // files of smaller size are also immutable, but can be removed after merge to bigger files. diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index b46410c6a3a..cece7b1848c 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -10,7 +10,7 @@ replace ( require ( github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed - github.com/erigontech/mdbx-go v0.38.4 + github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 github.com/erigontech/secp256k1 v1.1.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 ) @@ -62,7 +62,7 @@ require ( require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect + github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 65818353ba7..b08794204eb 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -154,8 +154,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed h1:un44S8Tuol4LBIC6R94t93GShM53BYjz7GsNPziDLQ8= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE= -github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= -github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U= @@ -271,8 +271,8 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index c70c671f0e9..b7a52c6d06b 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -202,7 +202,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) { } - env, err := mdbx.NewEnv() + env, err := mdbx.NewEnv(mdbx.Label(opts.label)) if err != nil { return nil, err } diff --git a/erigon-lib/kv/mdbx/kv_mdbx_test.go b/erigon-lib/kv/mdbx/kv_mdbx_test.go index 5b0d7fa9b7f..fad3e08d525 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx_test.go +++ b/erigon-lib/kv/mdbx/kv_mdbx_test.go @@ -458,64 +458,6 @@ func TestAppendFirstLast(t *testing.T) { require.Equal(t, []string{"value6.1"}, values) } -func TestNextPrevCurrent(t *testing.T) { - _, _, c := BaseCase(t) - - k, v, err := c.First() - require.Nil(t, err) - keys, values := iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Next() - require.Equal(t, []byte("key1"), k) - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - require.Equal(t, k, []byte("key1")) - require.Equal(t, v, []byte("value1.3")) - - k, v, err = c.Next() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key3", "key3"}, keys) - require.Equal(t, []string{"value3.1", "value3.3"}, values) - - k, v, err = c.Prev() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Prev() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values) - - err = c.DeleteCurrent() - require.Nil(t, err) - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - -} - func TestSeek(t *testing.T) { _, _, c := BaseCase(t) diff --git a/erigon-lib/state/history_stream.go b/erigon-lib/state/history_stream.go index 7f22adc7b20..083eaccefc0 100644 --- a/erigon-lib/state/history_stream.go +++ b/erigon-lib/state/history_stream.go @@ -283,10 +283,11 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error { } seek = next } - for k, _, err := hi.valsCDup.Seek(seek); k != nil; k, _, err = hi.valsCDup.NextNoDup() { - if err != nil { - return err - } + k, _, err := hi.valsCDup.Seek(seek) + if err != nil { + return err + } + for k != nil { if hi.toPrefix != nil && bytes.Compare(k, hi.toPrefix) >= 0 { break } @@ -295,8 +296,16 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error { return err } if v == nil { + seek, ok := kv.NextSubtree(k) + if !ok { + break + } + if k, _, err = hi.valsCDup.Seek(seek); err != nil { + panic(err) + } continue } + hi.nextKey = k hi.nextVal = v[8:] return nil @@ -566,24 +575,33 @@ func (hi *HistoryChangesIterDB) advanceSmallVals() (err error) { return err } } - for ; k != nil; k, _, err = hi.valsCDup.NextNoDup() { - if err != nil { - return err - } + for k != nil { v, err := hi.valsCDup.SeekBothRange(k, hi.startTxKey[:]) if err != nil { return err } if v == nil { + next, ok := kv.NextSubtree(k) + if !ok { + hi.nextKey = nil + return nil + } + k, _, err = hi.valsCDup.Seek(next) + if err != nil { + return err + } continue } foundTxNumVal := v[:8] - if hi.endTxNum >= 0 && int(binary.BigEndian.Uint64(foundTxNumVal)) >= hi.endTxNum { - continue + if hi.endTxNum < 0 || int(binary.BigEndian.Uint64(foundTxNumVal)) < hi.endTxNum { + hi.nextKey = k + hi.nextVal = v[8:] + return nil + } + k, _, err = hi.valsCDup.NextNoDup() + if err != nil { + return err } - hi.nextKey = k - hi.nextVal = v[8:] - return nil } hi.nextKey = nil return nil diff --git a/erigon-lib/state/history_test.go b/erigon-lib/state/history_test.go index ba5df93e6f8..d8462ca7e14 100644 --- a/erigon-lib/state/history_test.go +++ b/erigon-lib/state/history_test.go @@ -28,25 +28,23 @@ import ( "testing" "time" - "github.com/erigontech/erigon-lib/common/length" - "github.com/erigontech/erigon-lib/config3" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon-lib/seg" - - "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/common/datadir" - "github.com/stretchr/testify/require" btree2 "github.com/tidwall/btree" + "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/background" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/hexutility" + "github.com/erigontech/erigon-lib/common/length" + "github.com/erigontech/erigon-lib/config3" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/mdbx" "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/stream" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon-lib/recsplit/eliasfano32" + "github.com/erigontech/erigon-lib/seg" ) func testDbAndHistory(tb testing.TB, largeValues bool, logger log.Logger) (kv.RwDB, *History) { diff --git a/erigon-lib/state/inverted_index_stream.go b/erigon-lib/state/inverted_index_stream.go index 5f7b5808f16..191b38ea862 100644 --- a/erigon-lib/state/inverted_index_stream.go +++ b/erigon-lib/state/inverted_index_stream.go @@ -373,12 +373,20 @@ func (it *InvertedIterator1) advanceInDb() { if v, err = it.cursor.SeekBothRange(k, it.startTxKey[:]); err != nil { panic(err) } - if v != nil { - txNum := binary.BigEndian.Uint64(v) - if txNum < it.endTxNum { - it.nextDbKey = append(it.nextDbKey[:0], k...) - return + if v == nil { + seek, ok := kv.NextSubtree(k) + if !ok { + break } + if k, _, err = it.cursor.Seek(seek); err != nil { + panic(err) + } + continue + } + txNum := binary.BigEndian.Uint64(v) + if txNum < it.endTxNum { + it.nextDbKey = append(it.nextDbKey[:0], k...) + return } if k, _, err = it.cursor.NextNoDup(); err != nil { panic(err) diff --git a/erigon-lib/state/merge_test.go b/erigon-lib/state/merge_test.go index c7fa5124e06..3838440dec1 100644 --- a/erigon-lib/state/merge_test.go +++ b/erigon-lib/state/merge_test.go @@ -18,11 +18,12 @@ package state import ( "context" - "github.com/erigontech/erigon-lib/common/datadir" "os" "sort" "testing" + "github.com/erigontech/erigon-lib/common/datadir" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" btree2 "github.com/tidwall/btree" diff --git a/go.mod b/go.mod index b1d51ea37a2..721f6a8a9a2 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace ( require ( github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 - github.com/erigontech/mdbx-go v0.38.4 + github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 github.com/erigontech/secp256k1 v1.1.0 github.com/erigontech/silkworm-go v0.18.0 ) @@ -186,7 +186,7 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect + github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect diff --git a/go.sum b/go.sum index ff4b776ceda..d271898f3df 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 h1:KCFa2uXEfZoBjV4buzjWmCmoqVLXiGCq0ZmQ2OjeRvQ= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116/go.mod h1:8vQ+VjvLu2gkPs8EwdPrOTAAo++WuLuBi54N7NuAF0I= -github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= -github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= @@ -485,8 +485,8 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= From 8c182b2ce448c017b6a8215bd0ccef78b72f0f60 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 24 Dec 2024 09:13:51 +0700 Subject: [PATCH 18/94] bitmap v2 (#13161) --- cmd/hack/hack.go | 2 +- core/state/recon_state.go | 2 +- erigon-lib/downloader/mdbx_piece_completion.go | 2 +- erigon-lib/go.mod | 3 ++- erigon-lib/go.sum | 3 +++ erigon-lib/kv/bitmapdb/bitmapdb.go | 4 ++-- erigon-lib/kv/bitmapdb/bitmapdb_test.go | 2 +- erigon-lib/kv/bitmapdb/stream.go | 2 +- erigon-lib/state/aggregator.go | 2 +- erigon-lib/state/inverted_index_stream.go | 2 +- go.mod | 3 ++- go.sum | 3 +++ turbo/jsonrpc/erigon_receipts.go | 2 +- turbo/jsonrpc/eth_receipts.go | 2 +- turbo/jsonrpc/otterscan_search_backward.go | 2 +- turbo/jsonrpc/otterscan_search_backward_test.go | 2 +- turbo/jsonrpc/otterscan_search_forward.go | 2 +- turbo/jsonrpc/otterscan_search_forward_test.go | 2 +- turbo/jsonrpc/otterscan_search_test.go | 2 +- turbo/jsonrpc/overlay_api.go | 4 ++-- turbo/stages/bodydownload/body_data_struct.go | 2 +- 21 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go index 13a900a549a..237cad6d7cc 100644 --- a/cmd/hack/hack.go +++ b/cmd/hack/hack.go @@ -32,7 +32,7 @@ import ( "slices" "strings" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/holiman/uint256" "github.com/erigontech/erigon-lib/log/v3" diff --git a/core/state/recon_state.go b/core/state/recon_state.go index 1e7bbea1db9..89bfeb8f3eb 100644 --- a/core/state/recon_state.go +++ b/core/state/recon_state.go @@ -25,7 +25,7 @@ import ( "encoding/binary" "sync" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/google/btree" btree2 "github.com/tidwall/btree" diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index fe03cb207e8..97ee06f5963 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -21,7 +21,7 @@ import ( "encoding/binary" "sync" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/v2" "github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/types/infohash" diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index cece7b1848c..c63958639c5 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -16,7 +16,7 @@ require ( ) require ( - github.com/RoaringBitmap/roaring v1.9.4 + github.com/RoaringBitmap/roaring/v2 v2.4.2 github.com/anacrolix/dht/v2 v2.21.1 github.com/anacrolix/go-libutp v1.3.1 github.com/anacrolix/log v0.15.2 @@ -60,6 +60,7 @@ require ( ) require ( + github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index b08794204eb..b125a49fa91 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -14,6 +14,8 @@ github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrX github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/RoaringBitmap/roaring/v2 v2.4.2 h1:ew/INI7HLRyYK+dCbF6FcUwoe2Q0q5HCV7WafY9ljBk= +github.com/RoaringBitmap/roaring/v2 v2.4.2/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= @@ -693,6 +695,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/erigon-lib/kv/bitmapdb/bitmapdb.go b/erigon-lib/kv/bitmapdb/bitmapdb.go index 674f936da59..15dd2ef7ede 100644 --- a/erigon-lib/kv/bitmapdb/bitmapdb.go +++ b/erigon-lib/kv/bitmapdb/bitmapdb.go @@ -23,8 +23,8 @@ import ( "sort" "sync" - "github.com/RoaringBitmap/roaring" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common" diff --git a/erigon-lib/kv/bitmapdb/bitmapdb_test.go b/erigon-lib/kv/bitmapdb/bitmapdb_test.go index cb66ad05d18..9f5b3c88881 100644 --- a/erigon-lib/kv/bitmapdb/bitmapdb_test.go +++ b/erigon-lib/kv/bitmapdb/bitmapdb_test.go @@ -19,7 +19,7 @@ package bitmapdb_test import ( "testing" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/v2" "github.com/erigontech/erigon-lib/kv/bitmapdb" "github.com/stretchr/testify/require" ) diff --git a/erigon-lib/kv/bitmapdb/stream.go b/erigon-lib/kv/bitmapdb/stream.go index 890f1552671..24f32f3f337 100644 --- a/erigon-lib/kv/bitmapdb/stream.go +++ b/erigon-lib/kv/bitmapdb/stream.go @@ -17,7 +17,7 @@ package bitmapdb import ( - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" ) type BitmapStream struct { diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 7c879fb576e..495a49d0a01 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -31,7 +31,7 @@ import ( "sync/atomic" "time" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/c2h5oh/datasize" "github.com/tidwall/btree" rand2 "golang.org/x/exp/rand" diff --git a/erigon-lib/state/inverted_index_stream.go b/erigon-lib/state/inverted_index_stream.go index 191b38ea862..bf5aba0155e 100644 --- a/erigon-lib/state/inverted_index_stream.go +++ b/erigon-lib/state/inverted_index_stream.go @@ -21,7 +21,7 @@ import ( "container/heap" "encoding/binary" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/bitmapdb" diff --git a/go.mod b/go.mod index 721f6a8a9a2..cc50c4eb0a4 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/99designs/gqlgen v0.17.56 github.com/Giulio2002/bls v0.0.0-20241116091023-2ddcc8954ec0 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/RoaringBitmap/roaring v1.9.4 + github.com/RoaringBitmap/roaring/v2 v2.4.2 github.com/alecthomas/kong v0.8.1 github.com/anacrolix/sync v0.5.1 github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8 @@ -109,6 +109,7 @@ require ( ) require ( + github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b // indirect github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc // indirect diff --git a/go.sum b/go.sum index d271898f3df..79d6db3b838 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrX github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/RoaringBitmap/roaring/v2 v2.4.2 h1:ew/INI7HLRyYK+dCbF6FcUwoe2Q0q5HCV7WafY9ljBk= +github.com/RoaringBitmap/roaring/v2 v2.4.2/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= @@ -1384,6 +1386,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/turbo/jsonrpc/erigon_receipts.go b/turbo/jsonrpc/erigon_receipts.go index 8d98d53049f..91deb7506f1 100644 --- a/turbo/jsonrpc/erigon_receipts.go +++ b/turbo/jsonrpc/erigon_receipts.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/v2" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv/order" diff --git a/turbo/jsonrpc/eth_receipts.go b/turbo/jsonrpc/eth_receipts.go index 04a5fceeb7a..4a2592bbb34 100644 --- a/turbo/jsonrpc/eth_receipts.go +++ b/turbo/jsonrpc/eth_receipts.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/v2" "github.com/erigontech/erigon-lib/chain" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" diff --git a/turbo/jsonrpc/otterscan_search_backward.go b/turbo/jsonrpc/otterscan_search_backward.go index 1d2e1542192..f3f65e428df 100644 --- a/turbo/jsonrpc/otterscan_search_backward.go +++ b/turbo/jsonrpc/otterscan_search_backward.go @@ -19,7 +19,7 @@ package jsonrpc import ( "bytes" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" ) diff --git a/turbo/jsonrpc/otterscan_search_backward_test.go b/turbo/jsonrpc/otterscan_search_backward_test.go index d623070d6e3..c8cbdea5291 100644 --- a/turbo/jsonrpc/otterscan_search_backward_test.go +++ b/turbo/jsonrpc/otterscan_search_backward_test.go @@ -22,7 +22,7 @@ import ( "github.com/erigontech/erigon-lib/common/hexutil" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/stretchr/testify/require" diff --git a/turbo/jsonrpc/otterscan_search_forward.go b/turbo/jsonrpc/otterscan_search_forward.go index 169a5d0f2d5..195070b0752 100644 --- a/turbo/jsonrpc/otterscan_search_forward.go +++ b/turbo/jsonrpc/otterscan_search_forward.go @@ -19,7 +19,7 @@ package jsonrpc import ( "bytes" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" ) diff --git a/turbo/jsonrpc/otterscan_search_forward_test.go b/turbo/jsonrpc/otterscan_search_forward_test.go index 9a0dd14b5e7..346a829327d 100644 --- a/turbo/jsonrpc/otterscan_search_forward_test.go +++ b/turbo/jsonrpc/otterscan_search_forward_test.go @@ -20,7 +20,7 @@ import ( "bytes" "testing" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutil" "github.com/erigontech/erigon/cmd/rpcdaemon/rpcdaemontest" diff --git a/turbo/jsonrpc/otterscan_search_test.go b/turbo/jsonrpc/otterscan_search_test.go index 9f7e4f4efd6..50814c82801 100644 --- a/turbo/jsonrpc/otterscan_search_test.go +++ b/turbo/jsonrpc/otterscan_search_test.go @@ -19,7 +19,7 @@ package jsonrpc import ( "testing" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" ) func createBitmap(t *testing.T, blocks []uint64) []byte { diff --git a/turbo/jsonrpc/overlay_api.go b/turbo/jsonrpc/overlay_api.go index 0072579da9b..d39401d406c 100644 --- a/turbo/jsonrpc/overlay_api.go +++ b/turbo/jsonrpc/overlay_api.go @@ -24,8 +24,8 @@ import ( "sync" "time" - "github.com/RoaringBitmap/roaring" - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/erigontech/erigon-lib/chain" "github.com/erigontech/erigon-lib/common" diff --git a/turbo/stages/bodydownload/body_data_struct.go b/turbo/stages/bodydownload/body_data_struct.go index 9d233ed1700..b357a7eb9de 100644 --- a/turbo/stages/bodydownload/body_data_struct.go +++ b/turbo/stages/bodydownload/body_data_struct.go @@ -17,7 +17,7 @@ package bodydownload import ( - "github.com/RoaringBitmap/roaring/roaring64" + "github.com/RoaringBitmap/roaring/v2/roaring64" "github.com/google/btree" libcommon "github.com/erigontech/erigon-lib/common" From ec2b7b27cc60a9bcd032d3a57ff0a3baefe79251 Mon Sep 17 00:00:00 2001 From: Ilya Mikheev <54912776+JkLondon@users.noreply.github.com> Date: Tue, 24 Dec 2024 04:11:49 +0100 Subject: [PATCH 19/94] Reducing block calls (#13157) closes #11347 closes #13156 --------- Co-authored-by: JkLondon Co-authored-by: alex.sharov --- turbo/jsonrpc/debug_api_test.go | 6 +++ turbo/jsonrpc/eth_receipts.go | 42 +++++++++++------ turbo/jsonrpc/eth_txs.go | 49 ++++++++++++++------ turbo/jsonrpc/otterscan_search_v3.go | 2 +- turbo/jsonrpc/receipts/receipts_generator.go | 30 ++++++------ 5 files changed, 83 insertions(+), 46 deletions(-) diff --git a/turbo/jsonrpc/debug_api_test.go b/turbo/jsonrpc/debug_api_test.go index c982bf37d19..d6b55c27804 100644 --- a/turbo/jsonrpc/debug_api_test.go +++ b/turbo/jsonrpc/debug_api_test.go @@ -79,6 +79,12 @@ func TestTraceBlockByNumber(t *testing.T) { if err != nil { t.Errorf("traceBlock %s: %v", tt.txHash, err) } + if tx == nil { + t.Errorf("nil tx") + } + if tx.BlockHash == nil { + t.Errorf("nil block hash") + } txcount, err := ethApi.GetBlockTransactionCountByHash(m.Ctx, *tx.BlockHash) if err != nil { t.Errorf("traceBlock %s: %v", tt.txHash, err) diff --git a/turbo/jsonrpc/eth_receipts.go b/turbo/jsonrpc/eth_receipts.go index 4a2592bbb34..ba914a8552f 100644 --- a/turbo/jsonrpc/eth_receipts.go +++ b/turbo/jsonrpc/eth_receipts.go @@ -51,8 +51,8 @@ func (api *BaseAPI) getReceipts(ctx context.Context, tx kv.TemporalTx, block *ty return api.receiptsGenerator.GetReceipts(ctx, chainConfig, tx, block) } -func (api *BaseAPI) getReceipt(ctx context.Context, cc *chain.Config, tx kv.TemporalTx, block *types.Block, index int, txNum uint64) (*types.Receipt, error) { - return api.receiptsGenerator.GetReceipt(ctx, cc, tx, block, index, txNum) +func (api *BaseAPI) getReceipt(ctx context.Context, cc *chain.Config, tx kv.TemporalTx, header *types.Header, txn types.Transaction, index int, txNum uint64) (*types.Receipt, error) { + return api.receiptsGenerator.GetReceipt(ctx, cc, tx, header, txn, index, txNum) } func (api *BaseAPI) getCachedReceipt(ctx context.Context, hash common.Hash) (*types.Receipt, bool) { @@ -449,25 +449,37 @@ func (api *APIImpl) GetTransactionReceipt(ctx context.Context, txnHash common.Ha return nil, nil } - block, err := api.blockByNumberWithSenders(ctx, tx, blockNum) + header, err := api._blockReader.HeaderByNumber(ctx, tx, blockNum) if err != nil { return nil, err } - if block == nil { - return nil, nil // not error, see https://github.com/erigontech/erigon/issues/1645 + + txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, api._blockReader)) + + txNumMin, err := txNumsReader.Min(tx, blockNum) + if err != nil { + return nil, err } - var txnIndex uint64 - var txn types.Transaction - for idx, transaction := range block.Transactions() { - if transaction.Hash() == txnHash { - txn = transaction - txnIndex = uint64(idx) - break - } + if txNumMin+2 > txNum { //TODO: what a magic is this "2" and how to avoid it + return nil, fmt.Errorf("uint underflow txnums error txNum: %d, txNumMin: %d, blockNum: %d", txNum, txNumMin, blockNum) + } + + var txnIndex = int(txNum - txNumMin - 2) + + txn, err := api._blockReader.TxnByIdxInBlock(ctx, tx, header.Number.Uint64(), txnIndex) + if err != nil { + return nil, err } if txn == nil && chainConfig.Bor != nil { //TODO: add tx gran here to. + block, err := api.blockByNumberWithSenders(ctx, tx, blockNum) + if err != nil { + return nil, err + } + if block == nil { + return nil, nil // not error, see https://github.com/erigontech/erigon/issues/1645 + } receipts, err := api.getReceipts(ctx, tx, block) if err != nil { return nil, fmt.Errorf("getReceipts error: %w", err) @@ -490,12 +502,12 @@ func (api *APIImpl) GetTransactionReceipt(ctx context.Context, txnHash common.Ha return ethutils.MarshalReceipt(borReceipt, bortypes.NewBorTransaction(), chainConfig, block.HeaderNoCopy(), txnHash, false), nil } - receipt, err := api.getReceipt(ctx, chainConfig, tx, block, int(txnIndex), txNum) + receipt, err := api.getReceipt(ctx, chainConfig, tx, header, txn, txnIndex, txNum) if err != nil { return nil, fmt.Errorf("getReceipt error: %w", err) } - return ethutils.MarshalReceipt(receipt, block.Transactions()[txnIndex], chainConfig, block.HeaderNoCopy(), txnHash, true), nil + return ethutils.MarshalReceipt(receipt, txn, chainConfig, header, txnHash, true), nil } // GetBlockReceipts - receipts for individual block diff --git a/turbo/jsonrpc/eth_txs.go b/turbo/jsonrpc/eth_txs.go index 2232c9e5f29..980a9da3a98 100644 --- a/turbo/jsonrpc/eth_txs.go +++ b/turbo/jsonrpc/eth_txs.go @@ -19,8 +19,12 @@ package jsonrpc import ( "bytes" "context" + "fmt" "math/big" + "github.com/erigontech/erigon-lib/kv/rawdbv3" + "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks" + "github.com/erigontech/erigon-lib/common/hexutil" "github.com/erigontech/erigon-lib/common" @@ -49,10 +53,13 @@ func (api *APIImpl) GetTransactionByHash(ctx context.Context, txnHash common.Has } // https://infura.io/docs/ethereum/json-rpc/eth-getTransactionByHash - blockNum, _, ok, err := api.txnLookup(ctx, tx, txnHash) + blockNum, txNum, ok, err := api.txnLookup(ctx, tx, txnHash) if err != nil { return nil, err } + + txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, api._blockReader)) + // Private API returns 0 if transaction is not found. if blockNum == 0 && chainConfig.Bor != nil { if api.useBridgeReader { @@ -66,28 +73,36 @@ func (api *APIImpl) GetTransactionByHash(ctx context.Context, txnHash common.Has } } if ok { - block, err := api.blockByNumberWithSenders(ctx, tx, blockNum) + txNumMin, err := txNumsReader.Min(tx, blockNum) if err != nil { return nil, err } - if block == nil { - return nil, nil + + if txNumMin+2 > txNum { //TODO: what a magic is this "2" and how to avoid it + return nil, fmt.Errorf("uint underflow txnums error txNum: %d, txNumMin: %d, blockNum: %d", txNum, txNumMin, blockNum) } - blockHash := block.Hash() - var txnIndex uint64 - var txn types2.Transaction - for i, transaction := range block.Transactions() { - if transaction.Hash() == txnHash { - txn = transaction - txnIndex = uint64(i) - break - } + + var txnIndex uint64 = txNum - txNumMin - 2 + + txn, err := api._txnReader.TxnByIdxInBlock(ctx, tx, blockNum, int(txnIndex)) + if err != nil { + return nil, err } + header, err := api._blockReader.HeaderByNumber(ctx, tx, blockNum) + if err != nil { + return nil, err + } + if header == nil { + return nil, nil + } + + blockHash := header.Hash() + // Add GasPrice for the DynamicFeeTransaction var baseFee *big.Int if chainConfig.IsLondon(blockNum) && blockHash != (common.Hash{}) { - baseFee = block.BaseFee() + baseFee = header.BaseFee } // if no transaction was found then we return nil @@ -96,7 +111,11 @@ func (api *APIImpl) GetTransactionByHash(ctx context.Context, txnHash common.Has return nil, nil } borTx := bortypes.NewBorTransaction() - return newRPCBorTransaction(borTx, txnHash, blockHash, blockNum, uint64(len(block.Transactions())), baseFee, chainConfig.ChainID), nil + _, txCount, err := api._blockReader.Body(ctx, tx, blockHash, blockNum) + if err != nil { + return nil, err + } + return newRPCBorTransaction(borTx, txnHash, blockHash, blockNum, uint64(txCount), baseFee, chainConfig.ChainID), nil } return NewRPCTransaction(txn, blockHash, blockNum, txnIndex, baseFee), nil diff --git a/turbo/jsonrpc/otterscan_search_v3.go b/turbo/jsonrpc/otterscan_search_v3.go index dbc34b5a18a..b6c179025ed 100644 --- a/turbo/jsonrpc/otterscan_search_v3.go +++ b/turbo/jsonrpc/otterscan_search_v3.go @@ -96,7 +96,7 @@ func (api *OtterscanAPIImpl) buildSearchResults(ctx context.Context, tx kv.Tempo rpcTx := NewRPCTransaction(txn, block.Hash(), blockNum, uint64(txIndex), block.BaseFee()) txs = append(txs, rpcTx) - receipt, err := api.receiptsGenerator.GetReceipt(ctx, chainConfig, tx, block, txIndex, txNum+1) + receipt, err := api.receiptsGenerator.GetReceipt(ctx, chainConfig, tx, block.HeaderNoCopy(), txn, txIndex, txNum+1) if err != nil { return nil, nil, false, err } diff --git a/turbo/jsonrpc/receipts/receipts_generator.go b/turbo/jsonrpc/receipts/receipts_generator.go index 9a580abe9f7..08878274853 100644 --- a/turbo/jsonrpc/receipts/receipts_generator.go +++ b/turbo/jsonrpc/receipts/receipts_generator.go @@ -84,16 +84,16 @@ func (g *Generator) GetCachedReceipt(ctx context.Context, hash common.Hash) (*ty return g.receiptCache.Get(hash) } -func (g *Generator) PrepareEnv(ctx context.Context, block *types.Block, cfg *chain.Config, tx kv.TemporalTx, txIndex int) (*ReceiptEnv, error) { +func (g *Generator) PrepareEnv(ctx context.Context, header *types.Header, cfg *chain.Config, tx kv.TemporalTx, txIndex int) (*ReceiptEnv, error) { txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, g.blockReader)) - ibs, _, _, _, _, err := transactions.ComputeBlockContext(ctx, g.engine, block.HeaderNoCopy(), cfg, g.blockReader, txNumsReader, tx, txIndex) + ibs, _, _, _, _, err := transactions.ComputeBlockContext(ctx, g.engine, header, cfg, g.blockReader, txNumsReader, tx, txIndex) if err != nil { - return nil, fmt.Errorf("ReceiptsGen: PrepareEnv: bn=%d, %w", block.NumberU64(), err) + return nil, fmt.Errorf("ReceiptsGen: PrepareEnv: bn=%d, %w", header.Number.Uint64(), err) } usedGas := new(uint64) usedBlobGas := new(uint64) - gp := new(core.GasPool).AddGas(block.GasLimit()).AddBlobGas(cfg.GetMaxBlobGasPerBlock()) + gp := new(core.GasPool).AddGas(header.GasLimit).AddBlobGas(cfg.GetMaxBlobGasPerBlock()) noopWriter := state.NewNoopWriter() @@ -104,7 +104,6 @@ func (g *Generator) PrepareEnv(ctx context.Context, block *types.Block, cfg *cha } return h } - header := block.HeaderNoCopy() return &ReceiptEnv{ ibs: ibs, usedGas: usedGas, @@ -124,16 +123,17 @@ func (g *Generator) addToCacheReceipt(hash common.Hash, receipt *types.Receipt) g.receiptCache.Add(hash, receipt.Copy()) // .Copy() helps pprof to attribute memory to cache - instead of evm (where it was allocated). } -func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.TemporalTx, block *types.Block, index int, txNum uint64) (*types.Receipt, error) { - if receipt, ok := g.receiptCache.Get(block.Transactions()[index].Hash()); ok { +func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.TemporalTx, header *types.Header, txn types.Transaction, index int, txNum uint64) (*types.Receipt, error) { + if receipts, ok := g.receiptsCache.Get(header.Hash()); ok && len(receipts) > index { + return receipts[index], nil + } + if receipt, ok := g.receiptCache.Get(txn.Hash()); ok { return receipt, nil } - if receipts, ok := g.receiptsCache.Get(block.Hash()); ok && len(receipts) > index { - return receipts[index], nil - } var receipt *types.Receipt - genEnv, err := g.PrepareEnv(ctx, block, cfg, tx, index) + + genEnv, err := g.PrepareEnv(ctx, header, cfg, tx, index) if err != nil { return nil, err } @@ -143,12 +143,12 @@ func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.Tem return nil, err } - receipt, _, err = core.ApplyTransaction(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, block.Transactions()[index], genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}) + receipt, _, err = core.ApplyTransaction(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}) if err != nil { - return nil, fmt.Errorf("ReceiptGen.GetReceipt: bn=%d, txnIdx=%d, %w", block.NumberU64(), index, err) + return nil, fmt.Errorf("ReceiptGen.GetReceipt: bn=%d, txnIdx=%d, %w", header.Number.Uint64(), index, err) } - receipt.BlockHash = block.Hash() + receipt.BlockHash = header.Hash() receipt.CumulativeGasUsed = cumGasUsed receipt.TransactionIndex = uint(index) @@ -169,7 +169,7 @@ func (g *Generator) GetReceipts(ctx context.Context, cfg *chain.Config, tx kv.Te receipts := make(types.Receipts, len(block.Transactions())) - genEnv, err := g.PrepareEnv(ctx, block, cfg, tx, 0) + genEnv, err := g.PrepareEnv(ctx, block.HeaderNoCopy(), cfg, tx, 0) if err != nil { return nil, err } From ba0129d47372cbd763ed4fec4d4ca9e28a4bfecf Mon Sep 17 00:00:00 2001 From: lupin012 <58134934+lupin012@users.noreply.github.com> Date: Thu, 26 Dec 2024 06:16:19 +0100 Subject: [PATCH 20/94] debug_traceBlockByNumber with refund=false (#13216) The check on gasUsed should be done only if refund is true (default case) --- turbo/jsonrpc/tracing.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/turbo/jsonrpc/tracing.go b/turbo/jsonrpc/tracing.go index c368dbcecf3..d4361c07e75 100644 --- a/turbo/jsonrpc/tracing.go +++ b/turbo/jsonrpc/tracing.go @@ -224,7 +224,12 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp } if dbg.AssertEnabled { - if block.GasUsed() != usedGas { + var refunds = true + if config.NoRefunds != nil && *config.NoRefunds { + refunds = false + } + + if refunds == true && block.GasUsed() != usedGas { panic(fmt.Errorf("assert: block.GasUsed() %d != usedGas %d. blockNum=%d", block.GasUsed(), usedGas, blockNumber)) } } From 2b3ae9a572e33ac0386dde01fbeb288fcb97798c Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 26 Dec 2024 16:15:29 +0700 Subject: [PATCH 21/94] disable assert (#13237) --- erigon-lib/common/dbg/dbg_assert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/common/dbg/dbg_assert.go b/erigon-lib/common/dbg/dbg_assert.go index 618bebce464..6793c955032 100644 --- a/erigon-lib/common/dbg/dbg_assert.go +++ b/erigon-lib/common/dbg/dbg_assert.go @@ -1,3 +1,3 @@ package dbg -var AssertEnabled = EnvBool("ERIGON_ASSERT", true) +var AssertEnabled = EnvBool("ERIGON_ASSERT", false) From b11a83b15619487850cd2589263575d6ed8ba173 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 26 Dec 2024 16:39:55 +0700 Subject: [PATCH 22/94] agg: schema (#13232) - `extract domain compresscfg` from domain to schema - extract schema out of agg constructor --- erigon-lib/state/aggregator.go | 237 ++++++++++++++---------- erigon-lib/state/aggregator_test.go | 4 +- erigon-lib/state/domain.go | 25 +-- erigon-lib/state/domain_test.go | 91 ++++----- erigon-lib/state/history.go | 14 +- erigon-lib/state/history_test.go | 80 ++++---- erigon-lib/state/inverted_index.go | 15 +- erigon-lib/state/inverted_index_test.go | 34 ++-- erigon-lib/state/merge_test.go | 208 ++++++++++----------- migrations/reset_stage_txn_lookup.go | 1 + 10 files changed, 367 insertions(+), 342 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 495a49d0a01..ea739577034 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -100,106 +100,84 @@ type OnFreezeFunc func(frozenFileNames []string) const AggregatorSqueezeCommitmentValues = true const MaxNonFuriousDirtySpacePerTx = 64 * datasize.MB -func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { - tmpdir := dirs.Tmp - salt, err := getStateIndicesSalt(dirs.Snap) +func commitmentFileMustExist(dirs datadir.Dirs, fromStep, toStep uint64) bool { + fPath := filepath.Join(dirs.SnapDomain, fmt.Sprintf("v1-%s.%d-%d.kv", kv.CommitmentDomain, fromStep, toStep)) + exists, err := dir.FileExist(fPath) if err != nil { - return nil, err + panic(err) } + return exists +} - ctx, ctxCancel := context.WithCancel(ctx) - a := &Aggregator{ - ctx: ctx, - ctxCancel: ctxCancel, - onFreeze: func(frozenFileNames []string) {}, - dirs: dirs, - tmpdir: tmpdir, - aggregationStep: aggregationStep, - db: db, - leakDetector: dbg.NewLeakDetector("agg", dbg.SlowTx()), - ps: background.NewProgressSet(), - logger: logger, - collateAndBuildWorkers: 1, - mergeWorkers: 1, - - commitmentValuesTransform: AggregatorSqueezeCommitmentValues, - - produce: true, - } - commitmentFileMustExist := func(fromStep, toStep uint64) bool { - fPath := filepath.Join(dirs.SnapDomain, fmt.Sprintf("v1-%s.%d-%d.kv", kv.CommitmentDomain, fromStep, toStep)) - exists, err := dir.FileExist(fPath) - if err != nil { - panic(err) - } - return exists - } - - integrityCheck := func(name kv.Domain, fromStep, toStep uint64) bool { - // case1: `kill -9` during building new .kv - // - `accounts` domain may be at step X and `commitment` domain at step X-1 - // - not a problem because `commitment` domain still has step X in DB - // case2: `kill -9` during building new .kv and `rm -rf chaindata` - // - `accounts` domain may be at step X and `commitment` domain at step X-1 - // - problem! `commitment` domain doesn't have step X in DB - // solution: ignore step X files in both cases - switch name { - case kv.AccountsDomain, kv.StorageDomain, kv.CodeDomain: - if toStep-fromStep > 1 { // only recently built files - return true - } - return commitmentFileMustExist(fromStep, toStep) - default: +func domainIntegrityCheck(name kv.Domain, dirs datadir.Dirs, fromStep, toStep uint64) bool { + // case1: `kill -9` during building new .kv + // - `accounts` domain may be at step X and `commitment` domain at step X-1 + // - not a problem because `commitment` domain still has step X in DB + // case2: `kill -9` during building new .kv and `rm -rf chaindata` + // - `accounts` domain may be at step X and `commitment` domain at step X-1 + // - problem! `commitment` domain doesn't have step X in DB + // solution: ignore step X files in both cases + switch name { + case kv.AccountsDomain, kv.StorageDomain, kv.CodeDomain: + if toStep-fromStep > 1 { // only recently built files return true } + return commitmentFileMustExist(dirs, fromStep, toStep) + default: + return true } +} - cfg := domainCfg{ +var Schema = map[kv.Domain]domainCfg{ + kv.AccountsDomain: { name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, - restrictSubsetFileDeletions: a.commitmentValuesTransform, - integrity: integrityCheck, - compression: seg.CompressNone, + indexList: withBTree | withExistence, + crossDomainIntegrity: domainIntegrityCheck, + compression: seg.CompressNone, + compressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblAccountHistoryVals, compression: seg.CompressNone, historyLargeValues: false, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant - iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg, - aggregationStep: aggregationStep, keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx}, + iiCfg: iiCfg{ + keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant + }, }, - } - if a.d[kv.AccountsDomain], err = NewDomain(cfg, logger); err != nil { - return nil, err - } - cfg = domainCfg{ + }, + kv.StorageDomain: { name: kv.StorageDomain, valuesTable: kv.TblStorageVals, - restrictSubsetFileDeletions: a.commitmentValuesTransform, - integrity: integrityCheck, + indexList: withBTree | withExistence, compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblStorageHistoryVals, compression: seg.CompressNone, historyLargeValues: false, + filenameBase: kv.StorageDomain.String(), - iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg, - aggregationStep: aggregationStep, keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx}, + iiCfg: iiCfg{ + keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.StorageDomain.String(), + }, }, - } - if a.d[kv.StorageDomain], err = NewDomain(cfg, logger); err != nil { - return nil, err - } - cfg = domainCfg{ + }, + kv.CodeDomain: { name: kv.CodeDomain, valuesTable: kv.TblCodeVals, - restrictSubsetFileDeletions: a.commitmentValuesTransform, - integrity: integrityCheck, + indexList: withBTree | withExistence, compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet + compressCfg: DomainCompressCfg, largeValues: true, hist: histCfg{ @@ -207,21 +185,21 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 compression: seg.CompressKeys | seg.CompressVals, historyLargeValues: true, + filenameBase: kv.CodeDomain.String(), - iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg, - aggregationStep: aggregationStep, keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx}, + iiCfg: iiCfg{ + withExistence: false, compressorCfg: seg.DefaultCfg, + keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx, + filenameBase: kv.CodeDomain.String(), + }, }, - } - if a.d[kv.CodeDomain], err = NewDomain(cfg, logger); err != nil { - return nil, err - } - cfg = domainCfg{ + }, + kv.CommitmentDomain: { name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, - restrictSubsetFileDeletions: a.commitmentValuesTransform, - replaceKeysInValues: a.commitmentValuesTransform, - integrity: integrityCheck, - compression: seg.CompressKeys, + indexList: withBTree | withExistence, + compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblCommitmentHistoryVals, @@ -229,42 +207,90 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 snapshotsDisabled: true, historyLargeValues: false, + filenameBase: kv.CommitmentDomain.String(), - iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg, - aggregationStep: aggregationStep, keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx}, + iiCfg: iiCfg{ + keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.CommitmentDomain.String(), + }, }, - } - if a.d[kv.CommitmentDomain], err = NewDomain(cfg, logger); err != nil { - return nil, err - } - cfg = domainCfg{ + }, + kv.ReceiptDomain: { name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, + + indexList: withBTree | withExistence, compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, - integrity: integrityCheck, + compressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblReceiptHistoryVals, compression: seg.CompressNone, historyLargeValues: false, + filenameBase: kv.ReceiptDomain.String(), - iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg, - aggregationStep: aggregationStep, keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx}, + iiCfg: iiCfg{ + keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.ReceiptDomain.String(), + }, }, + }, +} + +func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { + tmpdir := dirs.Tmp + salt, err := getStateIndicesSalt(dirs.Snap) + if err != nil { + return nil, err } - if a.d[kv.ReceiptDomain], err = NewDomain(cfg, logger); err != nil { + + ctx, ctxCancel := context.WithCancel(ctx) + a := &Aggregator{ + ctx: ctx, + ctxCancel: ctxCancel, + onFreeze: func(frozenFileNames []string) {}, + dirs: dirs, + tmpdir: tmpdir, + aggregationStep: aggregationStep, + db: db, + leakDetector: dbg.NewLeakDetector("agg", dbg.SlowTx()), + ps: background.NewProgressSet(), + logger: logger, + collateAndBuildWorkers: 1, + mergeWorkers: 1, + + commitmentValuesTransform: AggregatorSqueezeCommitmentValues, + + produce: true, + } + + if err := a.registerDomain(kv.AccountsDomain, salt, dirs, aggregationStep, logger); err != nil { return nil, err } - if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, db, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { + if err := a.registerDomain(kv.StorageDomain, salt, dirs, aggregationStep, logger); err != nil { return nil, err } - if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, db, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { + if err := a.registerDomain(kv.CodeDomain, salt, dirs, aggregationStep, logger); err != nil { return nil, err } - if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, db, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { + if err := a.registerDomain(kv.CommitmentDomain, salt, dirs, aggregationStep, logger); err != nil { return nil, err } - if err := a.registerII(kv.TracesToIdxPos, salt, dirs, db, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { + if err := a.registerDomain(kv.ReceiptDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesToIdxPos, salt, dirs, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { return nil, err } a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history @@ -317,9 +343,26 @@ func getStateIndicesSalt(baseDir string) (salt *uint32, err error) { return salt, nil } -func (a *Aggregator) registerII(idx kv.InvertedIdxPos, salt *uint32, dirs datadir.Dirs, db kv.RoDB, aggregationStep uint64, filenameBase, indexKeysTable, indexTable string, logger log.Logger) error { +func (a *Aggregator) registerDomain(name kv.Domain, salt *uint32, dirs datadir.Dirs, aggregationStep uint64, logger log.Logger) (err error) { + cfg := Schema[name] + //TODO: move dynamic part of config to InvertedIndex + cfg.restrictSubsetFileDeletions = a.commitmentValuesTransform + if name == kv.CommitmentDomain { + cfg.replaceKeysInValues = a.commitmentValuesTransform + } + cfg.hist.iiCfg.salt = salt + cfg.hist.iiCfg.dirs = dirs + cfg.hist.iiCfg.aggregationStep = aggregationStep + a.d[name], err = NewDomain(cfg, logger) + if err != nil { + return err + } + return nil +} + +func (a *Aggregator) registerII(idx kv.InvertedIdxPos, salt *uint32, dirs datadir.Dirs, aggregationStep uint64, filenameBase, indexKeysTable, indexTable string, logger log.Logger) error { idxCfg := iiCfg{ - salt: salt, dirs: dirs, db: db, + salt: salt, dirs: dirs, aggregationStep: aggregationStep, filenameBase: filenameBase, keysTable: indexKeysTable, diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index a5d89ad3d57..f70f655d650 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1088,9 +1088,7 @@ func testDbAndAggregatorv3(tb testing.TB, aggStep uint64) (kv.RwDB, *Aggregator) tb.Helper() require, logger := require.New(tb), log.New() dirs := datadir.New(tb.TempDir()) - db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { - return kv.ChaindataTablesCfg - }).MustOpen() + db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).MustOpen() tb.Cleanup(db.Close) agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index d34e39be8c9..25e28680a8f 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -101,7 +101,7 @@ type domainCfg struct { valuesTable string // bucket to store domain values; key -> inverted_step + values (Dupsort) largeValues bool - integrity rangeDomainIntegrityChecker + crossDomainIntegrity rangeDomainIntegrityChecker // replaceKeysInValues allows to replace commitment branch values with shorter keys. // for commitment domain only @@ -129,12 +129,11 @@ var DomainCompressCfg = seg.Cfg{ func NewDomain(cfg domainCfg, logger log.Logger) (*Domain, error) { if cfg.hist.iiCfg.dirs.SnapDomain == "" { - panic("empty `dirs` variable") + panic("assert: empty `dirs`") } - if cfg.indexList == 0 { - cfg.indexList = withBTree | withExistence + if cfg.hist.filenameBase == "" { + panic("assert: emtpy `filenameBase`" + cfg.name.String()) } - cfg.compressCfg = DomainCompressCfg d := &Domain{ domainCfg: cfg, @@ -143,10 +142,6 @@ func NewDomain(cfg domainCfg, logger log.Logger) (*Domain, error) { } var err error - if cfg.hist.filenameBase == "" { - cfg.hist.filenameBase = cfg.name.String() - cfg.hist.iiCfg.filenameBase = cfg.hist.filenameBase - } if d.History, err = NewHistory(cfg.hist, logger); err != nil { return nil, err } @@ -298,10 +293,18 @@ func (d *Domain) closeFilesAfterStep(lowerBound uint64) { } func (d *Domain) scanDirtyFiles(fileNames []string) (garbageFiles []*filesItem) { - for _, dirtyFile := range scanDirtyFiles(fileNames, d.aggregationStep, d.filenameBase, "kv", d.logger) { + if d.filenameBase == "" { + panic("assert: empty `filenameBase`") + } + if d.aggregationStep == 0 { + panic("assert: empty `aggregationStep`") + } + + l := scanDirtyFiles(fileNames, d.aggregationStep, d.filenameBase, "kv", d.logger) + for _, dirtyFile := range l { startStep, endStep := dirtyFile.startTxNum/d.aggregationStep, dirtyFile.endTxNum/d.aggregationStep domainName, _ := kv.String2Domain(d.filenameBase) - if d.integrity != nil && !d.integrity(domainName, startStep, endStep) { + if d.crossDomainIntegrity != nil && !d.crossDomainIntegrity(domainName, d.dirs, startStep, endStep) { d.logger.Debug("[agg] skip garbage file", "name", d.filenameBase, "startStep", startStep, "endStep", endStep) continue } diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index ae5c32c7bc3..18451c4bc6b 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -34,10 +34,6 @@ import ( "testing" "time" - "github.com/holiman/uint256" - "github.com/stretchr/testify/require" - btree2 "github.com/tidwall/btree" - "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/background" datadir2 "github.com/erigontech/erigon-lib/common/datadir" @@ -51,6 +47,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/seg" "github.com/erigontech/erigon-lib/types" + "github.com/holiman/uint256" + "github.com/stretchr/testify/require" ) type rndGen struct { @@ -75,35 +73,16 @@ func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { func testDbAndDomainOfStep(t *testing.T, aggStep uint64, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() dirs := datadir2.New(t.TempDir()) - keysTable := "Keys" - valsTable := "Vals" - historyKeysTable := "HistoryKeys" - historyValsTable := "HistoryVals" - settingsTable := "Settings" //nolint - indexTable := "Index" - db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { - tcfg := kv.TableCfg{ - keysTable: kv.TableCfgItem{Flags: kv.DupSort}, - valsTable: kv.TableCfgItem{Flags: kv.DupSort}, - historyKeysTable: kv.TableCfgItem{Flags: kv.DupSort}, - historyValsTable: kv.TableCfgItem{Flags: kv.DupSort}, - settingsTable: kv.TableCfgItem{}, - indexTable: kv.TableCfgItem{Flags: kv.DupSort}, - kv.TblPruningProgress: kv.TableCfgItem{}, - } - return tcfg - }).MustOpen() + cfg := Schema[kv.AccountsDomain] + cfg.crossDomainIntegrity = nil //no other domains + + db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() t.Cleanup(db.Close) salt := uint32(1) - cfg := domainCfg{ - name: kv.AccountsDomain, valuesTable: valsTable, - hist: histCfg{ - valuesTable: historyValsTable, - compression: seg.CompressNone, historyLargeValues: true, - - iiCfg: iiCfg{salt: &salt, dirs: dirs, db: db, withExistence: false, - aggregationStep: aggStep, keysTable: historyKeysTable, valuesTable: indexTable}, - }} + + cfg.hist.iiCfg.aggregationStep = aggStep + cfg.hist.iiCfg.dirs = dirs + cfg.hist.iiCfg.salt = &salt d, err := NewDomain(cfg, logger) require.NoError(t, err) d.DisableFsync() @@ -532,9 +511,9 @@ func collateAndMerge(t *testing.T, db kv.RwDB, tx kv.RwTx, d *Domain, txs uint64 valuesOuts, indexOuts, historyOuts := dc.staticFilesInRange(r) valuesIn, indexIn, historyIn, err := dc.mergeFiles(ctx, valuesOuts, indexOuts, historyOuts, r, nil, background.NewProgressSet()) require.NoError(t, err) - if valuesIn != nil && valuesIn.decompressor != nil { - fmt.Printf("merge: %s\n", valuesIn.decompressor.FileName()) - } + //if valuesIn != nil && valuesIn.decompressor != nil { + //fmt.Printf("merge: %s\n", valuesIn.decompressor.FileName()) + //} d.integrateMergedDirtyFiles(valuesOuts, indexOuts, historyOuts, valuesIn, indexIn, historyIn) d.reCalcVisibleFiles(d.dirtyFilesEndTxNumMinimax()) return false @@ -1044,28 +1023,38 @@ func TestDomain_OpenFilesWithDeletions(t *testing.T) { dom.Close() } +func emptyTestDomain(aggStep uint64) *Domain { + cfg := Schema[kv.AccountsDomain] + cfg.crossDomainIntegrity = nil + + salt := uint32(1) + cfg.hist.iiCfg.salt = &salt + cfg.hist.iiCfg.dirs = datadir2.New(os.TempDir()) + cfg.hist.iiCfg.aggregationStep = aggStep + + d, err := NewDomain(cfg, log.New()) + if err != nil { + panic(err) + } + return d +} + func TestScanStaticFilesD(t *testing.T) { t.Parallel() - ii := &Domain{ - History: &History{ - histCfg: histCfg{ - filenameBase: "test", - }, - InvertedIndex: emptyTestInvertedIndex(1)}, - dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess), - } + d := emptyTestDomain(1) + files := []string{ - "v1-test.0-1.kv", - "v1-test.1-2.kv", - "v1-test.0-4.kv", - "v1-test.2-3.kv", - "v1-test.3-4.kv", - "v1-test.4-5.kv", - } - ii.scanDirtyFiles(files) + "v1-accounts.0-1.kv", + "v1-accounts.1-2.kv", + "v1-accounts.0-4.kv", + "v1-accounts.2-3.kv", + "v1-accounts.3-4.kv", + "v1-accounts.4-5.kv", + } + d.scanDirtyFiles(files) var found []string - ii.dirtyFiles.Walk(func(items []*filesItem) bool { + d.dirtyFiles.Walk(func(items []*filesItem) bool { for _, item := range items { found = append(found, fmt.Sprintf("%d-%d", item.startTxNum, item.endTxNum)) } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index c0c4be4d4bb..c23d11f85fb 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/erigontech/erigon-lib/common/datadir" btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" @@ -69,7 +70,7 @@ type History struct { _visibleFiles []visibleFile } -type rangeDomainIntegrityChecker func(d kv.Domain, fromStep, toStep uint64) bool +type rangeDomainIntegrityChecker func(d kv.Domain, dirs datadir.Dirs, fromStep, toStep uint64) bool type rangeIntegrityChecker func(fromStep, toStep uint64) bool type histCfg struct { @@ -136,8 +137,11 @@ func NewHistory(cfg histCfg, logger log.Logger) (*History, error) { return &h, nil } +func (h *History) vFileName(fromStep, toStep uint64) string { + return fmt.Sprintf("v1-%s.%d-%d.v", h.filenameBase, fromStep, toStep) +} func (h *History) vFilePath(fromStep, toStep uint64) string { - return filepath.Join(h.dirs.SnapHistory, fmt.Sprintf("v1-%s.%d-%d.v", h.filenameBase, fromStep, toStep)) + return filepath.Join(h.dirs.SnapHistory, h.vFileName(fromStep, toStep)) } func (h *History) vAccessorFilePath(fromStep, toStep uint64) string { return filepath.Join(h.dirs.SnapAccessors, fmt.Sprintf("v1-%s.%d-%d.vi", h.filenameBase, fromStep, toStep)) @@ -169,6 +173,12 @@ func (h *History) openFolder() error { } func (h *History) scanDirtyFiles(fileNames []string) { + if h.filenameBase == "" { + panic("assert: empty `filenameBase`") + } + if h.aggregationStep == 0 { + panic("assert: empty `aggregationStep`") + } for _, dirtyFile := range scanDirtyFiles(fileNames, h.aggregationStep, h.filenameBase, "v", h.logger) { startStep, endStep := dirtyFile.startTxNum/h.aggregationStep, dirtyFile.endTxNum/h.aggregationStep if h.integrity != nil && !h.integrity(startStep, endStep) { diff --git a/erigon-lib/state/history_test.go b/erigon-lib/state/history_test.go index d8462ca7e14..24272c8e01e 100644 --- a/erigon-lib/state/history_test.go +++ b/erigon-lib/state/history_test.go @@ -28,9 +28,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - btree2 "github.com/tidwall/btree" - "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/datadir" @@ -45,45 +42,28 @@ import ( "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon-lib/recsplit/eliasfano32" "github.com/erigontech/erigon-lib/seg" + "github.com/stretchr/testify/require" ) func testDbAndHistory(tb testing.TB, largeValues bool, logger log.Logger) (kv.RwDB, *History) { tb.Helper() dirs := datadir.New(tb.TempDir()) - keysTable := "AccountKeys" - indexTable := "AccountIndex" - valsTable := "AccountVals" - settingsTable := "Settings" - db := mdbx.New(kv.ChainDB, logger).InMem(dirs.SnapDomain).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { - if largeValues { - return kv.TableCfg{ - keysTable: kv.TableCfgItem{Flags: kv.DupSort}, - indexTable: kv.TableCfgItem{Flags: kv.DupSort}, - valsTable: kv.TableCfgItem{Flags: kv.DupSort}, - settingsTable: kv.TableCfgItem{}, - kv.TblPruningProgress: kv.TableCfgItem{}, - } - } - return kv.TableCfg{ - keysTable: kv.TableCfgItem{Flags: kv.DupSort}, - indexTable: kv.TableCfgItem{Flags: kv.DupSort}, - valsTable: kv.TableCfgItem{Flags: kv.DupSort}, - settingsTable: kv.TableCfgItem{}, - kv.TblPruningProgress: kv.TableCfgItem{}, - } - }).MustOpen() + db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() //TODO: tests will fail if set histCfg.compression = CompressKeys | CompressValues salt := uint32(1) - cfg := histCfg{ - filenameBase: "hist", - valuesTable: valsTable, - - iiCfg: iiCfg{salt: &salt, dirs: dirs, db: db, withExistence: false, - aggregationStep: 16, filenameBase: "hist", keysTable: keysTable, valuesTable: indexTable, - }, - compression: seg.CompressNone, historyLargeValues: largeValues, - } - h, err := NewHistory(cfg, logger) + cfg := Schema[kv.AccountsDomain] + + cfg.hist.iiCfg.aggregationStep = 16 + cfg.hist.iiCfg.dirs = dirs + cfg.hist.iiCfg.salt = &salt + + cfg.hist.historyLargeValues = largeValues + + //perf of tests + cfg.hist.iiCfg.withExistence = false + cfg.hist.iiCfg.compression = seg.CompressNone + cfg.hist.compression = seg.CompressNone + h, err := NewHistory(cfg.hist, logger) require.NoError(tb, err) h.DisableFsync() tb.Cleanup(db.Close) @@ -235,7 +215,8 @@ func TestHistoryCollationBuild(t *testing.T) { c, err := h.collate(ctx, 0, 0, 8, tx) require.NoError(err) - require.True(strings.HasSuffix(c.historyPath, "v1-hist.0-1.v")) + + require.True(strings.HasSuffix(c.historyPath, h.vFileName(0, 1))) require.Equal(6, c.historyCount) require.Equal(3, c.efHistoryComp.Count()/2) @@ -1387,20 +1368,23 @@ func TestIterateChanged2(t *testing.T) { func TestScanStaticFilesH(t *testing.T) { t.Parallel() - h := &History{ - histCfg: histCfg{ - filenameBase: "test", - }, - InvertedIndex: emptyTestInvertedIndex(1), - dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess), + newTestDomain := func() (*InvertedIndex, *History) { + d := emptyTestDomain(1) + d.History.InvertedIndex.integrity = nil + d.History.InvertedIndex.indexList = 0 + d.History.indexList = 0 + return d.History.InvertedIndex, d.History } + + _, h := newTestDomain() + files := []string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.0-4.v", - "v1-test.2-3.v", - "v1-test.3-4.v", - "v1-test.4-5.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.0-4.v", + "v1-accounts.2-3.v", + "v1-accounts.3-4.v", + "v1-accounts.4-5.v", } h.scanDirtyFiles(files) require.Equal(t, 6, h.dirtyFiles.Len()) diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 0c805f06e9b..e5ce42f88b9 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -77,7 +77,6 @@ type InvertedIndex struct { type iiCfg struct { salt *uint32 dirs datadir.Dirs - db kv.RoDB // global db pointer. mostly for background warmup. filenameBase string // filename base for all files of this inverted index aggregationStep uint64 // amount of transactions inside single aggregation step @@ -101,7 +100,13 @@ type iiVisible struct { func NewInvertedIndex(cfg iiCfg, logger log.Logger) (*InvertedIndex, error) { if cfg.dirs.SnapDomain == "" { - panic("empty `dirs` varialbe") + panic("assert: empty `dirs`") + } + if cfg.filenameBase == "" { + panic("assert: empty `filenameBase`") + } + if cfg.aggregationStep == 0 { + panic("assert: empty `aggregationStep`") } //if cfg.compressorCfg.MaxDictPatterns == 0 && cfg.compressorCfg.MaxPatternLen == 0 { cfg.compressorCfg = seg.DefaultCfg @@ -177,6 +182,12 @@ func (ii *InvertedIndex) openFolder() error { } func (ii *InvertedIndex) scanDirtyFiles(fileNames []string) { + if ii.filenameBase == "" { + panic("assert: empty `filenameBase`") + } + if ii.aggregationStep == 0 { + panic("assert: empty `aggregationStep`") + } for _, dirtyFile := range scanDirtyFiles(fileNames, ii.aggregationStep, ii.filenameBase, "ef", ii.logger) { startStep, endStep := dirtyFile.startTxNum/ii.aggregationStep, dirtyFile.endTxNum/ii.aggregationStep if ii.integrity != nil && !ii.integrity(startStep, endStep) { diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index e9e19640887..e173e4f7df4 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -56,7 +56,7 @@ func testDbAndInvertedIndex(tb testing.TB, aggStep uint64, logger log.Logger) (k }).MustOpen() tb.Cleanup(db.Close) salt := uint32(1) - cfg := iiCfg{salt: &salt, dirs: dirs, db: db, aggregationStep: aggStep, filenameBase: "inv", keysTable: keysTable, valuesTable: indexTable} + cfg := iiCfg{salt: &salt, dirs: dirs, aggregationStep: aggStep, filenameBase: "inv", keysTable: keysTable, valuesTable: indexTable} ii, err := NewInvertedIndex(cfg, logger) require.NoError(tb, err) ii.DisableFsync() @@ -636,12 +636,12 @@ func TestScanStaticFiles(t *testing.T) { ii := emptyTestInvertedIndex(1) files := []string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.0-4.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", - "v1-test.4-5.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.0-4.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", + "v1-accounts.4-5.ef", } ii.scanDirtyFiles(files) require.Equal(t, 6, ii.dirtyFiles.Len()) @@ -656,16 +656,16 @@ func TestScanStaticFiles(t *testing.T) { func TestCtxFiles(t *testing.T) { ii := emptyTestInvertedIndex(1) files := []string{ - "v1-test.0-1.ef", // overlap with same `endTxNum=4` - "v1-test.1-2.ef", - "v1-test.0-4.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", - "v1-test.4-5.ef", // no overlap - "v1-test.480-484.ef", // overlap with same `startTxNum=480` - "v1-test.480-488.ef", - "v1-test.480-496.ef", - "v1-test.480-512.ef", + "v1-accounts.0-1.ef", // overlap with same `endTxNum=4` + "v1-accounts.1-2.ef", + "v1-accounts.0-4.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", + "v1-accounts.4-5.ef", // no overlap + "v1-accounts.480-484.ef", // overlap with same `startTxNum=480` + "v1-accounts.480-488.ef", + "v1-accounts.480-496.ef", + "v1-accounts.480-512.ef", } ii.scanDirtyFiles(files) require.Equal(t, 10, ii.dirtyFiles.Len()) diff --git a/erigon-lib/state/merge_test.go b/erigon-lib/state/merge_test.go index 3838440dec1..794514f5f2e 100644 --- a/erigon-lib/state/merge_test.go +++ b/erigon-lib/state/merge_test.go @@ -23,20 +23,23 @@ import ( "testing" "github.com/erigontech/erigon-lib/common/datadir" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - btree2 "github.com/tidwall/btree" + "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/seg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/erigontech/erigon-lib/recsplit/eliasfano32" ) func emptyTestInvertedIndex(aggStep uint64) *InvertedIndex { salt := uint32(1) - cfg := iiCfg{salt: &salt, dirs: datadir.New(os.TempDir()), db: nil, aggregationStep: aggStep, filenameBase: "test"} + cfg := Schema[kv.AccountsDomain].hist.iiCfg + + cfg.salt = &salt + cfg.dirs = datadir.New(os.TempDir()) + cfg.aggregationStep = aggStep ii, err := NewInvertedIndex(cfg, log.New()) ii.indexList = 0 @@ -45,15 +48,23 @@ func emptyTestInvertedIndex(aggStep uint64) *InvertedIndex { } return ii } + func TestFindMergeRangeCornerCases(t *testing.T) { t.Parallel() + newTestDomain := func() (*InvertedIndex, *History) { + d := emptyTestDomain(1) + d.History.InvertedIndex.integrity = nil + d.History.InvertedIndex.indexList = 0 + d.History.indexList = 0 + return d.History.InvertedIndex, d.History + } t.Run("ii: > 2 unmerged files", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, _ := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -74,12 +85,12 @@ func TestFindMergeRangeCornerCases(t *testing.T) { assert.Equal(t, 3, len(idxF)) }) t.Run("hist: > 2 unmerged files", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -88,14 +99,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.2-3.v", - "v1-test.3-4.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.2-3.v", + "v1-accounts.3-4.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -112,12 +120,12 @@ func TestFindMergeRangeCornerCases(t *testing.T) { assert.Equal(t, 2, int(r.index.to)) }) t.Run("not equal amount of files", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -126,12 +134,9 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -151,11 +156,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { assert.Equal(t, 2, int(r.index.to)) }) t.Run("idx merged, history not yet", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -164,12 +169,9 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -188,13 +190,13 @@ func TestFindMergeRangeCornerCases(t *testing.T) { assert.Equal(t, 2, int(r.history.to)) }) t.Run("idx merged, history not yet, 2", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", - "v1-test.0-4.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", + "v1-accounts.0-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -203,14 +205,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.2-3.v", - "v1-test.3-4.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.2-3.v", + "v1-accounts.3-4.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -232,9 +231,9 @@ func TestFindMergeRangeCornerCases(t *testing.T) { require.Equal(t, 2, len(histFiles)) }) t.Run("idx merged and small files lost", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-4.ef", + "v1-accounts.0-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -243,14 +242,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.2-3.v", - "v1-test.3-4.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.2-3.v", + "v1-accounts.3-4.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -271,10 +267,10 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) t.Run("history merged, but index not and history garbage left", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -284,13 +280,10 @@ func TestFindMergeRangeCornerCases(t *testing.T) { ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) // `kill -9` may leave small garbage files, but if big one already exists we assume it's good(fsynced) and no reason to merge again - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.0-2.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.0-2.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -312,13 +305,13 @@ func TestFindMergeRangeCornerCases(t *testing.T) { require.Equal(t, 0, len(histFiles)) }) t.Run("history merge progress ahead of idx", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.0-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.0-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -327,15 +320,12 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.0-2.v", - "v1-test.2-3.v", - "v1-test.3-4.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.0-2.v", + "v1-accounts.2-3.v", + "v1-accounts.3-4.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -357,12 +347,12 @@ func TestFindMergeRangeCornerCases(t *testing.T) { require.Equal(t, 3, len(histFiles)) }) t.Run("idx merge progress ahead of history", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.0-2.ef", - "v1-test.2-3.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.0-2.ef", + "v1-accounts.2-3.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -371,13 +361,10 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{ - histCfg: histCfg{filenameBase: "test"}, - InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.2-3.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.2-3.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -399,11 +386,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { require.Equal(t, 2, len(histFiles)) }) t.Run("idx merged, but garbage left", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, h := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.0-2.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.0-2.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -412,12 +399,11 @@ func TestFindMergeRangeCornerCases(t *testing.T) { }) ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - h := &History{InvertedIndex: ii, dirtyFiles: btree2.NewBTreeG[*filesItem](filesItemLess)} h.scanDirtyFiles([]string{ - "v1-test.0-1.v", - "v1-test.1-2.v", - "v1-test.0-2.v", - "v1-test.2-3.v", + "v1-accounts.0-1.v", + "v1-accounts.1-2.v", + "v1-accounts.0-2.v", + "v1-accounts.2-3.v", }) h.dirtyFiles.Scan(func(item *filesItem) bool { fName := h.vFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) @@ -433,13 +419,13 @@ func TestFindMergeRangeCornerCases(t *testing.T) { assert.False(t, r.history.needMerge) }) t.Run("idx merged, but garbage left2", func(t *testing.T) { - ii := emptyTestInvertedIndex(1) + ii, _ := newTestDomain() ii.scanDirtyFiles([]string{ - "v1-test.0-1.ef", - "v1-test.1-2.ef", - "v1-test.0-2.ef", - "v1-test.2-3.ef", - "v1-test.3-4.ef", + "v1-accounts.0-1.ef", + "v1-accounts.1-2.ef", + "v1-accounts.0-2.ef", + "v1-accounts.2-3.ef", + "v1-accounts.3-4.ef", }) ii.dirtyFiles.Scan(func(item *filesItem) bool { fName := ii.efFilePath(item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep) diff --git a/migrations/reset_stage_txn_lookup.go b/migrations/reset_stage_txn_lookup.go index d4c9e7cb8b2..bccd9e2db50 100644 --- a/migrations/reset_stage_txn_lookup.go +++ b/migrations/reset_stage_txn_lookup.go @@ -18,6 +18,7 @@ package migrations import ( "context" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" From fe01f04a881d81ca3d7ec083a068378c8edc15a2 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Thu, 26 Dec 2024 18:53:26 +0100 Subject: [PATCH 23/94] Caplin: added `IPV6` support (#13214) --- .../forkchoice/fork_graph/fork_graph_disk.go | 7 +++++-- cl/sentinel/sentinel.go | 19 +++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index b15df89e319..721340a3a74 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -171,6 +171,9 @@ func (f *forkGraphDisk) AnchorSlot() uint64 { } func (f *forkGraphDisk) isBlockRootTheCurrentState(blockRoot libcommon.Hash) bool { + if f.currentState == nil { + return false + } blockRootState, _ := f.currentState.BlockRoot() return blockRoot == blockRootState } @@ -200,7 +203,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, } else { newState, err = f.getState(block.ParentRoot, false, true) if err != nil { - return nil, LogisticError, fmt.Errorf("AddChainSegment: %w, parentRoot; %x", err, block.ParentRoot) + return nil, LogisticError, fmt.Errorf("AddChainSegment: %w, parentRoot: %x", err, block.ParentRoot) } } @@ -263,7 +266,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, // Add block to list of invalid blocks log.Warn("Invalid beacon block", "slot", block.Slot, "blockRoot", blockRoot, "reason", invalidBlockErr) f.badBlocks.Store(libcommon.Hash(blockRoot), struct{}{}) - //f.currentState = nil + f.currentState = nil return nil, InvalidBlock, invalidBlockErr } diff --git a/cl/sentinel/sentinel.go b/cl/sentinel/sentinel.go index 8a0b94f85b3..4e6e13f2301 100644 --- a/cl/sentinel/sentinel.go +++ b/cl/sentinel/sentinel.go @@ -147,25 +147,20 @@ func (s *Sentinel) createListener() (*discover.UDPv5, error) { ) ip := net.ParseIP(ipAddr) - if ip.To4() == nil { - return nil, fmt.Errorf("IPV4 address not provided instead %s was provided", ipAddr) + if ip == nil { + return nil, fmt.Errorf("bad ip address provided, %s was provided", ipAddr) } var bindIP net.IP var networkVersion string - // check for our network version - switch { - // if we have 16 byte and 4 byte representation then we are in using udp6 - case ip.To16() != nil && ip.To4() == nil: - bindIP = net.IPv6zero - networkVersion = "udp6" - // only 4 bytes then we are using udp4 - case ip.To4() != nil: + // If the IP is an IPv4 address, bind to the correct zero address. + if ip.To4() != nil { bindIP = net.IPv4zero networkVersion = "udp4" - default: - return nil, fmt.Errorf("bad ip address provided, %s was provided", ipAddr) + } else { + bindIP = net.IPv6zero + networkVersion = "udp6" } udpAddr := &net.UDPAddr{ From abb60034e470efb38d8284334485457c89aac25c Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Fri, 27 Dec 2024 10:07:19 +0700 Subject: [PATCH 24/94] agg: less hardcode domains. step1 (#13244) `minimaxTxNumInDomainFiles` method - to accept list of domains from outsides --- erigon-lib/kv/tables.go | 2 ++ erigon-lib/state/aggregator.go | 39 ++++++++++++++++++-------- erigon-lib/state/domain_shared.go | 2 +- erigon-lib/state/domain_shared_test.go | 2 ++ erigon-lib/state/squeeze.go | 2 +- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 3752d92daf1..3300ce829f5 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -732,6 +732,8 @@ const ( DomainLen Domain = 5 ) +var StateDomains = []Domain{AccountsDomain, StorageDomain, CodeDomain, CommitmentDomain} + const ( AccountsHistoryIdx InvertedIdx = "AccountsHistoryIdx" StorageHistoryIdx InvertedIdx = "StorageHistoryIdx" diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index ea739577034..fac019852f5 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -484,6 +484,10 @@ func (a *Aggregator) EnableHistory(name kv.Domain) *Aggregator { return a } +func (a *Aggregator) HasBackgroundFilesBuild2() bool { + return a.buildingFiles.Load() || a.mergingFiles.Load() +} + func (a *Aggregator) HasBackgroundFilesBuild() bool { return a.ps.Has() } func (a *Aggregator) BackgroundProgress() string { return a.ps.String() } @@ -920,14 +924,25 @@ type flusher interface { Flush(ctx context.Context, tx kv.RwTx) error } -func (ac *AggregatorRoTx) minimaxTxNumInDomainFiles() uint64 { - m := min( - ac.d[kv.AccountsDomain].files.EndTxNum(), - ac.d[kv.CodeDomain].files.EndTxNum(), - ac.d[kv.StorageDomain].files.EndTxNum(), - ac.d[kv.CommitmentDomain].files.EndTxNum(), - ) - return m +func (ac *AggregatorRoTx) StepsInFiles(entitySet ...kv.Domain) uint64 { + txNumInFiles := ac.TxNumsInFiles(entitySet...) + if txNumInFiles > 0 { + txNumInFiles-- + } + return txNumInFiles / ac.a.StepSize() +} + +func (ac *AggregatorRoTx) TxNumsInFiles(entitySet ...kv.Domain) (minTxNum uint64) { + if len(entitySet) == 0 { + panic("assert: missed arguments") + } + for i, domain := range entitySet { + domainEnd := ac.d[domain].files.EndTxNum() + if i == 0 || domainEnd < minTxNum { + minTxNum = domainEnd + } + } + return minTxNum } func (ac *AggregatorRoTx) CanPrune(tx kv.Tx, untilTx uint64) bool { @@ -1043,7 +1058,7 @@ func (ac *AggregatorRoTx) PruneSmallBatchesDb(ctx context.Context, timeout time. ac.a.logger.Info("[snapshots] pruning state", "until commit", time.Until(started.Add(timeout)).String(), "pruneLimit", pruneLimit, - "aggregatedStep", (ac.minimaxTxNumInDomainFiles()-1)/ac.a.StepSize(), + "aggregatedStep", ac.StepsInFiles(), "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), "pruned", fullStat.String(), ) @@ -1137,7 +1152,7 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du ac.a.logger.Info("[snapshots] pruning state", "until commit", time.Until(started.Add(timeout)).String(), "pruneLimit", pruneLimit, - "aggregatedStep", (ac.minimaxTxNumInDomainFiles()-1)/ac.a.StepSize(), + "aggregatedStep", ac.StepsInFiles(), "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), "pruned", fullStat.String(), ) @@ -1315,7 +1330,7 @@ func (ac *AggregatorRoTx) Prune(ctx context.Context, tx kv.RwTx, limit uint64, l } func (ac *AggregatorRoTx) LogStats(tx kv.Tx, tx2block func(endTxNumMinimax uint64) (uint64, error)) { - maxTxNum := ac.minimaxTxNumInDomainFiles() + maxTxNum := ac.TxNumsInFiles(kv.StateDomains...) if maxTxNum == 0 { return } @@ -1453,7 +1468,7 @@ func (a *Aggregator) recalcVisibleFiles(toTxNum uint64) { func (a *Aggregator) recalcVisibleFilesMinimaxTxNum() { aggTx := a.BeginFilesRo() defer aggTx.Close() - a.visibleFilesMinimaxTxNum.Store(aggTx.minimaxTxNumInDomainFiles()) + a.visibleFilesMinimaxTxNum.Store(aggTx.TxNumsInFiles(kv.StateDomains...)) } type RangesV3 struct { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 6d79a7491f0..77d4636e55f 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -472,7 +472,7 @@ func (sd *SharedDomains) replaceShortenedKeysInBranch(prefix []byte, branch comm if !sd.aggTx.a.commitmentValuesTransform || len(branch) == 0 || - sd.aggTx.minimaxTxNumInDomainFiles() == 0 || + sd.aggTx.TxNumsInFiles(kv.StateDomains...) == 0 || bytes.Equal(prefix, keyCommitmentState) || ((fEndTxNum-fStartTxNum)/sd.aggTx.a.StepSize())%2 != 0 { // this checks if file has even number of steps, singular files does not transform values. diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index 008454bc819..e05c50b192a 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -320,6 +320,8 @@ func TestSharedDomain_IteratePrefix(t *testing.T) { ac = agg.BeginFilesRo() defer ac.Close() + require.Equal(int(stepSize*2), int(ac.TxNumsInFiles(kv.StateDomains...))) + rwTx, err = db.BeginRw(ctx) require.NoError(err) defer rwTx.Rollback() diff --git a/erigon-lib/state/squeeze.go b/erigon-lib/state/squeeze.go index f5ab3c80e68..db92ffff8d2 100644 --- a/erigon-lib/state/squeeze.go +++ b/erigon-lib/state/squeeze.go @@ -380,7 +380,7 @@ func (a *Aggregator) RebuildCommitmentFiles(ctx context.Context, rwDb kv.RwDB, t fromTxNumRange, toTxNumRange := r.FromTo() lastTxnumInShard := toTxNumRange - if acRo.minimaxTxNumInDomainFiles() >= toTxNumRange { + if acRo.TxNumsInFiles(kv.StateDomains...) >= toTxNumRange { a.logger.Info("skipping existing range", "range", r.String("", a.StepSize())) continue } From 34114fab0c82bd6e2c8aa106401c524dd079bc70 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Fri, 27 Dec 2024 10:07:25 +0700 Subject: [PATCH 25/94] buildSema: accept it as a constructor parameter (#13241) non-funcional PR to prepare for future move of `setUpBlockReader` out of `backend.go` package --- eth/backend.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 5b7a8584928..12ebf19aa1a 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -334,9 +334,11 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger os.Exit(1) } + segmentsBuildLimiter := semaphore.NewWeighted(int64(dbg.BuildSnapshotAllowance)) + // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, logger) + blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, logger, segmentsBuildLimiter) if err != nil { return nil, err } @@ -788,10 +790,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger ethBackendRPC := privateapi.NewEthBackendServer(ctx, backend, backend.chainDB, backend.notifications, blockReader, logger, latestBlockBuiltStore) // initialize engine backend - blockSnapBuildSema := semaphore.NewWeighted(int64(dbg.BuildSnapshotAllowance)) - - agg.SetSnapshotBuildSema(blockSnapBuildSema) - blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, heimdallStore, bridgeStore, backend.chainConfig, config, backend.notifications.Events, blockSnapBuildSema, logger) + blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, heimdallStore, bridgeStore, backend.chainConfig, config, backend.notifications.Events, segmentsBuildLimiter, logger) miningRPC = privateapi.NewMiningServer(ctx, backend, ethashApi, logger) @@ -988,7 +987,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger config.CaplinConfig.LoopBlockLimit = uint64(config.LoopBlockLimit) go func() { eth1Getter := getters.NewExecutionSnapshotReader(ctx, blockReader, backend.chainDB) - if err := caplin1.RunCaplinService(ctx, executionEngine, config.CaplinConfig, dirs, eth1Getter, backend.downloaderClient, creds, blockSnapBuildSema); err != nil { + if err := caplin1.RunCaplinService(ctx, executionEngine, config.CaplinConfig, dirs, eth1Getter, backend.downloaderClient, creds, segmentsBuildLimiter); err != nil { logger.Error("could not start caplin", "err", err) } ctxCancel() @@ -1440,7 +1439,7 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl return err } -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, logger log.Logger) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *libstate.Aggregator, error) { +func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, logger log.Logger, blockSnapBuildSema *semaphore.Weighted) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *libstate.Aggregator, error) { var minFrozenBlock uint64 if frozenLimit := snConfig.Sync.FrozenBlockLimit; frozenLimit != 0 { @@ -1471,6 +1470,7 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } + agg.SetSnapshotBuildSema(blockSnapBuildSema) agg.SetProduceMod(snConfig.Snapshot.ProduceE3) allSegmentsDownloadComplete, err := rawdb.AllSegmentsDownloadCompleteFromDB(db) From bd519f06c4ba71c26126571bc0077262740eaec9 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Fri, 27 Dec 2024 17:44:28 +0100 Subject: [PATCH 26/94] Caplin: simplified cache-query (#13247) This PR should simplify the cache system during reorgs by removing caches which should not be bottlenecks anymore --- cl/phase1/core/state/cache.go | 220 +----------------- cl/phase1/core/state/ssz_test.go | 8 - .../fork_graph/fork_graph_disk_fs.go | 10 +- 3 files changed, 13 insertions(+), 225 deletions(-) diff --git a/cl/phase1/core/state/cache.go b/cl/phase1/core/state/cache.go index a04090888c7..a5180932831 100644 --- a/cl/phase1/core/state/cache.go +++ b/cl/phase1/core/state/cache.go @@ -20,7 +20,6 @@ import ( "crypto/sha256" "encoding/binary" "io" - "math" "runtime" "github.com/erigontech/erigon/cl/cltypes/solid" @@ -289,37 +288,6 @@ func (b *CachingBeaconState) InitBeaconState() error { // EncodeCaches, encodes the beacon state caches into a byte slice func (b *CachingBeaconState) EncodeCaches(w io.Writer) error { - num := make([]byte, 8) - // activeValidatorsCaches - if err := b.encodeActiveValidatorsCache(w, num); err != nil { - return err - } - // shuffledSetsCache - if err := b.encodeShuffledSetsCache(w, num); err != nil { - return err - } - // Now do the extra caches - if b.totalActiveBalanceCache == nil { - if err := binary.Write(w, binary.BigEndian, uint64(math.MaxUint64)); err != nil { - return err - } - } else { - if err := binary.Write(w, binary.BigEndian, *b.totalActiveBalanceCache); err != nil { - return err - } - } - if err := binary.Write(w, binary.BigEndian, b.totalActiveBalanceRootCache); err != nil { - return err - } - if b.proposerIndex == nil { - if err := binary.Write(w, binary.BigEndian, uint64(math.MaxUint64)); err != nil { - return err - } - } else { - if err := binary.Write(w, binary.BigEndian, *b.proposerIndex); err != nil { - return err - } - } if _, err := w.Write(b.previousStateRoot[:]); err != nil { return err } @@ -353,37 +321,11 @@ func (b *CachingBeaconState) EncodeCaches(w io.Writer) error { } func (b *CachingBeaconState) DecodeCaches(r io.Reader) error { - num := make([]byte, 8) - // activeValidatorsCaches - if err := b.decodeActiveValidatorsCache(r, num); err != nil { - return err - } - // shuffledSetsCache - if err := b.decodeShuffledSetsCache(r, num); err != nil { - return err - } - // Now do the extra caches - var totalActiveBalanceCache uint64 - if err := binary.Read(r, binary.BigEndian, &totalActiveBalanceCache); err != nil { - return err - } - if totalActiveBalanceCache == math.MaxUint64 { - b.totalActiveBalanceCache = nil - } else { - b.totalActiveBalanceCache = &totalActiveBalanceCache - } - if err := binary.Read(r, binary.BigEndian, &b.totalActiveBalanceRootCache); err != nil { - return err - } - var proposerIndex uint64 - if err := binary.Read(r, binary.BigEndian, &proposerIndex); err != nil { - return err - } - if proposerIndex == math.MaxUint64 { - b.proposerIndex = nil - } else { - b.proposerIndex = &proposerIndex - } + b.shuffledSetsCache, _ = lru.New[common.Hash, []uint64]("beacon_shuffled_sets_cache", shuffledSetsCacheSize) + b.activeValidatorsCache, _ = lru.New[uint64, []uint64]("beacon_active_validators_cache", activeValidatorsCacheSize) + b.totalActiveBalanceCache = nil + b.proposerIndex = nil + if _, err := r.Read(b.previousStateRoot[:]); err != nil { return err } @@ -414,157 +356,5 @@ func (b *CachingBeaconState) DecodeCaches(r io.Reader) error { } } - // if err := b.BeaconState.RandaoMixes().MerkleTree.Read(r); err != nil { - // return err - // } - return nil -} - -func writeUint64WithBuffer(w io.Writer, num uint64, buf []byte) error { - binary.BigEndian.PutUint64(buf, num) - if _, err := w.Write(buf); err != nil { - return err - } - return nil -} - -func readUint64WithBuffer(r io.Reader, buf []byte, out *uint64) error { - if _, err := r.Read(buf); err != nil { - return err - } - *out = binary.BigEndian.Uint64(buf) - return nil -} - -// internal encoding/decoding algos -func (b *CachingBeaconState) encodeActiveValidatorsCache(w io.Writer, num []byte) error { - keysA := b.activeValidatorsCache.Keys() - keys := make([]uint64, 0, len(keysA)) - lists := make([][]uint64, 0, len(keys)) - for _, key := range keysA { - l, ok := b.activeValidatorsCache.Get(key) - if !ok || len(l) == 0 { - continue - } - keys = append(keys, key) - lists = append(lists, l) - } - // Write the total length - if err := writeUint64WithBuffer(w, uint64(len(keys)), num); err != nil { - return err - } - - for i, key := range keys { - if err := writeUint64WithBuffer(w, uint64(len(lists[i])), num); err != nil { - return err - } - if err := writeUint64WithBuffer(w, key, num); err != nil { - return err - } - for _, v := range lists[i] { - if err := writeUint64WithBuffer(w, v, num); err != nil { - return err - } - } - } - - return nil -} - -func (b *CachingBeaconState) decodeActiveValidatorsCache(r io.Reader, num []byte) error { - var err error - b.activeValidatorsCache, err = lru.New[uint64, []uint64]("beacon_active_validators_cache", activeValidatorsCacheSize) - if err != nil { - return err - } - var l uint64 - - if err := readUint64WithBuffer(r, num, &l); err != nil { - return err - } - for i := 0; i < int(l); i++ { - var l uint64 - - if err := readUint64WithBuffer(r, num, &l); err != nil { - return err - } - var key uint64 - if err := readUint64WithBuffer(r, num, &key); err != nil { - return err - } - list := make([]uint64, l) - for i := 0; i < int(l); i++ { - if err := readUint64WithBuffer(r, num, &list[i]); err != nil { - return err - } - } - b.activeValidatorsCache.Add(key, list) - } - return nil -} - -// internal encoding/decoding algos -func (b *CachingBeaconState) encodeShuffledSetsCache(w io.Writer, num []byte) error { - keysA := b.shuffledSetsCache.Keys() - keys := make([]common.Hash, 0, len(keysA)) - lists := make([][]uint64, 0, len(keys)) - - for _, key := range keysA { - l, ok := b.shuffledSetsCache.Get(key) - if !ok || len(l) == 0 { - continue - } - keys = append(keys, key) - lists = append(lists, l) - } - // Write the total length - if err := writeUint64WithBuffer(w, uint64(len(keys)), num); err != nil { - return err - } - for i, key := range keys { - if err := writeUint64WithBuffer(w, uint64(len(lists[i])), num); err != nil { - return err - } - if _, err := w.Write(key[:]); err != nil { - return err - } - for _, v := range lists[i] { - if err := writeUint64WithBuffer(w, v, num); err != nil { - return err - } - } - } - return nil -} - -func (b *CachingBeaconState) decodeShuffledSetsCache(r io.Reader, num []byte) error { - var err error - b.shuffledSetsCache, err = lru.New[common.Hash, []uint64]("beacon_shuffled_sets_cache", shuffledSetsCacheSize) - if err != nil { - return err - } - - var l uint64 - if err := readUint64WithBuffer(r, num, &l); err != nil { - return err - } - for i := 0; i < int(l); i++ { - var l uint64 - if err := readUint64WithBuffer(r, num, &l); err != nil { - return err - } - var key common.Hash - if _, err := r.Read(key[:]); err != nil { - return err - } - list := make([]uint64, l) - for i := 0; i < int(l); i++ { - if err := readUint64WithBuffer(r, num, &list[i]); err != nil { - return err - } - } - b.shuffledSetsCache.Add(key, list) - } - return nil } diff --git a/cl/phase1/core/state/ssz_test.go b/cl/phase1/core/state/ssz_test.go index 18ccb075a8c..3b12c0d642f 100644 --- a/cl/phase1/core/state/ssz_test.go +++ b/cl/phase1/core/state/ssz_test.go @@ -56,14 +56,6 @@ func TestBeaconStatePhase0EncodingDecoding(t *testing.T) { // Lets test the caches too var w bytes.Buffer require.NoError(t, state.EncodeCaches(&w)) - values1 := state.activeValidatorsCache.Values() - keys1 := state.activeValidatorsCache.Keys() - values2 := state.shuffledSetsCache.Values() - keys2 := state.shuffledSetsCache.Keys() require.NoError(t, state.DecodeCaches(&w)) - require.Equal(t, values1, state.activeValidatorsCache.Values()) - require.Equal(t, keys1, state.activeValidatorsCache.Keys()) - require.Equal(t, values2, state.shuffledSetsCache.Values()) - require.Equal(t, keys2, state.shuffledSetsCache.Keys()) } diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index 4757ea7307b..7f32c315129 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -111,6 +111,13 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot libcommon.Hash, bs *stat if !forced && bs.Slot()%dumpSlotFrequency != 0 { return } + f.stateDumpLock.Lock() + unlockOnDefer := true + defer func() { + if unlockOnDefer { + f.stateDumpLock.Unlock() + } + }() // Truncate and then grow the buffer to the size of the state. f.sszBuffer, err = bs.EncodeSSZ(f.sszBuffer[:0]) if err != nil { @@ -163,8 +170,7 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot libcommon.Hash, bs *stat log.Error("failed to sync dumped file", "err", err) return } - - f.stateDumpLock.Lock() + unlockOnDefer = false go func() { cacheFile, err := f.fs.OpenFile(getBeaconStateCacheFilename(blockRoot), os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0o755) if err != nil { From 9bfa6b6a661dc488aa3b6184c984b989550c1efa Mon Sep 17 00:00:00 2001 From: awskii Date: Fri, 27 Dec 2024 21:55:48 +0000 Subject: [PATCH 27/94] remove .go suffix from `eth1_chain_reader` package (#13257) --- cl/phase1/execution_client/execution_client_direct.go | 2 +- eth/backend.go | 2 +- turbo/app/import_cmd.go | 2 +- turbo/engineapi/engine_block_downloader/block_downloader.go | 2 +- turbo/engineapi/engine_server.go | 2 +- .../{eth1_chain_reader.go => eth1_chain_reader}/chain_reader.go | 0 turbo/stages/mock/mock_sentry.go | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename turbo/execution/eth1/{eth1_chain_reader.go => eth1_chain_reader}/chain_reader.go (100%) diff --git a/cl/phase1/execution_client/execution_client_direct.go b/cl/phase1/execution_client/execution_client_direct.go index 0955c942197..f1c3b4b0f2e 100644 --- a/cl/phase1/execution_client/execution_client_direct.go +++ b/cl/phase1/execution_client/execution_client_direct.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/turbo/engineapi/engine_types" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" ) type ExecutionClientDirect struct { diff --git a/eth/backend.go b/eth/backend.go index 12ebf19aa1a..4cd0655e5dc 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -122,7 +122,7 @@ import ( "github.com/erigontech/erigon/turbo/engineapi/engine_block_downloader" "github.com/erigontech/erigon/turbo/engineapi/engine_helpers" "github.com/erigontech/erigon/turbo/execution/eth1" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" "github.com/erigontech/erigon/turbo/jsonrpc" "github.com/erigontech/erigon/turbo/services" "github.com/erigontech/erigon/turbo/shards" diff --git a/turbo/app/import_cmd.go b/turbo/app/import_cmd.go index dd776226d03..4bab92e2fec 100644 --- a/turbo/app/import_cmd.go +++ b/turbo/app/import_cmd.go @@ -37,7 +37,7 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/wrap" "github.com/erigontech/erigon/consensus/merge" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" "github.com/erigontech/erigon/turbo/services" "github.com/erigontech/erigon-lib/rlp" diff --git a/turbo/engineapi/engine_block_downloader/block_downloader.go b/turbo/engineapi/engine_block_downloader/block_downloader.go index a516896d265..e04ba7567b2 100644 --- a/turbo/engineapi/engine_block_downloader/block_downloader.go +++ b/turbo/engineapi/engine_block_downloader/block_downloader.go @@ -39,7 +39,7 @@ import ( "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/turbo/adapter" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" "github.com/erigontech/erigon/turbo/services" "github.com/erigontech/erigon/turbo/stages/bodydownload" "github.com/erigontech/erigon/turbo/stages/headerdownload" diff --git a/turbo/engineapi/engine_server.go b/turbo/engineapi/engine_server.go index e79378c4355..e5d1378c7ce 100644 --- a/turbo/engineapi/engine_server.go +++ b/turbo/engineapi/engine_server.go @@ -47,7 +47,7 @@ import ( "github.com/erigontech/erigon/turbo/engineapi/engine_helpers" "github.com/erigontech/erigon/turbo/engineapi/engine_logs_spammer" "github.com/erigontech/erigon/turbo/engineapi/engine_types" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" "github.com/erigontech/erigon/turbo/jsonrpc" "github.com/erigontech/erigon/turbo/rpchelper" "github.com/erigontech/erigon/turbo/services" diff --git a/turbo/execution/eth1/eth1_chain_reader.go/chain_reader.go b/turbo/execution/eth1/eth1_chain_reader/chain_reader.go similarity index 100% rename from turbo/execution/eth1/eth1_chain_reader.go/chain_reader.go rename to turbo/execution/eth1/eth1_chain_reader/chain_reader.go diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 3be2cb7f03a..2e015df02c0 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -80,7 +80,7 @@ import ( "github.com/erigontech/erigon/turbo/builder" "github.com/erigontech/erigon/turbo/engineapi/engine_helpers" "github.com/erigontech/erigon/turbo/execution/eth1" - "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader.go" + "github.com/erigontech/erigon/turbo/execution/eth1/eth1_chain_reader" "github.com/erigontech/erigon/turbo/jsonrpc/receipts" "github.com/erigontech/erigon/turbo/rpchelper" "github.com/erigontech/erigon/turbo/services" From 033f86cfb3e43b2a4e723b76f006b9672bc05b53 Mon Sep 17 00:00:00 2001 From: sudeep Date: Sat, 28 Dec 2024 08:03:11 +0530 Subject: [PATCH 28/94] remove agg from SnapshotsCfg (#13253) https://github.com/erigontech/erigon/issues/13240 --- cmd/integration/commands/stages.go | 2 +- eth/backend.go | 9 ++++----- eth/stagedsync/stage_snapshots.go | 13 ++++++------- turbo/stages/mock/mock_sentry.go | 6 +++--- turbo/stages/stageloop.go | 12 ++++-------- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 8435fcdcc9a..478f50a3b41 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1486,7 +1486,7 @@ func newSync(ctx context.Context, db kv.TemporalRwDB, miningConfig *params.Minin bridgeStore = bridge.NewSnapshotStore(bridge.NewDbStore(db), borSn, chainConfig.Bor) heimdallStore = heimdall.NewSnapshotStore(heimdall.NewDbStore(db), borSn) } - stageList := stages2.NewDefaultStages(context.Background(), db, snapDb, p2p.Config{}, &cfg, sentryControlServer, notifications, nil, blockReader, blockRetire, agg, nil, nil, + stageList := stages2.NewDefaultStages(context.Background(), db, snapDb, p2p.Config{}, &cfg, sentryControlServer, notifications, nil, blockReader, blockRetire, nil, nil, heimdallClient, heimdallStore, bridgeStore, recents, signatures, logger) sync := stagedsync.New(cfg.Sync, stageList, stagedsync.DefaultUnwindOrder, stagedsync.DefaultPruneOrder, logger, stages.ModeApplyingBlocks) diff --git a/eth/backend.go b/eth/backend.go index 4cd0655e5dc..55677cc86f7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -580,7 +580,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger terseLogger.SetHandler(log.LvlFilterHandler(log.LvlWarn, log.StderrHandler)) // Needs its own notifications to not update RPC daemon and txpool about pending blocks stateSync := stages2.NewInMemoryExecution(backend.sentryCtx, backend.chainDB, config, backend.sentriesClient, - dirs, notifications, blockReader, blockWriter, backend.agg, backend.silkworm, terseLogger) + dirs, notifications, blockReader, blockWriter, backend.silkworm, terseLogger) chainReader := consensuschain.NewReader(chainConfig, txc.Tx, blockReader, logger) // We start the mining step if err := stages2.StateStep(ctx, chainReader, backend.engine, txc, stateSync, header, body, unwindPoint, headersChain, bodiesChain, config.ImportMode); err != nil { @@ -909,7 +909,6 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.downloaderClient, blockReader, blockRetire, - backend.agg, backend.silkworm, backend.forkValidator, heimdallClient, @@ -924,7 +923,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.syncPruneOrder = stagedsync.PolygonSyncPruneOrder } else { backend.syncStages = stages2.NewDefaultStages(backend.sentryCtx, backend.chainDB, snapDb, p2pConfig, config, backend.sentriesClient, backend.notifications, backend.downloaderClient, - blockReader, blockRetire, backend.agg, backend.silkworm, backend.forkValidator, heimdallClient, heimdallStore, bridgeStore, recents, signatures, logger) + blockReader, blockRetire, backend.silkworm, backend.forkValidator, heimdallClient, heimdallStore, bridgeStore, recents, signatures, logger) backend.syncUnwindOrder = stagedsync.DefaultUnwindOrder backend.syncPruneOrder = stagedsync.DefaultPruneOrder } @@ -956,7 +955,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger } checkStateRoot := true - pipelineStages := stages2.NewPipelineStages(ctx, backend.chainDB, config, p2pConfig, backend.sentriesClient, backend.notifications, backend.downloaderClient, blockReader, blockRetire, backend.agg, backend.silkworm, backend.forkValidator, logger, checkStateRoot) + pipelineStages := stages2.NewPipelineStages(ctx, backend.chainDB, config, p2pConfig, backend.sentriesClient, backend.notifications, backend.downloaderClient, blockReader, blockRetire, backend.silkworm, backend.forkValidator, logger, checkStateRoot) backend.pipelineStagedSync = stagedsync.New(config.Sync, pipelineStages, stagedsync.PipelineUnwindOrder, stagedsync.PipelinePruneOrder, logger, stages.ModeApplyingBlocks) backend.eth1ExecutionServer = eth1.NewEthereumExecutionModule(blockReader, backend.chainDB, backend.pipelineStagedSync, backend.forkValidator, chainConfig, assembleBlockPOS, hook, backend.notifications.Accumulator, backend.notifications.StateChangesConsumer, logger, backend.engine, config.Sync, ctx) executionRpc := direct.NewExecutionClientDirect(backend.eth1ExecutionServer) @@ -1015,7 +1014,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.polygonDownloadSync = stagedsync.New(backend.config.Sync, stagedsync.DownloadSyncStages( backend.sentryCtx, stagedsync.StageSnapshotsCfg( backend.chainDB, *backend.sentriesClient.ChainConfig, config.Sync, dirs, blockRetire, backend.downloaderClient, - blockReader, backend.notifications, backend.agg, false, false, false, backend.silkworm, config.Prune, + blockReader, backend.notifications, false, false, false, backend.silkworm, config.Prune, )), nil, nil, backend.logger, stages.ModeApplyingBlocks) // these range extractors set the db to the local db instead of the chain db diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 29e81f0e0e3..01090754d39 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -37,6 +37,7 @@ import ( "time" "github.com/erigontech/erigon-lib/common" + state2 "github.com/erigontech/erigon-lib/state" "github.com/anacrolix/torrent" "golang.org/x/sync/errgroup" @@ -96,7 +97,6 @@ type SnapshotsCfg struct { caplin bool blobs bool caplinState bool - agg *state.Aggregator silkworm *silkworm.Silkworm snapshotUploader *snapshotUploader syncConfig ethconfig.Sync @@ -111,7 +111,6 @@ func StageSnapshotsCfg(db kv.RwDB, snapshotDownloader protodownloader.DownloaderClient, blockReader services.FullBlockReader, notifier *shards.Notifications, - agg *state.Aggregator, caplin bool, blobs bool, caplinState bool, @@ -127,7 +126,6 @@ func StageSnapshotsCfg(db kv.RwDB, blockReader: blockReader, notifier: notifier, caplin: caplin, - agg: agg, silkworm: silkworm, syncConfig: syncConfig, blobs: blobs, @@ -279,8 +277,9 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R }) diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Download header-chain"}) + agg := cfg.db.(*temporal.DB).Agg().(*state2.Aggregator) // Download only the snapshots that are for the header chain. - if err := snapshotsync.WaitForDownloader(ctx, s.LogPrefix(), cfg.dirs, true /*headerChain=*/, cfg.blobs, cfg.caplinState, cfg.prune, cstate, cfg.agg, tx, cfg.blockReader, &cfg.chainConfig, cfg.snapshotDownloader, s.state.StagesIdsList()); err != nil { + if err := snapshotsync.WaitForDownloader(ctx, s.LogPrefix(), cfg.dirs, true /*headerChain=*/, cfg.blobs, cfg.caplinState, cfg.prune, cstate, agg, tx, cfg.blockReader, &cfg.chainConfig, cfg.snapshotDownloader, s.state.StagesIdsList()); err != nil { return err } @@ -289,7 +288,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R } diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Download snapshots"}) - if err := snapshotsync.WaitForDownloader(ctx, s.LogPrefix(), cfg.dirs, false /*headerChain=*/, cfg.blobs, cfg.caplinState, cfg.prune, cstate, cfg.agg, tx, cfg.blockReader, &cfg.chainConfig, cfg.snapshotDownloader, s.state.StagesIdsList()); err != nil { + if err := snapshotsync.WaitForDownloader(ctx, s.LogPrefix(), cfg.dirs, false /*headerChain=*/, cfg.blobs, cfg.caplinState, cfg.prune, cstate, agg, tx, cfg.blockReader, &cfg.chainConfig, cfg.snapshotDownloader, s.state.StagesIdsList()); err != nil { return err } if cfg.notifier.Events != nil { @@ -309,7 +308,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R indexWorkers := estimate.IndexSnapshot.Workers() diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "E3 Indexing"}) - if err := cfg.agg.BuildMissedIndices(ctx, indexWorkers); err != nil { + if err := agg.BuildMissedIndices(ctx, indexWorkers); err != nil { return err } @@ -331,7 +330,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R } diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Fill DB"}) - if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, cfg.agg, logger); err != nil { + if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, agg, logger); err != nil { return err } diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 2e015df02c0..529b4cbfb89 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -382,7 +382,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK terseLogger.SetHandler(log.LvlFilterHandler(log.LvlWarn, log.StderrHandler)) // Needs its own notifications to not update RPC daemon and txpool about pending blocks stateSync := stages2.NewInMemoryExecution(mock.Ctx, mock.DB, &cfg, mock.sentriesClient, - dirs, notifications, mock.BlockReader, blockWriter, mock.agg, nil, terseLogger) + dirs, notifications, mock.BlockReader, blockWriter, nil, terseLogger) chainReader := consensuschain.NewReader(mock.ChainConfig, txc.Tx, mock.BlockReader, logger) // We start the mining step if err := stages2.StateStep(ctx, chainReader, mock.Engine, txc, stateSync, header, body, unwindPoint, headersChain, bodiesChain, true); err != nil { @@ -524,7 +524,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK mock.agg.SetProduceMod(mock.BlockReader.FreezingCfg().ProduceE3) mock.Sync = stagedsync.New( cfg.Sync, - stagedsync.DefaultStages(mock.Ctx, stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, mock.BlockReader, mock.Notifications, mock.agg, false, false, false, nil, prune), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, cfg.Sync, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications), stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil, nil, nil, mock.BlockReader, nil, nil, recents, signatures, false, nil), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(mock.DB, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, blockWriter), stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, cfg.Sync, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), stagedsync.StageExecuteBlocksCfg( + stagedsync.DefaultStages(mock.Ctx, stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, mock.BlockReader, mock.Notifications, false, false, false, nil, prune), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, cfg.Sync, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications), stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil, nil, nil, mock.BlockReader, nil, nil, recents, signatures, false, nil), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(mock.DB, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, blockWriter), stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, cfg.Sync, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), stagedsync.StageExecuteBlocksCfg( mock.DB, prune, cfg.BatchSize, @@ -548,7 +548,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK cfg.Genesis = gspec pipelineStages := stages2.NewPipelineStages(mock.Ctx, db, &cfg, p2p.Config{}, mock.sentriesClient, mock.Notifications, - snapDownloader, mock.BlockReader, blockRetire, mock.agg, nil, forkValidator, logger, checkStateRoot) + snapDownloader, mock.BlockReader, blockRetire, nil, forkValidator, logger, checkStateRoot) mock.posStagedSync = stagedsync.New(cfg.Sync, pipelineStages, stagedsync.PipelineUnwindOrder, stagedsync.PipelinePruneOrder, logger, stages.ModeApplyingBlocks) mock.Eth1ExecutionService = eth1.NewEthereumExecutionModule(mock.BlockReader, mock.DB, mock.posStagedSync, forkValidator, mock.ChainConfig, assembleBlockPOS, nil, mock.Notifications.Accumulator, mock.Notifications.StateChangesConsumer, logger, engine, cfg.Sync, ctx) diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index acbb47371f3..716926b6a4c 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -673,7 +673,6 @@ func NewDefaultStages(ctx context.Context, snapDownloader proto_downloader.DownloaderClient, blockReader services.FullBlockReader, blockRetire services.BlockRetire, - agg *state.Aggregator, silkworm *silkworm.Silkworm, forkValidator *engine_helpers.ForkValidator, heimdallClient heimdall.Client, @@ -691,7 +690,7 @@ func NewDefaultStages(ctx context.Context, runInTestMode := cfg.ImportMode return stagedsync.DefaultStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, agg, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, heimdallStore, bridgeStore, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures, cfg.WithHeimdallWaypointRecording, nil), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), @@ -711,7 +710,6 @@ func NewPipelineStages(ctx context.Context, snapDownloader proto_downloader.DownloaderClient, blockReader services.FullBlockReader, blockRetire services.BlockRetire, - agg *state.Aggregator, silkworm *silkworm.Silkworm, forkValidator *engine_helpers.ForkValidator, logger log.Logger, @@ -732,7 +730,7 @@ func NewPipelineStages(ctx context.Context, if len(cfg.Sync.UploadLocation) == 0 { return stagedsync.PipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, agg, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg(db, cfg.Prune, cfg.BatchSize, controlServer.ChainConfig, controlServer.Engine, &vm.Config{}, notifications, cfg.StateStream, false, dirs, blockReader, controlServer.Hd, cfg.Genesis, cfg.Sync, SilkwormForExecutionStage(silkworm, cfg)), @@ -741,7 +739,7 @@ func NewPipelineStages(ctx context.Context, } return stagedsync.UploaderPipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, agg, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), @@ -751,7 +749,7 @@ func NewPipelineStages(ctx context.Context, } func NewInMemoryExecution(ctx context.Context, db kv.RwDB, cfg *ethconfig.Config, controlServer *sentry_multi_client.MultiClient, - dirs datadir.Dirs, notifications *shards.Notifications, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, agg *state.Aggregator, + dirs datadir.Dirs, notifications *shards.Notifications, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, silkworm *silkworm.Silkworm, logger log.Logger) *stagedsync.Sync { return stagedsync.New( cfg.Sync, @@ -776,7 +774,6 @@ func NewPolygonSyncStages( snapDownloader proto_downloader.DownloaderClient, blockReader services.FullBlockReader, blockRetire services.BlockRetire, - agg *state.Aggregator, silkworm *silkworm.Silkworm, forkValidator *engine_helpers.ForkValidator, heimdallClient heimdall.Client, @@ -798,7 +795,6 @@ func NewPolygonSyncStages( snapDownloader, blockReader, notifications, - agg, config.InternalCL && config.CaplinConfig.Backfilling, config.CaplinConfig.BlobBackfilling, config.CaplinConfig.Archive, From ab8c054a7179072bb12fa30c94dbb28f008c28d3 Mon Sep 17 00:00:00 2001 From: sudeep Date: Sat, 28 Dec 2024 13:18:53 +0530 Subject: [PATCH 29/94] remove agg from Ethereum struct (#13254) --- erigon-lib/kv/kv_interface.go | 6 ++++++ erigon-lib/kv/remotedb/kv_remote.go | 1 + erigon-lib/kv/temporal/kv_temporal.go | 7 +++++++ erigon-lib/state/aggregator.go | 8 +++----- eth/backend.go | 8 ++------ 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/erigon-lib/kv/kv_interface.go b/erigon-lib/kv/kv_interface.go index 0a5327c6458..f8832cbbb02 100644 --- a/erigon-lib/kv/kv_interface.go +++ b/erigon-lib/kv/kv_interface.go @@ -229,6 +229,11 @@ type Closer interface { Close() } +type OnFreezeFunc func(frozenFileNames []string) +type SnapshotNotifier interface { + OnFreeze(f OnFreezeFunc) +} + // RoDB - Read-only version of KV. type RoDB interface { Closer @@ -516,6 +521,7 @@ type TemporalPutDel interface { type TemporalRoDB interface { RoDB + SnapshotNotifier ViewTemporal(ctx context.Context, f func(tx TemporalTx) error) error BeginTemporalRo(ctx context.Context) (TemporalTx, error) } diff --git a/erigon-lib/kv/remotedb/kv_remote.go b/erigon-lib/kv/remotedb/kv_remote.go index 6e64863ebb4..43a3d34cccf 100644 --- a/erigon-lib/kv/remotedb/kv_remote.go +++ b/erigon-lib/kv/remotedb/kv_remote.go @@ -228,6 +228,7 @@ func (db *DB) Update(ctx context.Context, f func(tx kv.RwTx) error) (err error) func (db *DB) UpdateNosync(ctx context.Context, f func(tx kv.RwTx) error) (err error) { return errors.New("remote db provider doesn't support .UpdateNosync method") } +func (db *DB) OnFreeze(f kv.OnFreezeFunc) { panic("not implemented") } func (tx *tx) ViewID() uint64 { return tx.viewID } func (tx *tx) CollectMetrics() {} diff --git a/erigon-lib/kv/temporal/kv_temporal.go b/erigon-lib/kv/temporal/kv_temporal.go index ebe1367dd0b..399af03919e 100644 --- a/erigon-lib/kv/temporal/kv_temporal.go +++ b/erigon-lib/kv/temporal/kv_temporal.go @@ -152,6 +152,13 @@ func (db *DB) UpdateNosync(ctx context.Context, f func(tx kv.RwTx) error) error return tx.Commit() } +func (db *DB) Close() { + db.RwDB.Close() + db.agg.Close() +} + +func (db *DB) OnFreeze(f kv.OnFreezeFunc) { db.agg.OnFreeze(f) } + type Tx struct { *mdbx.MdbxTx db *DB diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index fac019852f5..ac1bccda2d1 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -82,7 +82,7 @@ type Aggregator struct { wg sync.WaitGroup // goroutines spawned by Aggregator, to ensure all of them are finish at agg.Close - onFreeze OnFreezeFunc + onFreeze kv.OnFreezeFunc ps *background.ProgressSet @@ -95,8 +95,6 @@ type Aggregator struct { produce bool } -type OnFreezeFunc func(frozenFileNames []string) - const AggregatorSqueezeCommitmentValues = true const MaxNonFuriousDirtySpacePerTx = 64 * datasize.MB @@ -377,8 +375,8 @@ func (a *Aggregator) registerII(idx kv.InvertedIdxPos, salt *uint32, dirs datadi return nil } -func (a *Aggregator) StepSize() uint64 { return a.aggregationStep } -func (a *Aggregator) OnFreeze(f OnFreezeFunc) { a.onFreeze = f } +func (a *Aggregator) StepSize() uint64 { return a.aggregationStep } +func (a *Aggregator) OnFreeze(f kv.OnFreezeFunc) { a.onFreeze = f } func (a *Aggregator) DisableFsync() { for _, d := range a.d { d.DisableFsync() diff --git a/eth/backend.go b/eth/backend.go index 55677cc86f7..e1b1edcb6d2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -204,7 +204,6 @@ type Ethereum struct { forkValidator *engine_helpers.ForkValidator downloader *downloader.Downloader - agg *libstate.Aggregator blockSnapshots *freezeblocks.RoSnapshots blockReader services.FullBlockReader blockWriter *blockio.BlockWriter @@ -343,7 +342,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger return nil, err } - backend.agg, backend.blockSnapshots, backend.blockReader, backend.blockWriter = agg, allSnapshots, blockReader, blockWriter + backend.blockSnapshots, backend.blockReader, backend.blockWriter = allSnapshots, blockReader, blockWriter backend.chainDB, err = temporal.New(rawChainDB, agg) if err != nil { @@ -1420,7 +1419,7 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl s.downloaderClient = direct.NewDownloaderClient(bittorrentServer) } - s.agg.OnFreeze(func(frozenFileNames []string) { + s.chainDB.OnFreeze(func(frozenFileNames []string) { events := s.notifications.Events events.OnNewSnapshot() if s.downloaderClient != nil { @@ -1643,9 +1642,6 @@ func (s *Ethereum) Stop() error { if s.txPoolDB != nil { s.txPoolDB.Close() } - if s.agg != nil { - s.agg.Close() - } s.chainDB.Close() if s.silkwormRPCDaemonService != nil { From 4730f008f2cf949622245f3bbabd80b3953b8f28 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Sun, 29 Dec 2024 03:31:05 +0100 Subject: [PATCH 30/94] E3: Remove unused "PruneNonEssentials" feauture (#13255) This is an unused feauture --- core/state/rw_v3.go | 11 ----------- core/state/txtask.go | 37 ++++++++++++++++++------------------- eth/stagedsync/exec3.go | 31 ++++++++++++++----------------- 3 files changed, 32 insertions(+), 47 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index c3f14e3bb6c..a0f92c4e475 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -210,30 +210,19 @@ func (rs *StateV3) ApplyState4(ctx context.Context, txTask *TxTask) error { } func (rs *StateV3) ApplyLogsAndTraces4(txTask *TxTask, domains *libstate.SharedDomains) error { - shouldPruneNonEssentials := txTask.PruneNonEssentials && txTask.Config != nil - for addr := range txTask.TraceFroms { - if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { - continue - } if err := domains.IndexAdd(kv.TblTracesFromIdx, addr[:]); err != nil { return err } } for addr := range txTask.TraceTos { - if shouldPruneNonEssentials && addr != txTask.Config.DepositContract { - continue - } if err := domains.IndexAdd(kv.TblTracesToIdx, addr[:]); err != nil { return err } } for _, lg := range txTask.Logs { - if shouldPruneNonEssentials && lg.Address != txTask.Config.DepositContract { - continue - } if err := domains.IndexAdd(kv.TblLogAddressIdx, lg.Address[:]); err != nil { return err } diff --git a/core/state/txtask.go b/core/state/txtask.go index 1d49438897e..61bd90d132a 100644 --- a/core/state/txtask.go +++ b/core/state/txtask.go @@ -42,25 +42,24 @@ import ( // which is processed by a single thread that writes into the ReconState1 and // flushes to the database type TxTask struct { - TxNum uint64 - BlockNum uint64 - Rules *chain.Rules - Header *types.Header - Txs types.Transactions - Uncles []*types.Header - Coinbase libcommon.Address - Withdrawals types.Withdrawals - BlockHash libcommon.Hash - sender *libcommon.Address - SkipAnalysis bool - PruneNonEssentials bool - TxIndex int // -1 for block initialisation - Final bool - Failed bool - Tx types.Transaction - GetHashFn func(n uint64) libcommon.Hash - TxAsMessage types.Message - EvmBlockContext evmtypes.BlockContext + TxNum uint64 + BlockNum uint64 + Rules *chain.Rules + Header *types.Header + Txs types.Transactions + Uncles []*types.Header + Coinbase libcommon.Address + Withdrawals types.Withdrawals + BlockHash libcommon.Hash + sender *libcommon.Address + SkipAnalysis bool + TxIndex int // -1 for block initialisation + Final bool + Failed bool + Tx types.Transaction + GetHashFn func(n uint64) libcommon.Hash + TxAsMessage types.Message + EvmBlockContext evmtypes.BlockContext HistoryExecution bool // use history reader for that txn instead of state reader diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 89710facaae..0c8c06fbe23 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -245,8 +245,6 @@ func ExecV3(ctx context.Context, agg.SetCollateAndBuildWorkers(1) } - pruneNonEssentials := cfg.prune.History.Enabled() && cfg.prune.History.PruneTo(execStage.BlockNumber) == execStage.BlockNumber - var err error inMemExec := txc.Doms != nil var doms *state2.SharedDomains @@ -525,21 +523,20 @@ Loop: for txIndex := -1; txIndex <= len(txs); txIndex++ { // Do not oversend, wait for the result heap to go under certain size txTask := &state.TxTask{ - BlockNum: blockNum, - Header: header, - Coinbase: b.Coinbase(), - Uncles: b.Uncles(), - Rules: rules, - Txs: txs, - TxNum: inputTxNum, - TxIndex: txIndex, - BlockHash: b.Hash(), - SkipAnalysis: skipAnalysis, - Final: txIndex == len(txs), - GetHashFn: getHashFn, - EvmBlockContext: blockContext, - Withdrawals: b.Withdrawals(), - PruneNonEssentials: pruneNonEssentials, + BlockNum: blockNum, + Header: header, + Coinbase: b.Coinbase(), + Uncles: b.Uncles(), + Rules: rules, + Txs: txs, + TxNum: inputTxNum, + TxIndex: txIndex, + BlockHash: b.Hash(), + SkipAnalysis: skipAnalysis, + Final: txIndex == len(txs), + GetHashFn: getHashFn, + EvmBlockContext: blockContext, + Withdrawals: b.Withdrawals(), // use history reader instead of state reader to catch up to the tx where we left off HistoryExecution: offsetFromBlockBeginning > 0 && txIndex < int(offsetFromBlockBeginning), From 282c8a67f13174b70651bc6045484213abb46126 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Sun, 29 Dec 2024 03:31:43 +0100 Subject: [PATCH 31/94] Caplin: fix race in observer for bandwidth control (#13265) --- cl/sentinel/sentinel.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cl/sentinel/sentinel.go b/cl/sentinel/sentinel.go index 4e6e13f2301..0bde7a755ff 100644 --- a/cl/sentinel/sentinel.go +++ b/cl/sentinel/sentinel.go @@ -93,6 +93,7 @@ type Sentinel struct { blockReader freezeblocks.BeaconSnapshotReader blobStorage blob_storage.BlobStorage + bwc *metrics.BandwidthCounter indiciesDB kv.RoDB @@ -238,9 +239,9 @@ func New( if err != nil { return nil, err } - bwc := metrics.NewBandwidthCounter() + s.bwc = metrics.NewBandwidthCounter() - opts = append(opts, libp2p.ConnectionGater(gater), libp2p.BandwidthReporter(bwc)) + opts = append(opts, libp2p.ConnectionGater(gater), libp2p.BandwidthReporter(s.bwc)) host, err := libp2p.New(opts...) signal.Reset(syscall.SIGINT) @@ -248,7 +249,6 @@ func New( return nil, err } s.host = host - go s.observeBandwidth(ctx, bwc) s.peers = peers.NewPool() mux := chi.NewRouter() @@ -267,7 +267,7 @@ func New( return s, nil } -func (s *Sentinel) observeBandwidth(ctx context.Context, bwc *metrics.BandwidthCounter) { +func (s *Sentinel) observeBandwidth(ctx context.Context) { ticker := time.NewTicker(200 * time.Millisecond) for { countSubnetsSubscribed := func() int { @@ -296,7 +296,7 @@ func (s *Sentinel) observeBandwidth(ctx context.Context, bwc *metrics.BandwidthC case <-ctx.Done(): return case <-ticker.C: - totals := bwc.GetBandwidthTotals() + totals := s.bwc.GetBandwidthTotals() monitor.ObserveTotalInBytes(totals.TotalIn) monitor.ObserveTotalOutBytes(totals.TotalOut) minBound := datasize.KB @@ -315,7 +315,7 @@ func (s *Sentinel) observeBandwidth(ctx context.Context, bwc *metrics.BandwidthC // Check which peers should be banned for _, p := range peers { // get peer bandwidth - peerBandwidth := bwc.GetBandwidthForPeer(p) + peerBandwidth := s.bwc.GetBandwidthForPeer(p) // check if peer is over limit if peerBandwidth.RateIn > maxRateIn || peerBandwidth.RateOut > maxRateOut { peersToBan = append(peersToBan, p) @@ -369,6 +369,7 @@ func (s *Sentinel) Start() error { go s.listenForPeers() go s.forkWatcher() + go s.observeBandwidth(s.ctx) return nil } From a11af15f2dd0b6f5c66901bbda97889af6b84f6b Mon Sep 17 00:00:00 2001 From: Willian Mitsuda Date: Mon, 30 Dec 2024 00:16:12 -0300 Subject: [PATCH 32/94] Fix panic (#13270) fix #13269 --- erigon-lib/state/aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index ac1bccda2d1..6123509a46d 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -1056,7 +1056,7 @@ func (ac *AggregatorRoTx) PruneSmallBatchesDb(ctx context.Context, timeout time. ac.a.logger.Info("[snapshots] pruning state", "until commit", time.Until(started.Add(timeout)).String(), "pruneLimit", pruneLimit, - "aggregatedStep", ac.StepsInFiles(), + "aggregatedStep", ac.StepsInFiles(kv.StateDomains...), "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), "pruned", fullStat.String(), ) @@ -1150,7 +1150,7 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du ac.a.logger.Info("[snapshots] pruning state", "until commit", time.Until(started.Add(timeout)).String(), "pruneLimit", pruneLimit, - "aggregatedStep", ac.StepsInFiles(), + "aggregatedStep", ac.StepsInFiles(kv.StateDomains...), "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), "pruned", fullStat.String(), ) From 6f33aa719a30462f2bdb807fb092e369755a7c75 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 30 Dec 2024 17:38:23 +0700 Subject: [PATCH 33/94] idx tooling (#13271) - `erigon seg meta` print details of .idx - `integration stage_exec --sync.mode.chaintip` supports `--no-commit` - --- cmd/integration/commands/stages.go | 18 ++++++++++++++-- core/snaptype/block_types.go | 16 +++++++------- erigon-lib/recsplit/eliasfano16/elias_fano.go | 2 ++ erigon-lib/recsplit/eliasfano32/elias_fano.go | 3 +++ .../recsplit/eliasfano32/elias_fano_test.go | 2 +- erigon-lib/recsplit/index.go | 21 +++++++++++++++---- erigon-lib/recsplit/recsplit.go | 5 +++++ erigon-lib/recsplit/recsplit_test.go | 3 +-- erigon-lib/state/domain.go | 12 +++++------ erigon-lib/state/history.go | 4 ++-- erigon-lib/state/inverted_index.go | 4 ++-- erigon-lib/state/merge.go | 4 ++-- polygon/heimdall/types.go | 8 +++---- turbo/app/snapshots_cmd.go | 17 +++++++++++---- turbo/snapshotsync/caplin_state_snapshots.go | 8 +++---- 15 files changed, 86 insertions(+), 41 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 478f50a3b41..6cd8da51232 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1133,12 +1133,26 @@ func stageExec(db kv.TemporalRwDB, ctx context.Context, logger log.Logger) error block = sendersProgress } - for bn := execProgress; bn < block; bn++ { - if err := db.Update(ctx, func(tx kv.RwTx) error { + if noCommit { + tx, err := db.BeginTemporalRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + for bn := execProgress; bn < block; bn++ { txc.Tx = tx if err := stagedsync.SpawnExecuteBlocksStage(s, sync, txc, bn, ctx, cfg, logger); err != nil { return err } + } + } else { + if err := db.Update(ctx, func(tx kv.RwTx) error { + for bn := execProgress; bn < block; bn++ { + txc.Tx = tx + if err := stagedsync.SpawnExecuteBlocksStage(s, sync, txc, bn, ctx, cfg, logger); err != nil { + return err + } + } return nil }); err != nil { return err diff --git a/core/snaptype/block_types.go b/core/snaptype/block_types.go index 62854c1a163..fd85477319d 100644 --- a/core/snaptype/block_types.go +++ b/core/snaptype/block_types.go @@ -117,8 +117,8 @@ var ( cfg := recsplit.RecSplitArgs{ Enums: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, Salt: &salt, BaseDataID: info.From, @@ -159,8 +159,8 @@ var ( cfg := recsplit.RecSplitArgs{ Enums: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, Salt: &salt, BaseDataID: info.From, @@ -231,8 +231,8 @@ var ( Enums: true, LessFalsePositives: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, IndexFile: filepath.Join(sn.Dir(), sn.Type.IdxFileName(sn.Version, sn.From, sn.To)), BaseDataID: baseTxnID.U64(), @@ -244,8 +244,8 @@ var ( txnHash2BlockNumIdx, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ KeyCount: d.Count(), Enums: false, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, IndexFile: filepath.Join(sn.Dir(), sn.Type.IdxFileName(sn.Version, sn.From, sn.To, Indexes.TxnHash2BlockNum)), BaseDataID: firstBlockNum, diff --git a/erigon-lib/recsplit/eliasfano16/elias_fano.go b/erigon-lib/recsplit/eliasfano16/elias_fano.go index a9baea2a1b2..0a5e819da73 100644 --- a/erigon-lib/recsplit/eliasfano16/elias_fano.go +++ b/erigon-lib/recsplit/eliasfano16/elias_fano.go @@ -24,6 +24,7 @@ import ( "math/bits" "unsafe" + "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common/bitutil" ) @@ -263,6 +264,7 @@ type DoubleEliasFano struct { posMinDelta uint64 } +func (ef *DoubleEliasFano) Size() datasize.ByteSize { return datasize.ByteSize(len(ef.data) * 8) } func (ef *DoubleEliasFano) deriveFields() (int, int) { if ef.uPosition/(ef.numBuckets+1) == 0 { ef.lPosition = 0 diff --git a/erigon-lib/recsplit/eliasfano32/elias_fano.go b/erigon-lib/recsplit/eliasfano32/elias_fano.go index fb00f12b300..45d2dad4106 100644 --- a/erigon-lib/recsplit/eliasfano32/elias_fano.go +++ b/erigon-lib/recsplit/eliasfano32/elias_fano.go @@ -25,6 +25,7 @@ import ( "sort" "unsafe" + "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common/bitutil" "github.com/erigontech/erigon-lib/kv/stream" ) @@ -73,6 +74,8 @@ func NewEliasFano(count uint64, maxOffset uint64) *EliasFano { return ef } +func (ef *EliasFano) Size() datasize.ByteSize { return datasize.ByteSize(len(ef.data) * 8) } + func (ef *EliasFano) AddOffset(offset uint64) { //fmt.Printf("0x%x,\n", offset) if ef.l != 0 { diff --git a/erigon-lib/recsplit/eliasfano32/elias_fano_test.go b/erigon-lib/recsplit/eliasfano32/elias_fano_test.go index 89976581072..2cc17430aad 100644 --- a/erigon-lib/recsplit/eliasfano32/elias_fano_test.go +++ b/erigon-lib/recsplit/eliasfano32/elias_fano_test.go @@ -464,7 +464,7 @@ func checkSeekReverse(t *testing.T, j int, ef *EliasFano, vals []uint64) { require.Equal(t, bits.TrailingZeros64(prevUpperMask), bits.TrailingZeros64(efi.upperMask)) } -func BenchmarkName(b *testing.B) { +func BenchmarkEF(b *testing.B) { count := uint64(1_000_000) maxOffset := (count - 1) * 123 ef := NewEliasFano(count, maxOffset) diff --git a/erigon-lib/recsplit/index.go b/erigon-lib/recsplit/index.go index ae52667a84d..0eb45eac60e 100644 --- a/erigon-lib/recsplit/index.go +++ b/erigon-lib/recsplit/index.go @@ -30,6 +30,7 @@ import ( "time" "unsafe" + "github.com/c2h5oh/datasize" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/dbg" "github.com/erigontech/erigon-lib/log/v3" @@ -248,7 +249,19 @@ func (idx *Index) DataHandle() unsafe.Pointer { return unsafe.Pointer(&idx.data[0]) } -func (idx *Index) Size() int64 { return idx.size } +func (idx *Index) Size() int64 { return idx.size } +func (idx *Index) Enums() bool { return idx.enums } +func (idx *Index) Sizes() (total, offsets, ef, golombRice, existence, layer1 datasize.ByteSize) { + total = datasize.ByteSize(idx.size) + if idx.offsetEf != nil { + offsets = idx.offsetEf.Size() + } + ef = idx.ef.Size() + golombRice = datasize.ByteSize(len(idx.grData) * 8) + existence = datasize.ByteSize(len(idx.existence)) + layer1 = total - offsets - golombRice - existence + return +} func (idx *Index) ModTime() time.Time { return idx.modTime } func (idx *Index) BaseDataID() uint64 { return idx.baseDataID } func (idx *Index) FilePath() string { return idx.filePath } @@ -287,9 +300,9 @@ func (idx *Index) Empty() bool { return idx.keyCount == 0 } -func (idx *Index) KeyCount() uint64 { - return idx.keyCount -} +func (idx *Index) KeyCount() uint64 { return idx.keyCount } +func (idx *Index) LeafSize() uint16 { return idx.leafSize } +func (idx *Index) BucketSize() int { return idx.bucketSize } // Lookup is not thread-safe because it used id.hasher func (idx *Index) Lookup(bucketHash, fingerprint uint64) (uint64, bool) { diff --git a/erigon-lib/recsplit/recsplit.go b/erigon-lib/recsplit/recsplit.go index 76e06d30cd0..a1274f355dd 100644 --- a/erigon-lib/recsplit/recsplit.go +++ b/erigon-lib/recsplit/recsplit.go @@ -140,6 +140,11 @@ type RecSplitArgs struct { NoFsync bool // fsync is enabled by default, but tests can manually disable } +// DefaultLeafSize - LeafSize=8 and BucketSize=100, use abount 1.8 bits per key. Increasing the leaf and bucket +// sizes gives more compact structures (1.56 bits per key), at the price of a slower construction time +const DefaultLeafSize = 8 +const DefaultBucketSize = 2000 // typical from 100 to 2000, with smaller buckets giving slightly larger but faster function + // NewRecSplit creates a new RecSplit instance with given number of keys and given bucket size // Typical bucket size is 100 - 2000, larger bucket sizes result in smaller representations of hash functions, at a cost of slower access // salt parameters is used to randomise the hash function construction, to ensure that different Erigon instances (nodes) diff --git a/erigon-lib/recsplit/recsplit_test.go b/erigon-lib/recsplit/recsplit_test.go index 6c770a876fe..68860782d24 100644 --- a/erigon-lib/recsplit/recsplit_test.go +++ b/erigon-lib/recsplit/recsplit_test.go @@ -22,9 +22,8 @@ import ( "path/filepath" "testing" - "github.com/stretchr/testify/assert" - "github.com/erigontech/erigon-lib/log/v3" + "github.com/stretchr/testify/assert" ) func TestRecSplit2(t *testing.T) { diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 25e28680a8f..a9caa460ad6 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -687,14 +687,14 @@ func (dt *DomainRoTx) getLatestFromFile(i int, filekey []byte) (v []byte, ok boo if reader.Empty() { return nil, false, 0, nil } - offset, ok := reader.Lookup(filekey) + offset, ok := reader.TwoLayerLookup(filekey) if !ok { return nil, false, 0, nil } g.Reset(offset) k, _ := g.Next(nil) - if !bytes.Equal(filekey, k) { + if !bytes.Equal(filekey, k) { // MPH false-positives protection return nil, false, 0, nil } v, _ := g.Next(nil) @@ -1201,11 +1201,11 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio func (d *Domain) buildAccessor(ctx context.Context, fromStep, toStep uint64, data *seg.Decompressor, ps *background.ProgressSet) error { idxPath := d.kvAccessorFilePath(fromStep, toStep) cfg := recsplit.RecSplitArgs{ - Enums: false, - LessFalsePositives: false, + Enums: true, + LessFalsePositives: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: d.dirs.Tmp, IndexFile: idxPath, Salt: d.salt, diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index c23d11f85fb..fd73cf3f7bd 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -360,8 +360,8 @@ func (h *History) buildVI(ctx context.Context, historyIdxPath string, hist, efHi rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ KeyCount: hist.Count(), Enums: false, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: h.dirs.Tmp, IndexFile: historyIdxPath, Salt: h.salt, diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index e5ce42f88b9..52d0de6e4ee 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -1200,8 +1200,8 @@ func (ii *InvertedIndex) buildMapAccessor(ctx context.Context, fromStep, toStep Enums: true, LessFalsePositives: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: ii.dirs.Tmp, IndexFile: idxPath, Salt: ii.salt, diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go index 3f3f40476d5..510e0a23af3 100644 --- a/erigon-lib/state/merge.go +++ b/erigon-lib/state/merge.go @@ -825,8 +825,8 @@ func (ht *HistoryRoTx) mergeFiles(ctx context.Context, indexFiles, historyFiles if rs, err = recsplit.NewRecSplit(recsplit.RecSplitArgs{ KeyCount: keyCount, Enums: false, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: ht.h.dirs.Tmp, IndexFile: idxPath, Salt: ht.h.salt, diff --git a/polygon/heimdall/types.go b/polygon/heimdall/types.go index e5749b97d83..06603bf4ae3 100644 --- a/polygon/heimdall/types.go +++ b/polygon/heimdall/types.go @@ -197,8 +197,8 @@ var ( rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ KeyCount: blockCount, Enums: blockCount > 0, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, IndexFile: filepath.Join(sn.Dir(), snaptype.IdxFileName(sn.Version, sn.From, sn.To, Enums.Events.String())), BaseDataID: baseEventId, @@ -519,8 +519,8 @@ func buildValueIndex(ctx context.Context, sn snaptype.FileInfo, salt uint32, d * rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ KeyCount: d.Count(), Enums: d.Count() > 0, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, IndexFile: filepath.Join(sn.Dir(), sn.Type.IdxFileName(sn.Version, sn.From, sn.To)), BaseDataID: baseId, diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index e85985e0f53..40b293fb76d 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -36,6 +36,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon/polygon/bridge" "github.com/urfave/cli/v2" @@ -915,18 +916,18 @@ func doMeta(cliCtx *cli.Context) error { return errors.New("expecting file path as a first argument") } fname := args.First() - if strings.Contains(fname, ".seg") || strings.Contains(fname, ".kv") || strings.Contains(fname, ".v") || strings.Contains(fname, ".ef") { + if strings.HasSuffix(fname, ".seg") || strings.HasSuffix(fname, ".kv") || strings.HasSuffix(fname, ".v") || strings.HasSuffix(fname, ".ef") { src, err := seg.NewDecompressor(fname) if err != nil { - return err + panic(err) } defer src.Close() log.Info("meta", "count", src.Count(), "size", datasize.ByteSize(src.Size()).HumanReadable(), "serialized_dict", datasize.ByteSize(src.SerializedDictSize()).HumanReadable(), "dict_words", src.DictWords(), "name", src.FileName()) - } else if strings.Contains(fname, ".bt") { + } else if strings.HasSuffix(fname, ".bt") { kvFPath := strings.TrimSuffix(fname, ".bt") + ".kv" src, err := seg.NewDecompressor(kvFPath) if err != nil { - return err + panic(err) } defer src.Close() bt, err := libstate.OpenBtreeIndexWithDecompressor(fname, libstate.DefaultBtreeM, src, seg.CompressNone) @@ -949,6 +950,14 @@ func doMeta(cliCtx *cli.Context) error { } log.Info("meta", "distances(*100K)", fmt.Sprintf("%v", distances)) + } else if strings.HasSuffix(fname, ".kvi") || strings.HasSuffix(fname, ".idx") || strings.HasSuffix(fname, ".efi") || strings.HasSuffix(fname, ".vi") { + idx, err := recsplit.OpenIndex(fname) + if err != nil { + panic(err) + } + defer idx.Close() + total, offsets, ef, golombRice, existence, layer1 := idx.Sizes() + log.Info("meta", "sz_total", total.HumanReadable(), "sz_offsets", offsets.HumanReadable(), "sz_double_ef", ef.HumanReadable(), "sz_golombRice", golombRice.HumanReadable(), "sz_existence", existence.HumanReadable(), "sz_l1", layer1.HumanReadable(), "keys_count", idx.KeyCount(), "leaf_size", idx.LeafSize(), "bucket_size", idx.BucketSize(), "enums", idx.Enums()) } return nil } diff --git a/turbo/snapshotsync/caplin_state_snapshots.go b/turbo/snapshotsync/caplin_state_snapshots.go index d5ecb82b57a..5579d71cb89 100644 --- a/turbo/snapshotsync/caplin_state_snapshots.go +++ b/turbo/snapshotsync/caplin_state_snapshots.go @@ -53,8 +53,8 @@ func BeaconSimpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmp num := make([]byte, binary.MaxVarintLen64) cfg := recsplit.RecSplitArgs{ Enums: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, Salt: &salt, BaseDataID: sn.From, @@ -647,8 +647,8 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st num := make([]byte, binary.MaxVarintLen64) cfg := recsplit.RecSplitArgs{ Enums: true, - BucketSize: 2000, - LeafSize: 8, + BucketSize: recsplit.DefaultBucketSize, + LeafSize: recsplit.DefaultLeafSize, TmpDir: tmpDir, Salt: &salt, BaseDataID: sn.From, From 0e548a45422bab4534f5d1ddb62252ee110899f4 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 30 Dec 2024 20:46:00 +0700 Subject: [PATCH 34/94] recsplit: change default BucketSize (#13272) Params: - change default BucketSize: at `100` i don't see any size grow. but build speed -30% and read speed -20% - keep same default LeafSize: it increasing build speed, but decreasing reads speed. so we do preoritize reads speed. --- erigon-lib/kv/mdbx/kv_mdbx.go | 3 +++ erigon-lib/recsplit/recsplit.go | 2 +- erigon-lib/state/aggregator_test.go | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index b7a52c6d06b..0ec79bc146f 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -1109,6 +1109,9 @@ func (tx *MdbxTx) stdCursor(bucket string) (kv.RwCursor, error) { c := &MdbxCursor{bucketName: bucket, tx: tx, bucketCfg: b, id: tx.ID} tx.ID++ + if tx.tx == nil { + panic("assert: tx.tx nil. seems this `tx` was Rollback'ed") + } var err error c.c, err = tx.tx.OpenCursor(mdbx.DBI(tx.db.buckets[c.bucketName].DBI)) if err != nil { diff --git a/erigon-lib/recsplit/recsplit.go b/erigon-lib/recsplit/recsplit.go index a1274f355dd..dd41a7f0347 100644 --- a/erigon-lib/recsplit/recsplit.go +++ b/erigon-lib/recsplit/recsplit.go @@ -143,7 +143,7 @@ type RecSplitArgs struct { // DefaultLeafSize - LeafSize=8 and BucketSize=100, use abount 1.8 bits per key. Increasing the leaf and bucket // sizes gives more compact structures (1.56 bits per key), at the price of a slower construction time const DefaultLeafSize = 8 -const DefaultBucketSize = 2000 // typical from 100 to 2000, with smaller buckets giving slightly larger but faster function +const DefaultBucketSize = 100 // typical from 100 to 2000, with smaller buckets giving slightly larger but faster function // NewRecSplit creates a new RecSplit instance with given number of keys and given bucket size // Typical bucket size is 100 - 2000, larger bucket sizes result in smaller representations of hash functions, at a cost of slower access diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index f70f655d650..444f3a57bbe 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -547,13 +547,13 @@ func TestAggregatorV3_PruneSmallBatches(t *testing.T) { require.NoError(t, err) codeRangeAfter = extractKVErrIterator(t, it) - its, err := ac.d[kv.AccountsDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, tx) + its, err := ac.d[kv.AccountsDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, afterTx) require.NoError(t, err) accountHistRangeAfter = extractKVSErrIterator(t, its) - its, err = ac.d[kv.CodeDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, tx) + its, err = ac.d[kv.CodeDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, afterTx) require.NoError(t, err) codeHistRangeAfter = extractKVSErrIterator(t, its) - its, err = ac.d[kv.StorageDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, tx) + its, err = ac.d[kv.StorageDomain].ht.HistoryRange(0, int(maxTx), order.Asc, maxInt, afterTx) require.NoError(t, err) storageHistRangeAfter = extractKVSErrIterator(t, its) } From dd3663a39c9922717a5f08077fdb59e422bdd50a Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 30 Dec 2024 21:17:35 +0700 Subject: [PATCH 35/94] commitment domain: make `.kvi` default index (#13276) --- erigon-lib/state/aggregator.go | 12 +++++++++++- erigon-lib/state/aggregator_test.go | 21 ++++++++++++++++++--- erigon-lib/state/domain.go | 4 ++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 6123509a46d..4eb8d9cd0fe 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -126,6 +126,16 @@ func domainIntegrityCheck(name kv.Domain, dirs datadir.Dirs, fromStep, toStep ui } } +var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) + +func init() { + if dbgCommBtIndex { + cfg := Schema[kv.CommitmentDomain] + cfg.indexList = withBTree | withExistence + Schema[kv.CommitmentDomain] = cfg + } +} + var Schema = map[kv.Domain]domainCfg{ kv.AccountsDomain: { name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, @@ -195,7 +205,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.CommitmentDomain: { name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, - indexList: withBTree | withExistence, + indexList: withHashMap, compression: seg.CompressKeys, compressCfg: DomainCompressCfg, diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 444f3a57bbe..2df5b21c9f6 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1273,9 +1273,24 @@ func TestAggregator_RebuildCommitmentBasedOnFiles(t *testing.T) { compression := ac.d[kv.CommitmentDomain].d.compression fnames := []string{} for _, f := range ac.d[kv.CommitmentDomain].files { - k, stateVal, _, found, err := f.src.bindex.Get(keyCommitmentState, seg.NewReader(f.src.decompressor.MakeGetter(), compression)) - require.NoError(t, err) - require.True(t, found) + var k, stateVal []byte + if ac.d[kv.CommitmentDomain].d.indexList&withHashMap != 0 { + idx := f.src.index.GetReaderFromPool() + r := seg.NewReader(f.src.decompressor.MakeGetter(), compression) + + offset, ok := idx.TwoLayerLookup(keyCommitmentState) + require.True(t, ok) + r.Reset(offset) + k, _ = r.Next(nil) + stateVal, _ = r.Next(nil) + } else { + var found bool + var err error + k, stateVal, _, found, err = f.src.bindex.Get(keyCommitmentState, seg.NewReader(f.src.decompressor.MakeGetter(), compression)) + require.NoError(t, err) + require.True(t, found) + require.EqualValues(t, keyCommitmentState, k) + } require.EqualValues(t, keyCommitmentState, k) rh, err := commitment.HexTrieExtractStateRoot(stateVal) require.NoError(t, err) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index a9caa460ad6..3d9fb444dc9 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1055,7 +1055,7 @@ func (d *Domain) buildFileRange(ctx context.Context, stepFrom, stepTo uint64, co if err = d.buildAccessor(ctx, stepFrom, stepTo, valuesDecomp, ps); err != nil { return StaticFiles{}, fmt.Errorf("build %s values idx: %w", d.filenameBase, err) } - valuesIdx, err = recsplit.OpenIndex(d.efAccessorFilePath(stepFrom, stepTo)) + valuesIdx, err = recsplit.OpenIndex(d.kvAccessorFilePath(stepFrom, stepTo)) if err != nil { return StaticFiles{}, err } @@ -1158,7 +1158,7 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio if err = d.buildAccessor(ctx, step, step+1, valuesDecomp, ps); err != nil { return StaticFiles{}, fmt.Errorf("build %s values idx: %w", d.filenameBase, err) } - valuesIdx, err = recsplit.OpenIndex(d.efAccessorFilePath(step, step+1)) + valuesIdx, err = recsplit.OpenIndex(d.kvAccessorFilePath(step, step+1)) if err != nil { return StaticFiles{}, err } From c72c0be8b5ede03fd1ce5eee019daf5090a3757b Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 31 Dec 2024 18:28:29 +0700 Subject: [PATCH 36/94] Set workers only from one sync (#13284) --- eth/stagedsync/exec3.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 0c8c06fbe23..67516febd4a 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -206,6 +206,8 @@ func ExecV3(ctx context.Context, initialCycle bool, isMining bool, ) error { + inMemExec := txc.Doms != nil + // TODO: e35 doesn't support parallel-exec yet parallel = false //nolint if parallel && cfg.chainConfig.ChainName == networkname.Gnosis { @@ -237,16 +239,17 @@ func ExecV3(ctx context.Context, } } agg := cfg.db.(state2.HasAgg).Agg().(*state2.Aggregator) - if initialCycle { - agg.SetCollateAndBuildWorkers(min(2, estimate.StateV3Collate.Workers())) - agg.SetCompressWorkers(estimate.CompressSnapshot.Workers()) - } else { - agg.SetCompressWorkers(1) - agg.SetCollateAndBuildWorkers(1) + if !inMemExec && !isMining { + if initialCycle { + agg.SetCollateAndBuildWorkers(min(2, estimate.StateV3Collate.Workers())) + agg.SetCompressWorkers(estimate.CompressSnapshot.Workers()) + } else { + agg.SetCompressWorkers(1) + agg.SetCollateAndBuildWorkers(1) + } } var err error - inMemExec := txc.Doms != nil var doms *state2.SharedDomains if inMemExec { doms = txc.Doms From 6e1bab7ea1dabd95556add338ee78b79f6fc830f Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Wed, 1 Jan 2025 05:05:52 +0100 Subject: [PATCH 37/94] Erigon: revert go-mdbx (#13286) --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 5 +++-- erigon-lib/kv/mdbx/kv_mdbx.go | 2 +- go.mod | 2 +- go.sum | 5 +++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index c63958639c5..9b374ea6817 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -10,7 +10,7 @@ replace ( require ( github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed - github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 + github.com/erigontech/mdbx-go v0.38.4 github.com/erigontech/secp256k1 v1.1.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index b125a49fa91..4d9209e71d9 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -156,8 +156,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed h1:un44S8Tuol4LBIC6R94t93GShM53BYjz7GsNPziDLQ8= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE= -github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= -github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= +github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= +github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U= @@ -273,6 +273,7 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 0ec79bc146f..123bcd1dd19 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -202,7 +202,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) { } - env, err := mdbx.NewEnv(mdbx.Label(opts.label)) + env, err := mdbx.NewEnv() if err != nil { return nil, err } diff --git a/go.mod b/go.mod index cc50c4eb0a4..d19137b65d0 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace ( require ( github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 - github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 + github.com/erigontech/mdbx-go v0.38.4 github.com/erigontech/secp256k1 v1.1.0 github.com/erigontech/silkworm-go v0.18.0 ) diff --git a/go.sum b/go.sum index 79d6db3b838..ed8f1651f93 100644 --- a/go.sum +++ b/go.sum @@ -271,8 +271,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 h1:KCFa2uXEfZoBjV4buzjWmCmoqVLXiGCq0ZmQ2OjeRvQ= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116/go.mod h1:8vQ+VjvLu2gkPs8EwdPrOTAAo++WuLuBi54N7NuAF0I= -github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= -github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= +github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= +github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= @@ -487,6 +487,7 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= From 68694ca33c8a6a3c1d3aae33064a8ff0814c50cc Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 2 Jan 2025 04:09:57 +0700 Subject: [PATCH 38/94] `.kvi` support in tooling (#13293) - `torrent_create --all` - seedable files list --- erigon-lib/downloader/snaptype/files.go | 2 +- erigon-lib/seg/seg_auto_rw.go | 4 +- erigon-lib/state/aggregator.go | 12 ++-- erigon-lib/state/aggregator_test.go | 2 +- erigon-lib/state/domain.go | 34 ++++----- erigon-lib/state/domain_committed.go | 4 +- erigon-lib/state/domain_shared.go | 6 +- erigon-lib/state/files_item.go | 8 +-- erigon-lib/state/history.go | 4 +- erigon-lib/state/inverted_index.go | 16 +++-- erigon-lib/state/merge.go | 6 +- erigon-lib/state/squeeze.go | 11 +-- turbo/app/snapshots_cmd.go | 93 ++++++++++++++++--------- turbo/rpchelper/helper.go | 1 + 14 files changed, 115 insertions(+), 88 deletions(-) diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go index b07e1e9f632..b75a402d7d2 100644 --- a/erigon-lib/downloader/snaptype/files.go +++ b/erigon-lib/downloader/snaptype/files.go @@ -232,7 +232,7 @@ func SeedableV3Extensions() []string { } func AllV3Extensions() []string { - return []string{".kv", ".v", ".ef", ".kvei", ".vi", ".efi", ".bt"} + return []string{".kv", ".v", ".ef", ".kvei", ".vi", ".efi", ".bt", ".kvi"} } func IsSeedableExtension(name string) bool { diff --git a/erigon-lib/seg/seg_auto_rw.go b/erigon-lib/seg/seg_auto_rw.go index 92136c8afc4..11487a277fb 100644 --- a/erigon-lib/seg/seg_auto_rw.go +++ b/erigon-lib/seg/seg_auto_rw.go @@ -175,7 +175,7 @@ func DetectCompressType(getter *Getter) (compressed FileCompression) { getter.Reset(0) for i := 0; i < 100; i++ { if getter.HasNext() { - _, _ = getter.NextUncompressed() + _, _ = getter.SkipUncompressed() } if getter.HasNext() { _, _ = getter.Skip() @@ -196,7 +196,7 @@ func DetectCompressType(getter *Getter) (compressed FileCompression) { _, _ = getter.Skip() } if getter.HasNext() { - _, _ = getter.NextUncompressed() + _, _ = getter.SkipUncompressed() } } return compressed diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 4eb8d9cd0fe..26f147c3958 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -131,7 +131,7 @@ var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) func init() { if dbgCommBtIndex { cfg := Schema[kv.CommitmentDomain] - cfg.indexList = withBTree | withExistence + cfg.IndexList = AccessorBTree | AccessorExistence Schema[kv.CommitmentDomain] = cfg } } @@ -140,7 +140,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.AccountsDomain: { name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, - indexList: withBTree | withExistence, + IndexList: AccessorBTree | AccessorExistence, crossDomainIntegrity: domainIntegrityCheck, compression: seg.CompressNone, compressCfg: DomainCompressCfg, @@ -162,7 +162,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.StorageDomain: { name: kv.StorageDomain, valuesTable: kv.TblStorageVals, - indexList: withBTree | withExistence, + IndexList: AccessorBTree | AccessorExistence, compression: seg.CompressKeys, compressCfg: DomainCompressCfg, @@ -183,7 +183,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.CodeDomain: { name: kv.CodeDomain, valuesTable: kv.TblCodeVals, - indexList: withBTree | withExistence, + IndexList: AccessorBTree | AccessorExistence, compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet compressCfg: DomainCompressCfg, largeValues: true, @@ -205,7 +205,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.CommitmentDomain: { name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, - indexList: withHashMap, + IndexList: AccessorHashMap, compression: seg.CompressKeys, compressCfg: DomainCompressCfg, @@ -227,7 +227,7 @@ var Schema = map[kv.Domain]domainCfg{ kv.ReceiptDomain: { name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, - indexList: withBTree | withExistence, + IndexList: AccessorBTree | AccessorExistence, compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, compressCfg: DomainCompressCfg, diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 2df5b21c9f6..816eb25c2e9 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1274,7 +1274,7 @@ func TestAggregator_RebuildCommitmentBasedOnFiles(t *testing.T) { fnames := []string{} for _, f := range ac.d[kv.CommitmentDomain].files { var k, stateVal []byte - if ac.d[kv.CommitmentDomain].d.indexList&withHashMap != 0 { + if ac.d[kv.CommitmentDomain].d.IndexList&AccessorHashMap != 0 { idx := f.src.index.GetReaderFromPool() r := seg.NewReader(f.src.decompressor.MakeGetter(), compression) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 3d9fb444dc9..7d9c2770f0a 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -97,8 +97,8 @@ type domainCfg struct { name kv.Domain compression seg.FileCompression compressCfg seg.Cfg - indexList idxList // list of indexes for given domain - valuesTable string // bucket to store domain values; key -> inverted_step + values (Dupsort) + IndexList Accessors // list of indexes for given domain + valuesTable string // bucket to store domain values; key -> inverted_step + values (Dupsort) largeValues bool crossDomainIntegrity rangeDomainIntegrityChecker @@ -358,7 +358,7 @@ func (d *Domain) openDirtyFiles() (err error) { } } - if item.index == nil && d.indexList&withHashMap != 0 { + if item.index == nil && d.IndexList&AccessorHashMap != 0 { fPath := d.kvAccessorFilePath(fromStep, toStep) exists, err := dir.FileExist(fPath) if err != nil { @@ -373,7 +373,7 @@ func (d *Domain) openDirtyFiles() (err error) { } } } - if item.bindex == nil && d.indexList&withBTree != 0 { + if item.bindex == nil && d.IndexList&AccessorBTree != 0 { fPath := d.kvBtFilePath(fromStep, toStep) exists, err := dir.FileExist(fPath) if err != nil { @@ -392,7 +392,7 @@ func (d *Domain) openDirtyFiles() (err error) { } } } - if item.existence == nil && d.indexList&withExistence != 0 { + if item.existence == nil && d.IndexList&AccessorExistence != 0 { fPath := d.kvExistenceIdxFilePath(fromStep, toStep) exists, err := dir.FileExist(fPath) if err != nil { @@ -443,7 +443,7 @@ func (d *Domain) closeWhatNotInList(fNames []string) { } func (d *Domain) reCalcVisibleFiles(toTxNum uint64) { - d._visible = newDomainVisible(d.name, calcVisibleFiles(d.dirtyFiles, d.indexList, false, toTxNum)) + d._visible = newDomainVisible(d.name, calcVisibleFiles(d.dirtyFiles, d.IndexList, false, toTxNum)) d.History.reCalcVisibleFiles(toTxNum) } @@ -674,7 +674,7 @@ func (dt *DomainRoTx) getLatestFromFile(i int, filekey []byte) (v []byte, ok boo } g := dt.statelessGetter(i) - if dt.d.indexList&withBTree != 0 { + if dt.d.IndexList&AccessorBTree != 0 { _, v, offset, ok, err = dt.statelessBtree(i).Get(filekey, g) if err != nil || !ok { return nil, false, 0, err @@ -682,7 +682,7 @@ func (dt *DomainRoTx) getLatestFromFile(i int, filekey []byte) (v []byte, ok boo //fmt.Printf("getLatestFromBtreeColdFiles key %x shard %d %x\n", filekey, exactColdShard, v) return v, true, offset, nil } - if dt.d.indexList&withHashMap != 0 { + if dt.d.IndexList&AccessorHashMap != 0 { reader := dt.statelessIdxReader(i) if reader.Empty() { return nil, false, 0, nil @@ -1051,7 +1051,7 @@ func (d *Domain) buildFileRange(ctx context.Context, stepFrom, stepTo uint64, co return StaticFiles{}, fmt.Errorf("open %s values decompressor: %w", d.filenameBase, err) } - if d.indexList&withHashMap != 0 { + if d.IndexList&AccessorHashMap != 0 { if err = d.buildAccessor(ctx, stepFrom, stepTo, valuesDecomp, ps); err != nil { return StaticFiles{}, fmt.Errorf("build %s values idx: %w", d.filenameBase, err) } @@ -1061,7 +1061,7 @@ func (d *Domain) buildFileRange(ctx context.Context, stepFrom, stepTo uint64, co } } - if d.indexList&withBTree != 0 { + if d.IndexList&AccessorBTree != 0 { btPath := d.kvBtFilePath(stepFrom, stepTo) btM := DefaultBtreeM if stepFrom == 0 && d.filenameBase == "commitment" { @@ -1073,7 +1073,7 @@ func (d *Domain) buildFileRange(ctx context.Context, stepFrom, stepTo uint64, co return StaticFiles{}, fmt.Errorf("build %s .bt idx: %w", d.filenameBase, err) } } - if d.indexList&withExistence != 0 { + if d.IndexList&AccessorExistence != 0 { fPath := d.kvExistenceIdxFilePath(stepFrom, stepTo) exists, err := dir.FileExist(fPath) if err != nil { @@ -1154,7 +1154,7 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio return StaticFiles{}, fmt.Errorf("open %s values decompressor: %w", d.filenameBase, err) } - if d.indexList&withHashMap != 0 { + if d.IndexList&AccessorHashMap != 0 { if err = d.buildAccessor(ctx, step, step+1, valuesDecomp, ps); err != nil { return StaticFiles{}, fmt.Errorf("build %s values idx: %w", d.filenameBase, err) } @@ -1164,7 +1164,7 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio } } - if d.indexList&withBTree != 0 { + if d.IndexList&AccessorBTree != 0 { btPath := d.kvBtFilePath(step, step+1) btM := DefaultBtreeM if step == 0 && d.filenameBase == "commitment" { @@ -1175,7 +1175,7 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio return StaticFiles{}, fmt.Errorf("build %s .bt idx: %w", d.filenameBase, err) } } - if d.indexList&withExistence != 0 { + if d.IndexList&AccessorExistence != 0 { fPath := d.kvExistenceIdxFilePath(step, step+1) exists, err := dir.FileExist(fPath) if err != nil { @@ -1263,7 +1263,7 @@ func (d *Domain) missedAccessors() (l []*filesItem) { func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps *background.ProgressSet) { d.History.BuildMissedAccessors(ctx, g, ps) for _, item := range d.missedBtreeAccessors() { - if d.indexList&withBTree == 0 { + if d.IndexList&AccessorBTree == 0 { continue } if item.decompressor == nil { @@ -1281,7 +1281,7 @@ func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps }) } for _, item := range d.missedAccessors() { - if d.indexList&withHashMap == 0 { + if d.IndexList&AccessorHashMap == 0 { continue } if item.decompressor == nil { @@ -1451,7 +1451,7 @@ func (dt *DomainRoTx) getFromFiles(filekey []byte, maxTxNum uint64) (v []byte, f if maxTxNum == 0 { maxTxNum = math.MaxUint64 } - useExistenceFilter := dt.d.indexList&withExistence != 0 + useExistenceFilter := dt.d.IndexList&AccessorExistence != 0 useCache := dt.name != kv.CommitmentDomain && maxTxNum == math.MaxUint64 hi, _ := dt.ht.iit.hashKey(filekey) diff --git a/erigon-lib/state/domain_committed.go b/erigon-lib/state/domain_committed.go index 33865be7415..8daef3f6b26 100644 --- a/erigon-lib/state/domain_committed.go +++ b/erigon-lib/state/domain_committed.go @@ -110,7 +110,7 @@ func (dt *DomainRoTx) findShortenedKey(fullKey []byte, itemGetter *seg.Reader, i // } //} - if dt.d.indexList&withHashMap != 0 { + if dt.d.IndexList&AccessorHashMap != 0 { reader := recsplit.NewIndexReader(item.index) defer reader.Close() @@ -135,7 +135,7 @@ func (dt *DomainRoTx) findShortenedKey(fullKey []byte, itemGetter *seg.Reader, i } return encodeShorterKey(nil, offset), true } - if dt.d.indexList&withBTree != 0 { + if dt.d.IndexList&AccessorBTree != 0 { if item.bindex == nil { dt.d.logger.Warn("[agg] commitment branch key replacement: file doesn't have index", "name", item.decompressor.FileName()) } diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 77d4636e55f..e570c8c7263 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -818,8 +818,8 @@ func (sd *SharedDomains) IterateStoragePrefix(prefix []byte, it func(k []byte, v } } case FILE_CURSOR: - indexList := sd.aggTx.d[kv.StorageDomain].d.indexList - if indexList&withBTree != 0 { + indexList := sd.aggTx.d[kv.StorageDomain].d.IndexList + if indexList&AccessorBTree != 0 { if ci1.btCursor.Next() { ci1.key = ci1.btCursor.Key() if ci1.key != nil && bytes.HasPrefix(ci1.key, prefix) { @@ -830,7 +830,7 @@ func (sd *SharedDomains) IterateStoragePrefix(prefix []byte, it func(k []byte, v ci1.btCursor.Close() } } - if indexList&withHashMap != 0 { + if indexList&AccessorHashMap != 0 { ci1.dg.Reset(ci1.latestOffset) if !ci1.dg.HasNext() { break diff --git a/erigon-lib/state/files_item.go b/erigon-lib/state/files_item.go index d59fcbebc4f..c782fb295be 100644 --- a/erigon-lib/state/files_item.go +++ b/erigon-lib/state/files_item.go @@ -243,7 +243,7 @@ type visibleFile struct { func (i *visibleFile) isSubSetOf(j *visibleFile) bool { return i.src.isSubsetOf(j.src) } //nolint func (i *visibleFile) isSubsetOf(j *visibleFile) bool { return i.src.isSubsetOf(j.src) } //nolint -func calcVisibleFiles(files *btree2.BTreeG[*filesItem], l idxList, trace bool, toTxNum uint64) (roItems []visibleFile) { +func calcVisibleFiles(files *btree2.BTreeG[*filesItem], l Accessors, trace bool, toTxNum uint64) (roItems []visibleFile) { newVisibleFiles := make([]visibleFile, 0, files.Len()) // trace = true if trace { @@ -271,21 +271,21 @@ func calcVisibleFiles(files *btree2.BTreeG[*filesItem], l idxList, trace bool, t } continue } - if (l&withBTree != 0) && item.bindex == nil { + if (l&AccessorBTree != 0) && item.bindex == nil { if trace { log.Warn("[dbg] calcVisibleFiles: BTindex not opened", "f", item.decompressor.FileName()) } //panic(fmt.Errorf("btindex nil: %s", item.decompressor.FileName())) continue } - if (l&withHashMap != 0) && item.index == nil { + if (l&AccessorHashMap != 0) && item.index == nil { if trace { log.Warn("[dbg] calcVisibleFiles: RecSplit not opened", "f", item.decompressor.FileName()) } //panic(fmt.Errorf("index nil: %s", item.decompressor.FileName())) continue } - if (l&withExistence != 0) && item.existence == nil { + if (l&AccessorExistence != 0) && item.existence == nil { if trace { log.Warn("[dbg] calcVisibleFiles: Existence not opened", "f", item.decompressor.FileName()) } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index fd73cf3f7bd..db2258b6046 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -96,7 +96,7 @@ type histCfg struct { snapshotsDisabled bool // don't produce .v and .ef files, keep in db table. old data will be pruned anyway. historyDisabled bool // skip all write operations to this History (even in DB) - indexList idxList + indexList Accessors compressorCfg seg.Cfg // compression settings for history files compression seg.FileCompression // defines type of compression for history files @@ -108,7 +108,7 @@ func NewHistory(cfg histCfg, logger log.Logger) (*History, error) { //if cfg.compressorCfg.MaxDictPatterns == 0 && cfg.compressorCfg.MaxPatternLen == 0 { cfg.compressorCfg = seg.DefaultCfg if cfg.indexList == 0 { - cfg.indexList = withHashMap + cfg.indexList = AccessorHashMap } if cfg.iiCfg.filenameBase == "" { cfg.iiCfg.filenameBase = cfg.filenameBase diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 52d0de6e4ee..3f1c1345662 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -89,7 +89,7 @@ type iiCfg struct { // external checker for integrity of inverted index ranges integrity rangeIntegrityChecker - indexList idxList + indexList Accessors } type iiVisible struct { @@ -111,7 +111,7 @@ func NewInvertedIndex(cfg iiCfg, logger log.Logger) (*InvertedIndex, error) { //if cfg.compressorCfg.MaxDictPatterns == 0 && cfg.compressorCfg.MaxPatternLen == 0 { cfg.compressorCfg = seg.DefaultCfg if cfg.indexList == 0 { - cfg.indexList = withHashMap + cfg.indexList = AccessorHashMap } ii := InvertedIndex{ @@ -200,12 +200,14 @@ func (ii *InvertedIndex) scanDirtyFiles(fileNames []string) { } } -type idxList int +type Accessors int -var ( - withBTree idxList = 0b1 - withHashMap idxList = 0b10 - withExistence idxList = 0b100 +func (l Accessors) Has(target Accessors) bool { return l&target != 0 } + +const ( + AccessorBTree Accessors = 0b1 + AccessorHashMap Accessors = 0b10 + AccessorExistence Accessors = 0b100 ) func (ii *InvertedIndex) reCalcVisibleFiles(toTxNum uint64) { diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go index 510e0a23af3..2c22e540a30 100644 --- a/erigon-lib/state/merge.go +++ b/erigon-lib/state/merge.go @@ -515,7 +515,7 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h return nil, nil, nil, fmt.Errorf("merge %s decompressor [%d-%d]: %w", dt.d.filenameBase, r.values.from, r.values.to, err) } - if dt.d.indexList&withBTree != 0 { + if dt.d.IndexList&AccessorBTree != 0 { btPath := dt.d.kvBtFilePath(fromStep, toStep) btM := DefaultBtreeM if toStep == 0 && dt.d.filenameBase == "commitment" { @@ -526,7 +526,7 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h return nil, nil, nil, fmt.Errorf("merge %s btindex [%d-%d]: %w", dt.d.filenameBase, r.values.from, r.values.to, err) } } - if dt.d.indexList&withHashMap != 0 { + if dt.d.IndexList&AccessorHashMap != 0 { if err = dt.d.buildAccessor(ctx, fromStep, toStep, valuesIn.decompressor, ps); err != nil { return nil, nil, nil, fmt.Errorf("merge %s buildAccessor [%d-%d]: %w", dt.d.filenameBase, r.values.from, r.values.to, err) } @@ -535,7 +535,7 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h } } - if dt.d.indexList&withExistence != 0 { + if dt.d.IndexList&AccessorExistence != 0 { bloomIndexPath := dt.d.kvExistenceIdxFilePath(fromStep, toStep) exists, err := dir.FileExist(bloomIndexPath) if err != nil { diff --git a/erigon-lib/state/squeeze.go b/erigon-lib/state/squeeze.go index db92ffff8d2..ad929203d53 100644 --- a/erigon-lib/state/squeeze.go +++ b/erigon-lib/state/squeeze.go @@ -57,14 +57,9 @@ func (a *Aggregator) Sqeeze(ctx context.Context, domain kv.Domain) error { strings.ReplaceAll(to, ".kv", ".bt"), strings.ReplaceAll(to, ".kv", ".bt.torrent"), strings.ReplaceAll(to, ".kv", ".kvei"), - strings.ReplaceAll(to, ".kv", ".kvei.torrent")) - - // _ = os.Remove(tempFileCopy) - // _ = os.Remove(strings.ReplaceAll(to, ".kv", ".bt")) - // _ = os.Remove(strings.ReplaceAll(to, ".kv", ".bt.torrent")) - // _ = os.Remove(strings.ReplaceAll(to, ".kv", ".kvei")) - // _ = os.Remove(strings.ReplaceAll(to, ".kv", ".kvei.torrent")) - // _ = os.Remove(strings.ReplaceAll(to, ".kv", ".kv.torrent")) + strings.ReplaceAll(to, ".kv", ".kvei.torrent"), + strings.ReplaceAll(to, ".kv", ".kvi"), + strings.ReplaceAll(to, ".kv", ".kvi.torrent")) } for _, f := range filesToRemove { diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 40b293fb76d..4835364c440 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -644,17 +644,17 @@ func checkIfBlockSnapshotsPublishable(snapDir string) error { return nil } -func checkIfStateSnapshotsPublishable(dir datadir.Dirs) error { +func checkIfStateSnapshotsPublishable(dirs datadir.Dirs) error { var stepSum uint64 var maxStep uint64 - if err := filepath.Walk(dir.SnapDomain, func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(dirs.SnapDomain, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if info.IsDir() && path != dir.SnapDomain { - return fmt.Errorf("unexpected directory in domain (%s) check %s", dir.SnapDomain, path) + if info.IsDir() && path != dirs.SnapDomain { + return fmt.Errorf("unexpected directory in domain (%s) check %s", dirs.SnapDomain, path) } - if path == dir.SnapDomain { + if path == dirs.SnapDomain { return nil } rangeString := strings.Split(info.Name(), ".")[1] @@ -677,38 +677,58 @@ func checkIfStateSnapshotsPublishable(dir datadir.Dirs) error { stepSum += to - from // do a range check over all snapshots types (sanitizes domain and history folder) - for _, snapType := range []string{"accounts", "storage", "code", "commitment"} { - expectedFileName := strings.Replace(info.Name(), "accounts", snapType, 1) - if _, err := os.Stat(filepath.Join(dir.SnapDomain, expectedFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", expectedFileName, filepath.Join(dir.SnapDomain, expectedFileName)) + for _, snapType := range kv.StateDomains { + expectedFileName := strings.Replace(info.Name(), "accounts", snapType.String(), 1) + if _, err := os.Stat(filepath.Join(dirs.SnapDomain, expectedFileName)); err != nil { + return fmt.Errorf("missing file %s at path %s", expectedFileName, filepath.Join(dirs.SnapDomain, expectedFileName)) } // check that the index file exist - btFileName := strings.Replace(expectedFileName, ".kv", ".bt", 1) - if _, err := os.Stat(filepath.Join(dir.SnapDomain, btFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", btFileName, filepath.Join(dir.SnapDomain, btFileName)) + if libstate.Schema[snapType].IndexList.Has(libstate.AccessorBTree) { + fileName := strings.Replace(expectedFileName, ".kv", ".bt", 1) + exists, err := dir.FileExist(filepath.Join(dirs.SnapDomain, fileName)) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("missing file %s", fileName) + } } - - kveiFileName := strings.Replace(expectedFileName, ".kv", ".kvei", 1) - if _, err := os.Stat(filepath.Join(dir.SnapDomain, kveiFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", kveiFileName, filepath.Join(dir.SnapDomain, kveiFileName)) + if libstate.Schema[snapType].IndexList.Has(libstate.AccessorExistence) { + fileName := strings.Replace(expectedFileName, ".kv", ".kvei", 1) + exists, err := dir.FileExist(filepath.Join(dirs.SnapDomain, fileName)) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("missing file %s", fileName) + } + } + if libstate.Schema[snapType].IndexList.Has(libstate.AccessorHashMap) { + fileName := strings.Replace(expectedFileName, ".kv", ".kvi", 1) + exists, err := dir.FileExist(filepath.Join(dirs.SnapDomain, fileName)) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("missing file %s", fileName) + } } - } return nil }); err != nil { return err } - if err := filepath.Walk(dir.SnapIdx, func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(dirs.SnapIdx, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if info.IsDir() && path != dir.SnapIdx { - return fmt.Errorf("unexpected directory in idx (%s) check %s", dir.SnapIdx, path) + if info.IsDir() && path != dirs.SnapIdx { + return fmt.Errorf("unexpected directory in idx (%s) check %s", dirs.SnapIdx, path) } - if path == dir.SnapIdx { + if path == dirs.SnapIdx { return nil } rangeString := strings.Split(info.Name(), ".")[1] @@ -729,25 +749,25 @@ func checkIfStateSnapshotsPublishable(dir datadir.Dirs) error { // do a range check over all snapshots types (sanitizes domain and history folder) for _, snapType := range []string{"accounts", "storage", "code", "logtopics", "logaddrs", "tracesfrom", "tracesto"} { expectedFileName := strings.Replace(info.Name(), "accounts", snapType, 1) - if _, err := os.Stat(filepath.Join(dir.SnapIdx, expectedFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", expectedFileName, filepath.Join(dir.SnapIdx, expectedFileName)) + if _, err := os.Stat(filepath.Join(dirs.SnapIdx, expectedFileName)); err != nil { + return fmt.Errorf("missing file %s at path %s", expectedFileName, filepath.Join(dirs.SnapIdx, expectedFileName)) } // Check accessors efiFileName := strings.Replace(expectedFileName, ".ef", ".efi", 1) - if _, err := os.Stat(filepath.Join(dir.SnapAccessors, efiFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", efiFileName, filepath.Join(dir.SnapAccessors, efiFileName)) + if _, err := os.Stat(filepath.Join(dirs.SnapAccessors, efiFileName)); err != nil { + return fmt.Errorf("missing file %s at path %s", efiFileName, filepath.Join(dirs.SnapAccessors, efiFileName)) } if !slices.Contains(viTypes, snapType) { continue } viFileName := strings.Replace(expectedFileName, ".ef", ".vi", 1) - if _, err := os.Stat(filepath.Join(dir.SnapAccessors, viFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", viFileName, filepath.Join(dir.SnapAccessors, viFileName)) + if _, err := os.Stat(filepath.Join(dirs.SnapAccessors, viFileName)); err != nil { + return fmt.Errorf("missing file %s at path %s", viFileName, filepath.Join(dirs.SnapAccessors, viFileName)) } // check that .v vFileName := strings.Replace(expectedFileName, ".ef", ".v", 1) - if _, err := os.Stat(filepath.Join(dir.SnapHistory, vFileName)); err != nil { - return fmt.Errorf("missing file %s at path %s", vFileName, filepath.Join(dir.SnapHistory, vFileName)) + if _, err := os.Stat(filepath.Join(dirs.SnapHistory, vFileName)); err != nil { + return fmt.Errorf("missing file %s at path %s", vFileName, filepath.Join(dirs.SnapHistory, vFileName)) } } return nil @@ -814,10 +834,19 @@ func doPublishable(cliCtx *cli.Context) error { return err } // check if salt-state.txt and salt-block.txt exist - if _, err := os.Stat(filepath.Join(dat.Snap, "salt-state.txt")); err != nil { + exists, err := dir.FileExist(filepath.Join(dat.Snap, "salt-state.txt")) + if err != nil { + return err + } + if !exists { return fmt.Errorf("missing file %s", filepath.Join(dat.Snap, "salt-state.txt")) } - if _, err := os.Stat(filepath.Join(dat.Snap, "salt-blocks.txt")); err != nil { + + exists, err = dir.FileExist(filepath.Join(dat.Snap, "salt-blocks.txt")) + if err != nil { + return err + } + if !exists { return fmt.Errorf("missing file %s", filepath.Join(dat.Snap, "salt-blocks.txt")) } log.Info("All snapshots are publishable") @@ -836,7 +865,7 @@ func doClearIndexing(cliCtx *cli.Context) error { } // Delete all files in domainDir with extensions .bt and .bt.torrent - if err := deleteFilesWithExtensions(domainDir, []string{".bt", ".bt.torrent", ".kvei", ".kvei.torrent"}); err != nil { + if err := deleteFilesWithExtensions(domainDir, []string{".bt", ".bt.torrent", ".kvei", ".kvei.torrent", ".kvi", ".kvi.torrent"}); err != nil { return fmt.Errorf("failed to delete files in domainDir: %w", err) } diff --git a/turbo/rpchelper/helper.go b/turbo/rpchelper/helper.go index a544749b5aa..b83bfe2462a 100644 --- a/turbo/rpchelper/helper.go +++ b/turbo/rpchelper/helper.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + state2 "github.com/erigontech/erigon-lib/state" libcommon "github.com/erigontech/erigon-lib/common" From e525debec3436b352792e69b049a7ef3c78bb71e Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 2 Jan 2025 19:25:22 +0700 Subject: [PATCH 39/94] etl: cursor leak (#13299) --- erigon-lib/etl/collector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/etl/collector.go b/erigon-lib/etl/collector.go index a0890b0f334..9d509ad718b 100644 --- a/erigon-lib/etl/collector.go +++ b/erigon-lib/etl/collector.go @@ -189,6 +189,7 @@ func (c *Collector) Load(db kv.RwTx, toBucket string, loadFunc LoadFunc, args Tr if err != nil { return err } + defer cursor.Close() var errLast error lastKey, _, errLast = cursor.Last() if errLast != nil { From acc36b38d819f776143ae8b646bf7966da0da1e3 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 2 Jan 2025 19:49:06 +0700 Subject: [PATCH 40/94] add .kvi to downloader whitelist (#13304) --- erigon-lib/downloader/webseed.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index f7fca735c98..5139caecfe7 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -589,6 +589,7 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi strings.HasSuffix(name, ".idx.torrent") || strings.HasSuffix(name, ".kvei.torrent") || strings.HasSuffix(name, ".bt.torrent") || + strings.HasSuffix(name, ".kvi.torrent") || strings.HasSuffix(name, ".vi.torrent") || strings.HasSuffix(name, ".txt.torrent") || strings.HasSuffix(name, ".efi.torrent") From 85489c50f819c8571c34fc482249619eb0fe03e0 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 2 Jan 2025 20:18:40 +0700 Subject: [PATCH 41/94] add more info to header decode error (#13292) --- eth/stagedsync/stage_snapshots.go | 2 +- turbo/snapshotsync/freezeblocks/block_snapshots.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 01090754d39..c5df53527c1 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -331,7 +331,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Fill DB"}) if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, agg, logger); err != nil { - return err + return fmt.Errorf("FillDBFromSnapshots: %w", err) } if temporal, ok := tx.(*temporal.Tx); ok { diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index 64e24b51b89..464d5bf4d99 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -901,12 +901,12 @@ func ForEachHeader(ctx context.Context, s *RoSnapshots, walker func(header *type for _, sn := range view.Headers() { if err := sn.Src().WithReadAhead(func() error { g := sn.Src().MakeGetter() - for g.HasNext() { + for i := 0; g.HasNext(); i++ { word, _ = g.Next(word[:0]) var header types.Header r.Reset(word[1:]) if err := rlp.Decode(r, &header); err != nil { - return err + return fmt.Errorf("%w, file=%s, record=%d", err, sn.Src().FileName(), i) } if err := walker(&header); err != nil { return err From 7316b099fc796a497883bac7e69e7384ae710870 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 2 Jan 2025 20:46:07 +0700 Subject: [PATCH 42/94] linter: close cursor (#13298) --- cl/persistence/beacon_indicies/indicies.go | 5 ++ cmd/evm/internal/t8ntool/transition.go | 1 + cmd/integration/commands/refetence_db.go | 4 ++ cmd/verkle/main.go | 3 + erigon-lib/kv/backup/backup.go | 2 + erigon-lib/kv/mdbx/kv_mdbx.go | 58 +++++++++---------- .../kv/membatchwithdb/memory_mutation.go | 7 ++- erigon-lib/kv/remotedb/kv_remote.go | 7 ++- .../kv/remotedbserver/remotedbserver.go | 6 +- erigon-lib/rules.go | 31 +++++++++- erigon-lib/state/domain_stream.go | 4 +- erigon-lib/state/history.go | 4 +- erigon-lib/state/inverted_index_stream.go | 4 +- eth/stagedsync/stage_txlookup.go | 8 +-- eth/stagedsync/witness_util.go | 1 + p2p/enode/nodedb.go | 4 ++ polygon/bridge/mdbx_store.go | 1 + rules.go | 31 +++++++++- turbo/stages/headerdownload/header_algos.go | 1 + 19 files changed, 131 insertions(+), 51 deletions(-) diff --git a/cl/persistence/beacon_indicies/indicies.go b/cl/persistence/beacon_indicies/indicies.go index c83fa08b211..ebdd51efdd2 100644 --- a/cl/persistence/beacon_indicies/indicies.go +++ b/cl/persistence/beacon_indicies/indicies.go @@ -252,6 +252,7 @@ func PruneSignedHeaders(tx kv.RwTx, from uint64) error { if err != nil { return err } + defer cursor.Close() for k, _, err := cursor.Seek(base_encoding.Encode64ToBytes4(from)); err == nil && k != nil; k, _, err = cursor.Prev() { if err != nil { //nolint:govet return err @@ -268,6 +269,7 @@ func RangeBlockRoots(ctx context.Context, tx kv.Tx, fromSlot, toSlot uint64, fn if err != nil { return err } + defer cursor.Close() for k, v, err := cursor.Seek(base_encoding.Encode64ToBytes4(fromSlot)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= toSlot; k, v, err = cursor.Next() { if !fn(base_encoding.Decode64FromBytes4(k), libcommon.BytesToHash(v)) { break @@ -281,6 +283,7 @@ func PruneBlockRoots(ctx context.Context, tx kv.RwTx, fromSlot, toSlot uint64) e if err != nil { return err } + defer cursor.Close() for k, _, err := cursor.Seek(base_encoding.Encode64ToBytes4(fromSlot)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= toSlot; k, _, err = cursor.Next() { if err := cursor.DeleteCurrent(); err != nil { return err @@ -296,6 +299,7 @@ func ReadBeaconBlockRootsInSlotRange(ctx context.Context, tx kv.Tx, fromSlot, co if err != nil { return nil, nil, err } + defer cursor.Close() currentCount := uint64(0) for k, v, err := cursor.Seek(base_encoding.Encode64ToBytes4(fromSlot)); err == nil && k != nil && currentCount != count; k, v, err = cursor.Next() { currentCount++ @@ -373,6 +377,7 @@ func PruneBlocks(ctx context.Context, tx kv.RwTx, to uint64) error { if err != nil { return err } + defer cursor.Close() for k, _, err := cursor.First(); err == nil && k != nil; k, _, err = cursor.Prev() { if len(k) != 40 { continue diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 0ac1968dc66..7821505ab1d 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -627,6 +627,7 @@ func CalculateStateRoot(tx kv.RwTx) (*libcommon.Hash, error) { if err != nil { return nil, err } + defer c.Close() h := libcommon.NewHasher() defer libcommon.ReturnHasherToPool(h) domains, err := libstate.NewSharedDomains(tx, log.New()) diff --git a/cmd/integration/commands/refetence_db.go b/cmd/integration/commands/refetence_db.go index 9897adb70f6..1e2af3f41b3 100644 --- a/cmd/integration/commands/refetence_db.go +++ b/cmd/integration/commands/refetence_db.go @@ -263,6 +263,7 @@ func compareBuckets(ctx context.Context, tx kv.Tx, b string, refTx kv.Tx, refB s if err != nil { return err } + defer c.Close() k, v, e := c.First() if e != nil { return e @@ -271,6 +272,7 @@ func compareBuckets(ctx context.Context, tx kv.Tx, b string, refTx kv.Tx, refB s if err != nil { return err } + defer refC.Close() refK, refV, revErr := refC.First() if revErr != nil { return revErr @@ -386,6 +388,7 @@ MainLoop: if err != nil { return err } + defer c.Close() for { if !fileScanner.Scan() { @@ -419,6 +422,7 @@ MainLoop: logger.Info("Progress", "bucket", bucket, "key", hex.EncodeToString(k)) } } + c.Close() err = fileScanner.Err() if err != nil { panic(err) diff --git a/cmd/verkle/main.go b/cmd/verkle/main.go index e28d23e34c6..bc90eb2ba0d 100644 --- a/cmd/verkle/main.go +++ b/cmd/verkle/main.go @@ -259,6 +259,7 @@ func dump(ctx context.Context, cfg optionsCfg) error { if err != nil { return err } + defer verkleCursor.Close() for k, v, err := verkleCursor.First(); k != nil; k, v, err = verkleCursor.Next() { if err != nil { return err @@ -324,6 +325,7 @@ func dump_acc_preimages(ctx context.Context, cfg optionsCfg) error { if err != nil { return err } + defer stateCursor.Close() num, err := stages.GetStageProgress(tx, stages.Execution) if err != nil { return err @@ -380,6 +382,7 @@ func dump_storage_preimages(ctx context.Context, cfg optionsCfg, logger log.Logg if err != nil { return err } + defer stateCursor.Close() num, err := stages.GetStageProgress(tx, stages.Execution) if err != nil { return err diff --git a/erigon-lib/kv/backup/backup.go b/erigon-lib/kv/backup/backup.go index f533c7f90ed..52894e96b77 100644 --- a/erigon-lib/kv/backup/backup.go +++ b/erigon-lib/kv/backup/backup.go @@ -111,6 +111,7 @@ func backupTable(ctx context.Context, src kv.RoDB, srcTx kv.Tx, dst kv.RwDB, tab if err != nil { return err } + defer srcC.Close() total, _ = srcTx.Count(table) if err := dst.Update(ctx, func(tx kv.RwTx) error { @@ -128,6 +129,7 @@ func backupTable(ctx context.Context, src kv.RoDB, srcTx kv.Tx, dst kv.RwDB, tab if err != nil { return err } + defer c.Close() casted, isDupsort := c.(kv.RwCursorDupSort) i := uint64(0) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 123bcd1dd19..ac07a325f0e 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -585,7 +585,7 @@ func (db *MdbxKV) BeginRo(ctx context.Context) (txn kv.Tx, err error) { db: db, tx: tx, readOnly: true, - id: db.leakDetector.Add(), + traceID: db.leakDetector.Add(), }, nil } @@ -616,31 +616,32 @@ func (db *MdbxKV) beginRw(ctx context.Context, flags uint) (txn kv.RwTx, err err } return &MdbxTx{ - db: db, - tx: tx, - ctx: ctx, - id: db.leakDetector.Add(), + db: db, + tx: tx, + ctx: ctx, + traceID: db.leakDetector.Add(), }, nil } type MdbxTx struct { tx *mdbx.Txn - id uint64 // set only if TRACE_TX=true + traceID uint64 // set only if TRACE_TX=true db *MdbxKV statelessCursors map[string]kv.RwCursor readOnly bool ctx context.Context toCloseMap map[uint64]kv.Closer - ID uint64 + cursorID uint64 } type MdbxCursor struct { - tx *MdbxTx + toCloseMap map[uint64]kv.Closer c *mdbx.Cursor bucketName string - bucketCfg kv.TableCfgItem + isDupSort bool id uint64 + label kv.Label // marker to distinct db instances - one process may open many databases. for example to collect metrics of only 1 database } func (db *MdbxKV) Env() *mdbx.Env { return db.env } @@ -872,7 +873,7 @@ func (tx *MdbxTx) Commit() error { } else { runtime.UnlockOSThread() } - tx.db.leakDetector.Del(tx.id) + tx.db.leakDetector.Del(tx.traceID) }() tx.closeCursors() @@ -923,7 +924,7 @@ func (tx *MdbxTx) Rollback() { } else { runtime.UnlockOSThread() } - tx.db.leakDetector.Del(tx.id) + tx.db.leakDetector.Del(tx.traceID) }() tx.closeCursors() //tx.printDebugInfo() @@ -956,7 +957,7 @@ func (tx *MdbxTx) statelessCursor(bucket string) (kv.RwCursor, error) { c, ok := tx.statelessCursors[bucket] if !ok { var err error - c, err = tx.RwCursor(bucket) + c, err = tx.RwCursor(bucket) //nolint:gocritic if err != nil { return nil, err } @@ -1105,9 +1106,8 @@ func (tx *MdbxTx) Cursor(bucket string) (kv.Cursor, error) { } func (tx *MdbxTx) stdCursor(bucket string) (kv.RwCursor, error) { - b := tx.db.buckets[bucket] - c := &MdbxCursor{bucketName: bucket, tx: tx, bucketCfg: b, id: tx.ID} - tx.ID++ + c := &MdbxCursor{bucketName: bucket, toCloseMap: tx.toCloseMap, label: tx.db.opts.label, isDupSort: tx.db.buckets[bucket].Flags&mdbx.DupSort != 0, id: tx.cursorID} + tx.cursorID++ if tx.tx == nil { panic("assert: tx.tx nil. seems this `tx` was Rollback'ed") @@ -1220,7 +1220,7 @@ func (c *MdbxCursor) Delete(k []byte) error { return err } - if c.bucketCfg.Flags&mdbx.DupSort != 0 { + if c.isDupSort { return c.c.Del(mdbx.AllDups) } @@ -1237,7 +1237,7 @@ func (c *MdbxCursor) PutNoOverwrite(k, v []byte) error { return c.c.Put(k, v, md func (c *MdbxCursor) Put(key []byte, value []byte) error { if err := c.c.Put(key, value, 0); err != nil { - return fmt.Errorf("label: %s, table: %s, err: %w", c.tx.db.opts.label, c.bucketName, err) + return fmt.Errorf("label: %s, table: %s, err: %w", c.label, c.bucketName, err) } return nil } @@ -1258,7 +1258,7 @@ func (c *MdbxCursor) SeekExact(key []byte) ([]byte, []byte, error) { // Return error - if provided data will not sorted (or bucket have old records which mess with new in sorting manner). func (c *MdbxCursor) Append(k []byte, v []byte) error { if err := c.c.Put(k, v, mdbx.Append); err != nil { - return fmt.Errorf("label: %s, bucket: %s, %w", c.tx.db.opts.label, c.bucketName, err) + return fmt.Errorf("label: %s, bucket: %s, %w", c.label, c.bucketName, err) } return nil } @@ -1266,7 +1266,7 @@ func (c *MdbxCursor) Append(k []byte, v []byte) error { func (c *MdbxCursor) Close() { if c.c != nil { c.c.Close() - delete(c.tx.toCloseMap, c.id) + delete(c.toCloseMap, c.id) c.c = nil } } @@ -1381,21 +1381,21 @@ func (c *MdbxDupSortCursor) LastDup() ([]byte, error) { func (c *MdbxDupSortCursor) Append(k []byte, v []byte) error { if err := c.c.Put(k, v, mdbx.Append|mdbx.AppendDup); err != nil { - return fmt.Errorf("label: %s, in Append: bucket=%s, %w", c.tx.db.opts.label, c.bucketName, err) + return fmt.Errorf("label: %s, in Append: bucket=%s, %w", c.label, c.bucketName, err) } return nil } func (c *MdbxDupSortCursor) AppendDup(k []byte, v []byte) error { if err := c.c.Put(k, v, mdbx.AppendDup); err != nil { - return fmt.Errorf("label: %s, in AppendDup: bucket=%s, %w", c.tx.db.opts.label, c.bucketName, err) + return fmt.Errorf("label: %s, in AppendDup: bucket=%s, %w", c.label, c.bucketName, err) } return nil } func (c *MdbxDupSortCursor) PutNoDupData(k, v []byte) error { if err := c.c.Put(k, v, mdbx.NoDupData); err != nil { - return fmt.Errorf("label: %s, in PutNoDupData: %w", c.tx.db.opts.label, err) + return fmt.Errorf("label: %s, in PutNoDupData: %w", c.label, err) } return nil @@ -1404,7 +1404,7 @@ func (c *MdbxDupSortCursor) PutNoDupData(k, v []byte) error { // DeleteCurrentDuplicates - delete all of the data items for the current key. func (c *MdbxDupSortCursor) DeleteCurrentDuplicates() error { if err := c.c.Del(mdbx.AllDups); err != nil { - return fmt.Errorf("label: %s,in DeleteCurrentDuplicates: %w", c.tx.db.opts.label, err) + return fmt.Errorf("label: %s,in DeleteCurrentDuplicates: %w", c.label, err) } return nil } @@ -1456,8 +1456,8 @@ func (tx *MdbxTx) Prefix(table string, prefix []byte) (stream.KV, error) { } func (tx *MdbxTx) Range(table string, fromPrefix, toPrefix []byte, asc order.By, limit int) (stream.KV, error) { - s := &cursor2iter{ctx: tx.ctx, tx: tx, fromPrefix: fromPrefix, toPrefix: toPrefix, orderAscend: asc, limit: int64(limit), id: tx.ID} - tx.ID++ + s := &cursor2iter{ctx: tx.ctx, tx: tx, fromPrefix: fromPrefix, toPrefix: toPrefix, orderAscend: asc, limit: int64(limit), id: tx.cursorID} + tx.cursorID++ if tx.toCloseMap == nil { tx.toCloseMap = make(map[uint64]kv.Closer) } @@ -1487,7 +1487,7 @@ func (s *cursor2iter) init(table string, tx kv.Tx) error { if !s.orderAscend && s.fromPrefix != nil && s.toPrefix != nil && bytes.Compare(s.fromPrefix, s.toPrefix) <= 0 { return fmt.Errorf("tx.Dual: %x must be lexicographicaly before %x", s.toPrefix, s.fromPrefix) } - c, err := tx.Cursor(table) + c, err := tx.Cursor(table) //nolint:gocritic if err != nil { return err } @@ -1619,8 +1619,8 @@ func (s *cursor2iter) Next() (k, v []byte, err error) { } func (tx *MdbxTx) RangeDupSort(table string, key []byte, fromPrefix, toPrefix []byte, asc order.By, limit int) (stream.KV, error) { - s := &cursorDup2iter{ctx: tx.ctx, tx: tx, key: key, fromPrefix: fromPrefix, toPrefix: toPrefix, orderAscend: bool(asc), limit: int64(limit), id: tx.ID} - tx.ID++ + s := &cursorDup2iter{ctx: tx.ctx, tx: tx, key: key, fromPrefix: fromPrefix, toPrefix: toPrefix, orderAscend: bool(asc), limit: int64(limit), id: tx.cursorID} + tx.cursorID++ if tx.toCloseMap == nil { tx.toCloseMap = make(map[uint64]kv.Closer) } @@ -1651,7 +1651,7 @@ func (s *cursorDup2iter) init(table string, tx kv.Tx) error { if !s.orderAscend && s.fromPrefix != nil && s.toPrefix != nil && bytes.Compare(s.fromPrefix, s.toPrefix) <= 0 { return fmt.Errorf("tx.Dual: %x must be lexicographicaly before %x", s.toPrefix, s.fromPrefix) } - c, err := tx.CursorDupSort(table) + c, err := tx.CursorDupSort(table) //nolint:gocritic if err != nil { return err } diff --git a/erigon-lib/kv/membatchwithdb/memory_mutation.go b/erigon-lib/kv/membatchwithdb/memory_mutation.go index 60cfbb68dda..fcebd96ab78 100644 --- a/erigon-lib/kv/membatchwithdb/memory_mutation.go +++ b/erigon-lib/kv/membatchwithdb/memory_mutation.go @@ -121,6 +121,7 @@ func initSequences(db kv.Tx, memTx kv.RwTx) error { if err != nil { return err } + defer cursor.Close() for k, v, err := cursor.First(); k != nil; k, v, err = cursor.Next() { if err != nil { return err @@ -169,7 +170,7 @@ func (m *MemoryMutation) statelessCursor(table string) (kv.RwCursor, error) { c, ok := m.statelessCursors[table] if !ok { var err error - c, err = m.RwCursor(table) + c, err = m.RwCursor(table) // nolint:gocritic if err != nil { return nil, err } @@ -664,11 +665,11 @@ func (m *MemoryMutation) makeCursor(bucket string) (kv.RwCursorDupSort, error) { c.table = bucket var err error - c.cursor, err = m.db.CursorDupSort(bucket) + c.cursor, err = m.db.CursorDupSort(bucket) //nolint:gocritic if err != nil { return nil, err } - c.memCursor, err = m.memTx.RwCursorDupSort(bucket) + c.memCursor, err = m.memTx.RwCursorDupSort(bucket) //nolint:gocritic if err != nil { return nil, err } diff --git a/erigon-lib/kv/remotedb/kv_remote.go b/erigon-lib/kv/remotedb/kv_remote.go index 43a3d34cccf..529659a0e05 100644 --- a/erigon-lib/kv/remotedb/kv_remote.go +++ b/erigon-lib/kv/remotedb/kv_remote.go @@ -262,7 +262,7 @@ func (tx *tx) statelessCursor(bucket string) (kv.Cursor, error) { c, ok := tx.statelessCursors[bucket] if !ok { var err error - c, err = tx.Cursor(bucket) + c, err = tx.Cursor(bucket) // nolint:gocritic if err != nil { return nil, err } @@ -282,6 +282,7 @@ func (tx *tx) ForEach(bucket string, fromPrefix []byte, walker func(k, v []byte) if err != nil { return err } + defer it.Close() for it.HasNext() { k, v, err := it.Next() if err != nil { @@ -689,9 +690,9 @@ func (tx *tx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc or func (tx *tx) Prefix(table string, prefix []byte) (stream.KV, error) { nextPrefix, ok := kv.NextSubtree(prefix) if !ok { - return tx.Range(table, prefix, nil, order.Asc, kv.Unlim) + return tx.Range(table, prefix, nil, order.Asc, kv.Unlim) //nolint:gocritic } - return tx.Range(table, prefix, nextPrefix, order.Asc, kv.Unlim) + return tx.Range(table, prefix, nextPrefix, order.Asc, kv.Unlim) //nolint:gocritic } func (tx *tx) rangeOrderLimit(table string, fromPrefix, toPrefix []byte, asc order.By, limit int) (stream.KV, error) { diff --git a/erigon-lib/kv/remotedbserver/remotedbserver.go b/erigon-lib/kv/remotedbserver/remotedbserver.go index 54ad31537ea..b7e46c598f0 100644 --- a/erigon-lib/kv/remotedbserver/remotedbserver.go +++ b/erigon-lib/kv/remotedbserver/remotedbserver.go @@ -270,7 +270,7 @@ func (s *KvServer) Tx(stream remote.KV_TxServer) error { if err := s.with(id, func(tx kv.Tx) error { for _, c := range cursors { // restore all cursors position var err error - c.c, err = tx.Cursor(c.bucket) + c.c, err = tx.Cursor(c.bucket) //nolint:gocritic if err != nil { return err } @@ -311,7 +311,7 @@ func (s *KvServer) Tx(stream remote.KV_TxServer) error { CursorID++ var err error if err := s.with(id, func(tx kv.Tx) error { - c, err = tx.Cursor(in.BucketName) + c, err = tx.Cursor(in.BucketName) //nolint:gocritic if err != nil { return err } @@ -331,7 +331,7 @@ func (s *KvServer) Tx(stream remote.KV_TxServer) error { CursorID++ var err error if err := s.with(id, func(tx kv.Tx) error { - c, err = tx.CursorDupSort(in.BucketName) + c, err = tx.CursorDupSort(in.BucketName) //nolint:gocritic if err != nil { return err } diff --git a/erigon-lib/rules.go b/erigon-lib/rules.go index 5d7c14fbbdb..55ac6ecbbc0 100644 --- a/erigon-lib/rules.go +++ b/erigon-lib/rules.go @@ -19,7 +19,6 @@ package gorules -// https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module // to apply changes in this file, please do: ./build/bin/golangci-lint cache clean import ( "github.com/quasilyte/go-ruleguard/dsl" @@ -68,6 +67,36 @@ func txDeferRollback(m dsl.Matcher) { `) } +func cursorDeferClose(m dsl.Matcher) { + m.Match( + `$c, $err = $db.Cursor($table); $chk; $close`, + `$c, $err := $db.Cursor($table); $chk; $close`, + `$c, $err = $db.RwCursor($table); $chk; $close`, + `$c, $err := $db.RwCursor($table); $chk; $close`, + `$c, $err = $db.CursorDupSort($table); $chk; $close`, + `$c, $err := $db.CursorDupSort($table); $chk; $close`, + `$c, $err = $db.RwCursorDupSort($table); $chk; $close`, + `$c, $err := $db.RwCursorDupSort($table); $chk; $close`, + ). + Where(!m["close"].Text.Matches(`defer .*\.Close()`)). + //At(m["rollback"]). + Report(`Add "defer $c.Close()" right after cursor creation error check`) +} + +func streamDeferClose(m dsl.Matcher) { + m.Match( + `$c, $err = $db.Range($params); $chk; $close`, + `$c, $err := $db.Range($params); $chk; $close`, + `$c, $err = $db.RangeDupSort($params); $chk; $close`, + `$c, $err := $db.RangeDupSort($params); $chk; $close`, + `$c, $err = $db.Prefix($params); $chk; $close`, + `$c, $err := $db.Prefix($params); $chk; $close`, + ). + Where(!m["close"].Text.Matches(`defer .*\.Close()`)). + //At(m["rollback"]). + Report(`Add "defer $c.Close()" right after cursor creation error check`) +} + func closeCollector(m dsl.Matcher) { m.Match(`$c := etl.NewCollector($*_); $close`). Where(!m["close"].Text.Matches(`defer .*\.Close()`)). diff --git a/erigon-lib/state/domain_stream.go b/erigon-lib/state/domain_stream.go index 67d7605f849..1156be7b3fa 100644 --- a/erigon-lib/state/domain_stream.go +++ b/erigon-lib/state/domain_stream.go @@ -130,7 +130,7 @@ func (hi *DomainLatestIterFile) init(dc *DomainRoTx) error { var key, value []byte if dc.d.largeValues { - valsCursor, err := hi.roTx.Cursor(dc.d.valuesTable) + valsCursor, err := hi.roTx.Cursor(dc.d.valuesTable) //nolint:gocritic if err != nil { return err } @@ -146,7 +146,7 @@ func (hi *DomainLatestIterFile) init(dc *DomainRoTx) error { heap.Push(hi.h, &CursorItem{t: DB_CURSOR, key: common.Copy(k), val: common.Copy(value), cNonDup: valsCursor, endTxNum: endTxNum, reverse: true}) } } else { - valsCursor, err := hi.roTx.CursorDupSort(dc.d.valuesTable) + valsCursor, err := hi.roTx.CursorDupSort(dc.d.valuesTable) //nolint:gocritic if err != nil { return err } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index db2258b6046..c6832962ce5 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1221,7 +1221,7 @@ func (ht *HistoryRoTx) valsCursor(tx kv.Tx) (c kv.Cursor, err error) { if ht.valsC != nil { return ht.valsC, nil } - ht.valsC, err = tx.Cursor(ht.h.valuesTable) + ht.valsC, err = tx.Cursor(ht.h.valuesTable) //nolint:gocritic if err != nil { return nil, err } @@ -1231,7 +1231,7 @@ func (ht *HistoryRoTx) valsCursorDup(tx kv.Tx) (c kv.CursorDupSort, err error) { if ht.valsCDup != nil { return ht.valsCDup, nil } - ht.valsCDup, err = tx.CursorDupSort(ht.h.valuesTable) + ht.valsCDup, err = tx.CursorDupSort(ht.h.valuesTable) //nolint:gocritic if err != nil { return nil, err } diff --git a/erigon-lib/state/inverted_index_stream.go b/erigon-lib/state/inverted_index_stream.go index bf5aba0155e..6df0a465153 100644 --- a/erigon-lib/state/inverted_index_stream.go +++ b/erigon-lib/state/inverted_index_stream.go @@ -192,7 +192,7 @@ func (it *RecentInvertedIdxIter) advanceInDB() { var v []byte var err error if it.cursor == nil { - if it.cursor, err = it.roTx.CursorDupSort(it.indexTable); err != nil { + if it.cursor, err = it.roTx.CursorDupSort(it.indexTable); err != nil { //nolint:gocritic // TODO pass error properly around panic(err) } @@ -356,7 +356,7 @@ func (it *InvertedIterator1) advanceInDb() { var k, v []byte var err error if it.cursor == nil { - if it.cursor, err = it.roTx.CursorDupSort(it.indexTable); err != nil { + if it.cursor, err = it.roTx.CursorDupSort(it.indexTable); err != nil { //nolint:gocritic // TODO pass error properly around panic(err) } diff --git a/eth/stagedsync/stage_txlookup.go b/eth/stagedsync/stage_txlookup.go index 8083945b318..a198ffff997 100644 --- a/eth/stagedsync/stage_txlookup.go +++ b/eth/stagedsync/stage_txlookup.go @@ -23,17 +23,15 @@ import ( "math/big" "time" - "github.com/erigontech/erigon-lib/kv/rawdbv3" - - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/eth/stagedsync/stages" - "github.com/erigontech/erigon-lib/chain" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/etl" "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/rawdbv3" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/rawdb" + "github.com/erigontech/erigon/eth/stagedsync/stages" "github.com/erigontech/erigon/ethdb/prune" "github.com/erigontech/erigon/polygon/bor/borcfg" bortypes "github.com/erigontech/erigon/polygon/bor/types" diff --git a/eth/stagedsync/witness_util.go b/eth/stagedsync/witness_util.go index cebda023e47..ebfeff0a715 100644 --- a/eth/stagedsync/witness_util.go +++ b/eth/stagedsync/witness_util.go @@ -119,6 +119,7 @@ func FindOldestWitness(tx kv.Tx, tableName string) (uint64, error) { if err != nil { return 0, err } + defer cursor.Close() k, _, err := cursor.First() if err != nil { diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index 5466ebf8e2c..857d2f0b48a 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -141,6 +141,7 @@ func newPersistentDB(ctx context.Context, logger log.Logger, path string) (*DB, if err != nil { return err } + defer c.Close() _, v, errGet := c.SeekExact([]byte(dbVersionKey)) if errGet != nil { return errGet @@ -381,6 +382,7 @@ func deleteRangeInBucket(tx kv.RwTx, prefix []byte, bucket string) error { if err != nil { return err } + defer c.Close() var k []byte for k, _, err = c.Seek(prefix); (err == nil) && (k != nil) && bytes.HasPrefix(k, prefix); k, _, err = c.Next() { if err = c.DeleteCurrent(); err != nil { @@ -431,6 +433,7 @@ func (db *DB) expireNodes() { if err != nil { return err } + defer c.Close() p := []byte(dbNodePrefix) var prevId ID var empty = true @@ -564,6 +567,7 @@ func (db *DB) QuerySeeds(n int, maxAge time.Duration) []*Node { if err != nil { return err } + defer c.Close() seek: for seeks := 0; len(nodes) < n && seeks < n*5; seeks++ { // seekInFiles to a random entry. The first byte is incremented by a diff --git a/polygon/bridge/mdbx_store.go b/polygon/bridge/mdbx_store.go index 17742bc6f39..660ca54b206 100644 --- a/polygon/bridge/mdbx_store.go +++ b/polygon/bridge/mdbx_store.go @@ -571,6 +571,7 @@ func (s txStore) blockEventIdsRange(ctx context.Context, blockNum uint64, lastFr if err != nil { return start, end, false, err } + defer cursor.Close() _, v, err := cursor.SeekExact(kByte) if err != nil { diff --git a/rules.go b/rules.go index 9a7e62eed6d..6f378fa5579 100644 --- a/rules.go +++ b/rules.go @@ -18,7 +18,6 @@ package gorules -// https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module // to apply changes in this file, please do: ./build/bin/golangci-lint cache clean import ( "github.com/quasilyte/go-ruleguard/dsl" @@ -67,6 +66,36 @@ func txDeferRollback(m dsl.Matcher) { `) } +func cursorDeferClose(m dsl.Matcher) { + m.Match( + `$c, $err = $db.Cursor($table); $chk; $close`, + `$c, $err := $db.Cursor($table); $chk; $close`, + `$c, $err = $db.RwCursor($table); $chk; $close`, + `$c, $err := $db.RwCursor($table); $chk; $close`, + `$c, $err = $db.CursorDupSort($table); $chk; $close`, + `$c, $err := $db.CursorDupSort($table); $chk; $close`, + `$c, $err = $db.RwCursorDupSort($table); $chk; $close`, + `$c, $err := $db.RwCursorDupSort($table); $chk; $close`, + ). + Where(!m["close"].Text.Matches(`defer .*\.Close()`)). + //At(m["rollback"]). + Report(`Add "defer $c.Close()" right after cursor creation error check`) +} + +func streamDeferClose(m dsl.Matcher) { + m.Match( + `$c, $err = $db.Range($params); $chk; $close`, + `$c, $err := $db.Range($params); $chk; $close`, + `$c, $err = $db.RangeDupSort($params); $chk; $close`, + `$c, $err := $db.RangeDupSort($params); $chk; $close`, + `$c, $err = $db.Prefix($params); $chk; $close`, + `$c, $err := $db.Prefix($params); $chk; $close`, + ). + Where(!m["close"].Text.Matches(`defer .*\.Close()`)). + //At(m["rollback"]). + Report(`Add "defer $c.Close()" right after cursor creation error check`) +} + func closeCollector(m dsl.Matcher) { m.Match(`$c := etl.NewCollector($*_); $close`). Where(!m["close"].Text.Matches(`defer .*\.Close()`)). diff --git a/turbo/stages/headerdownload/header_algos.go b/turbo/stages/headerdownload/header_algos.go index 7bc99c39788..54d46861a68 100644 --- a/turbo/stages/headerdownload/header_algos.go +++ b/turbo/stages/headerdownload/header_algos.go @@ -333,6 +333,7 @@ func (hd *HeaderDownload) RecoverFromDb(db kv.RoDB) error { if err != nil { return err } + defer c.Close() hd.highestInDb, err = stages.GetStageProgress(tx, stages.Headers) if err != nil { return err From ef62be7a5d6714c5acb76cb537cea3e7c7d5ab27 Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Thu, 2 Jan 2025 18:40:36 +0000 Subject: [PATCH 43/94] erigon-lib: move event notifier and observers to erigon-lib (#13307) moving `EventNotifier` and `Observers` structs from `polygon/polygoncommon` pkg to `erigon-lib/event` pkg since those will be used in other packages outside of polygon (e.g. using in `shutter` pkg as part of https://github.com/erigontech/erigon/pull/13306) --- .../event/notifier.go | 18 ++++----- .../event}/observers.go | 2 +- polygon/heimdall/scraper.go | 15 ++++--- polygon/heimdall/service.go | 8 ++-- polygon/p2p/message_listener.go | 39 +++++++++---------- polygon/p2p/peer_event_registrar.go | 8 ++-- polygon/p2p/peer_event_registrar_mock.go | 21 +++++----- polygon/p2p/peer_tracker.go | 10 ++--- polygon/p2p/peer_tracker_test.go | 8 ++-- polygon/p2p/publisher_test.go | 8 ++-- polygon/p2p/service.go | 6 +-- polygon/sync/tip_events.go | 11 +++--- 12 files changed, 75 insertions(+), 79 deletions(-) rename polygon/polygoncommon/event_notifier.go => erigon-lib/event/notifier.go (83%) rename {polygon/polygoncommon => erigon-lib/event}/observers.go (99%) diff --git a/polygon/polygoncommon/event_notifier.go b/erigon-lib/event/notifier.go similarity index 83% rename from polygon/polygoncommon/event_notifier.go rename to erigon-lib/event/notifier.go index 1965e362a01..63a4e576536 100644 --- a/polygon/polygoncommon/event_notifier.go +++ b/erigon-lib/event/notifier.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with Erigon. If not, see . -package polygoncommon +package event import ( "context" @@ -22,36 +22,36 @@ import ( "sync/atomic" ) -// EventNotifier notifies waiters about an event. +// Notifier notifies waiters about an event. // It supports a single "producer" and multiple waiters. // A producer can set the event state to "signaled" or "non-signaled". // Waiters can wait for the "signaled" event state. -type EventNotifier struct { +type Notifier struct { mutex sync.Mutex cond *sync.Cond hasEvent atomic.Bool } -func NewEventNotifier() *EventNotifier { - instance := &EventNotifier{} +func NewNotifier() *Notifier { + instance := &Notifier{} instance.cond = sync.NewCond(&instance.mutex) return instance } // Reset to the "non-signaled" state. -func (en *EventNotifier) Reset() { +func (en *Notifier) Reset() { en.hasEvent.Store(false) } // SetAndBroadcast sets the "signaled" state and notifies all waiters. -func (en *EventNotifier) SetAndBroadcast() { +func (en *Notifier) SetAndBroadcast() { en.hasEvent.Store(true) en.cond.Broadcast() } // Wait for the "signaled" state. // If the event is already "signaled" it returns immediately. -func (en *EventNotifier) Wait(ctx context.Context) error { +func (en *Notifier) Wait(ctx context.Context) error { waitCtx, waitCancel := context.WithCancel(ctx) defer waitCancel() @@ -74,7 +74,7 @@ func (en *EventNotifier) Wait(ctx context.Context) error { <-waitCtx.Done() // if the parent context is done, force the waiting goroutine to exit - // this might lead to spurious wake ups for other waiters, + // this might lead to spurious wake-ups for other waiters, // but it is ok due to the waiting loop conditions en.cond.Broadcast() diff --git a/polygon/polygoncommon/observers.go b/erigon-lib/event/observers.go similarity index 99% rename from polygon/polygoncommon/observers.go rename to erigon-lib/event/observers.go index d05c5fd3d7f..46e7785e7ab 100644 --- a/polygon/polygoncommon/observers.go +++ b/erigon-lib/event/observers.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with Erigon. If not, see . -package polygoncommon +package event import ( "sync" diff --git a/polygon/heimdall/scraper.go b/polygon/heimdall/scraper.go index 060b42e9cdf..3cfb97450dc 100644 --- a/polygon/heimdall/scraper.go +++ b/polygon/heimdall/scraper.go @@ -22,12 +22,11 @@ import ( "fmt" "time" + libcommon "github.com/erigontech/erigon-lib/common" commonerrors "github.com/erigontech/erigon-lib/common/errors" "github.com/erigontech/erigon-lib/common/generics" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/log/v3" - - libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/polygon/polygoncommon" ) type Scraper[TEntity Entity] struct { @@ -35,8 +34,8 @@ type Scraper[TEntity Entity] struct { store EntityStore[TEntity] fetcher entityFetcher[TEntity] pollDelay time.Duration - observers *polygoncommon.Observers[[]TEntity] - syncEvent *polygoncommon.EventNotifier + observers *event.Observers[[]TEntity] + syncEvent *event.Notifier transientErrors []error logger log.Logger } @@ -54,8 +53,8 @@ func NewScraper[TEntity Entity]( store: store, fetcher: fetcher, pollDelay: pollDelay, - observers: polygoncommon.NewObservers[[]TEntity](), - syncEvent: polygoncommon.NewEventNotifier(), + observers: event.NewObservers[[]TEntity](), + syncEvent: event.NewNotifier(), transientErrors: transientErrors, logger: logger, } @@ -146,7 +145,7 @@ func (s *Scraper[TEntity]) Run(ctx context.Context) error { return ctx.Err() } -func (s *Scraper[TEntity]) RegisterObserver(observer func([]TEntity)) polygoncommon.UnregisterFunc { +func (s *Scraper[TEntity]) RegisterObserver(observer func([]TEntity)) event.UnregisterFunc { return s.observers.Register(observer) } diff --git a/polygon/heimdall/service.go b/polygon/heimdall/service.go index 5012619b6b1..dd60dc69bbf 100644 --- a/polygon/heimdall/service.go +++ b/polygon/heimdall/service.go @@ -26,10 +26,10 @@ import ( "golang.org/x/sync/errgroup" libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/polygon/bor/borcfg" "github.com/erigontech/erigon/polygon/bor/valset" - "github.com/erigontech/erigon/polygon/polygoncommon" ) type ServiceConfig struct { @@ -222,7 +222,7 @@ func (s *Service) Producers(ctx context.Context, blockNum uint64) (*valset.Valid return s.reader.Producers(ctx, blockNum) } -func (s *Service) RegisterMilestoneObserver(callback func(*Milestone), opts ...ObserverOption) polygoncommon.UnregisterFunc { +func (s *Service) RegisterMilestoneObserver(callback func(*Milestone), opts ...ObserverOption) event.UnregisterFunc { options := NewObserverOptions(opts...) return s.milestoneScraper.RegisterObserver(func(entities []*Milestone) { for _, entity := range libcommon.SliceTakeLast(entities, options.eventsLimit) { @@ -231,7 +231,7 @@ func (s *Service) RegisterMilestoneObserver(callback func(*Milestone), opts ...O }) } -func (s *Service) RegisterCheckpointObserver(callback func(*Checkpoint), opts ...ObserverOption) polygoncommon.UnregisterFunc { +func (s *Service) RegisterCheckpointObserver(callback func(*Checkpoint), opts ...ObserverOption) event.UnregisterFunc { options := NewObserverOptions(opts...) return s.checkpointScraper.RegisterObserver(func(entities []*Checkpoint) { for _, entity := range libcommon.SliceTakeLast(entities, options.eventsLimit) { @@ -240,7 +240,7 @@ func (s *Service) RegisterCheckpointObserver(callback func(*Checkpoint), opts .. }) } -func (s *Service) RegisterSpanObserver(callback func(*Span), opts ...ObserverOption) polygoncommon.UnregisterFunc { +func (s *Service) RegisterSpanObserver(callback func(*Span), opts ...ObserverOption) event.UnregisterFunc { options := NewObserverOptions(opts...) return s.spanScraper.RegisterObserver(func(entities []*Span) { for _, entity := range libcommon.SliceTakeLast(entities, options.eventsLimit) { diff --git a/polygon/p2p/message_listener.go b/polygon/p2p/message_listener.go index 3d32038bccd..18efcd9d123 100644 --- a/polygon/p2p/message_listener.go +++ b/polygon/p2p/message_listener.go @@ -23,13 +23,12 @@ import ( "google.golang.org/grpc" - "github.com/erigontech/erigon-lib/log/v3" - + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/p2p/sentry" "github.com/erigontech/erigon-lib/rlp" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" ) type DecodedInboundMessage[TPacket any] struct { @@ -38,7 +37,7 @@ type DecodedInboundMessage[TPacket any] struct { PeerId *PeerId } -type UnregisterFunc = polygoncommon.UnregisterFunc +type UnregisterFunc = event.UnregisterFunc func NewMessageListener( logger log.Logger, @@ -51,11 +50,11 @@ func NewMessageListener( sentryClient: sentryClient, statusDataFactory: statusDataFactory, peerPenalizer: peerPenalizer, - newBlockObservers: polygoncommon.NewObservers[*DecodedInboundMessage[*eth.NewBlockPacket]](), - newBlockHashesObservers: polygoncommon.NewObservers[*DecodedInboundMessage[*eth.NewBlockHashesPacket]](), - blockHeadersObservers: polygoncommon.NewObservers[*DecodedInboundMessage[*eth.BlockHeadersPacket66]](), - blockBodiesObservers: polygoncommon.NewObservers[*DecodedInboundMessage[*eth.BlockBodiesPacket66]](), - peerEventObservers: polygoncommon.NewObservers[*sentryproto.PeerEvent](), + newBlockObservers: event.NewObservers[*DecodedInboundMessage[*eth.NewBlockPacket]](), + newBlockHashesObservers: event.NewObservers[*DecodedInboundMessage[*eth.NewBlockHashesPacket]](), + blockHeadersObservers: event.NewObservers[*DecodedInboundMessage[*eth.BlockHeadersPacket66]](), + blockBodiesObservers: event.NewObservers[*DecodedInboundMessage[*eth.BlockBodiesPacket66]](), + peerEventObservers: event.NewObservers[*sentryproto.PeerEvent](), } } @@ -64,11 +63,11 @@ type MessageListener struct { sentryClient sentryproto.SentryClient statusDataFactory sentry.StatusDataFactory peerPenalizer *PeerPenalizer - newBlockObservers *polygoncommon.Observers[*DecodedInboundMessage[*eth.NewBlockPacket]] - newBlockHashesObservers *polygoncommon.Observers[*DecodedInboundMessage[*eth.NewBlockHashesPacket]] - blockHeadersObservers *polygoncommon.Observers[*DecodedInboundMessage[*eth.BlockHeadersPacket66]] - blockBodiesObservers *polygoncommon.Observers[*DecodedInboundMessage[*eth.BlockBodiesPacket66]] - peerEventObservers *polygoncommon.Observers[*sentryproto.PeerEvent] + newBlockObservers *event.Observers[*DecodedInboundMessage[*eth.NewBlockPacket]] + newBlockHashesObservers *event.Observers[*DecodedInboundMessage[*eth.NewBlockHashesPacket]] + blockHeadersObservers *event.Observers[*DecodedInboundMessage[*eth.BlockHeadersPacket66]] + blockBodiesObservers *event.Observers[*DecodedInboundMessage[*eth.BlockBodiesPacket66]] + peerEventObservers *event.Observers[*sentryproto.PeerEvent] stopWg sync.WaitGroup } @@ -98,23 +97,23 @@ func (ml *MessageListener) Run(ctx context.Context) error { return ctx.Err() } -func (ml *MessageListener) RegisterNewBlockObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { +func (ml *MessageListener) RegisterNewBlockObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { return ml.newBlockObservers.Register(observer) } -func (ml *MessageListener) RegisterNewBlockHashesObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { +func (ml *MessageListener) RegisterNewBlockHashesObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { return ml.newBlockHashesObservers.Register(observer) } -func (ml *MessageListener) RegisterBlockHeadersObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.BlockHeadersPacket66]]) UnregisterFunc { +func (ml *MessageListener) RegisterBlockHeadersObserver(observer event.Observer[*DecodedInboundMessage[*eth.BlockHeadersPacket66]]) UnregisterFunc { return ml.blockHeadersObservers.Register(observer) } -func (ml *MessageListener) RegisterBlockBodiesObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.BlockBodiesPacket66]]) UnregisterFunc { +func (ml *MessageListener) RegisterBlockBodiesObserver(observer event.Observer[*DecodedInboundMessage[*eth.BlockBodiesPacket66]]) UnregisterFunc { return ml.blockBodiesObservers.Register(observer) } -func (ml *MessageListener) RegisterPeerEventObserver(observer polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc { +func (ml *MessageListener) RegisterPeerEventObserver(observer event.Observer[*sentryproto.PeerEvent]) UnregisterFunc { return ml.peerEventObservers.Register(observer) } @@ -193,7 +192,7 @@ func notifyInboundMessageObservers[TPacket any]( ctx context.Context, logger log.Logger, peerPenalizer *PeerPenalizer, - observers *polygoncommon.Observers[*DecodedInboundMessage[TPacket]], + observers *event.Observers[*DecodedInboundMessage[TPacket]], message *sentryproto.InboundMessage, ) error { peerId := PeerIdFromH512(message.PeerId) diff --git a/polygon/p2p/peer_event_registrar.go b/polygon/p2p/peer_event_registrar.go index 9a5c80497a1..b9c900f625e 100644 --- a/polygon/p2p/peer_event_registrar.go +++ b/polygon/p2p/peer_event_registrar.go @@ -17,14 +17,14 @@ package p2p import ( + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" ) //go:generate mockgen -typed=true -source=./peer_event_registrar.go -destination=./peer_event_registrar_mock.go -package=p2p type peerEventRegistrar interface { - RegisterPeerEventObserver(observer polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc - RegisterNewBlockObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc - RegisterNewBlockHashesObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc + RegisterPeerEventObserver(observer event.Observer[*sentryproto.PeerEvent]) UnregisterFunc + RegisterNewBlockObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc + RegisterNewBlockHashesObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc } diff --git a/polygon/p2p/peer_event_registrar_mock.go b/polygon/p2p/peer_event_registrar_mock.go index 866ed27cacf..189c9f3a03f 100644 --- a/polygon/p2p/peer_event_registrar_mock.go +++ b/polygon/p2p/peer_event_registrar_mock.go @@ -12,9 +12,9 @@ package p2p import ( reflect "reflect" + event "github.com/erigontech/erigon-lib/event" sentryproto "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" eth "github.com/erigontech/erigon/eth/protocols/eth" - polygoncommon "github.com/erigontech/erigon/polygon/polygoncommon" gomock "go.uber.org/mock/gomock" ) @@ -22,7 +22,6 @@ import ( type MockpeerEventRegistrar struct { ctrl *gomock.Controller recorder *MockpeerEventRegistrarMockRecorder - isgomock struct{} } // MockpeerEventRegistrarMockRecorder is the mock recorder for MockpeerEventRegistrar. @@ -43,7 +42,7 @@ func (m *MockpeerEventRegistrar) EXPECT() *MockpeerEventRegistrarMockRecorder { } // RegisterNewBlockHashesObserver mocks base method. -func (m *MockpeerEventRegistrar) RegisterNewBlockHashesObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { +func (m *MockpeerEventRegistrar) RegisterNewBlockHashesObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegisterNewBlockHashesObserver", observer) ret0, _ := ret[0].(UnregisterFunc) @@ -69,19 +68,19 @@ func (c *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall) Return(arg0 U } // Do rewrite *gomock.Call.Do -func (c *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall) Do(f func(polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall { +func (c *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall) Do(f func(event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall) DoAndReturn(f func(polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall { +func (c *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall) DoAndReturn(f func(event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockHashesObserverCall { c.Call = c.Call.DoAndReturn(f) return c } // RegisterNewBlockObserver mocks base method. -func (m *MockpeerEventRegistrar) RegisterNewBlockObserver(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { +func (m *MockpeerEventRegistrar) RegisterNewBlockObserver(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegisterNewBlockObserver", observer) ret0, _ := ret[0].(UnregisterFunc) @@ -107,19 +106,19 @@ func (c *MockpeerEventRegistrarRegisterNewBlockObserverCall) Return(arg0 Unregis } // Do rewrite *gomock.Call.Do -func (c *MockpeerEventRegistrarRegisterNewBlockObserverCall) Do(f func(polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockObserverCall { +func (c *MockpeerEventRegistrarRegisterNewBlockObserverCall) Do(f func(event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockObserverCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpeerEventRegistrarRegisterNewBlockObserverCall) DoAndReturn(f func(polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockObserverCall { +func (c *MockpeerEventRegistrarRegisterNewBlockObserverCall) DoAndReturn(f func(event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc) *MockpeerEventRegistrarRegisterNewBlockObserverCall { c.Call = c.Call.DoAndReturn(f) return c } // RegisterPeerEventObserver mocks base method. -func (m *MockpeerEventRegistrar) RegisterPeerEventObserver(observer polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc { +func (m *MockpeerEventRegistrar) RegisterPeerEventObserver(observer event.Observer[*sentryproto.PeerEvent]) UnregisterFunc { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegisterPeerEventObserver", observer) ret0, _ := ret[0].(UnregisterFunc) @@ -145,13 +144,13 @@ func (c *MockpeerEventRegistrarRegisterPeerEventObserverCall) Return(arg0 Unregi } // Do rewrite *gomock.Call.Do -func (c *MockpeerEventRegistrarRegisterPeerEventObserverCall) Do(f func(polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc) *MockpeerEventRegistrarRegisterPeerEventObserverCall { +func (c *MockpeerEventRegistrarRegisterPeerEventObserverCall) Do(f func(event.Observer[*sentryproto.PeerEvent]) UnregisterFunc) *MockpeerEventRegistrarRegisterPeerEventObserverCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpeerEventRegistrarRegisterPeerEventObserverCall) DoAndReturn(f func(polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc) *MockpeerEventRegistrarRegisterPeerEventObserverCall { +func (c *MockpeerEventRegistrarRegisterPeerEventObserverCall) DoAndReturn(f func(event.Observer[*sentryproto.PeerEvent]) UnregisterFunc) *MockpeerEventRegistrarRegisterPeerEventObserverCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/polygon/p2p/peer_tracker.go b/polygon/p2p/peer_tracker.go index c54f29b6204..acc23088e1e 100644 --- a/polygon/p2p/peer_tracker.go +++ b/polygon/p2p/peer_tracker.go @@ -24,10 +24,10 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" ) func NewPeerTracker( @@ -65,7 +65,7 @@ type PeerTracker struct { func (pt *PeerTracker) Run(ctx context.Context) error { pt.logger.Info(peerTrackerLogPrefix("running peer tracker component")) - var peerEventUnreg polygoncommon.UnregisterFunc + var peerEventUnreg event.UnregisterFunc defer func() { peerEventUnreg() }() err := func() error { @@ -214,7 +214,7 @@ func (pt *PeerTracker) updatePeerSyncProgress(peerId *PeerId, update func(psp *p update(peerSyncProgress) } -func newPeerEventObserver(pt *PeerTracker) polygoncommon.Observer[*sentryproto.PeerEvent] { +func newPeerEventObserver(pt *PeerTracker) event.Observer[*sentryproto.PeerEvent] { return func(message *sentryproto.PeerEvent) { peerId := PeerIdFromH512(message.PeerId) switch message.EventId { @@ -226,7 +226,7 @@ func newPeerEventObserver(pt *PeerTracker) polygoncommon.Observer[*sentryproto.P } } -func newBlockHashAnnouncesObserver(pt *PeerTracker) polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]] { +func newBlockHashAnnouncesObserver(pt *PeerTracker) event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]] { return func(message *DecodedInboundMessage[*eth.NewBlockHashesPacket]) { for _, hashOrNum := range *message.Decoded { pt.BlockHashPresent(message.PeerId, hashOrNum.Hash) @@ -234,7 +234,7 @@ func newBlockHashAnnouncesObserver(pt *PeerTracker) polygoncommon.Observer[*Deco } } -func newBlockAnnouncesObserver(pt *PeerTracker) polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]] { +func newBlockAnnouncesObserver(pt *PeerTracker) event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]] { return func(message *DecodedInboundMessage[*eth.NewBlockPacket]) { pt.BlockHashPresent(message.PeerId, message.Decoded.Block.Hash()) } diff --git a/polygon/p2p/peer_tracker_test.go b/polygon/p2p/peer_tracker_test.go index 28489077c41..86830fca5bc 100644 --- a/polygon/p2p/peer_tracker_test.go +++ b/polygon/p2p/peer_tracker_test.go @@ -29,12 +29,12 @@ import ( "go.uber.org/mock/gomock" libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/gointerfaces/typesproto" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" "github.com/erigontech/erigon/turbo/testlog" ) @@ -286,7 +286,7 @@ func (ptt *peerTrackerTest) mockPeerProvider(peerReply *sentryproto.PeersReply) func (ptt *peerTrackerTest) mockPeerEvents(events <-chan *sentryproto.PeerEvent) { ptt.peerEventRegistrar.EXPECT(). RegisterPeerEventObserver(gomock.Any()). - DoAndReturn(func(observer polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc { + DoAndReturn(func(observer event.Observer[*sentryproto.PeerEvent]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -308,7 +308,7 @@ func (ptt *peerTrackerTest) mockNewBlockHashesEvents(events <-chan *DecodedInbou ptt.peerEventRegistrar.EXPECT(). RegisterNewBlockHashesObserver(gomock.Any()). DoAndReturn( - func(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { + func(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -331,7 +331,7 @@ func (ptt *peerTrackerTest) mockNewBlockEvents(events <-chan *DecodedInboundMess ptt.peerEventRegistrar.EXPECT(). RegisterNewBlockObserver(gomock.Any()). DoAndReturn( - func(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { + func(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { diff --git a/polygon/p2p/publisher_test.go b/polygon/p2p/publisher_test.go index 9cb35448202..8c6f016bb6e 100644 --- a/polygon/p2p/publisher_test.go +++ b/polygon/p2p/publisher_test.go @@ -31,12 +31,12 @@ import ( "google.golang.org/grpc" "github.com/erigontech/erigon-lib/direct" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/gointerfaces/typesproto" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" "github.com/erigontech/erigon/turbo/testlog" ) @@ -234,7 +234,7 @@ func (pt publisherTest) mockPeerProvider(peerReply *sentryproto.PeersReply) { func (pt publisherTest) mockPeerEvents(events <-chan *sentryproto.PeerEvent) { pt.peerEventRegistrar.EXPECT(). RegisterPeerEventObserver(gomock.Any()). - DoAndReturn(func(observer polygoncommon.Observer[*sentryproto.PeerEvent]) UnregisterFunc { + DoAndReturn(func(observer event.Observer[*sentryproto.PeerEvent]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -260,7 +260,7 @@ func (pt publisherTest) mockNewBlockHashesEvents(events <-chan *DecodedInboundMe pt.peerEventRegistrar.EXPECT(). RegisterNewBlockHashesObserver(gomock.Any()). DoAndReturn( - func(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { + func(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -287,7 +287,7 @@ func (pt publisherTest) mockNewBlockEvents(events <-chan *DecodedInboundMessage[ pt.peerEventRegistrar.EXPECT(). RegisterNewBlockObserver(gomock.Any()). DoAndReturn( - func(observer polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { + func(observer event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) UnregisterFunc { ctx, cancel := context.WithCancel(context.Background()) go func() { for { diff --git a/polygon/p2p/service.go b/polygon/p2p/service.go index b21bf38ae99..15c01a15033 100644 --- a/polygon/p2p/service.go +++ b/polygon/p2p/service.go @@ -24,12 +24,12 @@ import ( "golang.org/x/sync/errgroup" libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/p2p/sentry" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/polygon/polygoncommon" ) func NewService(logger log.Logger, maxPeers int, sc sentryproto.SentryClient, sdf sentry.StatusDataFactory) *Service { @@ -124,10 +124,10 @@ func (s *Service) Penalize(ctx context.Context, peerId *PeerId) error { return s.peerPenalizer.Penalize(ctx, peerId) } -func (s *Service) RegisterNewBlockObserver(o polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) polygoncommon.UnregisterFunc { +func (s *Service) RegisterNewBlockObserver(o event.Observer[*DecodedInboundMessage[*eth.NewBlockPacket]]) event.UnregisterFunc { return s.messageListener.RegisterNewBlockObserver(o) } -func (s *Service) RegisterNewBlockHashesObserver(o polygoncommon.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) polygoncommon.UnregisterFunc { +func (s *Service) RegisterNewBlockHashesObserver(o event.Observer[*DecodedInboundMessage[*eth.NewBlockHashesPacket]]) event.UnregisterFunc { return s.messageListener.RegisterNewBlockHashesObserver(o) } diff --git a/polygon/sync/tip_events.go b/polygon/sync/tip_events.go index b839266c97f..c9175a518fb 100644 --- a/polygon/sync/tip_events.go +++ b/polygon/sync/tip_events.go @@ -20,17 +20,16 @@ import ( "context" "fmt" + lru "github.com/hashicorp/golang-lru/v2" "golang.org/x/sync/errgroup" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/event" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/eth/protocols/eth" "github.com/erigontech/erigon/polygon/heimdall" "github.com/erigontech/erigon/polygon/p2p" - "github.com/erigontech/erigon/polygon/polygoncommon" - - lru "github.com/hashicorp/golang-lru/v2" ) type EventType string @@ -107,12 +106,12 @@ func (e Event) AsNewMilestone() EventNewMilestone { } type p2pObserverRegistrar interface { - RegisterNewBlockObserver(polygoncommon.Observer[*p2p.DecodedInboundMessage[*eth.NewBlockPacket]]) polygoncommon.UnregisterFunc - RegisterNewBlockHashesObserver(polygoncommon.Observer[*p2p.DecodedInboundMessage[*eth.NewBlockHashesPacket]]) polygoncommon.UnregisterFunc + RegisterNewBlockObserver(event.Observer[*p2p.DecodedInboundMessage[*eth.NewBlockPacket]]) event.UnregisterFunc + RegisterNewBlockHashesObserver(event.Observer[*p2p.DecodedInboundMessage[*eth.NewBlockHashesPacket]]) event.UnregisterFunc } type heimdallObserverRegistrar interface { - RegisterMilestoneObserver(callback func(*heimdall.Milestone), opts ...heimdall.ObserverOption) polygoncommon.UnregisterFunc + RegisterMilestoneObserver(callback func(*heimdall.Milestone), opts ...heimdall.ObserverOption) event.UnregisterFunc } func NewTipEvents(logger log.Logger, p2pReg p2pObserverRegistrar, heimdallReg heimdallObserverRegistrar) *TipEvents { From e72764983d02999e689f7e7775c6a7ebcc2e3cf8 Mon Sep 17 00:00:00 2001 From: Ostroukhov Nikita Date: Fri, 3 Jan 2025 12:15:54 +0000 Subject: [PATCH 44/94] Updated metrics file with new polygon related panels (#13309) --- .../dashboards/erigon_internals.json | 3937 ++++++++++------- 1 file changed, 2343 insertions(+), 1594 deletions(-) diff --git a/cmd/prometheus/dashboards/erigon_internals.json b/cmd/prometheus/dashboards/erigon_internals.json index 3a580c3eadc..fcc9dccff33 100644 --- a/cmd/prometheus/dashboards/erigon_internals.json +++ b/cmd/prometheus/dashboards/erigon_internals.json @@ -11,16 +11,34 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.4.2" + "version": "11.5.0-80207" }, { "type": "panel", - "id": "graph", - "name": "Graph (old)", + "id": "piechart", + "name": "Pie chart", "version": "" }, { @@ -29,6 +47,12 @@ "name": "Prometheus", "version": "1.0.0" }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, { "type": "panel", "id": "timeseries", @@ -61,7 +85,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": null, "links": [], "panels": [ { @@ -100,20 +124,27 @@ "h": 8, "w": 12, "x": 0, - "y": 1 + "y": 2 }, "id": 240, "options": { - "displayLabels": ["percent", "value"], + "displayLabels": [ + "percent", + "value" + ], "legend": { "displayMode": "list", "placement": "right", "showLegend": true, - "values": ["value"] + "values": [ + "value" + ] }, "pieType": "pie", "reduceOptions": { - "calcs": ["max"], + "calcs": [ + "max" + ], "fields": "", "values": false }, @@ -122,7 +153,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { "editorMode": "code", @@ -131,7 +162,11 @@ "instant": true, "legendFormat": "Aggregate verification", "range": false, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -179,8 +214,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -195,7 +229,7 @@ "h": 4, "w": 12, "x": 0, - "y": 10 + "y": 168 }, "id": 219, "options": { @@ -203,7 +237,9 @@ "minVizWidth": 75, "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -211,14 +247,18 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "editorMode": "code", "expr": "committee_size{instance=~\"$instance\"}", "legendFormat": "{{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Attestating Committee Size", @@ -241,6 +281,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "none", @@ -271,8 +312,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -288,12 +328,14 @@ "h": 12, "w": 12, "x": 12, - "y": 10 + "y": 168 }, "id": 220, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -310,7 +352,11 @@ "expr": "aggregate_quality_50{instance=\"dev-bm-e3-ethmainnet-n1\"}*100", "legendFormat": "{{label_name}} {{instance}} 50th", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -383,8 +429,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +444,7 @@ "h": 4, "w": 12, "x": 0, - "y": 14 + "y": 292 }, "id": 222, "options": { @@ -409,7 +454,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -417,14 +464,18 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "editorMode": "code", "expr": "current_slot{instance=~\"$instance\"}", "legendFormat": "{{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Current slot", @@ -445,8 +496,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -461,7 +511,7 @@ "h": 4, "w": 12, "x": 0, - "y": 18 + "y": 296 }, "id": 223, "options": { @@ -471,7 +521,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -479,14 +531,18 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "editorMode": "code", "expr": "current_epoch{instance=~\"$instance\"}", "legendFormat": "Epoch {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Current epoch", @@ -532,20 +588,26 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 169 }, "id": 232, "options": { - "displayLabels": ["percent"], + "displayLabels": [ + "percent" + ], "legend": { "displayMode": "list", "placement": "right", "showLegend": true, - "values": ["value"] + "values": [ + "value" + ] }, "pieType": "pie", "reduceOptions": { - "calcs": ["last"], + "calcs": [ + "last" + ], "fields": "", "values": false }, @@ -561,7 +623,11 @@ "expr": "execution_time{instance=\"dev-bm-e3-ethmainnet-n1\"}", "legendFormat": "Block Execution", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -636,6 +702,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 22, "gradientMode": "none", @@ -666,8 +733,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -683,12 +749,14 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 169 }, "id": 230, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -705,7 +773,11 @@ "expr": "blob_verification_time{instance=~\"$instance\"}", "legendFormat": "{{label_name}} {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Blob sidecar computation time", @@ -728,6 +800,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 18, "gradientMode": "none", @@ -758,8 +831,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -775,12 +847,14 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 204 }, "id": 229, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -797,7 +871,11 @@ "expr": "block_importing_latency{instance=~\"$instance\"}", "legendFormat": "{{label_name}} {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Block importing latency", @@ -818,8 +896,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -835,7 +912,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 204 }, "id": 215, "options": { @@ -845,7 +922,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -853,7 +932,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "disableTextWrap": false, @@ -864,7 +943,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Total SyncCommitteeAggregate bandwidth", @@ -906,8 +989,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -923,7 +1005,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 212 }, "id": 236, "options": { @@ -958,7 +1040,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Caplin: Total bandwidth", @@ -979,8 +1065,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -996,7 +1081,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 212 }, "id": 214, "options": { @@ -1006,7 +1091,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -1014,7 +1101,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "disableTextWrap": false, @@ -1025,7 +1112,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Total AggregateAndProof bandwidth", @@ -1046,8 +1137,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1063,7 +1153,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 220 }, "id": 235, "options": { @@ -1073,7 +1163,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1081,7 +1173,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "disableTextWrap": false, @@ -1092,7 +1184,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Caplin: Outgoing bandwidth", @@ -1115,6 +1211,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 23, "gradientMode": "none", @@ -1145,8 +1242,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1162,7 +1258,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 220 }, "id": 210, "options": { @@ -1188,7 +1284,11 @@ "legendFormat": " {{ method }} {{ instance }}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Caplin: Attestation in block latency", @@ -1209,8 +1309,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1226,7 +1325,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 228 }, "id": 234, "options": { @@ -1236,7 +1335,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -1244,7 +1345,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "disableTextWrap": false, @@ -1255,7 +1356,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Caplin: Ingoing bandwidth", @@ -1278,6 +1383,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 37, "gradientMode": "none", @@ -1308,8 +1414,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1326,7 +1431,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [" dev-bm-e3-ethmainnet-n1"], + "names": [ + " dev-bm-e3-ethmainnet-n1" + ], "prefix": "All except:", "readOnly": true } @@ -1348,7 +1455,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 228 }, "id": 211, "options": { @@ -1370,7 +1477,11 @@ "expr": "full_block_processing_time{instance=~\"$instance\"}", "legendFormat": " {{ method }} {{ instance }}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Caplin: Block processing time", @@ -1393,6 +1504,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 27, "gradientMode": "none", @@ -1423,8 +1535,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1440,12 +1551,14 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 236 }, "id": 231, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -1462,7 +1575,11 @@ "expr": "execution_time{instance=~\"$instance\"}", "legendFormat": "{{label_name}} {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "ValidateChain: time spent", @@ -1504,8 +1621,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1521,7 +1637,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 236 }, "id": 221, "options": { @@ -1530,7 +1646,9 @@ "fullHighlight": false, "groupWidth": 0.7, "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -1552,7 +1670,11 @@ "expr": "process_justification_bits_and_finality_time{instance=~\"$instance\"}", "legendFormat": "{{label_name}} {{instance}} process_justification_bits_and_finality", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -1653,6 +1775,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 27, "gradientMode": "none", @@ -1683,8 +1806,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1700,12 +1822,14 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 244 }, "id": 216, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -1726,7 +1850,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Single attestation subnet bandwidth", @@ -1749,6 +1877,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1779,8 +1908,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1797,7 +1925,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [" snapshotter-bm-e3-ethmainnet-n1"], + "names": [ + " snapshotter-bm-e3-ethmainnet-n1" + ], "prefix": "All except:", "readOnly": true } @@ -1819,7 +1949,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 244 }, "id": 217, "options": { @@ -1841,7 +1971,11 @@ "expr": "epoch_processing_time{instance=~\"$instance\"}", "legendFormat": " {{ method }} {{ instance }}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Epoch processing time", @@ -1911,7 +2045,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 252 }, "id": 228, "options": { @@ -1933,7 +2067,11 @@ "expr": "compute_shuffled_indices{instance=~\"$instance\"} > 0", "legendFormat": "{{label_name}} {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Shuffling validator set", @@ -1970,7 +2108,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 252 }, "id": 213, "options": { @@ -1980,7 +2118,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1999,7 +2139,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Beacon Block Gossip bandwidth", @@ -2036,7 +2180,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 260 }, "id": 226, "options": { @@ -2046,7 +2190,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -2065,7 +2211,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Total blobs bandwidth", @@ -2135,7 +2285,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 260 }, "id": 227, "options": { @@ -2161,7 +2311,11 @@ "legendFormat": "kb/s {{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Total attestation subnets bandwidth", @@ -2231,7 +2385,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [" snapshotter-bm-e3-ethmainnet-n1 blobs"], + "names": [ + " snapshotter-bm-e3-ethmainnet-n1 blobs" + ], "prefix": "All except:", "readOnly": true } @@ -2253,7 +2409,7 @@ "h": 9, "w": 24, "x": 0, - "y": 83 + "y": 268 }, "id": 224, "options": { @@ -2275,7 +2431,11 @@ "expr": "frozen_blobs{instance=~\"$instance\"} > 0", "legendFormat": "{{label_name}} {{instance}} blobs", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -2383,12 +2543,14 @@ "h": 12, "w": 24, "x": 0, - "y": 92 + "y": 277 }, "id": 209, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -2410,7 +2572,11 @@ "legendFormat": "{{instance}}", "range": true, "refId": "A", - "useBackend": false + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -2458,6 +2624,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2488,8 +2655,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2505,12 +2671,14 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 170 }, "id": 204, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -2527,7 +2695,11 @@ "expr": "rate(kv_get_count{instance=~\"$instance\"}[$rate_interval])", "legendFormat": "{{domain}} {{level}}: {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Files Level", @@ -2550,6 +2722,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2580,8 +2753,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2597,12 +2769,14 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 170 }, "id": 203, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -2619,7 +2793,11 @@ "expr": "kv_get{quantile=\"$quantile\",instance=~\"$instance\"}", "legendFormat": "{{domain}} {{level}}: {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "Files Level", @@ -2642,6 +2820,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2672,8 +2851,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2688,7 +2866,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 205 }, "id": 238, "options": { @@ -2710,7 +2888,11 @@ "expr": "sum by(instance, level) (rate(trie_state_levelled_load_rate{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "loaded {{level}} {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } }, { "datasource": { @@ -2758,8 +2940,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2775,7 +2956,7 @@ "h": 3, "w": 12, "x": 0, - "y": 20 + "y": 213 }, "id": 239, "options": { @@ -2785,7 +2966,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -2793,14 +2976,18 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.4.0-77868", "targets": [ { "editorMode": "code", "expr": "100*(trie_state_skip_rate{instance=~\"$instance\"}/(trie_state_skip_rate{instance=~\"$instance\"}+trie_state_load_rate{instance=~\"$instance\"})) ", "legendFormat": "skip {{instance}}", "range": true, - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } } ], "title": "state load Skip Ratio", @@ -2837,6 +3024,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2918,7 +3106,9 @@ "id": 196, "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -2928,7 +3118,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -2963,6 +3153,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3015,7 +3206,9 @@ "id": 206, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -3025,7 +3218,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3063,6 +3256,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3111,7 +3305,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": ["dev-bm-e3-ethmainnet-n1 blocks "], + "names": [ + "dev-bm-e3-ethmainnet-n1 blocks " + ], "prefix": "All except:", "readOnly": true } @@ -3148,7 +3344,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3184,6 +3380,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3234,7 +3431,7 @@ "h": 8, "w": 8, "x": 0, - "y": 11 + "y": 19 }, "id": 197, "options": { @@ -3249,7 +3446,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3344,6 +3541,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3391,12 +3589,14 @@ "h": 6, "w": 8, "x": 8, - "y": 11 + "y": 19 }, "id": 207, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -3406,7 +3606,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3484,7 +3684,7 @@ "h": 6, "w": 8, "x": 16, - "y": 11 + "y": 19 }, "id": 202, "options": { @@ -3501,7 +3701,9 @@ "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -3509,7 +3711,7 @@ "sizing": "manual", "valueMode": "color" }, - "pluginVersion": "10.4.2", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3557,6 +3759,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3604,12 +3807,14 @@ "h": 5, "w": 8, "x": 8, - "y": 17 + "y": 25 }, "id": 158, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -3619,7 +3824,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3657,6 +3862,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3704,12 +3910,15 @@ "h": 5, "w": 8, "x": 16, - "y": 17 + "y": 25 }, "id": 199, "options": { "legend": { - "calcs": ["mean", "lastNotNull"], + "calcs": [ + "mean", + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -3719,7 +3928,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3756,6 +3965,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3804,7 +4014,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": ["commitment took: dev-bm-e3-ethmainnet-n1"], + "names": [ + "commitment took: dev-bm-e3-ethmainnet-n1" + ], "prefix": "All except:", "readOnly": true } @@ -3826,12 +4038,14 @@ "h": 12, "w": 8, "x": 0, - "y": 19 + "y": 27 }, "id": 112, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -3841,7 +4055,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -3927,6 +4141,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3973,7 +4188,7 @@ "h": 9, "w": 8, "x": 8, - "y": 22 + "y": 30 }, "id": 198, "options": { @@ -3988,7 +4203,7 @@ "sort": "desc" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -4084,6 +4299,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4131,7 +4347,7 @@ "h": 9, "w": 8, "x": 16, - "y": 22 + "y": 30 }, "id": 201, "options": { @@ -4146,7 +4362,7 @@ "sort": "desc" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -4200,91 +4416,6 @@ "title": "Block execution delays", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlPu" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mgas/sec" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 208, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "exec_mgas{instance=~\"$instance\"} < 5000", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mgas: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "mgas/s ", - "type": "stat" - } - ], - "title": "Blocks execution", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 17, - "panels": [ { "datasource": { "type": "prometheus", @@ -4302,6 +4433,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4317,7 +4449,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -4328,7 +4460,6 @@ } }, "mappings": [], - "min": 0.001, "thresholds": { "mode": "absolute", "steps": [ @@ -4341,18 +4472,17 @@ "value": 80 } ] - }, - "unit": "ops" + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 6 + "y": 39 }, - "id": 141, + "id": 245, "options": { "legend": { "calcs": [], @@ -4361,27 +4491,29 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{type=\"fork_depth\", instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "fork_depth $instance", + "range": true, + "refId": "A", + "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(db_commit_seconds_count{phase=\"total\",instance=~\"$instance\"}[$rate_interval]) > 0 ", - "interval": "", - "legendFormat": "commit: {{instance}}", - "range": true, - "refId": "A" + } } ], - "title": "Commit", + "title": "Update Fork Choice depth", "type": "timeseries" }, { @@ -4389,7 +4521,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4402,6 +4533,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4413,13 +4545,12 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, - "showPoints": "never", - "spanNulls": 3600000, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -4447,97 +4578,60 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 16, - "x": 8, - "y": 6 + "h": 8, + "w": 12, + "x": 12, + "y": 39 }, - "id": 166, + "id": 246, "options": { "legend": { - "calcs": ["mean"], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"total\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "interval": "", - "legendFormat": "total: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"gc_wall_clock\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "gc_wall_clock: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"write\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "write: {{instance}}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{type=\"arrival_delay\", instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "arrival_delay $instance", "range": true, - "refId": "C" - }, - { + "refId": "A", + "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"sync\",quantile=\"$quantile\",instance=~\"$instance\"} > 0.002", - "hide": false, - "interval": "", - "legendFormat": "sync: {{instance}}", - "range": true, - "refId": "D" + } }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"gc_cpu_time\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{instance=\"$instance\", quantile=\"$quantile\", type=\"execution_duration\"}", + "fullMetaSearch": false, "hide": false, - "interval": "", - "legendFormat": "gc_cpu_time: {{instance}}", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "execution_duration $instance", "range": true, - "refId": "I" + "refId": "B", + "useBackend": false } ], - "title": "Commit speed", + "title": "Update Fork Choice delays", "type": "timeseries" }, { @@ -4548,15 +4642,119 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "continuous-BlPu" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mgas/sec" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "mgas: snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 208, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_mgas{instance=~\"$instance\"} < 5000", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mgas: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "mgas/s ", + "type": "stat" + } + ], + "title": "Blocks execution", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 242, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4572,8 +4770,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -4595,18 +4793,17 @@ "value": 80 } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 11 + "y": 6 }, - "id": 159, + "id": 243, "options": { "legend": { "calcs": [], @@ -4615,37 +4812,46 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "waypoint_length{type=\"milestone\", instance=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "milestone_length $instance", + "range": true, + "refId": "A", + "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - }, - "expr": "db_size{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "size: {{instance}}", - "refId": "A" + } }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "db_mi_last_pgno{instance=~\"$instance\"}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "waypoint_length{type=\"checkpoint\", instance=\"$instance\"}", + "fullMetaSearch": false, "hide": false, - "interval": "", - "legendFormat": "db_mi_last_pgno: {{instance}}", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "checkpoint_length $instance", "range": true, - "refId": "B" + "refId": "B", + "useBackend": false } ], - "title": "DB Size", + "title": "Waypoint length", "type": "timeseries" }, { @@ -4665,6 +4871,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4680,7 +4887,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -4704,81 +4911,147 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 16, - "x": 8, - "y": 15 + "h": 8, + "w": 12, + "x": 12, + "y": 6 }, - "id": 168, + "id": 244, "options": { "legend": { - "calcs": ["mean"], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"newly\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "newly: {{instance}}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "wiggle_duration{instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "wiggle_duration $instance", "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"cow\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "cow: {{instance}}", - "refId": "B" - }, - { + "refId": "A", + "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" + } + } + ], + "title": "Wiggle Duration", + "type": "timeseries" + } + ], + "title": "Polygon", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 17, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"clone\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "clone: {{instance}}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"split\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "split: {{instance}}", - "refId": "D" + "mappings": [], + "min": 0.001, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 149 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", @@ -4786,49 +5059,102 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"merge\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, + "expr": "rate(db_commit_seconds_count{phase=\"total\",instance=~\"$instance\"}[$rate_interval]) > 0 ", "interval": "", - "legendFormat": "merge: {{instance}}", + "legendFormat": "commit: {{instance}}", "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "refId": "A" + } + ], + "title": "Commit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"spill\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "spill: {{instance}}", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"wops\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "wops: {{instance}}", - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"unspill\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "unspill: {{instance}}", - "refId": "H" + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 149 + }, + "id": 166, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", @@ -4836,12 +5162,11 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcrloops\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, + "expr": "db_commit_seconds{phase=\"total\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", "interval": "", - "legendFormat": "gcrloops: {{instance}}", + "legendFormat": "total: {{instance}}", "range": true, - "refId": "I" + "refId": "A" }, { "datasource": { @@ -4850,12 +5175,12 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcwloops\", instance=~\"$instance\"}[$rate_interval])", + "expr": "db_commit_seconds{phase=\"gc_wall_clock\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", "hide": false, "interval": "", - "legendFormat": "gcwloops: {{instance}}", + "legendFormat": "gc_wall_clock: {{instance}}", "range": true, - "refId": "J" + "refId": "B" }, { "datasource": { @@ -4864,12 +5189,12 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcxpages\", instance=~\"$instance\"}[$rate_interval])", + "expr": "db_commit_seconds{phase=\"write\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", "hide": false, "interval": "", - "legendFormat": "gcxpages: {{instance}}", + "legendFormat": "write: {{instance}}", "range": true, - "refId": "K" + "refId": "C" }, { "datasource": { @@ -4878,12 +5203,12 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"msync\", instance=~\"$instance\"}[$rate_interval])", + "expr": "db_commit_seconds{phase=\"sync\",quantile=\"$quantile\",instance=~\"$instance\"} > 0.002", "hide": false, "interval": "", - "legendFormat": "msync: {{instance}}", + "legendFormat": "sync: {{instance}}", "range": true, - "refId": "L" + "refId": "D" }, { "datasource": { @@ -4892,39 +5217,15 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_pgops{phase=\"fsync\", instance=~\"$instance\"}[$rate_interval])", + "expr": "db_commit_seconds{phase=\"gc_cpu_time\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", "hide": false, "interval": "", - "legendFormat": "fsync: {{instance}}", + "legendFormat": "gc_cpu_time: {{instance}}", "range": true, - "refId": "M" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"minicore\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "minicore: {{instance}}", - "refId": "N" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"prefault\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "prefault: {{instance}}", - "refId": "O" + "refId": "I" } ], - "title": "DB Pages Ops/sec", + "title": "Commit speed", "type": "timeseries" }, { @@ -4944,6 +5245,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4988,15 +5290,15 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 5, "w": 8, "x": 0, - "y": 16 + "y": 162 }, - "id": 167, + "id": 159, "options": { "legend": { - "calcs": ["mean"], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5006,18 +5308,16 @@ "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "tx_limit{instance=~\"$instance\"}", + "expr": "db_size{instance=~\"$instance\"}", "interval": "", - "legendFormat": "limit: {{instance}}", - "range": true, + "legendFormat": "size: {{instance}}", "refId": "A" }, { @@ -5026,15 +5326,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "tx_dirty{instance=~\"$instance\"}", + "expr": "db_mi_last_pgno{instance=~\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "dirty: {{instance}}", + "legendFormat": "db_mi_last_pgno: {{instance}}", "range": true, "refId": "B" } ], - "title": "Tx Size", + "title": "DB Size", "type": "timeseries" }, { @@ -5054,6 +5354,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5065,7 +5366,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 0, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -5098,15 +5399,17 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 22 + "h": 7, + "w": 16, + "x": 8, + "y": 166 }, - "id": 169, + "id": 168, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5116,7 +5419,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5125,9 +5428,10 @@ }, "editorMode": "code", "exemplar": true, - "expr": "db_gc_leaf{instance=~\"$instance\"} > 0", + "expr": "rate(db_pgops{phase=\"newly\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, "interval": "", - "legendFormat": "gc_leaf: {{instance}}", + "legendFormat": "newly: {{instance}}", "range": true, "refId": "A" }, @@ -5136,15 +5440,37 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", "exemplar": true, - "expr": "db_gc_overflow{instance=~\"$instance\"} > 0", + "expr": "rate(db_pgops{phase=\"cow\", instance=~\"$instance\"}[$rate_interval])", "hide": false, "interval": "", - "legendFormat": "gc_overflow: {{instance}}", - "range": true, + "legendFormat": "cow: {{instance}}", "refId": "B" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"clone\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "clone: {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"split\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "split: {{instance}}", + "refId": "D" + }, { "datasource": { "type": "prometheus", @@ -5152,15 +5478,145 @@ }, "editorMode": "code", "exemplar": true, - "expr": "exec_steps_in_db{instance=~\"$instance\"}/100 > 0", + "expr": "rate(db_pgops{phase=\"merge\", instance=~\"$instance\"}[$rate_interval])", "hide": false, "interval": "", - "legendFormat": "exec_steps_in_db: {{instance}}", + "legendFormat": "merge: {{instance}}", "range": true, "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"spill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "spill: {{instance}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"wops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "wops: {{instance}}", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"unspill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "unspill: {{instance}}", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcrloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcrloops: {{instance}}", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcwloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcwloops: {{instance}}", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcxpages\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcxpages: {{instance}}", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"msync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "msync: {{instance}}", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"fsync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "fsync: {{instance}}", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"minicore\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "minicore: {{instance}}", + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"prefault\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "prefault: {{instance}}", + "refId": "O" } ], - "title": "GC and State", + "title": "DB Pages Ops/sec", "type": "timeseries" }, { @@ -5168,7 +5624,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -5181,6 +5636,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5197,7 +5653,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -5220,20 +5676,22 @@ } ] }, - "unit": "short" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 16, - "x": 8, - "y": 22 + "w": 8, + "x": 0, + "y": 167 }, - "id": 150, + "id": 167, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5243,7 +5701,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.0-80050", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5251,10 +5709,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(process_minor_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "expr": "tx_limit{instance=~\"$instance\"}", "interval": "", - "legendFormat": "soft: {{instance}}", + "legendFormat": "limit: {{instance}}", "range": true, "refId": "A" }, @@ -5264,16 +5721,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(process_major_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "expr": "tx_dirty{instance=~\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "hard: {{instance}}", + "legendFormat": "dirty: {{instance}}", "range": true, "refId": "B" } ], - "title": "getrusage: minflt - soft page faults (reclaims), majflt - hard faults", + "title": "Tx Size", "type": "timeseries" }, { @@ -5293,6 +5749,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5304,12 +5761,12 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 0, "scaleDistribution": { "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5318,7 +5775,6 @@ "mode": "off" } }, - "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -5333,20 +5789,22 @@ } ] }, - "unit": "percentunit" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 5, + "h": 6, "w": 8, "x": 0, - "y": 28 + "y": 173 }, - "id": 194, + "id": 169, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5356,7 +5814,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5365,11 +5823,9 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(exec_repeats{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", + "expr": "db_gc_leaf{instance=~\"$instance\"} > 0", "interval": "", - "intervalFactor": 1, - "legendFormat": "repeats: {{instance}}", + "legendFormat": "gc_leaf: {{instance}}", "range": true, "refId": "A" }, @@ -5380,33 +5836,31 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(exec_triggers{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", + "expr": "db_gc_overflow{instance=~\"$instance\"} > 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "triggers: {{instance}}", + "legendFormat": "gc_overflow: {{instance}}", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_steps_in_db{instance=~\"$instance\"}/100 > 0", + "hide": false, + "interval": "", + "legendFormat": "exec_steps_in_db: {{instance}}", + "range": true, + "refId": "E" } ], - "title": "Exec v3", + "title": "GC and State", "type": "timeseries" - } - ], - "title": "Database", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 134, - "panels": [ + }, { "datasource": { "type": "prometheus", @@ -5425,6 +5879,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5464,20 +5919,22 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 8, + "w": 16, "x": 8, - "y": 7 + "y": 173 }, - "id": 155, + "id": 150, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5487,7 +5944,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5496,8 +5953,258 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(process_io_write_syscalls_total{instance=~\"$instance\"}[$rate_interval]) > 0", - "format": "time_series", + "expr": "rate(process_minor_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "interval": "", + "legendFormat": "soft: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_major_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "interval": "", + "legendFormat": "hard: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "getrusage: minflt - soft page faults (reclaims), majflt - hard faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 179 + }, + "id": 194, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_repeats{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "repeats: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_triggers{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "triggers: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Exec v3", + "type": "timeseries" + } + ], + "title": "Database", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 134, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 155, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_io_write_syscalls_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, @@ -5543,6 +6250,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5590,12 +6298,14 @@ "h": 5, "w": 8, "x": 16, - "y": 7 + "y": 16 }, "id": 148, "options": { "legend": { - "calcs": ["max"], + "calcs": [ + "max" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5605,7 +6315,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5709,6 +6419,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5756,12 +6467,14 @@ "h": 5, "w": 8, "x": 16, - "y": 12 + "y": 21 }, "id": 106, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5771,7 +6484,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5809,6 +6522,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -5857,7 +6571,9 @@ "id": "byNames", "options": { "mode": "exclude", - "names": ["write: dev-bm-e3-ethmainnet-n1"], + "names": [ + "write: dev-bm-e3-ethmainnet-n1" + ], "prefix": "All except:", "readOnly": true } @@ -5879,12 +6595,14 @@ "h": 6, "w": 8, "x": 8, - "y": 13 + "y": 22 }, "id": 85, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -5894,7 +6612,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5948,6 +6666,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -5995,12 +6714,14 @@ "h": 6, "w": 8, "x": 16, - "y": 17 + "y": 26 }, "id": 153, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6010,7 +6731,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6047,6 +6768,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6094,12 +6816,14 @@ "h": 6, "w": 8, "x": 8, - "y": 19 + "y": 28 }, "id": 154, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6109,7 +6833,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6227,6 +6951,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6274,7 +6999,7 @@ "h": 6, "w": 8, "x": 16, - "y": 23 + "y": 32 }, "id": 128, "options": { @@ -6289,7 +7014,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6337,6 +7062,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6384,12 +7110,14 @@ "h": 5, "w": 8, "x": 8, - "y": 25 + "y": 34 }, "id": 86, "options": { "legend": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6399,7 +7127,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6453,6 +7181,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6500,7 +7229,7 @@ "h": 5, "w": 8, "x": 16, - "y": 29 + "y": 38 }, "id": 124, "options": { @@ -6515,7 +7244,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6544,7 +7273,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 8 }, "id": 183, "panels": [ @@ -6565,6 +7294,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6612,12 +7342,15 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "id": 185, "options": { "legend": { - "calcs": ["mean", "last"], + "calcs": [ + "mean", + "last" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6627,7 +7360,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6673,6 +7406,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6720,12 +7454,15 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, "id": 186, "options": { "legend": { - "calcs": ["mean", "last"], + "calcs": [ + "mean", + "last" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6735,7 +7472,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6769,6 +7506,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6816,12 +7554,15 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, "id": 187, "options": { "legend": { - "calcs": ["mean", "last"], + "calcs": [ + "mean", + "last" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -6831,7 +7572,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6865,6 +7606,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6912,7 +7654,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 17 }, "id": 188, "options": { @@ -6927,7 +7669,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6972,6 +7714,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -7019,12 +7762,15 @@ "h": 6, "w": 8, "x": 8, - "y": 24 + "y": 25 }, "id": 189, "options": { "legend": { - "calcs": ["mean", "last"], + "calcs": [ + "mean", + "last" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -7034,7 +7780,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7105,6 +7851,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -7151,12 +7898,15 @@ "h": 6, "w": 8, "x": 16, - "y": 24 + "y": 25 }, "id": 184, "options": { "legend": { - "calcs": ["mean", "last"], + "calcs": [ + "mean", + "last" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -7166,7 +7916,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7203,1024 +7953,1051 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 9 }, "id": 75, - "panels": [ + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "ingress: snapshotter-bm-e3-bormainnet-n1", + "ingress: snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "exemplar": true, + "expr": "rate(p2p_ingress{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ingress: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(p2p_egress{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "egress: {{instance}}", + "range": true, + "refId": "C" + } + ], + "title": "Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "Bps" - }, - "overrides": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "ingress: snapshotter-bm-e3-bormainnet-n1", - "ingress: snapshotter-bm-e3-ethmainnet-n1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 96, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "expr": "p2p_peers{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "peers: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(p2p_ingress{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ingress: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "expr": "rate(p2p_dials{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "dials: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(p2p_serves{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "serves: {{instance}}", + "refId": "C" + } + ], + "title": "Peers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 173, + "panels": [], + "title": "TxPool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(p2p_egress{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "egress: {{instance}}", - "range": true, - "refId": "C" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 175, + "options": { + "legend": { + "calcs": [ + "mean", + "last" ], - "title": "Traffic", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 77, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "p2p_peers{instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "peers: {{instance}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(p2p_dials{instance=~\"$instance\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "dials: {{instance}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(p2p_serves{instance=~\"$instance\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "serves: {{instance}}", - "refId": "C" - } - ], - "title": "Peers", - "type": "timeseries" - } - ], - "title": "Network", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 173, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 175, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_process_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_remote_txs: {{ instance }}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_add_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "add_remote_txs: {{ instance }}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_new_block{quantile=\"$quantile\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "new_block: {{ instance }}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_write_to_db{quantile=\"$quantile\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "write_to_db: {{ instance }}", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_propagate_to_new_peer{quantile=\"$quantile\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "propagate_to_new_peer: {{ instance }}", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "pool_propagate_new_txs{quantile=\"$quantile\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "propagate_new_txs: {{ instance }}", - "refId": "F" - } - ], - "title": "Timings", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 177, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(pool_process_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "pool_process_remote_txs_count: {{ instance }}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(pool_add_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "pool_add_remote_txs_count: {{ instance }}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(pool_new_block_count{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "pool_new_block_count: {{ instance }}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(pool_write_to_db_count{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "pool_write_to_db_count: {{ instance }}", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(pool_p2p_out{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "E" - } - ], - "title": "RPS", - "type": "timeseries" + "exemplar": true, + "expr": "pool_process_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_remote_txs: {{ instance }}", + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] + "exemplar": true, + "expr": "pool_add_remote_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "add_remote_txs: {{ instance }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 18 + "exemplar": true, + "expr": "pool_new_block{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "new_block: {{ instance }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 176, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "exemplar": true, + "expr": "pool_write_to_db{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "write_to_db: {{ instance }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(delta(cache_total{result=\"hit\",name=\"txpool\",instance=~\"$instance\"}[1m]))/sum(delta(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])) ", - "hide": false, - "interval": "", - "legendFormat": "hit rate: {{ instance }} ", - "refId": "A" - } - ], - "title": "Cache hit-rate", - "type": "timeseries" + "exemplar": true, + "expr": "pool_propagate_to_new_peer{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "propagate_to_new_peer: {{ instance }}", + "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "exemplar": true, + "expr": "pool_propagate_new_txs{quantile=\"$quantile\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "propagate_new_txs: {{ instance }}", + "refId": "F" + } + ], + "title": "Timings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + { + "color": "red", + "value": 80 } - }, - "overrides": [] + ] }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 18 + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 177, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 180, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "exemplar": true, + "expr": "rate(pool_process_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_process_remote_txs_count: {{ instance }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "exemplar": true, + "expr": "rate(pool_add_remote_txs_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_add_remote_txs_count: {{ instance }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(pool_new_block_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_new_block_count: {{ instance }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(pool_write_to_db_count{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_write_to_db_count: {{ instance }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(pool_p2p_out{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "E" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "exemplar": true, - "expr": "rate(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "{{ result }}: {{ instance }} ", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum(delta(cache_total{result=\"hit\",name=\"txpool\",instance=~\"$instance\"}[1m]))/sum(delta(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])) ", + "hide": false, + "interval": "", + "legendFormat": "hit rate: {{ instance }} ", + "refId": "A" + } + ], + "title": "Cache hit-rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "exemplar": true, - "expr": "rate(cache_timeout_total{name=\"txpool\",instance=~\"$instance\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "timeout: {{ instance }} ", - "refId": "B" - } + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 180, + "options": { + "legend": { + "calcs": [ + "mean", + "last" ], - "title": "Cache rps", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 181, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "cache_keys_total{name=\"txpool\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "keys: {{ instance }} ", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "cache_list_total{name=\"txpool\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "list: {{ instance }} ", - "refId": "B" - } - ], - "title": "Cache keys", - "type": "timeseries" + "exemplar": true, + "expr": "rate(cache_total{name=\"txpool\",instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "{{ result }}: {{ instance }} ", + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "exemplar": true, + "expr": "rate(cache_timeout_total{name=\"txpool\",instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "timeout: {{ instance }} ", + "refId": "B" + } + ], + "title": "Cache rps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "unit": "binBps" - }, - "overrides": [] + { + "color": "red", + "value": 80 + } + ] }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 24 + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 28 + }, + "id": 181, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 178, - "options": { - "legend": { - "calcs": ["mean", "last"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "exemplar": true, + "expr": "cache_keys_total{name=\"txpool\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "keys: {{ instance }} ", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "exemplar": true, + "expr": "cache_list_total{name=\"txpool\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "list: {{ instance }} ", + "refId": "B" + } + ], + "title": "Cache keys", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "exemplar": true, - "expr": "rate(pool_write_to_db_bytes{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "pool_write_to_db_bytes: {{ instance }}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 178, + "options": { + "legend": { + "calcs": [ + "mean", + "last" ], - "title": "DB", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-77868", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(pool_write_to_db_bytes{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "pool_write_to_db_bytes: {{ instance }}", + "refId": "A" } ], - "title": "TxPool", - "type": "row" + "title": "DB", + "type": "timeseries" } ], "refresh": "10s", - "schemaVersion": 39, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { "current": { - "selected": false, "text": "0.97", "value": "0.97" }, - "hide": 0, "includeAll": false, - "multi": false, "name": "quantile", "options": [ { @@ -8260,21 +9037,15 @@ } ], "query": "0.0,0.25,0.5, 0.9, 0.97, 0.99, 1", - "skipUrlSync": false, "type": "custom" }, { - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "go_goroutines", - "hide": 0, "includeAll": true, "label": "instance", "multi": true, @@ -8287,8 +9058,6 @@ }, "refresh": 1, "regex": "/.*instance=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { @@ -8296,11 +9065,9 @@ "auto_count": 30, "auto_min": "10s", "current": { - "selected": false, "text": "1m", "value": "1m" }, - "hide": 0, "label": "Rate Interval", "name": "rate_interval", "options": [ @@ -8362,25 +9129,7 @@ ], "query": "1m,10m,30m,1h,3h,6h,12h,1d,7d,14d,30d", "refresh": 2, - "skipUrlSync": false, "type": "interval" - }, - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "PBFA97CFB590B2093" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" } ] }, @@ -8404,6 +9153,6 @@ "timezone": "", "title": "1. Erigon CUSTOM METRICS", "uid": "b42a61d7-02b1-416c-8ab4-b9c864356174", - "version": 21, + "version": 278, "weekStart": "" -} +} \ No newline at end of file From 0a57d69853794e189481c3927b442f6a09f5c74a Mon Sep 17 00:00:00 2001 From: teenager-ETH Date: Fri, 3 Jan 2025 16:08:16 +0100 Subject: [PATCH 45/94] typo fix Update wmake.ps1 (#13228) # Typo Fix in `wmake.ps1` ## Description This pull request addresses a typo in the `wmake.ps1` file: - Corrected "insterted" to "inserted" in a comment. ## Changes - Improved the accuracy of comments by fixing a minor spelling error. ## Testing This change only updates a comment in the script and does not affect the script's functionality. No additional testing is required. ## Notes for Reviewers Please review the corrected comment to ensure compliance with the project's documentation standards. Let me know if further updates are necessary. Thank you! --- wmake.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wmake.ps1 b/wmake.ps1 index f1397750efb..8ebcfec54fd 100644 --- a/wmake.ps1 +++ b/wmake.ps1 @@ -160,7 +160,7 @@ function Get-Uninstall-Item { param ([string]$pattern = $(throw "A search pattern must be provided")) # Trying to get the enumerable of all installed programs using Get-ItemProperty may cause - # exceptions due to possible garbage values insterted into the registry by installers. + # exceptions due to possible garbage values inserted into the registry by installers. # Specifically an invalid cast exception throws when registry keys contain invalid DWORD data. # See https://github.com/PowerShell/PowerShell/issues/9552 # Due to this all items must be parsed one by one From 98b4911b0bf5ffd6ea90f9ee6bc8035c13e096b6 Mon Sep 17 00:00:00 2001 From: linchizhen Date: Fri, 3 Jan 2025 23:08:33 +0800 Subject: [PATCH 46/94] chore: fix some struct names in comment (#13213) fix some struct names in comment Signed-off-by: linchizhen --- eth/protocols/eth/protocol.go | 14 +++++++------- turbo/stages/bodydownload/body_algos.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index f9875371694..4c2509024a7 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -157,7 +157,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket represents a block header query over eth/66 +// GetBlockHeadersPacket66 represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket @@ -202,7 +202,7 @@ func (hn *HashOrNumber) DecodeRLP(s *rlp.Stream) error { // BlockHeadersPacket represents a block header response. type BlockHeadersPacket []*types.Header -// BlockHeadersPacket represents a block header response over eth/66. +// BlockHeadersPacket66 represents a block header response over eth/66. type BlockHeadersPacket66 struct { RequestId uint64 BlockHeadersPacket @@ -300,13 +300,13 @@ type BlockBodiesPacket []*types.Body // BlockRawBodiesPacket is the network packet for block content distribution. type BlockRawBodiesPacket []*types.RawBody -// BlockBodiesPacket is the network packet for block content distribution over eth/66. +// BlockBodiesPacket66 is the network packet for block content distribution over eth/66. type BlockBodiesPacket66 struct { RequestId uint64 BlockBodiesPacket } -// BlockBodiesPacket is the network packet for block content distribution over eth/66. +// BlockRawBodiesPacket66 is the network packet for block content distribution over eth/66. type BlockRawBodiesPacket66 struct { RequestId uint64 BlockRawBodiesPacket @@ -340,7 +340,7 @@ func (p *BlockRawBodiesPacket) Unpack() ([][][]byte, [][]*types.Header, []types. // GetReceiptsPacket represents a block receipts query. type GetReceiptsPacket []libcommon.Hash -// GetReceiptsPacket represents a block receipts query over eth/66. +// GetReceiptsPacket66 represents a block receipts query over eth/66. type GetReceiptsPacket66 struct { RequestId uint64 GetReceiptsPacket @@ -349,7 +349,7 @@ type GetReceiptsPacket66 struct { // ReceiptsPacket is the network packet for block receipts distribution. type ReceiptsPacket [][]*types.Receipt -// ReceiptsPacket is the network packet for block receipts distribution over eth/66. +// ReceiptsPacket66 is the network packet for block receipts distribution over eth/66. type ReceiptsPacket66 struct { RequestId uint64 ReceiptsPacket @@ -358,7 +358,7 @@ type ReceiptsPacket66 struct { // ReceiptsRLPPacket is used for receipts, when we already have it encoded type ReceiptsRLPPacket []rlp.RawValue -// ReceiptsPacket66 is the eth-66 version of ReceiptsRLPPacket +// ReceiptsRLPPacket66 is the eth-66 version of ReceiptsRLPPacket type ReceiptsRLPPacket66 struct { RequestId uint64 ReceiptsRLPPacket diff --git a/turbo/stages/bodydownload/body_algos.go b/turbo/stages/bodydownload/body_algos.go index b6580584ce5..1ecc34481c2 100644 --- a/turbo/stages/bodydownload/body_algos.go +++ b/turbo/stages/bodydownload/body_algos.go @@ -227,7 +227,7 @@ func (bd *BodyDownload) DeliverBodies(txs [][][]byte, uncles [][]*types.Header, } } -// RawTransaction implements core/types.DerivableList interface for hashing +// RawTransactions implements core/types.DerivableList interface for hashing type RawTransactions [][]byte func (rt RawTransactions) Len() int { From 6fbd1e218db7690707b65cd9ef069a067c175fe6 Mon Sep 17 00:00:00 2001 From: Tristav <124001124+Pricstas@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:08:50 +0100 Subject: [PATCH 47/94] Fix article usage in code comments and documentation (#13195) This pull request corrects the use of articles in multiple code files, specifically replacing the incorrect "a" with "an" and vice versa in places where the next word starts with a vowel sound or a consonant sound. These changes improve the clarity and grammatical accuracy of comments and documentation. --- accounts/abi/event.go | 2 +- accounts/abi/reflect.go | 2 +- accounts/abi/type_test.go | 2 +- cl/cltypes/block_production.go | 2 +- cmd/erigoncustom/main.go | 2 +- consensus/aura/config.go | 2 +- consensus/merge/merge.go | 2 +- turbo/node/node.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/accounts/abi/event.go b/accounts/abi/event.go index 84aa6e02c6a..7e7b76b3186 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -32,7 +32,7 @@ import ( // don't get the signature canonical representation as the first LOG topic. type Event struct { // Name is the event name used for internal representation. It's derived from - // the raw name and a suffix will be added in the case of a event overload. + // the raw name and a suffix will be added in the case of an event overload. // // e.g. // These are two events that have the same name: diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 4e381f3847f..3d5edcec6de 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -27,7 +27,7 @@ import ( "strings" ) -// ConvertType converts an interface of a runtime type into a interface of the +// ConvertType converts an interface of a runtime type into an interface of the // given type // e.g. turn // var fields []reflect.StructField diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index 9a0816d691e..ed45449bf3f 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -28,7 +28,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" ) -// typeWithoutStringer is a alias for the Type type which simply doesn't implement +// typeWithoutStringer is an alias for the Type type which simply doesn't implement // the stringer interface to allow printing type details in the tests below. type typeWithoutStringer Type diff --git a/cl/cltypes/block_production.go b/cl/cltypes/block_production.go index 2d623366a3a..197b5ef9150 100644 --- a/cl/cltypes/block_production.go +++ b/cl/cltypes/block_production.go @@ -25,7 +25,7 @@ import ( ) // BlindOrExecutionBeaconBlock is a union type that can be either a BlindedBeaconBlock or a BeaconBlock, depending on the context. -// It's a intermediate type used in the block production process. +// It's an intermediate type used in the block production process. type BlindOrExecutionBeaconBlock struct { Slot uint64 `json:"-"` ProposerIndex uint64 `json:"-"` diff --git a/cmd/erigoncustom/main.go b/cmd/erigoncustom/main.go index 8318fb40219..4c93a12e8ec 100644 --- a/cmd/erigoncustom/main.go +++ b/cmd/erigoncustom/main.go @@ -61,7 +61,7 @@ func runErigon(ctx *cli.Context) error { //err := eri.Serve() //if err != nil { - // log.Error("error while serving a Erigon node", "err", err) + // log.Error("error while serving an Erigon node", "err", err) // return err //} return nil diff --git a/consensus/aura/config.go b/consensus/aura/config.go index 2f4d775870f..9c6e1c4abbb 100644 --- a/consensus/aura/config.go +++ b/consensus/aura/config.go @@ -30,7 +30,7 @@ import ( "github.com/erigontech/erigon/consensus" ) -// Draws an validator nonce modulo number of validators. +// Draws a validator nonce modulo number of validators. func GetFromValidatorSet(set ValidatorSet, parent libcommon.Hash, nonce uint, call consensus.Call) (libcommon.Address, error) { //d, err := set.defaultCaller(parent) //if err != nil { diff --git a/consensus/merge/merge.go b/consensus/merge/merge.go index 1151b255549..e50d91ab447 100644 --- a/consensus/merge/merge.go +++ b/consensus/merge/merge.go @@ -52,7 +52,7 @@ var ( // errInvalidNonce is returned if the nonce is non-zero. errInvalidNonce = errors.New("invalid nonce") - // errInvalidUncleHash is returned if a block contains an non-empty uncle list. + // errInvalidUncleHash is returned if a block contains a non-empty uncle list. errInvalidUncleHash = errors.New("non empty uncle hash") errOlderBlockTime = errors.New("timestamp older than parent") diff --git a/turbo/node/node.go b/turbo/node/node.go index 7e98e7f57c9..2c839487439 100644 --- a/turbo/node/node.go +++ b/turbo/node/node.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with Erigon. If not, see . -// Package node contains classes for running a Erigon node. +// Package node contains classes for running an Erigon node. package node import ( From 5dc66507dfc92a46423642675cfe9915be32135d Mon Sep 17 00:00:00 2001 From: leo Date: Fri, 3 Jan 2025 23:09:49 +0800 Subject: [PATCH 48/94] docs: Spelling error. (#13199) Spelling error. Signed-off-by: RiceChuan --- cmd/sentry/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/sentry/README.md b/cmd/sentry/README.md index d6209975027..1fe9566f8df 100644 --- a/cmd/sentry/README.md +++ b/cmd/sentry/README.md @@ -19,7 +19,7 @@ combined). Ethereum mainnet configuration is currently hard-coded. ./build/bin/sentry --datadir= ``` -The command above specifies `--datadir` option - directory where the database files will be written (it doesn't need access to Erion's datadir). These two options +The command above specifies `--datadir` option - directory where the database files will be written (it doesn't need access to Erigon's datadir). These two options will need to be specified regardless of the mode the program is run. This specific command above assumes and external p2p sentry running on the same computer listening to the port `9091`. In order to use a p2p sentry on a different computer, or a different port (or both), the option `--sentry.api.addr` can be used. For example: From 2f96df369b2c01a200cba3fac0e8fd1594f8c101 Mon Sep 17 00:00:00 2001 From: Yi Song Date: Fri, 3 Jan 2025 23:11:00 +0800 Subject: [PATCH 49/94] chore: fix struct field name in comment (#13176) Signed-off-by: goodfirm --- cl/validator/sync_contribution_pool/sync_contribution_pool.go | 2 +- turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cl/validator/sync_contribution_pool/sync_contribution_pool.go b/cl/validator/sync_contribution_pool/sync_contribution_pool.go index c56481928e0..925b77dce49 100644 --- a/cl/validator/sync_contribution_pool/sync_contribution_pool.go +++ b/cl/validator/sync_contribution_pool/sync_contribution_pool.go @@ -37,7 +37,7 @@ type syncContributionKey struct { } type syncContributionPoolImpl struct { - // syncContributionPool is a map of sync contributions, indexed by slot, subcommittee index and block root. + // syncContributionPoolForBlocks is a map of sync contributions, indexed by slot, subcommittee index and block root. syncContributionPoolForBlocks map[syncContributionKey]*cltypes.Contribution // Used for block publishing. syncContributionPoolForAggregates map[syncContributionKey]*cltypes.Contribution // Used for sync committee messages aggregation. beaconCfg *clparams.BeaconChainConfig diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go index cd69fa09421..8d076a852e6 100644 --- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go +++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go @@ -48,7 +48,7 @@ var decompressorPool = sync.Pool{ } type BeaconSnapshotReader interface { - // ReadBlock reads the block at the given slot. + // ReadBlockBySlot reads the block at the given slot. // If the block is not present, it returns nil. ReadBlockBySlot(ctx context.Context, tx kv.Tx, slot uint64) (*cltypes.SignedBeaconBlock, error) ReadBlockByRoot(ctx context.Context, tx kv.Tx, blockRoot libcommon.Hash) (*cltypes.SignedBeaconBlock, error) From 8945a32131b8d6d0c0a344d43ed6cc04b0f5944c Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Sat, 4 Jan 2025 01:54:04 +0800 Subject: [PATCH 50/94] caplin: Use `HandleEndpointFunc` for `node` endpoints (#13308) This fixes an issue where correct response headers were not set for these endpoints. Specifically, `Content-Type` was being set to `text/plain` instead of `application/json`. --- cl/beacon/beaconhttp/api.go | 13 ++-- cl/beacon/handler/handler.go | 12 ++-- cl/beacon/handler/node.go | 121 ++++++++++++++--------------------- 3 files changed, 60 insertions(+), 86 deletions(-) diff --git a/cl/beacon/beaconhttp/api.go b/cl/beacon/beaconhttp/api.go index 8b8f0bf2292..8a266a449cb 100644 --- a/cl/beacon/beaconhttp/api.go +++ b/cl/beacon/beaconhttp/api.go @@ -99,14 +99,13 @@ func HandleEndpointFunc[T any](h EndpointHandlerFunc[T]) http.HandlerFunc { } func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { ans, err := h.Handle(w, r) if err != nil { var endpointError *EndpointError - if e, ok := err.(*EndpointError); ok { + var e *EndpointError + if errors.As(err, &e) { endpointError = e - } else { - endpointError = WrapEndpointError(err) } endpointError.WriteTo(w) return @@ -131,14 +130,14 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc { w.WriteHeader(200) } case strings.Contains(contentType, "application/octet-stream"): - sszMarshaler, ok := any(ans).(ssz.Marshaler) + sizeMarshaller, ok := any(ans).(ssz.Marshaler) if !ok { NewEndpointError(http.StatusBadRequest, ErrorSszNotSupported).WriteTo(w) return } w.Header().Set("Content-Type", "application/octet-stream") // TODO: we should probably figure out some way to stream this in the future :) - encoded, err := sszMarshaler.EncodeSSZ(nil) + encoded, err := sizeMarshaller.EncodeSSZ(nil) if err != nil { WrapEndpointError(err).WriteTo(w) return @@ -149,7 +148,7 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc { default: http.Error(w, "content type must include application/json, application/octet-stream, or text/event-stream, got "+contentType, http.StatusBadRequest) } - }) + } } func isNil[T any](t T) bool { diff --git a/cl/beacon/handler/handler.go b/cl/beacon/handler/handler.go index 264e7195cda..b097f93bbda 100644 --- a/cl/beacon/handler/handler.go +++ b/cl/beacon/handler/handler.go @@ -228,12 +228,12 @@ func (a *ApiHandler) init() { if a.routerCfg.Node { r.Route("/node", func(r chi.Router) { r.Get("/health", a.GetEthV1NodeHealth) - r.Get("/version", a.GetEthV1NodeVersion) - r.Get("/peer_count", a.GetEthV1NodePeerCount) - r.Get("/peers", a.GetEthV1NodePeersInfos) - r.Get("/peers/{peer_id}", a.GetEthV1NodePeerInfos) - r.Get("/identity", a.GetEthV1NodeIdentity) - r.Get("/syncing", a.GetEthV1NodeSyncing) + r.Get("/version", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeVersion)) + r.Get("/peer_count", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeerCount)) + r.Get("/peers", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeersInfos)) + r.Get("/peers/{peer_id}", beaconhttp.HandleEndpointFunc(a.GetEthV1NodePeerInfos)) + r.Get("/identity", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeIdentity)) + r.Get("/syncing", beaconhttp.HandleEndpointFunc(a.GetEthV1NodeSyncing)) }) } diff --git a/cl/beacon/handler/node.go b/cl/beacon/handler/node.go index c1f900a66f9..9ace8716cb5 100644 --- a/cl/beacon/handler/node.go +++ b/cl/beacon/handler/node.go @@ -17,7 +17,7 @@ package handler import ( - "encoding/json" + "errors" "fmt" "net/http" "runtime" @@ -60,37 +60,29 @@ func (a *ApiHandler) GetEthV1NodeHealth(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusOK) } -func (a *ApiHandler) GetEthV1NodeVersion(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodeVersion(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { // Get OS and Arch - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": map[string]interface{}{ - "version": fmt.Sprintf("Caplin/%s %s/%s", a.version, runtime.GOOS, runtime.GOARCH), - }, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } + return newBeaconResponse(map[string]interface{}{ + "version": fmt.Sprintf("Caplin/%s %s/%s", a.version, runtime.GOOS, runtime.GOARCH), + }), nil } -func (a *ApiHandler) GetEthV1NodePeerCount(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodePeerCount(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { ret, err := a.sentinel.GetPeers(r.Context(), &sentinel.EmptyMessage{}) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err) } + // all fields should be converted to string - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": map[string]interface{}{ - "connected": strconv.FormatUint(ret.Connected, 10), - "disconnected": strconv.FormatUint(ret.Disconnected, 10), - "connecting": strconv.FormatUint(ret.Connecting, 10), - "disconnecting": strconv.FormatUint(ret.Disconnecting, 10), - }, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } + return newBeaconResponse(map[string]interface{}{ + "connected": strconv.FormatUint(ret.Connected, 10), + "disconnected": strconv.FormatUint(ret.Disconnected, 10), + "connecting": strconv.FormatUint(ret.Connecting, 10), + "disconnecting": strconv.FormatUint(ret.Disconnecting, 10), + }), nil } -func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { state := r.URL.Query().Get("state") direction := r.URL.Query().Get("direction") @@ -107,8 +99,7 @@ func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Reque State: stateIn, }) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err) } peers := make([]peer, 0, len(ret.Peers)) for i := range ret.Peers { @@ -121,80 +112,64 @@ func (a *ApiHandler) GetEthV1NodePeersInfos(w http.ResponseWriter, r *http.Reque AgentVersion: ret.Peers[i].AgentVersion, }) } - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": peers, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } + + return newBeaconResponse(peers), nil } -func (a *ApiHandler) GetEthV1NodePeerInfos(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodePeerInfos(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { pid, err := beaconhttp.StringFromRequest(r, "peer_id") if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err) } ret, err := a.sentinel.PeersInfo(r.Context(), &sentinel.PeersInfoRequest{}) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err) } // find the peer with matching enr for _, p := range ret.Peers { if p.Pid == pid { - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": peer{ - PeerID: p.Pid, - State: p.State, - Enr: p.Enr, - LastSeenP2PAddress: p.Address, - Direction: p.Direction, - AgentVersion: p.AgentVersion, - }, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return + return newBeaconResponse(peer{ + PeerID: p.Pid, + State: p.State, + Enr: p.Enr, + LastSeenP2PAddress: p.Address, + Direction: p.Direction, + AgentVersion: p.AgentVersion, + }), nil } } - http.Error(w, "peer not found", http.StatusNotFound) + + return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("peer not found")) } -func (a *ApiHandler) GetEthV1NodeIdentity(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodeIdentity(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { id, err := a.sentinel.Identity(r.Context(), &sentinel.EmptyMessage{}) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err) } - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": map[string]interface{}{ - "peer_id": id.Pid, - "enr": id.Enr, - "p2p_addresses": id.P2PAddresses, - "discovery_addresses": id.DiscoveryAddresses, - "metadata": map[string]interface{}{ - "seq": strconv.FormatUint(id.Metadata.Seq, 10), - "attnets": id.Metadata.Attnets, - "syncnets": id.Metadata.Syncnets, - }, + + return newBeaconResponse(map[string]interface{}{ + "peer_id": id.Pid, + "enr": id.Enr, + "p2p_addresses": id.P2PAddresses, + "discovery_addresses": id.DiscoveryAddresses, + "metadata": map[string]interface{}{ + "seq": strconv.FormatUint(id.Metadata.Seq, 10), + "attnets": id.Metadata.Attnets, + "syncnets": id.Metadata.Syncnets, }, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } + }), nil } -func (a *ApiHandler) GetEthV1NodeSyncing(w http.ResponseWriter, r *http.Request) { +func (a *ApiHandler) GetEthV1NodeSyncing(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) { currentSlot := a.ethClock.GetCurrentSlot() - if err := json.NewEncoder(w).Encode(map[string]interface{}{ - "data": map[string]interface{}{ + return newBeaconResponse( + map[string]interface{}{ "head_slot": strconv.FormatUint(a.syncedData.HeadSlot(), 10), "sync_distance": strconv.FormatUint(currentSlot-a.syncedData.HeadSlot(), 10), "is_syncing": a.syncedData.Syncing(), "is_optimistic": false, // needs to change "el_offline": false, - }, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } + }), nil } From 1337da1acb317aa65e398c3b33aad1f107e98764 Mon Sep 17 00:00:00 2001 From: bloxster <40316187+bloxster@users.noreply.github.com> Date: Sat, 4 Jan 2025 17:01:39 +0100 Subject: [PATCH 51/94] Corrected "accessor" folder mistyping (#13318) Corrected from "accessors" to "accessor" --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f7355131e8a..627a2c2dd4c 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ datadir domain # Latest State history # Historical values idx # InvertedIndices: can search/filtering/union/intersect them - to find historical data. like eth_getLogs or trace_transaction - accessors # Additional (generated) indices of history - have "random-touch" read-pattern. They can serve only `Get` requests (no search/filters). + accessor # Additional (generated) indices of history - have "random-touch" read-pattern. They can serve only `Get` requests (no search/filters). txpool # pending transactions. safe to remove. nodes # p2p peers. safe to remove. temp # used to sort data bigger than RAM. can grow to ~100gb. cleaned at startup. @@ -155,11 +155,11 @@ datadir domain # link to fast disk history idx - accessors + accessor temp # buffers to sort data >> RAM. sequential-buffered IO - is slow-disk-friendly # Example: how to speedup history access: -# - go step-by-step - first try store `accessors` on fast disk +# - go step-by-step - first try store `accessor` on fast disk # - if speed is not good enough: `idx` # - if still not enough: `history` ``` From 3820a8da1c0c4a2621a15cf83d7a405f78f95f42 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Sat, 4 Jan 2025 23:38:12 +0100 Subject: [PATCH 52/94] Caplin: added ability to do recent chain sync + change in flags name (#13263) Here are some changes to flags names: `--caplin.archive` -> `--caplin.states-archive` `--caplin.backfiiling` -> `--caplin.blocks-archive` `--caplin.backfiiling.blobs` -> `--caplin.blobs-archive` Lastly added a new flag: `--caplin.blobs-immediate-backfill` which will backfill the last 18 days worth of blobs for an op-node or other uses. --- cl/clparams/config.go | 15 +++++--- cl/persistence/blob_storage/blob_db.go | 9 +++-- cl/phase1/stages/cleanup_and_pruning.go | 2 +- cl/phase1/stages/clstages.go | 37 ++++++++++--------- cl/phase1/stages/stage_history_download.go | 42 +++++++++++++++------- cmd/capcli/cli.go | 2 +- cmd/caplin/caplin1/run.go | 14 +++----- cmd/utils/flags.go | 33 ++++++++++------- turbo/cli/default_flags.go | 8 +++-- turbo/stages/stageloop.go | 12 +++---- 10 files changed, 101 insertions(+), 73 deletions(-) diff --git a/cl/clparams/config.go b/cl/clparams/config.go index 2be5df351b1..20682c3ec4c 100644 --- a/cl/clparams/config.go +++ b/cl/clparams/config.go @@ -41,12 +41,15 @@ import ( var LatestStateFileName = "latest.ssz_snappy" type CaplinConfig struct { - Backfilling bool - BlobBackfilling bool + // Archive related config + ArchiveBlocks bool + ArchiveBlobs bool + ArchiveStates bool + ImmediateBlobsBackfilling bool BlobPruningDisabled bool - Archive bool SnapshotGenerationEnabled bool - NetworkId NetworkType + // Network related config + NetworkId NetworkType // DisableCheckpointSync is optional and is used to disable checkpoint sync used by default in the node DisabledCheckpointSync bool // CaplinMeVRelayUrl is optional and is used to connect to the external builder service. @@ -341,7 +344,11 @@ var ConfigurableCheckpointsURLs = []string{} // MinEpochsForBlockRequests equal to MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT / 2 func (b *BeaconChainConfig) MinEpochsForBlockRequests() uint64 { return b.MinValidatorWithdrawabilityDelay + (b.ChurnLimitQuotient)/2 +} +// MinSlotsForBlobRequests equal to MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUEST * SLOTS_PER_EPOCH +func (b *BeaconChainConfig) MinSlotsForBlobsSidecarsRequest() uint64 { + return b.MinEpochsForBlobsSidecarsRequest * b.SlotsPerEpoch } type ConfigByte byte diff --git a/cl/persistence/blob_storage/blob_db.go b/cl/persistence/blob_storage/blob_db.go index 0377bde4d7a..0df212207c5 100644 --- a/cl/persistence/blob_storage/blob_db.go +++ b/cl/persistence/blob_storage/blob_db.go @@ -39,7 +39,9 @@ import ( "github.com/spf13/afero" ) -const subdivisionSlot = 10_000 +const ( + subdivisionSlot = 10_000 +) type BlobStorage interface { WriteBlobSidecars(ctx context.Context, blockRoot libcommon.Hash, blobSidecars []*cltypes.BlobSidecar) error @@ -160,8 +162,9 @@ func (bs *BlobStore) Prune() error { currentSlot -= bs.slotsKept currentSlot = (currentSlot / subdivisionSlot) * subdivisionSlot var startPrune uint64 - if currentSlot >= 1_000_000 { - startPrune = currentSlot - 1_000_000 + minSlotsForBlobSidecarRequest := bs.beaconChainConfig.MinSlotsForBlobsSidecarsRequest() + if currentSlot >= minSlotsForBlobSidecarRequest { + startPrune = currentSlot - minSlotsForBlobSidecarRequest } // delete all the folders that are older than slotsKept for i := startPrune; i < currentSlot; i += subdivisionSlot { diff --git a/cl/phase1/stages/cleanup_and_pruning.go b/cl/phase1/stages/cleanup_and_pruning.go index cdd14cb7f55..2982f3ff6ff 100644 --- a/cl/phase1/stages/cleanup_and_pruning.go +++ b/cl/phase1/stages/cleanup_and_pruning.go @@ -16,7 +16,7 @@ func cleanupAndPruning(ctx context.Context, logger log.Logger, cfg *Cfg, args Ar defer tx.Rollback() pruneDistance := uint64(1_000_000) - if !cfg.backfilling { + if !cfg.caplinConfig.ArchiveBlocks { if err := beacon_indicies.PruneBlocks(ctx, tx, args.seenSlot-pruneDistance); err != nil { return err } diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index ab842a29ec4..dbcd73642c7 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -64,8 +64,8 @@ type Cfg struct { blobStore blob_storage.BlobStorage attestationDataProducer attestation_producer.AttestationDataProducer validatorMonitor monitor.ValidatorMonitor - - hasDownloaded, backfilling, blobBackfilling bool + caplinConfig clparams.CaplinConfig + hasDownloaded bool } type Args struct { @@ -91,8 +91,7 @@ func ClStagesCfg( blockReader freezeblocks.BeaconSnapshotReader, dirs datadir.Dirs, syncBackLoopLimit uint64, - backfilling bool, - blobBackfilling bool, + caplinConfig clparams.CaplinConfig, syncedData *synced_data.SyncedDataManager, emitters *beaconevents.EventEmitter, blobStore blob_storage.BlobStorage, @@ -100,24 +99,24 @@ func ClStagesCfg( validatorMonitor monitor.ValidatorMonitor, ) *Cfg { return &Cfg{ - rpc: rpc, - antiquary: antiquary, - ethClock: ethClock, - beaconCfg: beaconCfg, - state: state, - executionClient: executionClient, - gossipManager: gossipManager, - forkChoice: forkChoice, - dirs: dirs, - indiciesDB: indiciesDB, - sn: sn, - blockReader: blockReader, - backfilling: backfilling, + rpc: rpc, + antiquary: antiquary, + ethClock: ethClock, + caplinConfig: caplinConfig, + beaconCfg: beaconCfg, + state: state, + executionClient: executionClient, + gossipManager: gossipManager, + forkChoice: forkChoice, + dirs: dirs, + indiciesDB: indiciesDB, + sn: sn, + blockReader: blockReader, + syncedData: syncedData, emitter: emitters, blobStore: blobStore, blockCollector: block_collector.NewBlockCollector(log.Root(), executionClient, beaconCfg, syncBackLoopLimit, dirs.Tmp), - blobBackfilling: blobBackfilling, attestationDataProducer: attestationDataProducer, validatorMonitor: validatorMonitor, } @@ -254,7 +253,7 @@ func ConsensusClStages(ctx context.Context, startingSlot := cfg.state.LatestBlockHeader().Slot downloader := network2.NewBackwardBeaconDownloader(ctx, cfg.rpc, cfg.sn, cfg.executionClient, cfg.indiciesDB) - if err := SpawnStageHistoryDownload(StageHistoryReconstruction(downloader, cfg.antiquary, cfg.sn, cfg.indiciesDB, cfg.executionClient, cfg.beaconCfg, cfg.backfilling, cfg.blobBackfilling, false, startingRoot, startingSlot, cfg.dirs.Tmp, 600*time.Millisecond, cfg.blockCollector, cfg.blockReader, cfg.blobStore, logger), context.Background(), logger); err != nil { + if err := SpawnStageHistoryDownload(StageHistoryReconstruction(downloader, cfg.antiquary, cfg.sn, cfg.indiciesDB, cfg.executionClient, cfg.beaconCfg, cfg.caplinConfig, false, startingRoot, startingSlot, cfg.dirs.Tmp, 600*time.Millisecond, cfg.blockCollector, cfg.blockReader, cfg.blobStore, logger), context.Background(), logger); err != nil { cfg.hasDownloaded = false return err } diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index 3f3c3ad3bda..276522c4dfd 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -44,8 +44,7 @@ type StageHistoryReconstructionCfg struct { downloader *network.BackwardBeaconDownloader sn *freezeblocks.CaplinSnapshots startingRoot libcommon.Hash - backfilling bool - blobsBackfilling bool + caplinConfig clparams.CaplinConfig waitForAllRoutines bool startingSlot uint64 tmpdir string @@ -61,7 +60,7 @@ type StageHistoryReconstructionCfg struct { const logIntervalTime = 30 * time.Second -func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, antiquary *antiquary.Antiquary, sn *freezeblocks.CaplinSnapshots, indiciesDB kv.RwDB, engine execution_client.ExecutionEngine, beaconCfg *clparams.BeaconChainConfig, backfilling, blobsBackfilling, waitForAllRoutines bool, startingRoot libcommon.Hash, startinSlot uint64, tmpdir string, backfillingThrottling time.Duration, executionBlocksCollector block_collector.BlockCollector, blockReader freezeblocks.BeaconSnapshotReader, blobStorage blob_storage.BlobStorage, logger log.Logger) StageHistoryReconstructionCfg { +func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, antiquary *antiquary.Antiquary, sn *freezeblocks.CaplinSnapshots, indiciesDB kv.RwDB, engine execution_client.ExecutionEngine, beaconCfg *clparams.BeaconChainConfig, caplinConfig clparams.CaplinConfig, waitForAllRoutines bool, startingRoot libcommon.Hash, startinSlot uint64, tmpdir string, backfillingThrottling time.Duration, executionBlocksCollector block_collector.BlockCollector, blockReader freezeblocks.BeaconSnapshotReader, blobStorage blob_storage.BlobStorage, logger log.Logger) StageHistoryReconstructionCfg { return StageHistoryReconstructionCfg{ beaconCfg: beaconCfg, downloader: downloader, @@ -70,7 +69,7 @@ func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, an startingSlot: startinSlot, waitForAllRoutines: waitForAllRoutines, logger: logger, - backfilling: backfilling, + caplinConfig: caplinConfig, indiciesDB: indiciesDB, antiquary: antiquary, engine: engine, @@ -78,7 +77,6 @@ func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, an backfillingThrottling: backfillingThrottling, executionBlocksCollector: executionBlocksCollector, blockReader: blockReader, - blobsBackfilling: blobsBackfilling, blobStorage: blobStorage, } } @@ -90,7 +88,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co currentSlot := cfg.startingSlot if !clparams.SupportBackfilling(cfg.beaconCfg.DepositNetworkID) { - cfg.backfilling = false // disable backfilling if not on a supported network + cfg.caplinConfig.ArchiveBlocks = false // disable backfilling if not on a supported network } // Start the procedure @@ -112,17 +110,26 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co return false, err } defer tx.Rollback() + // handle the case where the block is a CL block including an execution payload if blk.Version() >= clparams.BellatrixVersion { currEth1Progress.Store(int64(blk.Block.Body.ExecutionPayload.BlockNumber)) } slot := blk.Block.Slot isInCLSnapshots := cfg.sn.SegmentsMax() > blk.Block.Slot + // Skip blocks that are already in the snapshots if !isInCLSnapshots { if err := beacon_indicies.WriteBeaconBlockAndIndicies(ctx, tx, blk, true); err != nil { return false, err } } + // we need to backfill an equivalent number of blobs to the blocks + hasDownloadEnoughForImmediateBlobsBackfilling := true + if cfg.caplinConfig.ImmediateBlobsBackfilling { + // download twice the number of blocks needed for good measure + blocksToDownload := cfg.beaconCfg.MinSlotsForBlobsSidecarsRequest() * 2 + hasDownloadEnoughForImmediateBlobsBackfilling = cfg.startingSlot < blocksToDownload || slot > cfg.startingSlot-blocksToDownload + } if cfg.engine != nil && cfg.engine.SupportInsertion() && blk.Version() >= clparams.BellatrixVersion { frozenBlocksInEL := cfg.engine.FrozenBlocks(ctx) @@ -143,8 +150,8 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co return false, tx.Commit() } } - if hasELBlock && !cfg.backfilling { - return true, tx.Commit() + if hasELBlock && !cfg.caplinConfig.ArchiveBlocks { + return hasDownloadEnoughForImmediateBlobsBackfilling, tx.Commit() } } isInElSnapshots := true @@ -159,7 +166,10 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co if slot == 0 || (isInCLSnapshots && isInElSnapshots) { return true, tx.Commit() } - return (!cfg.backfilling || slot <= cfg.sn.SegmentsMax()) && (slot <= destinationSlotForEL || isInElSnapshots), tx.Commit() + return hasDownloadEnoughForImmediateBlobsBackfilling && + (!cfg.caplinConfig.ArchiveBlocks || slot <= cfg.sn.SegmentsMax()) && + (slot <= destinationSlotForEL || isInElSnapshots), + tx.Commit() }) prevProgress := cfg.downloader.Progress() @@ -228,14 +238,15 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co } } cfg.antiquary.NotifyBackfilled() - if cfg.backfilling { + fmt.Println(cfg.caplinConfig.ArchiveBlocks) + if cfg.caplinConfig.ArchiveBlocks { cfg.logger.Info("Full backfilling finished") } else { cfg.logger.Info("Missing blocks download finished (note: this does not mean that the history is complete, only that the missing blocks need for sync have been downloaded)") } close(finishCh) - if cfg.blobsBackfilling { + if cfg.caplinConfig.ArchiveBlobs || cfg.caplinConfig.ImmediateBlobsBackfilling { go func() { if err := downloadBlobHistoryWorker(cfg, ctx, true, logger); err != nil { logger.Error("Error downloading blobs", "err", err) @@ -288,6 +299,11 @@ func downloadBlobHistoryWorker(cfg StageHistoryReconstructionCfg, ctx context.Co prevLogSlot := currentSlot prevTime := time.Now() targetSlot := cfg.beaconCfg.DenebForkEpoch * cfg.beaconCfg.SlotsPerEpoch + // in case of immediate blobs backfilling we need to backfill the blobs for the last relevant epochs + if !cfg.caplinConfig.ArchiveBlobs && cfg.caplinConfig.ImmediateBlobsBackfilling { + targetSlot = currentSlot - min(currentSlot, cfg.beaconCfg.MinSlotsForBlobsSidecarsRequest()) + } + logger.Info("[Blobs-Downloader] Downloading blobs backwards", "slot", currentSlot) for currentSlot >= targetSlot { if currentSlot <= cfg.sn.FrozenBlobs() { @@ -346,7 +362,7 @@ func downloadBlobHistoryWorker(cfg StageHistoryReconstructionCfg, ctx context.Co prevLogSlot = currentSlot prevTime = time.Now() - logger.Info("Downloading blobs backwards", "slot", currentSlot, "blks/sec", blkSecStr) + logger.Info("[Blobs-Downloader] Downloading blobs backwards", "slot", currentSlot, "blks/sec", blkSecStr) default: } // Generate the request @@ -381,7 +397,7 @@ func downloadBlobHistoryWorker(cfg StageHistoryReconstructionCfg, ctx context.Co } } if shouldLog { - logger.Info("Blob history download finished successfully") + logger.Info("[Blobs-Downloader] Blob history download finished successfully") } cfg.antiquary.NotifyBlobBackfilled() return nil diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index ae5250199be..75f5f59f6ea 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -184,7 +184,7 @@ func (c *Chain) Run(ctx *Context) error { } downloader := network.NewBackwardBeaconDownloader(ctx, beacon, nil, nil, db) - cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, true, false, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root()) + cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, clparams.CaplinConfig{}, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root()) return stages.SpawnStageHistoryDownload(cfg, ctx, log.Root()) } diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index 81d4cbd331a..ed2bb0730c4 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -133,11 +133,6 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi dirs datadir.Dirs, eth1Getter snapshot_format.ExecutionBlockReaderByNumber, snDownloader proto_downloader.DownloaderClient, creds credentials.TransportCredentials, snBuildSema *semaphore.Weighted) error { - var ( - backfilling = config.Backfilling - blobBackfilling = config.BlobBackfilling - states = config.Archive - ) var networkConfig *clparams.NetworkConfig var beaconConfig *clparams.BeaconChainConfig @@ -214,7 +209,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi ethClock := eth_clock.NewEthereumClock(state.GenesisTime(), state.GenesisValidatorsRoot(), beaconConfig) pruneBlobDistance := uint64(128600) - if config.BlobBackfilling || config.BlobPruningDisabled { + if config.ArchiveBlobs || config.BlobPruningDisabled { pruneBlobDistance = math.MaxUint64 } @@ -391,7 +386,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi vTables := state_accessors.NewStaticValidatorTable() // Read the current table - if states { + if config.ArchiveStates { if err := state_accessors.ReadValidatorsTable(tx, vTables); err != nil { return err } @@ -400,7 +395,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi if err := stateSnapshots.OpenFolder(); err != nil { return err } - antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, stateSnapshots, csn, rcsn, syncedDataManager, logger, states, backfilling, blobBackfilling, config.SnapshotGenerationEnabled, snBuildSema) + antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, stateSnapshots, csn, rcsn, syncedDataManager, logger, config.ArchiveStates, config.ArchiveBlocks, config.ArchiveBlobs, config.SnapshotGenerationEnabled, snBuildSema) // Create the antiquary go func() { if err := antiq.Loop(); err != nil { @@ -470,8 +465,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi rcsn, dirs, config.LoopBlockLimit, - backfilling, - blobBackfilling, + config, syncedDataManager, emitters, blobStorage, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7e63d1cb44d..97ae92d9205 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1006,18 +1006,28 @@ var ( Usage: "Print in logs RPC requests slower than given threshold: 100ms, 1s, 1m. Exluded methods: " + strings.Join(rpccfg.SlowLogBlackList, ","), Value: 0, } - CaplinBackfillingFlag = cli.BoolFlag{ - Name: "caplin.backfilling", + CaplinArchiveBlocksFlag = cli.BoolFlag{ + Name: "caplin.blocks-archive", Usage: "sets whether backfilling is enabled for caplin", Value: false, } - CaplinBlobBackfillingFlag = cli.BoolFlag{ - Name: "caplin.backfilling.blob", + CaplinArchiveStatesFlag = cli.BoolFlag{ + Name: "caplin.states-archive", + Usage: "enables archival node for historical states in caplin (it will enable block archival as well)", + Value: false, + } + CaplinArchiveBlobsFlag = cli.BoolFlag{ + Name: "caplin.blobs-archive", Usage: "sets whether backfilling is enabled for caplin", Value: false, } + CaplinImmediateBlobBackfillFlag = cli.BoolFlag{ + Name: "caplin.blobs-immediate-backfill", + Usage: "sets whether caplin should immediatelly backfill blobs (4096 epochs)", + Value: false, + } CaplinDisableBlobPruningFlag = cli.BoolFlag{ - Name: "caplin.backfilling.blob.no-pruning", + Name: "caplin.blobs-no-pruning", Usage: "disable blob pruning in caplin", Value: false, } @@ -1026,11 +1036,7 @@ var ( Usage: "disable checkpoint sync in caplin", Value: false, } - CaplinArchiveFlag = cli.BoolFlag{ - Name: "caplin.archive", - Usage: "enables archival node in caplin", - Value: false, - } + CaplinEnableSnapshotGeneration = cli.BoolFlag{ Name: "caplin.snapgen", Usage: "enables snapshot generation in caplin", @@ -1753,13 +1759,14 @@ func setBeaconAPI(ctx *cli.Context, cfg *ethconfig.Config) error { func setCaplin(ctx *cli.Context, cfg *ethconfig.Config) { // Caplin's block's backfilling is enabled if any of the following flags are set - cfg.CaplinConfig.Backfilling = ctx.Bool(CaplinBackfillingFlag.Name) || ctx.Bool(CaplinArchiveFlag.Name) || ctx.Bool(CaplinBlobBackfillingFlag.Name) + cfg.CaplinConfig.ArchiveBlocks = ctx.Bool(CaplinArchiveBlocksFlag.Name) || ctx.Bool(CaplinArchiveStatesFlag.Name) || ctx.Bool(CaplinArchiveBlobsFlag.Name) cfg.CaplinConfig.SnapshotGenerationEnabled = ctx.Bool(CaplinEnableSnapshotGeneration.Name) // More granularity here. - cfg.CaplinConfig.BlobBackfilling = ctx.Bool(CaplinBlobBackfillingFlag.Name) + cfg.CaplinConfig.ArchiveBlobs = ctx.Bool(CaplinArchiveBlobsFlag.Name) + cfg.CaplinConfig.ImmediateBlobsBackfilling = ctx.Bool(CaplinImmediateBlobBackfillFlag.Name) cfg.CaplinConfig.BlobPruningDisabled = ctx.Bool(CaplinDisableBlobPruningFlag.Name) cfg.CaplinConfig.DisabledCheckpointSync = ctx.Bool(CaplinDisableCheckpointSyncFlag.Name) - cfg.CaplinConfig.Archive = ctx.Bool(CaplinArchiveFlag.Name) + cfg.CaplinConfig.ArchiveStates = ctx.Bool(CaplinArchiveStatesFlag.Name) cfg.CaplinConfig.MevRelayUrl = ctx.String(CaplinMevRelayUrl.Name) cfg.CaplinConfig.EnableValidatorMonitor = ctx.Bool(CaplinValidatorMonitorFlag.Name) if checkpointUrls := ctx.StringSlice(CaplinCheckpointSyncUrlFlag.Name); len(checkpointUrls) > 0 { diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index f62cfe22636..e05e469cf24 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -213,11 +213,13 @@ var DefaultFlags = []cli.Flag{ &utils.BeaconApiProtocolFlag, &utils.BeaconApiIdleTimeoutFlag, - &utils.CaplinBackfillingFlag, - &utils.CaplinBlobBackfillingFlag, + &utils.CaplinArchiveBlocksFlag, + &utils.CaplinArchiveBlobsFlag, + &utils.CaplinArchiveStatesFlag, + &utils.CaplinImmediateBlobBackfillFlag, + &utils.CaplinDisableBlobPruningFlag, &utils.CaplinDisableCheckpointSyncFlag, - &utils.CaplinArchiveFlag, &utils.CaplinEnableSnapshotGeneration, &utils.CaplinMevRelayUrl, &utils.CaplinValidatorMonitorFlag, diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 716926b6a4c..3c3db298934 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -690,7 +690,7 @@ func NewDefaultStages(ctx context.Context, runInTestMode := cfg.ImportMode return stagedsync.DefaultStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, heimdallStore, bridgeStore, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures, cfg.WithHeimdallWaypointRecording, nil), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), @@ -730,7 +730,7 @@ func NewPipelineStages(ctx context.Context, if len(cfg.Sync.UploadLocation) == 0 { return stagedsync.PipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg(db, cfg.Prune, cfg.BatchSize, controlServer.ChainConfig, controlServer.Engine, &vm.Config{}, notifications, cfg.StateStream, false, dirs, blockReader, controlServer.Hd, cfg.Genesis, cfg.Sync, SilkwormForExecutionStage(silkworm, cfg)), @@ -739,7 +739,7 @@ func NewPipelineStages(ctx context.Context, } return stagedsync.UploaderPipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), @@ -795,9 +795,9 @@ func NewPolygonSyncStages( snapDownloader, blockReader, notifications, - config.InternalCL && config.CaplinConfig.Backfilling, - config.CaplinConfig.BlobBackfilling, - config.CaplinConfig.Archive, + config.InternalCL && config.CaplinConfig.ArchiveBlocks, + config.CaplinConfig.ArchiveBlobs, + config.CaplinConfig.ArchiveStates, silkworm, config.Prune, ), From 32da1c983b8f20536e1ee8f6ac112c788baea60e Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:56:25 +0000 Subject: [PATCH 53/94] prometheus: remove outdated charts (#13331) removes outdated charts we added as part of https://github.com/erigontech/erigon/pull/12927 because: 1. these were changed and superseded by new versions in erigon_internals as part of https://github.com/erigontech/erigon/pull/13309 2. no need to duplicate these both in `prometheus/dashboards/erigon.json` and `prometheus/dashboards/erigon_internals.json` - only `erigon_internals.json` is enough --- cmd/prometheus/dashboards/erigon.json | 327 -------------------------- 1 file changed, 327 deletions(-) diff --git a/cmd/prometheus/dashboards/erigon.json b/cmd/prometheus/dashboards/erigon.json index 732cbe2338a..78dc8885059 100644 --- a/cmd/prometheus/dashboards/erigon.json +++ b/cmd/prometheus/dashboards/erigon.json @@ -1199,333 +1199,6 @@ "title": "Block consume delay", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 201, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.2.0-74515", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "update_fork_choice{type=\"arrival_delay\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "arrival_delay: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "update_fork_choice{type=\"execution_duration\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "execution_duration: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "wiggle_duration{instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "wiggle_duration: {{instance}}", - "range": true, - "refId": "C" - } - ], - "title": "Update fork delays", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 201, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.2.0-74515", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "update_fork_choice{type=\"fork_depth\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "fork_depth: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Update fork depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 201, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.2.0-74515", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "waypoint_length{type=\"checkpoint\",instance=~\"$instance\"}", - "hide": false, - "legendFormat": "checkpoint_length: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "waypoint_length{type=\"milestone\",instance=~\"$instance\"}", - "hide": false, - "legendFormat": "milestone_length: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Block finalisation", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", From dc547a8f1d556b330f87eabf6ab338c0027cf206 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Tue, 7 Jan 2025 00:20:02 +0800 Subject: [PATCH 54/94] caplin: Fix `nil` panic during error handling (#13333) Fixes #13332 --- cl/beacon/beaconhttp/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cl/beacon/beaconhttp/api.go b/cl/beacon/beaconhttp/api.go index 8a266a449cb..1aef8a84a39 100644 --- a/cl/beacon/beaconhttp/api.go +++ b/cl/beacon/beaconhttp/api.go @@ -106,6 +106,8 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc { var e *EndpointError if errors.As(err, &e) { endpointError = e + } else { + endpointError = WrapEndpointError(err) } endpointError.WriteTo(w) return From dc4ffceeb8752a2155c5b250d5580e3f22baa591 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:10:54 +0100 Subject: [PATCH 55/94] [automation] Backup of erigon_custom_metrics dashboard - 07 Jan 2025 (#13339) The erigon_custom_metrics dashboard got changed so the new version should overtake the currently stored one, which should in turn get archived in a compressed form Co-authored-by: Michele Modolo --- .../erigon_custom_metrics.internal.json | 4907 +++++++++-------- .../history/20250107.tar.gz | Bin 0 -> 12240 bytes 2 files changed, 2684 insertions(+), 2223 deletions(-) create mode 100644 dashboards/erigon_custom_metrics/history/20250107.tar.gz diff --git a/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json b/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json index 085644e9e49..b8a482cfeb1 100644 --- a/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json +++ b/dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json @@ -28,7 +28,7 @@ "links": [], "panels": [ { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -36,108 +36,109 @@ "y": 0 }, "id": 241, - "panels": [], - "title": "Agg CL", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "ms" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - " dev-bm-e3-ethmainnet-n1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 240, - "options": { - "displayLabels": [ - "percent", - "value" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "max" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-79146", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "((aggregate_and_proof_signatures{instance=\"validator-bm-ethgiuweak-e3caplin-c1-n1\"}/577)*3)*0.4", - "instant": true, - "legendFormat": "Aggregate verification", - "range": false, - "refId": "A" - }, + "panels": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", - "expr": "3000", - "hide": false, - "instant": false, - "legendFormat": "Spare time", - "range": true, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + " dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 240, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "((aggregate_and_proof_signatures{instance=\"validator-bm-ethgiuweak-e3caplin-c1-n1\"}/577)*3)*0.4", + "instant": true, + "legendFormat": "Aggregate verification", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "3000", + "hide": false, + "instant": false, + "legendFormat": "Spare time", + "range": true, + "refId": "B" + } + ], + "title": "Aggregate verificiation per slot", + "type": "piechart" } ], - "title": "Aggregate verificiation per slot", - "type": "piechart" + "title": "Agg CL", + "type": "row" }, { "collapsed": true, @@ -145,7 +146,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 1 }, "id": 218, "panels": [ @@ -179,7 +180,7 @@ "h": 4, "w": 12, "x": 0, - "y": 17 + "y": 160 }, "id": 219, "options": { @@ -274,7 +275,7 @@ "h": 12, "w": 12, "x": 12, - "y": 17 + "y": 160 }, "id": 220, "options": { @@ -386,7 +387,7 @@ "h": 4, "w": 12, "x": 0, - "y": 141 + "y": 284 }, "id": 222, "options": { @@ -449,7 +450,7 @@ "h": 4, "w": 12, "x": 0, - "y": 145 + "y": 288 }, "id": 223, "options": { @@ -492,7 +493,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 2 }, "id": 212, "panels": [ @@ -522,7 +523,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 161 }, "id": 232, "options": { @@ -679,7 +680,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 161 }, "id": 230, "options": { @@ -773,7 +774,7 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 196 }, "id": 229, "options": { @@ -834,7 +835,7 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 196 }, "id": 215, "options": { @@ -923,7 +924,7 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 204 }, "id": 236, "options": { @@ -995,7 +996,7 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 204 }, "id": 214, "options": { @@ -1063,7 +1064,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 212 }, "id": 235, "options": { @@ -1164,7 +1165,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 212 }, "id": 210, "options": { @@ -1227,7 +1228,7 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 220 }, "id": 234, "options": { @@ -1353,7 +1354,7 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 220 }, "id": 211, "options": { @@ -1445,7 +1446,7 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 228 }, "id": 231, "options": { @@ -1527,7 +1528,7 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 228 }, "id": 221, "options": { @@ -1708,7 +1709,7 @@ "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 236 }, "id": 216, "options": { @@ -1831,7 +1832,7 @@ "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 236 }, "id": 217, "options": { @@ -1923,7 +1924,7 @@ "h": 8, "w": 12, "x": 0, - "y": 101 + "y": 244 }, "id": 228, "options": { @@ -1982,7 +1983,7 @@ "h": 8, "w": 12, "x": 12, - "y": 101 + "y": 244 }, "id": 213, "options": { @@ -2050,7 +2051,7 @@ "h": 8, "w": 12, "x": 0, - "y": 109 + "y": 252 }, "id": 226, "options": { @@ -2151,7 +2152,7 @@ "h": 8, "w": 12, "x": 12, - "y": 109 + "y": 252 }, "id": 227, "options": { @@ -2271,7 +2272,7 @@ "h": 9, "w": 24, "x": 0, - "y": 117 + "y": 260 }, "id": 224, "options": { @@ -2401,7 +2402,7 @@ "h": 12, "w": 24, "x": 0, - "y": 126 + "y": 269 }, "id": 209, "options": { @@ -2457,7 +2458,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 3 }, "id": 205, "panels": [ @@ -2525,7 +2526,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 162 }, "id": 204, "options": { @@ -2603,7 +2604,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2619,7 +2621,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 162 }, "id": 203, "options": { @@ -2636,7 +2638,7 @@ "sort": "desc" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "editorMode": "code", @@ -2712,7 +2714,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 197 }, "id": 238, "options": { @@ -2798,7 +2800,7 @@ "h": 3, "w": 12, "x": 0, - "y": 62 + "y": 205 }, "id": 239, "options": { @@ -2836,1517 +2838,752 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 4 }, "id": 171, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "panels": [], + "title": "Blocks execution", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "execution: validator-bm-holesky-e3caplin-cluster1-n1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 196, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sync{instance=~\"$instance\",stage=\"execution\"} > 0 ", - "instant": false, - "legendFormat": "{{ stage }}: {{instance}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "execution: validator-bm-holesky-e3caplin-cluster1-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 196, + "options": { + "legend": { + "calcs": [ + "lastNotNull" ], - "title": "Sync Stages progress", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 20 + "editorMode": "code", + "expr": "sync{instance=~\"$instance\",stage=\"execution\"} > 0 ", + "instant": false, + "legendFormat": "{{ stage }}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync Stages progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 206, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "multi", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(exec_gas{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "gas/s {{instance}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 206, + "options": { + "legend": { + "calcs": [ + "mean" ], - "title": "Exec throughput", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 60 - } - ] - }, - "unit": "s" + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_gas{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "gas/s {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Exec throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "dev-bm-e3-ethmainnet-n1 blocks " - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 } ] }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 200, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "dev-bm-e3-ethmainnet-n1 blocks " + ], + "prefix": "All except:", + "readOnly": true + } }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "prune_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", - "instant": false, - "legendFormat": "{{instance}} {{type}} ", - "range": true, - "refId": "A" - } - ], - "title": "Prune, seconds", - "transparent": true, - "type": "timeseries" + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 200, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 53 - }, - "id": 197, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "editorMode": "code", + "expr": "prune_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}} {{type}} ", + "range": true, + "refId": "A" + } + ], + "title": "Prune, seconds", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "irate(domain_collation_size{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "collated [domain]: {{instance}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "irate(domain_collation_hist_size{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "collated [history]: {{instance}}", - "range": true, - "refId": "E" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "sum(rate(domain_commitment_keys{instance=~\"$instance\"}[$rate_interval])) by (instance) > 0", - "hide": false, - "legendFormat": "keys committed: {{instance}}", - "range": true, - "refId": "A" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "irate(domain_commitment_updates{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "commitment node updates: {{instance}}", - "range": true, - "refId": "C" + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "irate(domain_commitment_updates_applied{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "commitment trie node updates: {{instance}}", - "range": true, - "refId": "F" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "irate(domain_prune_size{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "pruned keys [{{type}}]: {{instance}}", - "range": true, - "refId": "G" + "thresholdsStyle": { + "mode": "off" } - ], - "title": "State: Collate/Prune/Merge/Commitment", - "type": "timeseries" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 197, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 53 - }, - "id": 207, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "txs apply: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(exec_txns{instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "txn/s {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "C" - } - ], - "title": "Exec v3: txs/s ", - "type": "timeseries" + "editorMode": "code", + "expr": "irate(domain_collation_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "collated [domain]: {{instance}}", + "range": true, + "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 53 - }, - "id": 202, - "options": { - "displayMode": "lcd", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 81, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_prunable{instance=~\"$instance\",type=\"domain\"}", - "hide": false, - "legendFormat": "{{instance}} {{type}}-{{table}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_prunable{instance=~\"$instance\",type=\"history\",table!=\"commitment\"}/1562500", - "hide": false, - "legendFormat": "{{instance}} {{type}}-{{table}}", - "range": true, - "refId": "C" - } - ], - "title": "pruning availability, steps", - "transparent": true, - "type": "bargauge" + "editorMode": "code", + "expr": "irate(domain_collation_hist_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "collated [history]: {{instance}}", + "range": true, + "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": 3600000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 59 + "editorMode": "code", + "expr": "sum(rate(domain_commitment_keys{instance=~\"$instance\"}[$rate_interval])) by (instance) > 0", + "hide": false, + "legendFormat": "keys committed: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - "id": 158, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "editorMode": "code", + "expr": "irate(domain_commitment_updates{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment node updates: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(sync{instance=~\"$instance\",stage=\"execution\"}[$rate_interval]) ", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ stage }}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Sync Stages progress rate", - "type": "timeseries" + "editorMode": "code", + "expr": "irate(domain_commitment_updates_applied{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment trie node updates: {{instance}}", + "range": true, + "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 59 + "editorMode": "code", + "expr": "irate(domain_prune_size{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "pruned keys [{{type}}]: {{instance}}", + "range": true, + "refId": "G" + } + ], + "title": "State: Collate/Prune/Merge/Commitment", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 199, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "multi", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "exemplar": true, - "expr": "chain_execution_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "execution: {{instance}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 207, + "options": { + "legend": { + "calcs": [ + "mean" ], - "title": "Block Execution speed ", - "type": "timeseries" + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "commitment took: dev-bm-e3-ethmainnet-n1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "txs apply: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - "gridPos": { - "h": 12, - "w": 8, - "x": 0, - "y": 61 + "editorMode": "code", + "expr": "rate(exec_txns{instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "txn/s {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - "id": 112, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "editorMode": "code", + "expr": "rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Exec v3: txs/s ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(domain_collate_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", - "format": "time_series", - "instant": false, - "legendFormat": "collation took: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(domain_step_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "step took: {{instance}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(domain_prune_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "prune took [{{type}}]: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "rate(domain_commitment_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "legendFormat": "commitment took: {{instance}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(domain_commitment_write_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "instant": false, - "legendFormat": "commitment update write took: {{instance}}", - "range": true, - "refId": "F" - } + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 202, + "options": { + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 81, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "State: timins", - "type": "timeseries" + "fields": "", + "values": false }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.5.0-80207", + "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 64 - }, - "id": 198, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_running_merges{instance=~\"$instance\"}", - "legendFormat": "running merges: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_running_collations{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "running collations: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_pruning_progress{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "running prunes: {{instance}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_running_commitment{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "running commitment: {{instance}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_running_files_building{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "running files building: {{instance}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "domain_wal_flushes{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "WAL flushes {{instance}}", - "range": true, - "refId": "F" - } - ], - "title": "State: running collate/merge/prune", - "type": "timeseries" + "editorMode": "code", + "expr": "domain_prunable{instance=~\"$instance\",type=\"domain\"}", + "hide": false, + "legendFormat": "{{instance}} {{type}}-{{table}}", + "range": true, + "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 64 - }, - "id": 201, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "block_consumer_delay{type=\"header_download\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "header: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "block_consumer_delay{type=\"body_download\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "body: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "block_consumer_delay{type=\"pre_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "execution_start: {{instance}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "expr": "block_consumer_delay{type=\"post_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", - "hide": false, - "legendFormat": "execution_end: {{instance}}", - "range": true, - "refId": "D" - } - ], - "title": "Block execution delays", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlPu" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mgas/sec" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "mgas: snapshotter-bm-e3-ethmainnet-n1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 73 - }, - "id": 208, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0-77868", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "exec_mgas{instance=~\"$instance\"} < 5000", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mgas: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "mgas/s ", - "type": "stat" + "editorMode": "code", + "expr": "domain_prunable{instance=~\"$instance\",type=\"history\",table!=\"commitment\"}/1562500", + "hide": false, + "legendFormat": "{{instance}} {{type}}-{{table}}", + "range": true, + "refId": "C" } ], - "title": "Blocks execution", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 17, - "panels": [], - "title": "Database", - "type": "row" + "title": "pruning availability, steps", + "transparent": true, + "type": "bargauge" }, { "datasource": { @@ -4382,7 +3619,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": 3600000, "stacking": { "group": "A", "mode": "none" @@ -4392,7 +3629,6 @@ } }, "mappings": [], - "min": 0.001, "thresholds": { "mode": "absolute", "steps": [ @@ -4413,13 +3649,15 @@ "gridPos": { "h": 5, "w": 8, - "x": 0, - "y": 14 + "x": 8, + "y": 17 }, - "id": 141, + "id": 158, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -4429,7 +3667,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -4438,14 +3676,16 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(db_commit_seconds_count{phase=\"total\",instance=~\"$instance\"}[$rate_interval]) > 0 ", + "expr": "rate(sync{instance=~\"$instance\",stage=\"execution\"}[$rate_interval]) ", + "format": "time_series", "interval": "", - "legendFormat": "commit: {{instance}}", + "intervalFactor": 1, + "legendFormat": "{{ stage }}: {{instance}}", "range": true, "refId": "A" } ], - "title": "Commit", + "title": "Sync Stages progress rate", "type": "timeseries" }, { @@ -4453,7 +3693,6 @@ "type": "prometheus", "uid": "grafanacloud-prom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4478,13 +3717,12 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, "showPoints": "never", - "spanNulls": 3600000, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -4512,16 +3750,17 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 16, - "x": 8, - "y": 14 + "h": 5, + "w": 8, + "x": 16, + "y": 17 }, - "id": 166, + "id": 199, "options": { "legend": { "calcs": [ - "mean" + "mean", + "lastNotNull" ], "displayMode": "list", "placement": "bottom", @@ -4532,95 +3771,40 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", "exemplar": true, - "expr": "db_commit_seconds{phase=\"total\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "expr": "chain_execution_seconds{quantile=\"$quantile\",instance=~\"$instance\"}", + "format": "time_series", "interval": "", - "legendFormat": "total: {{instance}}", - "range": true, + "intervalFactor": 1, + "legendFormat": "execution: {{instance}}", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"gc_wall_clock\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "gc_wall_clock: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"write\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "write: {{instance}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"sync\",quantile=\"$quantile\",instance=~\"$instance\"} > 0.002", - "hide": false, - "interval": "", - "legendFormat": "sync: {{instance}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_commit_seconds{phase=\"gc_cpu_time\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "gc_cpu_time: {{instance}}", - "range": true, - "refId": "I" - } - ], - "title": "Commit speed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + } + ], + "title": "Block Execution speed ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisGridShow": true, "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -4641,7 +3825,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -4664,20 +3848,47 @@ } ] }, - "unit": "decbytes" + "unit": "s" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "commitment took: dev-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { - "h": 5, + "h": 12, "w": 8, "x": 0, "y": 19 }, - "id": 159, + "id": 112, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -4687,16 +3898,19 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "expr": "db_size{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "size: {{instance}}", + "editorMode": "code", + "expr": "rate(domain_collate_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "collation took: {{instance}}", + "range": true, "refId": "A" }, { @@ -4705,15 +3919,52 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "expr": "db_mi_last_pgno{instance=~\"$instance\"}", + "expr": "rate(domain_step_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", "hide": false, - "interval": "", - "legendFormat": "db_mi_last_pgno: {{instance}}", + "legendFormat": "step took: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_prune_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "prune took [{{type}}]: {{instance}}", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "rate(domain_commitment_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "legendFormat": "commitment took: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(domain_commitment_write_took_sum{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "commitment update write took: {{instance}}", + "range": true, + "refId": "F" } ], - "title": "DB Size", + "title": "State: timins", "type": "timeseries" }, { @@ -4743,13 +3994,13 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -4772,33 +4023,30 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 16, + "h": 9, + "w": 8, "x": 8, - "y": 23 + "y": 22 }, - "id": 168, + "id": 198, "options": { "legend": { - "calcs": [ - "mean" - ], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -4806,113 +4054,22 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"newly\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "newly: {{instance}}", + "expr": "domain_running_merges{instance=~\"$instance\"}", + "legendFormat": "running merges: {{instance}}", "range": true, "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"cow\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "cow: {{instance}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"clone\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "clone: {{instance}}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"split\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "split: {{instance}}", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"merge\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "merge: {{instance}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"spill\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "spill: {{instance}}", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"wops\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "wops: {{instance}}", - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"unspill\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "unspill: {{instance}}", - "refId": "H" - }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcrloops\", instance=~\"$instance\"}[$rate_interval])", + "expr": "domain_running_collations{instance=~\"$instance\"}", "hide": false, - "interval": "", - "legendFormat": "gcrloops: {{instance}}", + "legendFormat": "running collations: {{instance}}", "range": true, - "refId": "I" + "refId": "B" }, { "datasource": { @@ -4920,13 +4077,11 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcwloops\", instance=~\"$instance\"}[$rate_interval])", + "expr": "domain_pruning_progress{instance=~\"$instance\"}", "hide": false, - "interval": "", - "legendFormat": "gcwloops: {{instance}}", + "legendFormat": "running prunes: {{instance}}", "range": true, - "refId": "J" + "refId": "C" }, { "datasource": { @@ -4934,13 +4089,11 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"gcxpages\", instance=~\"$instance\"}[$rate_interval])", + "expr": "domain_running_commitment{instance=~\"$instance\"}", "hide": false, - "interval": "", - "legendFormat": "gcxpages: {{instance}}", + "legendFormat": "running commitment: {{instance}}", "range": true, - "refId": "K" + "refId": "D" }, { "datasource": { @@ -4948,13 +4101,12 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"msync\", instance=~\"$instance\"}[$rate_interval])", + "expr": "domain_running_files_building{instance=~\"$instance\"}", "hide": false, - "interval": "", - "legendFormat": "msync: {{instance}}", + "instant": false, + "legendFormat": "running files building: {{instance}}", "range": true, - "refId": "L" + "refId": "E" }, { "datasource": { @@ -4962,40 +4114,15 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(db_pgops{phase=\"fsync\", instance=~\"$instance\"}[$rate_interval])", + "expr": "domain_wal_flushes{instance=~\"$instance\"}", "hide": false, - "interval": "", - "legendFormat": "fsync: {{instance}}", + "instant": false, + "legendFormat": "WAL flushes {{instance}}", "range": true, - "refId": "M" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"minicore\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "minicore: {{instance}}", - "refId": "N" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "exemplar": true, - "expr": "rate(db_pgops{phase=\"prefault\", instance=~\"$instance\"}[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "prefault: {{instance}}", - "refId": "O" + "refId": "F" } ], - "title": "DB Pages Ops/sec", + "title": "State: running collate/merge/prune", "type": "timeseries" }, { @@ -5025,14 +4152,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5055,32 +4182,30 @@ } ] }, - "unit": "decbytes" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 9, "w": 8, - "x": 0, - "y": 24 + "x": 16, + "y": 22 }, - "id": 167, + "id": 201, "options": { "legend": { - "calcs": [ - "mean" - ], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.4.0-79146", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5088,9 +4213,9 @@ "uid": "grafanacloud-prom" }, "editorMode": "code", - "expr": "tx_limit{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "limit: {{instance}}", + "expr": "block_consumer_delay{type=\"header_download\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "header: {{instance}}", "range": true, "refId": "A" }, @@ -5099,382 +4224,1695 @@ "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", - "expr": "tx_dirty{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "dirty: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Tx Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" + "editorMode": "code", + "expr": "block_consumer_delay{type=\"body_download\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "body: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"pre_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "execution_start: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "block_consumer_delay{type=\"post_execution\",instance=~\"$instance\",quantile=\"$quantile\"}", + "hide": false, + "legendFormat": "execution_end: {{instance}}", + "range": true, + "refId": "D" + } + ], + "title": "Block execution delays", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 245, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{type=\"fork_depth\", instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "fork_depth $instance", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Fork Choice depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 246, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{type=\"arrival_delay\", instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "arrival_delay $instance", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "update_fork_choice{instance=\"$instance\", quantile=\"$quantile\", type=\"execution_duration\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "execution_duration $instance", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Update Fork Choice delays", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mgas/sec" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "mgas: snapshotter-bm-e3-ethmainnet-n1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 208, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_mgas{instance=~\"$instance\"} < 5000", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mgas: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "mgas/s ", + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 242, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 243, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "waypoint_length{type=\"milestone\", instance=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "milestone_length $instance", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "waypoint_length{type=\"checkpoint\", instance=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "checkpoint_length $instance", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Waypoint length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 244, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "wiggle_duration{instance=\"$instance\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "wiggle_duration $instance", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Wiggle Duration", + "type": "timeseries" + } + ], + "title": "Polygon", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 17, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0.001, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 141 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_commit_seconds_count{phase=\"total\",instance=~\"$instance\"}[$rate_interval]) > 0 ", + "interval": "", + "legendFormat": "commit: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Commit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 141 + }, + "id": 166, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"total\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "total: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"gc_wall_clock\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_wall_clock: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"write\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "write: {{instance}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"sync\",quantile=\"$quantile\",instance=~\"$instance\"} > 0.002", + "hide": false, + "interval": "", + "legendFormat": "sync: {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_commit_seconds{phase=\"gc_cpu_time\",quantile=\"$quantile\",instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_cpu_time: {{instance}}", + "range": true, + "refId": "I" + } + ], + "title": "Commit speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 154 + }, + "id": 159, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "expr": "db_size{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "size: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "db_mi_last_pgno{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "db_mi_last_pgno: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "DB Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 158 + }, + "id": 168, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"newly\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "newly: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"cow\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "cow: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"clone\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "clone: {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"split\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "split: {{instance}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"merge\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "merge: {{instance}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"spill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "spill: {{instance}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"wops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "wops: {{instance}}", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"unspill\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "unspill: {{instance}}", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcrloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcrloops: {{instance}}", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcwloops\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcwloops: {{instance}}", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"gcxpages\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "gcxpages: {{instance}}", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"msync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "msync: {{instance}}", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(db_pgops{phase=\"fsync\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "fsync: {{instance}}", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"minicore\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "minicore: {{instance}}", + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "rate(db_pgops{phase=\"prefault\", instance=~\"$instance\"}[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "prefault: {{instance}}", + "refId": "O" + } + ], + "title": "DB Pages Ops/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 159 + }, + "id": 167, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 30 - }, - "id": 169, - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "expr": "tx_limit{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "limit: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "expr": "tx_dirty{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "dirty: {{instance}}", + "range": true, + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-79146", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "editorMode": "code", - "exemplar": true, - "expr": "db_gc_leaf{instance=~\"$instance\"} > 0", - "interval": "", - "legendFormat": "gc_leaf: {{instance}}", - "range": true, - "refId": "A" + "title": "Tx Size", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", - "exemplar": true, - "expr": "db_gc_overflow{instance=~\"$instance\"} > 0", - "hide": false, - "interval": "", - "legendFormat": "gc_overflow: {{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": true, - "expr": "exec_steps_in_db{instance=~\"$instance\"}/100 > 0", - "hide": false, - "interval": "", - "legendFormat": "exec_steps_in_db: {{instance}}", - "range": true, - "refId": "E" - } - ], - "title": "GC and State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 165 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 169, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 30 - }, - "id": 150, - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "exemplar": true, + "expr": "db_gc_leaf{instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "gc_leaf: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "db_gc_overflow{instance=~\"$instance\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "gc_overflow: {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "exec_steps_in_db{instance=~\"$instance\"}/100 > 0", + "hide": false, + "interval": "", + "legendFormat": "exec_steps_in_db: {{instance}}", + "range": true, + "refId": "E" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "title": "GC and State", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-79146", - "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(process_minor_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", - "interval": "", - "legendFormat": "soft: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(process_major_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", - "hide": false, - "interval": "", - "legendFormat": "hard: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "getrusage: minflt - soft page faults (reclaims), majflt - hard faults", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 165 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "id": 150, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 36 - }, - "id": 194, - "options": { - "legend": { - "calcs": [ - "mean" + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_minor_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "interval": "", + "legendFormat": "soft: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_major_pagefaults_total{instance=~\"$instance\"}[$rate_interval]) > 0", + "hide": false, + "interval": "", + "legendFormat": "hard: {{instance}}", + "range": true, + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "title": "getrusage: minflt - soft page faults (reclaims), majflt - hard faults", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0-79146", - "targets": [ { "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(exec_repeats{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "repeats: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(exec_triggers{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "triggers: {{instance}}", - "range": true, - "refId": "B" + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 171 + }, + "id": 194, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0-80207", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_repeats{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "repeats: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(exec_triggers{instance=~\"$instance\"}[$rate_interval])/rate(exec_txs_done{instance=~\"$instance\"}[$rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "triggers: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Exec v3", + "type": "timeseries" } ], - "title": "Exec v3", - "type": "timeseries" + "title": "Database", + "type": "row" }, { "collapsed": false, @@ -5482,7 +5920,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 49 }, "id": 134, "panels": [], @@ -5538,7 +5976,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5554,7 +5993,7 @@ "h": 6, "w": 8, "x": 8, - "y": 42 + "y": 50 }, "id": 155, "options": { @@ -5571,7 +6010,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5658,7 +6097,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5674,7 +6114,7 @@ "h": 5, "w": 8, "x": 16, - "y": 42 + "y": 50 }, "id": 148, "options": { @@ -5691,7 +6131,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5826,7 +6266,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5842,7 +6283,7 @@ "h": 5, "w": 8, "x": 16, - "y": 47 + "y": 55 }, "id": 106, "options": { @@ -5859,7 +6300,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -5928,7 +6369,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5969,7 +6411,7 @@ "h": 6, "w": 8, "x": 8, - "y": 48 + "y": 56 }, "id": 85, "options": { @@ -5986,7 +6428,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6071,7 +6513,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6087,7 +6530,7 @@ "h": 6, "w": 8, "x": 16, - "y": 52 + "y": 60 }, "id": 153, "options": { @@ -6104,7 +6547,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6172,7 +6615,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6188,7 +6632,7 @@ "h": 6, "w": 8, "x": 8, - "y": 54 + "y": 62 }, "id": 154, "options": { @@ -6205,7 +6649,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6354,7 +6798,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6370,7 +6815,7 @@ "h": 6, "w": 8, "x": 16, - "y": 58 + "y": 66 }, "id": 128, "options": { @@ -6385,7 +6830,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6464,7 +6909,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6480,7 +6926,7 @@ "h": 5, "w": 8, "x": 8, - "y": 60 + "y": 68 }, "id": 86, "options": { @@ -6497,7 +6943,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6582,7 +7028,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6598,7 +7045,7 @@ "h": 5, "w": 8, "x": 16, - "y": 64 + "y": 72 }, "id": 124, "options": { @@ -6613,7 +7060,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6638,7 +7085,7 @@ "h": 1, "w": 24, "x": 0, - "y": 69 + "y": 77 }, "id": 183, "panels": [], @@ -6693,7 +7140,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6709,7 +7157,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 78 }, "id": 185, "options": { @@ -6727,7 +7175,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6804,7 +7252,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6820,7 +7269,7 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 78 }, "id": 186, "options": { @@ -6838,7 +7287,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -6903,7 +7352,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -6919,7 +7369,7 @@ "h": 8, "w": 12, "x": 0, - "y": 78 + "y": 86 }, "id": 187, "options": { @@ -6937,7 +7387,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7002,7 +7452,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7018,7 +7469,7 @@ "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 86 }, "id": 188, "options": { @@ -7033,7 +7484,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7109,7 +7560,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7125,7 +7577,7 @@ "h": 6, "w": 8, "x": 8, - "y": 86 + "y": 94 }, "id": 189, "options": { @@ -7143,7 +7595,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7245,7 +7697,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7260,7 +7713,7 @@ "h": 6, "w": 8, "x": 16, - "y": 86 + "y": 94 }, "id": 184, "options": { @@ -7278,7 +7731,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7316,7 +7769,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 100 }, "id": 75, "panels": [], @@ -7371,7 +7824,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7413,7 +7867,7 @@ "h": 9, "w": 12, "x": 0, - "y": 93 + "y": 101 }, "id": 96, "options": { @@ -7433,7 +7887,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7518,7 +7972,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7534,7 +7989,7 @@ "h": 9, "w": 12, "x": 12, - "y": 93 + "y": 101 }, "id": 77, "options": { @@ -7554,7 +8009,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7602,7 +8057,7 @@ "h": 1, "w": 24, "x": 0, - "y": 102 + "y": 110 }, "id": 173, "panels": [], @@ -7657,7 +8112,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7673,7 +8129,7 @@ "h": 8, "w": 12, "x": 0, - "y": 103 + "y": 111 }, "id": 175, "options": { @@ -7691,7 +8147,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7816,7 +8272,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7832,7 +8289,7 @@ "h": 8, "w": 12, "x": 12, - "y": 103 + "y": 111 }, "id": 177, "options": { @@ -7850,7 +8307,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -7966,7 +8423,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -7981,7 +8439,7 @@ "h": 6, "w": 8, "x": 0, - "y": 111 + "y": 119 }, "id": 176, "options": { @@ -7999,7 +8457,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -8065,7 +8523,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -8080,7 +8539,7 @@ "h": 6, "w": 8, "x": 8, - "y": 111 + "y": 119 }, "id": 180, "options": { @@ -8098,7 +8557,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -8176,7 +8635,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -8192,7 +8652,7 @@ "h": 6, "w": 8, "x": 16, - "y": 111 + "y": 119 }, "id": 181, "options": { @@ -8210,7 +8670,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -8288,7 +8748,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -8304,7 +8765,7 @@ "h": 6, "w": 8, "x": 0, - "y": 117 + "y": 125 }, "id": 178, "options": { @@ -8322,7 +8783,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0-77868", + "pluginVersion": "11.5.0-80207", "targets": [ { "datasource": { @@ -8508,6 +8969,6 @@ "timezone": "", "title": "1. Erigon CUSTOM METRICS", "uid": "b42a61d7-02b1-416c-8ab4-b9c864356174", - "version": 276, + "version": 279, "weekStart": "" } diff --git a/dashboards/erigon_custom_metrics/history/20250107.tar.gz b/dashboards/erigon_custom_metrics/history/20250107.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..ee1da02347adb33fbc70c01e482834217f577366 GIT binary patch literal 12240 zcmY+KRZtv2mxTv+7=lA^26tzG;O_3uV8I=Ny99R~AV|=_U;%e^Y1?F zZddi~`ud*g``D*$H)T93;J*jLpTNh|wU$NOzjs>Z+V^=dkB*I!#k0Cm?G>kCfongq zRu|`uSRU3mX(eT?(!RlsxbbVJzlxxto7;;A=RR*yM;#&Cu& zxX`z`UQgcC*%!(iIlR8Npfm&mQHIGk7d7B9Q@MRxT(wd8T!-up%T-yiF#F>T*>gmg z%Sv8O1=$$~y&qG-;HE=dwP2=gwO|#k?@pcGO^0=CLfTG%y&P_}W`Tb&z_ z)uijlUw+T)4`;qEW!cNFB?|9m%PI_`ifi82Kv2{Sr<;>9(`YPRQgRe45AFgIoqyeJ6T#^?o%B}Ns=M(r_P!(reTT8Z zKC;(mwguZ)mC~OIbG`p&LchhZ*Fv2p$-uZ5@AHK2hInK5fp4#Ruiq|Aju(6^7_!D3 z%JP2b?BqDU-ezJ4x9OWTsMwGv9#pX*WI8T zI9K!S#^`=E?Q$})I5g4JKtp%<&TmZVeOk^Qu9auiFw1;^HBL^VYDM8sU|hgYs>Ee($a$`rhX9 zxZo$r(Q>@skQK*|u0$RipL)hbS3uej8M>!7<`mIwrSs=OUvS{v&c#a7>%kXK=*z0{ zucoVmtEo(ty8-dKr&meq&4qK%4C*RRuNKQmE}sBvjHgJaB98BQe*T>?0NWgO8_*b{ zNBN%O=W9P6IzyEw6@cb-f5K62a2DwFk75It`z>f6k7qb1CX?fOE*B?e_VXOc0dQqK z*-GD;f;lBn+yn?59ad5RW}g~`6JIFS=O|@eR;ftMTjQ)eo2b#?j$I4)%lT6a3E3re z8p>+TdV0+@iX6P8-MQ=}6l6S;SQsN#2f(wA=)&q>CcYP0K=$>c4&31|dbL?_%}4ce zNSTXShHNN)xFTzo@zYcYl3POpNaSr)yQe;~;XoZM3i07zO$7OTPr^EjK3b($@?dx` zXXWtqby*eJGCT6>bLv%c$WS%6(!s00ols_5Bz@#1w{_$C^u}Rgr3g7|mbqv;s$On7n7Lzn`s1T&pAvFy<_1>EScfLCkGWlKC^4I>jbYgFPk`%Y- z5ASe!aSuame>FOmlXW85@olHGKGxT7zf-Z>HSGtq@dHZ9UE}C;rRqPD=#2a=`}N!xw${<+$XQ+N4wh5kjT;K2kt7W(wH>5m0wjKL z8t$q0NdQ9&G@7*=S;zfO{()X@bhA1~s3|5J$9lh0XEBrL=}x#Bv8vE1 zZDwqqAkeigimdzEUaZ}W4oE(a3nz&hF+GZ2?3M#04Fkec&*M@@=w-GqOQLs{cgsCz z$T(tVM9icq2fCSXG?W<9YZ56+j(}5|0E(#0;WQ$)Jx-k-W+1wD$=K&Rk zvUEET@9t_435OEmPJM~DE`IMMdW6Pl*_bydn*aL1gE4x&=><@>`Y4^65+4}iUH{cqw_rY#CA!j`O zxuc^ae`^mnzY^qfx)4%=sp{#7Sf+x!ue!QO3U*;##widr#8O8_cK!t~247;*i_ND% z_$gCkdXYz-%`aiLhuvRVI`zX4_-jr_ zg{fL0=HowT4b#F^hCQZ-zxs4|2tU<_zC*AdP1dM$FZjTh@f-6Hs!;$`I3u!WN(}&{ z?B3Fjf+j!3(Al0-r@nmMkc+ku-;qnrGMuJLeUYXa9mj`a^`S+ixO7|}a`s0{EoN6O z%Qt7TOx-KHry}M1?~?UQC-$lzrD`FXGxn&56z31YMNuGP?mfB<_rJpjf6Wd@P`9fp z?w2ke9C{9qEIt;Ei;Bcd#5vf7v$L!~$!W07xrFj;{7GHjGUQO)i{h=L7og%KgtyWl zPm!GlOi;Y$cwU)V+r)@}5j|CYW~x7`;$Wp0)3)gbB%J)&3YfBwLzGMFsy zXoGb1dfI0%k9p9_fznNyL zNSVxy-H-nZg1~nAijiS+MTp8a@x|V_PS!MNijj*+_(@nf$8T`Uy?hxu;(q^LRjT+h zyURa_Z{kN8?1AzrzyeW7#!oPo%fTNm255}Sgb&EUF?KxGmt!TkK zjvMQK@z`ID#to(I1sWe_Dq<||b-7NG6UKQl#igPNX4!re$Z_w)%>OHT>9JAs`jI-S zuwhQ0n}H|bb~QMCXsQs-d4Ff0ER*An^zoq$k#UMKL6vHzO0D9UTIN{R{Y+mQJfw{?B_C!=6V4`>sdY)!=~nfKz@IF^^Q8nw%rdf z6v|@pCDaWdqPFD+W`*l)n8XIV#j3vadK?g9Jp+2le!#zJ$XO-MRStyFpq$l(Wn0Av zA>&W6d604q2kaPH%CwT6adTyzPKvySholZLE+I0nTKY53aO$H?%0 zsTceD1@IuFY&xqqz=+8;J*sCO%Up-WssXPw-*n1ePyGMz@_@f^fOfFBji>mz`S$_Ob}1MkI%qH(o{#6?wsT%+x-%Vt$7{Od*7l#2+_h*;?h|0FEUjnMwfU0`EHQ zraloTPV$<$QfM^c)!}< z>s+Psxbl8`MuNNHkl;{;EkMq72ovMXD* zQO8~L6lEAKhGQ8qhOPYsd8wP5SERijmR`6c`2U3#2z1SdZ*>$z~ zI~ir{Rgoz9HZ&rfGy52lt^h<7IwtsDI2nR)w9Yj8n>xtC?jVYBhAF8Q*SCAz+=e8o z59SNXpKH@@XcyMMOPH&%8x%r{ys%CJZ2tl}t*-*QzX6 zqRHKyzb+v=tgUO^g)3p#A1W(2JwocjMNw^>rA)H8C2pnHG+?N)JD(ryX-i_J=c5^- zh{UX;@A{271JN0Mx-u0aUIAag?2pYi3q2@JeuC%)V`<(Zz(l02D-Fj942ocl7iVXf zHKXm7M-CF(QeEGQ(kcjCR zX5W3QloD^>b%V!gbT*)q>qpX(smNC~B5uJ?O0qLl9t&i$Zt}?QyWri7jmcuHQZjYD zqkk4zvXaPKB@fwc&g#G4x6$WWq2AFU=uiz=+hoyU&`}P_TWZ&?X=n8^UPlLT2i-lSTd1ntTge#0#$f#@ZA)wdZ`)Kd|I{K$YNUzT-aZ= z_p)ed<1XXRDxcq9O|3k-rS)<6-e{zdF7SH8{%s{|=M;hZ_H~^Hs>J{{;Vc=YMec zpTO;(VErGqwn+}xlE(h4xbv^#_kSgvJt7m=jEmF$Vd+1#_lQ(=sF~*2=OKN%9E{jX z+KSHEN(vaNoK;WO677VCqbA2dC6N$89SElz6aUX5t>*UheuNn`$*hyAJ(ffiIjYNvnXdTzKt?1$EEmayqzP!l~vXfg3C3 zsh2oW;lQrm`N?ee?`M_Sb()Qpbd!~}?RQW_|MLVG%>!?$dudRW+*;8I&9qlX^@*fE zWZEkYBEE!VBH&*I4b+$P-Ye3tqNd=>XpT%ooUcN5LqDr;nf zGBN3ufahj|^?5Xtf%eWhyur+^e$!t(kW^)pCJT_OD&%qK`)mSb+M$~Bzq_DCJ18a2 zt#04tzjrK*Jgxr_;Pc#ecl%VV*d`_Laj6yoE}CrdU`=zv40u!glgt0{jxa^+R@mfL zbWR&f&f1~G&Y`5@+!b0@@vFz1;h3(J;9=@>N>p7XpvrdZTc} zl*6`Dee=|jm)#iUcit85T&`YKFlEfw{eU#NubOPJ>q~>A1Qs>^jM&PAW&UqE^nBx0 zdpXXx^7G8hpKUaZa6N2Et$Fyng1didfv=%n2*QVKe-H(SfB+3ZNi>{vK^X+H-GVoW zGr+ZoONCz8@E{aKyb1!2siEq|NwSzm0vLH$CgDodDMNyu=d_fwwwj=*SD0^qXN9Ys z8&Lr00DOR56)`})i5w6)!rB835&L2ruOntrlG}6b`Le?{K3i_-FOZoaQ z1m0s(s2`^L*{zNfGMbbmVz1z>s_!&=EvcFE%5PfW9lh2Hubwak)=z&6D!^) z7L_kmRCl`5xZSGtg3mHN+dq8bJ&G05Q!|{5GbMD5%MvM)3CQ{w1wW-l=>%5R7a!E& zA(FM4m3I=yBQREE%<}brpzr^UY$QaZ1;TDLa#}_cuYXODtPdr4lS4KfJ*|9x? zEuPMbuKCLxn8UFefMA~NaEYy&B>B>5oToniKB#t#9QMX^uh^I)V+@*dO)F-tV6qu9WhwtFlxlx z+3$LA%mXfvR(x|qW)skrCq_Kp;f6d3LVd#C`h>If2{z@?kONlzY4+v!w~}Yg`A;aJ zABQ+F>qxY<_R?WU)kpbE09UY_7Z}=mOEs@xW@5ZjsE{GNz|Cj-jBsGIfp_(XtPJzf z$_IT@4*bG{|5>cxWtOqmq(!W=ku4mIdTd@t^)UaPk-aOX%f}X7^P@zk>)#EQ%6nErg@Sg1=>YLuY5+W=$Fbdcas`k z?ebN?xo_#2hN?vrhswAuX{PX1Q@0fSB{Y0TmipP{c-Vc8XJP4Zh-iW6=TF~DY&`W( zZg>;Cs2mdJYNvIgxM3aC--UG$9fQrE;|mj4{%c=0)WNXW z*mAy~N#T=Rk@|c=A1J3iY-C2u>7zY@O3Jyu&Ya&pLzi62gEe?a>DD*pM0kRX?q)(U z#;tc^$7aDGY;@@*&m2^WA|n1MgGQ1oo)e;X@_Pa9Tdn)|RTAm3gjd$+(#myNOZnWv z1P68~B?klRKGyl*mR77LG54A!I_ zH2CQq{sHpFA}q~=)_H1_vI#F_Cl$hPrM;D2VK%+}78yVwoMl&MU@t=Q`FZV0*l^n_ z&{*zBk{_uwK*v)m=P2r|ZYmjvdbay3p_(XdyvgPMLlc0?SL_OjI+rw`5-sj@=#eY` z<_O_$k};*KG*m?SP0q%C;W{{dU6>^yuLJKhPiW6u3UN>6u{7TcWwVy_er@%}Wn}4< zsyE{$T?rim85?{IHjtdkzm*C+z~J@_co2DPN_#7Zcuw9UW%OQG$7R(yVNdzxq3=42 zq1~9`=2T-i&?c$-c#z24A^pVj@klSanh3pD$`wHG#5R(gM7P4Cp5I6jQzD&;h z-bFI}+yn|eC1p4=g_&p4!3$b9X z2Eb)2BiJw@^%q$hnmTT;#aO+<&k;_@aJUnLER<Ld|u(SmW2{tUAUA6Uh#Hdx!_%ac6+Tpc*VB@}c>c%Pyqc(KS zOu|CRyu5Olwxr%duF$W&I3-Etva+riv`c$}h5iYFX}|jNN`h-wxZi`Yd0O%d=F3ui zE8}5b&>u_toH!2RyEgM@0pf8W@pz9u@(?#?W@rQJYg#`~<{o2ff0~TWAG`H58Aw!< zgjn;}-elw`-n((|_w0=X)p@nKOVqzK@)C~f z9;oQMPS?-_(gG3p>-EO%qHq)Q3ipH#I1-X5O7niTrq<`POh$$`hQQ?YMzp;wQ3@D) z>u}wbAu=|FCzj8Qi9*fpP0>D}J1P&j&I`M#CSEkZK|u^Yd+-gBX1NedsE*JAEfOeb zw1qR}cH1FuP?MaMrx%fI1cEpdY6HqJbG7yD;|j(TyHv2x9ir1nm4Cf*35j$a+GMWL zFn>s{-L8eThzN*vN;EKa{7R75#A?Yo8UQ4lpDoo-k&~ z1>oijtsTb9#Tk%8JHN}TXE=#I@FAVzROAGce%#y2cPs1|zT=iXU;mZ6p|?`V9QnzC z_dYDU`7o~Gmm*g~O4PsBm}v?Ux7K@OEB!D&ogbNsOX};vxP}DOI&g}$+gM62;_w{$ zARK*}b3M?MUhz_O>TvbInojSW&hZ}eovY5JGl5@=;txXz1zWKI3(KBjr#|163%=jC zI1#;!AG(ngEOn>sKN~gn%bDFZ2-+&pjkSat{VJ6dPZQte4M(nmIx8bpZi0r;G>Y6Z z?eip6(!zPUN$F7_5v0Wm%bLinaim);1(uPGRhB&yVg-zS_hN(3?dpf1MXANjfh@l- z#v5Hf$z<~PjQi{&Kwip5juIGp({q?xeVZ&}yAR9m#kOxEcJ>fEUqw#Vyl1sqL!wf{KMrPiu*Ta zAxS#am~ZskABoRFmY<1#DGj}Y;2aSkbSd17CjiW%Zb`10dwO?5!-eRw`$PEfFb=r2 z=K*0gqx-H~D@^tFb+cc9Q8k(#j(DfG)@=xSnC_k4ZogThsZEel*4YPNLZ zD%~s}2l2VTKWAX+SGTZo9IWcbW){@rje_A5W<4L*GzV)XxTGcOJVDE*{p5{?RfRY3 zMYgsfY9;n+9dO3Ea{Pvr%a2c>HjdX3&|*XCJPD+Vs{3Y2LoWl@EPD%ZNQehGV1eCV zzTWa$wgOrfBWfe`Th>@@3h0l&?lzOfNTxApC|soD)Z}Ne-MczSY4s}s08ZdQAK8bzRa^>PsFUz6;A<6U{Z^SJ3Gs| zs=>%$3@$BX`(UfD+VkojwC8bXKpx6!FVs;)}w#x3@a zrM*W-Ow{WO>P&q|5Zvn-#p}NNpg1pUj~PWmB0Fgd`yKh6hZoASPY0fHNMI7evo3g4 zOb>#<;mb@-3`eka;SPtTda{l8Dwv?L1TjS_CX8mz?MG0WeAz0YS{m#gf|oBI;jp z6TV=f@X}1jjFpJW^DukA_p8jyWSawsQ7VkEj+X097%DVtBw^&zPKdq?M8AY{|+_Bg4G(`rT_J?y6SP}&mV4M*Rqg( zq+Fuof3Yt8j<=>Ja}Gqn96m}mT`*=j3@c2X5vN)lnjwYTu=m)o_ZMmtLE2zy>OaZ+ z=`503z_+25fQ)%r5YpS;Mh-Z$jA|$Jb+M8W=3g}WmHD?-FC+b*Gxj8UEBT0mgm}7N z`B9xz+#XN{=Ou1W+?Oy`QOS3D@T*9BTBrD!(}5SI;|eub3WYBk{g4we%FJu+7GKJA zyA1LT!tZlJPB6)*A&MI9NKuMCCnLcDJd!En6*BR3o~QB`JWD=C&xw7e5q0X|-sten zw8m(65x;V>oTSOs1Nji~&4rX&qjEcaind3R|CsT0aq!0hS``=mcVSq=KJ!d%+UT%V z`^@-aLyK_V;~$+8V}`V72AlP8*y`^H+FG5G)jY)(q82?_;fnCl;N)vs9kncFR+kuL zJl^40eMDMJVheX#jJ$uUsAU^MNP?+5hz49w4G`M z8;I0Q{9xfagjyDPoeXwaJM;UUQE@q5IG#nzGgV1cG7+r` z$DH%x_zwq!Br2e|_z9_9FDlG6F?>1Txu_*#n6J9Jy2m#%U3Jj3x`dQWQ87h&R8^gK zz@zFU4i{k|4OtjFpj})iMzGa7N{K_Ofa0Rhox>ppe^UR@Q!0NaSt4%4E73N0KaRPi%G4I z#s|o@`q6UkTzy{=p^bC4RF$w_gkA|9Kev27+g@pmBI)cIQ`E)m*ThMhaocQ#+)QP#x_V$Bd7c!a zVY51!+ao9$IS-iID=mVI|1B6$cAKrA!fHrv>9gh?B4BfQVfU{@|v$VV`fhBa1K=l_`J=|ZMzfBLJck$5XO+w z;{ki{V+0GXAwE}`d(0!?G=?x1KMrawIPonoZdij(9tp-rDA@`$IA_ztm0!>5o_qze zW_ySIo-`kh?doI}&-I=k{Dm^tUm8ESzZB;)JZUiir+-dLg{slr05+AP+t&aEc+w z<9gApFN(ih%_H+XF=UaJPx=d4iHg!)mL)5I-yFoMOk?St?fi@bNBArG>1anK$of+U zIe|Dsh$^69B2@L)OUT^Gk0@0Px7CNK@~LkO8v6(cNuZLC%$%x969=#{cHKcYVV4H) z5o7*~Sh$5M@p;od3%kqTw>-mImDGvm^IIH2{2QeA)DH{lEHt2yvf}ei9(oy1@!t2s z)UO34WQA<<*6Tgw1QM@#7p1vE_jAKWoC`z$tVH5JD}fD$|0*4{ixP|grC(T7rRaIt5zs5}!28)p*d39*q(BOT6dVe!?q1JP1SAsl2bpEh<`=MjrH zex(Spal;Li@zE?ln83$*AA25>~MjL~b#2qSumH8{Tni)5&{ z;QT$pXr|R|0rCTF<%E+(#R9E+6F-z4y==0S*`rixw#WO;vS{T}D6PE>B>HP)?3*a- znndXR7Jf zO!5Ue5qKBh%k|Hzq8;~s{Ks%~`14(E`ePA2Sfdm3hDkwB`x zbCHAzKa&Q59G+99!JK-mkfX_LJxdll-Sh4cbRd4Ba!EK33L6JaXHP}JM33M)#4K;M z9iFMkBk(V)`(D`jb$EC>`S1Ipu@GjDo+?afjj}ai2_1Tw*lF8am}Xz%00)nD+L4zt zyJ=9L&xhmL3pTtcM-^Ln`3Sz0vPTmzzFz0blc)r)K?I$;`?dqZvMVfbU_urdi<8j(sX@!LM3T{dVjd0;42ratjgGa#VgUD(BhI)534rGuMTwKO35?}Pip(-InoqV zn)_kA7DigLYn}y;SU;YLG6YupRAmXvy$m=tHol%7eTU=U)YkkhhDX0~M7Fz%$VSv; zJnb$qj>L9f-@Lv*g`RN?-_|He12J!pPg^g-lqMHZhfo1R@SkfrkAghzdj*gW^}DfO zKWj$R^A*I%bM;Iz>cyggphuKegtLj~(rp&sf>$R(N$hkYHH(}vv@uqYj4@Uq&YSH7v~d}J1={D!)) zn-L5xcibu4vs?*WH)yX37+<+)mYOYQPBofo8j{}-zcFtYCOFzu7_6dsRe7PB!c^+q z6h8V=s9pLLO7YoBJGkBp+|O!|_ov`2woAM8r5d4J1j|=rKTgaNy!W-z;ap_vu8Ky$ zloCncu&9Fy;4WG<5(%Z&0m09qX3O^Q!k6QkeljOr*&K`!Lh%d@L`;nh4=fejO?IJU z>MH)mir#>>r91y^hre_nKp~H#y%*0Enz&Z~?wS^6?-@>zbN0Tq7_=|h*EfUpVHfP# z6O}b2xw7(CRDU$HRqgMHyJ~YaFyL={>K%?xNBJ?_gcv(~Zt!$U_|gltmUeq5p7x8M zWP7u+M%yxq!k7-l;hk&r$!)AnoP!`<`m_AZYUXGWvo5SVK>gO4dW~NQxFyo#7jg|a zfSW|f*V_w1;hOCD?2YxGy=@;ftF(qQLi2Y<6Fz^qj`sdmdE=bO)MiZ!cvdA6H7bX}u#&3T>3wC1b?jx`L=N6H7!uChT z%hy;O>K+&C&<#6Qi}+NZjDM%&skN6Q>j}22MCr|&-rXgAZZumW7$>1SvxUD zhtBxP{3U_nc5cPJ@<85FyVDY+$<&JkAs}F|fIPchu^kg0E5U$b6?s4f-aOt&ZyRWe z4c>B8H&bym`yF0(lOce zq#BxC0iPKkt1VB)e)hbj{e9sRW^^ljN>m%rca^VnImq5X_PBf*mcI%*5`Fy>9DMZp zcc<|{%=E(MCv?=t7cYv5?NOyL=ntg|HAnR8DOoP5AoWt19;ig#5FIgvE?Z?r`$75w z@d*@x@_wiSFw75MX>OF}o<;12GX0Zqj>E2I)AM70*kV*dD?QtQlo78ex0_La`C4ww zS}yNKCNCq^cG2Y`?8&W!tM+$G`F2aWN-HP<9hu*uz1hNSoitF794tT114GMCDf6Re zn~aKY9Nqrx`l>uTS-whg^k~pt8zHS=1CI)kZgTmc(4v}vyBQcLZ8lV2>~lc&E>`!& zl|7+9;mdF->MJbkX+u%(9B=7Iq-gBj@$O{)JBA3lSe)3-Rsq!)Hjo%yYr`)4oDyrM zU0-?hJ(13(R5On*eA&TabL|ux-p0%I-F39j;l{}nG^p)T?J{!Oj-QkfPa&`H9);&9 zfKFxpS_#4rhMH7nrOmjCq?FUXp_tKjx@ Date: Tue, 7 Jan 2025 15:57:14 +0000 Subject: [PATCH 56/94] txpool: streamline initialisation (#13202) Attempt to simplify how we initialise and run the`TxPool`. Observations: 1. Currently the package exports many sub-components to backend.go such as `Fetch`, `Sender`, `PoolDB`, `newTxs chan txpool.Announcements`. I see these all as internal components of the `TxPool` that `backend.go` and other initialisers should not care about. In reality, the only things `backend.go` cares about are the `TxPool` and the `TxPoolGrpcServer`. 2. Additionally, logic about starting and closing the tx pool leaks to backend.go (ie the creator of tx pool needs to know what sub-components need to be started and disposed). This PR addresses the above by: 1. Moving the p2p fetcher, sender and pool db as internal attributes of TxPool. 2. Introducing the `Run(ctx)` pattern that we are using in Astrid and Shutter where each runnable component is responsible for starting its own background goroutines, loops, listeners, etc. and also is responsible for shutting them all down by being ctx-aware (ie when ctx is cancelled all sub components gets disposed of/shutdown) --- cmd/txpool/main.go | 36 ++-- eth/backend.go | 105 +++++------ turbo/stages/mock/mock_sentry.go | 79 ++++---- txnprovider/txpool/assemble.go | 123 +++++++++++++ txnprovider/txpool/fetch.go | 20 +- txnprovider/txpool/fetch_test.go | 19 +- txnprovider/txpool/options.go | 69 +++++++ txnprovider/txpool/pool.go | 61 ++++-- txnprovider/txpool/pool_db.go | 74 ++++++++ txnprovider/txpool/pool_fuzz_test.go | 6 +- txnprovider/txpool/pool_test.go | 64 +++---- txnprovider/txpool/send.go | 10 +- txnprovider/txpool/txpool_grpc_server.go | 8 +- .../txpool/txpoolutil/all_components.go | 174 ------------------ 14 files changed, 475 insertions(+), 373 deletions(-) create mode 100644 txnprovider/txpool/assemble.go create mode 100644 txnprovider/txpool/options.go delete mode 100644 txnprovider/txpool/txpoolutil/all_components.go diff --git a/cmd/txpool/main.go b/cmd/txpool/main.go index c476f29a2db..eba6b7a2632 100644 --- a/cmd/txpool/main.go +++ b/cmd/txpool/main.go @@ -28,6 +28,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/datadir" + "github.com/erigontech/erigon-lib/common/paths" "github.com/erigontech/erigon-lib/direct" "github.com/erigontech/erigon-lib/gointerfaces" "github.com/erigontech/erigon-lib/gointerfaces/grpcutil" @@ -38,16 +39,12 @@ import ( "github.com/erigontech/erigon-lib/kv/remotedbserver" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/cmd/rpcdaemon/rpcdaemontest" - "github.com/erigontech/erigon/consensus/misc" - "github.com/erigontech/erigon/ethdb/privateapi" - "github.com/erigontech/erigon/txnprovider/txpool" - "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" - "github.com/erigontech/erigon/txnprovider/txpool/txpoolutil" - - "github.com/erigontech/erigon-lib/common/paths" "github.com/erigontech/erigon/cmd/utils" + "github.com/erigontech/erigon/ethdb/privateapi" "github.com/erigontech/erigon/turbo/debug" "github.com/erigontech/erigon/turbo/logging" + "github.com/erigontech/erigon/txnprovider/txpool" + "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" ) var ( @@ -183,26 +180,31 @@ func doTxpool(ctx context.Context, logger log.Logger) error { cfg.TracedSenders[i] = string(sender[:]) } - newTxs := make(chan txpool.Announcements, 1024) - defer close(newTxs) - txPoolDB, txPool, fetch, send, txpoolGrpcServer, err := txpoolutil.AllComponents(ctx, cfg, - kvcache.New(cacheConfig), newTxs, coreDB, sentryClients, kvClient, misc.Eip1559FeeCalculator, logger) + notifyMiner := func() {} + txPool, txpoolGrpcServer, err := txpool.Assemble( + ctx, + cfg, + coreDB, + kvcache.New(cacheConfig), + sentryClients, + kvClient, + notifyMiner, + logger, + ) if err != nil { return err } - defer txPoolDB.Close() - fetch.ConnectCore() - fetch.ConnectSentries() miningGrpcServer := privateapi.NewMiningServer(ctx, &rpcdaemontest.IsMiningMock{}, nil, logger) - grpcServer, err := txpool.StartGrpc(txpoolGrpcServer, miningGrpcServer, txpoolApiAddr, nil, logger) if err != nil { return err } - notifyMiner := func() {} - txpool.MainLoop(ctx, txPool, newTxs, send, txpoolGrpcServer.NewSlotsStreams, notifyMiner) + err = txPool.Run(ctx) + if err != nil { + return err + } grpcServer.GracefulStop() return nil diff --git a/eth/backend.go b/eth/backend.go index e1b1edcb6d2..08da323e42c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -87,7 +87,6 @@ import ( "github.com/erigontech/erigon/consensus/clique" "github.com/erigontech/erigon/consensus/ethash" "github.com/erigontech/erigon/consensus/merge" - "github.com/erigontech/erigon/consensus/misc" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/rawdb/blockio" @@ -134,7 +133,6 @@ import ( "github.com/erigontech/erigon/txnprovider/shutter" "github.com/erigontech/erigon/txnprovider/txpool" "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" - "github.com/erigontech/erigon/txnprovider/txpool/txpoolutil" ) // Config contains the configuration options of the ETH protocol. @@ -193,16 +191,12 @@ type Ethereum struct { waitForStageLoopStop chan struct{} waitForMiningStop chan struct{} - txPoolDB kv.RwDB - txPool *txpool.TxPool - newTxs chan txpool.Announcements - txPoolFetch *txpool.Fetch - txPoolSend *txpool.Send - txPoolGrpcServer txpoolproto.TxpoolServer - shutterPool *shutter.Pool - notifyMiningAboutNewTxs chan struct{} - forkValidator *engine_helpers.ForkValidator - downloader *downloader.Downloader + txPool *txpool.TxPool + txPoolGrpcServer txpoolproto.TxpoolServer + shutterPool *shutter.Pool + blockBuilderNotifyNewTxns chan struct{} + forkValidator *engine_helpers.ForkValidator + downloader *downloader.Downloader blockSnapshots *freezeblocks.RoSnapshots blockReader services.FullBlockReader @@ -276,14 +270,17 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger // kv_remote architecture does blocks on stream.Send - means current architecture require unlimited amount of txs to provide good throughput backend := &Ethereum{ - sentryCtx: ctx, - sentryCancel: ctxCancel, - config: config, - networkID: config.NetworkID, - etherbase: config.Miner.Etherbase, - waitForStageLoopStop: make(chan struct{}), - waitForMiningStop: make(chan struct{}), - logger: logger, + sentryCtx: ctx, + sentryCancel: ctxCancel, + config: config, + networkID: config.NetworkID, + etherbase: config.Miner.Etherbase, + waitForStageLoopStop: make(chan struct{}), + waitForMiningStop: make(chan struct{}), + blockBuilderNotifyNewTxns: make(chan struct{}, 1), + miningSealingQuit: make(chan struct{}), + minedBlocks: make(chan *types.Block, 1), + logger: logger, stopNode: func() error { return stack.Close() }, @@ -645,34 +642,43 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger return nil, err } - var txnProvider txnprovider.TxnProvider - var miningRPC txpoolproto.MiningServer stateDiffClient := direct.NewStateDiffClientDirect(kvRPC) + var txnProvider txnprovider.TxnProvider if config.TxPool.Disable { backend.txPoolGrpcServer = &txpool.GrpcDisabled{} } else { - backend.newTxs = make(chan txpool.Announcements, 1024) - backend.txPoolDB, backend.txPool, backend.txPoolFetch, backend.txPoolSend, backend.txPoolGrpcServer, err = txpoolutil.AllComponents( - ctx, config.TxPool, kvcache.NewDummy(), backend.newTxs, backend.chainDB, backend.sentriesClient.Sentries(), stateDiffClient, misc.Eip1559FeeCalculator, logger, + sentries := backend.sentriesClient.Sentries() + blockBuilderNotifyNewTxns := func() { + select { + case backend.blockBuilderNotifyNewTxns <- struct{}{}: + default: + } + } + backend.txPool, backend.txPoolGrpcServer, err = txpool.Assemble( + ctx, + config.TxPool, + backend.chainDB, + kvcache.NewDummy(), + sentries, + stateDiffClient, + blockBuilderNotifyNewTxns, + logger, ) if err != nil { return nil, err } + txnProvider = backend.txPool } if config.Shutter.Enabled { if config.TxPool.Disable { panic("can't enable shutter pool when devp2p txpool is disabled") } - backend.shutterPool = shutter.NewPool(logger, config.Shutter, txnProvider) + + backend.shutterPool = shutter.NewPool(logger, config.Shutter, backend.txPool) txnProvider = backend.shutterPool } - backend.notifyMiningAboutNewTxs = make(chan struct{}, 1) - backend.miningSealingQuit = make(chan struct{}) - backend.pendingBlocks = make(chan *types.Block, 1) - backend.minedBlocks = make(chan *types.Block, 1) - miner := stagedsync.NewMiningState(&config.Miner) backend.pendingBlocks = miner.PendingResultCh @@ -791,7 +797,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, heimdallStore, bridgeStore, backend.chainConfig, config, backend.notifications.Events, segmentsBuildLimiter, logger) - miningRPC = privateapi.NewMiningServer(ctx, backend, ethashApi, logger) + var miningRPC txpoolproto.MiningServer = privateapi.NewMiningServer(ctx, backend, ethashApi, logger) var creds credentials.TransportCredentials if stack.Config().PrivateApiAddr != "" { @@ -821,26 +827,6 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger if currentBlock == nil { currentBlock = genesis } - // We start the transaction pool on startup, for a couple of reasons: - // 1) Hive tests requires us to do so and starting it from eth_sendRawTransaction is not viable as we have not enough data - // to initialize it properly. - // 2) we cannot propose for block 1 regardless. - - if !config.TxPool.Disable { - backend.txPoolFetch.ConnectCore() - backend.txPoolFetch.ConnectSentries() - var newTxsBroadcaster *txpool.NewSlotsStreams - if casted, ok := backend.txPoolGrpcServer.(*txpool.GrpcServer); ok { - newTxsBroadcaster = casted.NewSlotsStreams - } - go txpool.MainLoop(backend.sentryCtx, backend.txPool, backend.newTxs, backend.txPoolSend, newTxsBroadcaster, - func() { - select { - case backend.notifyMiningAboutNewTxs <- struct{}{}: - default: - } - }) - } go func() { defer debug.LogPanic() @@ -1279,8 +1265,8 @@ func (s *Ethereum) StartMining(ctx context.Context, db kv.RwDB, stateDiffClient // block info in the state channel hasWork = true - case <-s.notifyMiningAboutNewTxs: - //log.Warn("[dbg] notifyMiningAboutNewTxs") + case <-s.blockBuilderNotifyNewTxns: + //log.Warn("[dbg] blockBuilderNotifyNewTxns") // Skip mining based on new txn notif for bor consensus hasWork = s.chainConfig.Bor == nil @@ -1598,6 +1584,14 @@ func (s *Ethereum) Start() error { } } + if s.txPool != nil { + // We start the transaction pool on startup, for a couple of reasons: + // 1) Hive tests requires us to do so and starting it from eth_sendRawTransaction is not viable as we have not enough data + // to initialize it properly. + // 2) we cannot propose for block 1 regardless. + s.bgComponentsEg.Go(func() error { return s.txPool.Run(s.sentryCtx) }) + } + if s.shutterPool != nil { s.bgComponentsEg.Go(func() error { return s.shutterPool.Run(s.sentryCtx) }) } @@ -1639,9 +1633,6 @@ func (s *Ethereum) Stop() error { for _, sentryServer := range s.sentryServers { sentryServer.Close() } - if s.txPoolDB != nil { - s.txPoolDB.Close() - } s.chainDB.Close() if s.silkwormRPCDaemonService != nil { diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 529b4cbfb89..db6203bd52a 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -29,8 +29,9 @@ import ( "github.com/c2h5oh/datasize" lru "github.com/hashicorp/golang-lru/arc/v2" - "github.com/holiman/uint256" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "google.golang.org/protobuf/types/known/emptypb" @@ -44,6 +45,7 @@ import ( proto_downloader "github.com/erigontech/erigon-lib/gointerfaces/downloaderproto" execution "github.com/erigontech/erigon-lib/gointerfaces/executionproto" proto_sentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon-lib/gointerfaces/txpoolproto" ptypes "github.com/erigontech/erigon-lib/gointerfaces/typesproto" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" @@ -125,11 +127,8 @@ type MockSentry struct { Notifications *shards.Notifications // TxPool - TxPoolFetch *txpool.Fetch - TxPoolSend *txpool.Send - TxPoolGrpcServer *txpool.GrpcServer TxPool *txpool.TxPool - txPoolDB kv.RwDB + TxPoolGrpcServer txpoolproto.TxpoolServer HistoryV3 bool agg *libstate.Aggregator @@ -137,13 +136,11 @@ type MockSentry struct { BlockReader services.FullBlockReader ReceiptsReader *receipts.Generator posStagedSync *stagedsync.Sync + bgComponentsEg errgroup.Group } func (ms *MockSentry) Close() { ms.cancel() - if ms.txPoolDB != nil { - ms.txPoolDB.Close() - } if ms.Engine != nil { ms.Engine.Close() } @@ -156,6 +153,9 @@ func (ms *MockSentry) Close() { if ms.DB != nil { ms.DB.Close() } + if err := ms.bgComponentsEg.Wait(); err != nil { + require.Equal(ms.tb, context.Canceled, err) // upon waiting for clean exit we should get ctx cancelled + } } // Stream returns stream, waiting if necessary @@ -318,6 +318,17 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK if tb != nil { tb.Cleanup(mock.Close) } + + // Committed genesis will be shared between download and mock sentry + _, mock.Genesis, err = core.CommitGenesisBlock(mock.DB, gspec, datadir.New(tmpdir), mock.Log) + if _, ok := err.(*chain.ConfigCompatError); err != nil && !ok { + if tb != nil { + tb.Fatal(err) + } else { + panic(err) + } + } + blockWriter := blockio.NewBlockWriter() mock.Address = crypto.PubkeyToAddress(mock.Key.PublicKey) @@ -333,49 +344,33 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK blockPropagator := func(Ctx context.Context, header *types.Header, body *types.RawBody, td *big.Int) {} if !cfg.TxPool.Disable { poolCfg := txpoolcfg.DefaultConfig - newTxs := make(chan txpool.Announcements, 1024) - if tb != nil { - tb.Cleanup(func() { - close(newTxs) - }) - } - chainID, _ := uint256.FromBig(mock.ChainConfig.ChainID) - shanghaiTime := mock.ChainConfig.ShanghaiTime - cancunTime := mock.ChainConfig.CancunTime - pragueTime := mock.ChainConfig.PragueTime - maxBlobsPerBlock := mock.ChainConfig.GetMaxBlobsPerBlock() - mock.txPoolDB = memdb.NewWithLabel(tmpdir, kv.TxPoolDB) - mock.TxPool, err = txpool.New(newTxs, mock.txPoolDB, mock.DB, poolCfg, kvcache.NewDummy(), *chainID, shanghaiTime, nil /* agraBlock */, cancunTime, pragueTime, maxBlobsPerBlock, nil, logger) + stateChangesClient := direct.NewStateDiffClientDirect(erigonGrpcServeer) + mock.TxPool, mock.TxPoolGrpcServer, err = txpool.Assemble( + ctx, + poolCfg, + mock.DB, + kvcache.NewDummy(), + sentries, + stateChangesClient, + func() {}, /* builderNotifyNewTxns */ + logger, + txpool.WithP2PFetcherWg(&mock.ReceiveWg), + txpool.WithP2PSenderWg(nil), + txpool.WithFeeCalculator(nil), + txpool.WithPoolDBInitializer(func(_ context.Context, _ txpoolcfg.Config, _ log.Logger) (kv.RwDB, error) { + return memdb.NewWithLabel(tmpdir, kv.TxPoolDB), nil + }), + ) if err != nil { tb.Fatal(err) } - stateChangesClient := direct.NewStateDiffClientDirect(erigonGrpcServeer) - - mock.TxPoolFetch = txpool.NewFetch(mock.Ctx, sentries, mock.TxPool, stateChangesClient, mock.DB, mock.txPoolDB, *chainID, logger) - mock.TxPoolFetch.SetWaitGroup(&mock.ReceiveWg) - mock.TxPoolSend = txpool.NewSend(mock.Ctx, sentries, mock.TxPool, logger) - mock.TxPoolGrpcServer = txpool.NewGrpcServer(mock.Ctx, mock.TxPool, mock.txPoolDB, *chainID, logger) - - mock.TxPoolFetch.ConnectCore() mock.StreamWg.Add(1) - mock.TxPoolFetch.ConnectSentries() + mock.bgComponentsEg.Go(func() error { return mock.TxPool.Run(ctx) }) mock.StreamWg.Wait() - - go txpool.MainLoop(mock.Ctx, mock.TxPool, newTxs, mock.TxPoolSend, mock.TxPoolGrpcServer.NewSlotsStreams, func() {}) } - // Committed genesis will be shared between download and mock sentry - _, mock.Genesis, err = core.CommitGenesisBlock(mock.DB, gspec, datadir.New(tmpdir), mock.Log) - if _, ok := err.(*chain.ConfigCompatError); err != nil && !ok { - if tb != nil { - tb.Fatal(err) - } else { - panic(err) - } - } latestBlockBuiltStore := builder.NewLatestBlockBuiltStore() - inMemoryExecution := func(txc wrap.TxContainer, header *types.Header, body *types.RawBody, unwindPoint uint64, headersChain []*types.Header, bodiesChain []*types.RawBody, notifications *shards.Notifications) error { terseLogger := log.New() diff --git a/txnprovider/txpool/assemble.go b/txnprovider/txpool/assemble.go new file mode 100644 index 00000000000..77332be8baf --- /dev/null +++ b/txnprovider/txpool/assemble.go @@ -0,0 +1,123 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package txpool + +import ( + "context" + "math/big" + + "github.com/c2h5oh/datasize" + "github.com/holiman/uint256" + + "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon-lib/gointerfaces/txpoolproto" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/kvcache" + "github.com/erigontech/erigon-lib/kv/mdbx" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" +) + +func Assemble( + ctx context.Context, + cfg txpoolcfg.Config, + chainDB kv.RwDB, + cache kvcache.Cache, + sentryClients []sentryproto.SentryClient, + stateChangesClient StateChangesClient, + builderNotifyNewTxns func(), + logger log.Logger, + opts ...Option, +) (*TxPool, txpoolproto.TxpoolServer, error) { + options := applyOpts(opts...) + poolDB, err := options.poolDBInitializer(ctx, cfg, logger) + if err != nil { + return nil, nil, err + } + + chainConfig, _, err := SaveChainConfigIfNeed(ctx, chainDB, poolDB, true, logger) + if err != nil { + return nil, nil, err + } + + chainID, _ := uint256.FromBig(chainConfig.ChainID) + maxBlobsPerBlock := chainConfig.GetMaxBlobsPerBlock() + + shanghaiTime := chainConfig.ShanghaiTime + var agraBlock *big.Int + if chainConfig.Bor != nil { + agraBlock = chainConfig.Bor.GetAgraBlock() + } + cancunTime := chainConfig.CancunTime + pragueTime := chainConfig.PragueTime + if cfg.OverridePragueTime != nil { + pragueTime = cfg.OverridePragueTime + } + + newTxns := make(chan Announcements, 1024) + newSlotsStreams := &NewSlotsStreams{} + pool, err := New( + ctx, + newTxns, + poolDB, + chainDB, + cfg, + cache, + *chainID, + shanghaiTime, + agraBlock, + cancunTime, + pragueTime, + maxBlobsPerBlock, + sentryClients, + stateChangesClient, + builderNotifyNewTxns, + newSlotsStreams, + logger, + opts..., + ) + if err != nil { + return nil, nil, err + } + + grpcServer := NewGrpcServer(ctx, pool, poolDB, newSlotsStreams, *chainID, logger) + return pool, grpcServer, nil +} + +type poolDBInitializer func(ctx context.Context, cfg txpoolcfg.Config, logger log.Logger) (kv.RwDB, error) + +var defaultPoolDBInitializer = func(ctx context.Context, cfg txpoolcfg.Config, logger log.Logger) (kv.RwDB, error) { + opts := mdbx.New(kv.TxPoolDB, logger). + Path(cfg.DBDir). + WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }). + WriteMergeThreshold(3 * 8192). + PageSize(16 * datasize.KB). + GrowthStep(16 * datasize.MB). + DirtySpace(uint64(128 * datasize.MB)). + MapSize(1 * datasize.TB). + WriteMap(cfg.MdbxWriteMap) + if cfg.MdbxPageSize > 0 { + opts = opts.PageSize(cfg.MdbxPageSize) + } + if cfg.MdbxDBSizeLimit > 0 { + opts = opts.MapSize(cfg.MdbxDBSizeLimit) + } + if cfg.MdbxGrowthStep > 0 { + opts = opts.GrowthStep(cfg.MdbxGrowthStep) + } + return opts.Open(ctx) +} diff --git a/txnprovider/txpool/fetch.go b/txnprovider/txpool/fetch.go index 21838e85798..1e11b37f706 100644 --- a/txnprovider/txpool/fetch.go +++ b/txnprovider/txpool/fetch.go @@ -43,7 +43,6 @@ import ( type Fetch struct { ctx context.Context // Context used for cancellation and closing of the fetcher pool Pool // Transaction pool implementation - coreDB kv.RoDB db kv.RwDB stateChangesClient StateChangesClient wg *sync.WaitGroup // used for synchronisation in the tests (nil when not in tests) @@ -62,17 +61,26 @@ type StateChangesClient interface { // NewFetch creates a new fetch object that will work with given sentry clients. Since the // SentryClient here is an interface, it is suitable for mocking in tests (mock will need // to implement all the functions of the SentryClient interface). -func NewFetch(ctx context.Context, sentryClients []sentry.SentryClient, pool Pool, stateChangesClient StateChangesClient, coreDB kv.RoDB, db kv.RwDB, - chainID uint256.Int, logger log.Logger) *Fetch { +func NewFetch( + ctx context.Context, + sentryClients []sentry.SentryClient, + pool Pool, + stateChangesClient StateChangesClient, + db kv.RwDB, + chainID uint256.Int, + logger log.Logger, + opts ...Option, +) *Fetch { + options := applyOpts(opts...) f := &Fetch{ ctx: ctx, sentryClients: sentryClients, pool: pool, - coreDB: coreDB, db: db, stateChangesClient: stateChangesClient, stateChangesParseCtx: NewTxnParseContext(chainID).ChainIDRequired(), //TODO: change ctx if rules changed pooledTxnsParseCtx: NewTxnParseContext(chainID).ChainIDRequired(), + wg: options.p2pFetcherWg, logger: logger, } f.pooledTxnsParseCtx.ValidateRLP(f.pool.ValidateSerializedTxn) @@ -81,10 +89,6 @@ func NewFetch(ctx context.Context, sentryClients []sentry.SentryClient, pool Poo return f } -func (f *Fetch) SetWaitGroup(wg *sync.WaitGroup) { - f.wg = wg -} - func (f *Fetch) threadSafeParsePooledTxn(cb func(*TxnParseContext) error) error { f.pooledTxnsParseCtxLock.Lock() defer f.pooledTxnsParseCtxLock.Unlock() diff --git a/txnprovider/txpool/fetch_test.go b/txnprovider/txpool/fetch_test.go index 441404fa743..3f6f0e00f1a 100644 --- a/txnprovider/txpool/fetch_test.go +++ b/txnprovider/txpool/fetch_test.go @@ -24,7 +24,6 @@ import ( "sync" "testing" - "github.com/erigontech/erigon-lib/kv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -37,6 +36,7 @@ import ( remote "github.com/erigontech/erigon-lib/gointerfaces/remoteproto" "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/gointerfaces/typesproto" + "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/memdb" "github.com/erigontech/erigon-lib/log/v3" ) @@ -53,9 +53,8 @@ func TestFetch(t *testing.T) { m := NewMockSentry(ctx, sentryServer) sentryClient := direct.NewSentryClientDirect(direct.ETH67, m) - fetch := NewFetch(ctx, []sentryproto.SentryClient{sentryClient}, pool, remoteKvClient, nil, nil, *u256.N1, log.New()) var wg sync.WaitGroup - fetch.SetWaitGroup(&wg) + fetch := NewFetch(ctx, []sentryproto.SentryClient{sentryClient}, pool, remoteKvClient, nil, *u256.N1, log.New(), WithP2PFetcherWg(&wg)) m.StreamWg.Add(2) fetch.ConnectSentries() m.StreamWg.Wait() @@ -74,7 +73,7 @@ func TestFetch(t *testing.T) { wg.Wait() } -func TestSendTxPropagate(t *testing.T) { +func TestSendTxnPropagate(t *testing.T) { ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() t.Run("few remote byHash", func(t *testing.T) { @@ -102,7 +101,7 @@ func TestSendTxPropagate(t *testing.T) { }).AnyTimes() m := NewMockSentry(ctx, sentryServer) - send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, nil, log.New()) + send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, log.New()) send.BroadcastPooledTxns(testRlps(2), 100) send.AnnouncePooledTxns([]byte{0, 1}, []uint32{10, 15}, toHashes(1, 42), 100) @@ -132,7 +131,7 @@ func TestSendTxPropagate(t *testing.T) { Times(times) m := NewMockSentry(ctx, sentryServer) - send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, nil, log.New()) + send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, log.New()) list := make(Hashes, p2pTxPacketLimit*3) for i := 0; i < len(list); i += 32 { b := []byte(fmt.Sprintf("%x", i)) @@ -167,7 +166,7 @@ func TestSendTxPropagate(t *testing.T) { Times(times) m := NewMockSentry(ctx, sentryServer) - send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, nil, log.New()) + send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, log.New()) send.BroadcastPooledTxns(testRlps(2), 100) send.AnnouncePooledTxns([]byte{0, 1}, []uint32{10, 15}, toHashes(1, 42), 100) @@ -207,7 +206,7 @@ func TestSendTxPropagate(t *testing.T) { }).AnyTimes() m := NewMockSentry(ctx, sentryServer) - send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, nil, log.New()) + send := NewSend(ctx, []sentryproto.SentryClient{direct.NewSentryClientDirect(direct.ETH68, m)}, log.New()) expectPeers := toPeerIDs(1, 2, 42) send.PropagatePooledTxnsToPeersList(expectPeers, []byte{0, 1}, []uint32{10, 15}, toHashes(1, 42)) @@ -231,7 +230,7 @@ func decodeHex(in string) []byte { func TestOnNewBlock(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - coreDB, db := memdb.NewTestDB(t, kv.ChainDB), memdb.NewTestDB(t, kv.TxPoolDB) + _, db := memdb.NewTestDB(t, kv.ChainDB), memdb.NewTestDB(t, kv.TxPoolDB) ctrl := gomock.NewController(t) stream := remote.NewMockKV_StateChangesClient(ctrl) @@ -286,7 +285,7 @@ func TestOnNewBlock(t *testing.T) { }). Times(1) - fetch := NewFetch(ctx, nil, pool, stateChanges, coreDB, db, *u256.N1, log.New()) + fetch := NewFetch(ctx, nil, pool, stateChanges, db, *u256.N1, log.New()) err := fetch.handleStateChanges(ctx, stateChanges) assert.ErrorIs(t, io.EOF, err) assert.Equal(t, 3, len(minedTxns.Txns)) diff --git a/txnprovider/txpool/options.go b/txnprovider/txpool/options.go new file mode 100644 index 00000000000..79f47369511 --- /dev/null +++ b/txnprovider/txpool/options.go @@ -0,0 +1,69 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package txpool + +import ( + "sync" + + "github.com/erigontech/erigon/consensus/misc" +) + +type Option func(*options) + +func WithFeeCalculator(f FeeCalculator) Option { + return func(o *options) { + o.feeCalculator = f + } +} + +func WithPoolDBInitializer(init poolDBInitializer) Option { + return func(o *options) { + o.poolDBInitializer = init + } +} + +func WithP2PFetcherWg(wg *sync.WaitGroup) Option { + return func(o *options) { + o.p2pFetcherWg = wg + } +} + +func WithP2PSenderWg(wg *sync.WaitGroup) Option { + return func(o *options) { + o.p2pSenderWg = wg + } +} + +type options struct { + feeCalculator FeeCalculator + poolDBInitializer poolDBInitializer + p2pSenderWg *sync.WaitGroup + p2pFetcherWg *sync.WaitGroup +} + +func applyOpts(opts ...Option) options { + overriddenDefaults := defaultOptions + for _, opt := range opts { + opt(&overriddenDefaults) + } + return overriddenDefaults +} + +var defaultOptions = options{ + poolDBInitializer: defaultPoolDBInitializer, + feeCalculator: misc.Eip1559FeeCalculator, +} diff --git a/txnprovider/txpool/pool.go b/txnprovider/txpool/pool.go index 2402f0de5f2..6a6b8bbadf5 100644 --- a/txnprovider/txpool/pool.go +++ b/txnprovider/txpool/pool.go @@ -48,6 +48,7 @@ import ( "github.com/erigontech/erigon-lib/gointerfaces" "github.com/erigontech/erigon-lib/gointerfaces/grpcutil" remote "github.com/erigontech/erigon-lib/gointerfaces/remoteproto" + "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" "github.com/erigontech/erigon-lib/gointerfaces/txpoolproto" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" @@ -144,6 +145,10 @@ type TxPool struct { isPostPrague atomic.Bool maxBlobsPerBlock uint64 feeCalculator FeeCalculator + p2pFetcher *Fetch + p2pSender *Send + newSlotsStreams *NewSlotsStreams + builderNotifyNewTxns func() logger log.Logger } @@ -152,6 +157,7 @@ type FeeCalculator interface { } func New( + ctx context.Context, newTxns chan Announcements, poolDB kv.RwDB, coreDB kv.RoDB, @@ -163,9 +169,14 @@ func New( cancunTime *big.Int, pragueTime *big.Int, maxBlobsPerBlock uint64, - feeCalculator FeeCalculator, + sentryClients []sentryproto.SentryClient, + stateChangesClient StateChangesClient, + builderNotifyNewTxns func(), + newSlotsStreams *NewSlotsStreams, logger log.Logger, + opts ...Option, ) (*TxPool, error) { + options := applyOpts(opts...) localsHistory, err := simplelru.NewLRU[string, struct{}](10_000, nil) if err != nil { return nil, err @@ -211,7 +222,9 @@ func New( minedBlobTxnsByBlock: map[uint64][]*metaTxn{}, minedBlobTxnsByHash: map[string]*metaTxn{}, maxBlobsPerBlock: maxBlobsPerBlock, - feeCalculator: feeCalculator, + feeCalculator: options.feeCalculator, + builderNotifyNewTxns: builderNotifyNewTxns, + newSlotsStreams: newSlotsStreams, logger: logger, } @@ -244,10 +257,12 @@ func New( res.pragueTime = &pragueTimeU64 } + res.p2pFetcher = NewFetch(ctx, sentryClients, res, stateChangesClient, poolDB, chainID, logger, opts...) + res.p2pSender = NewSend(ctx, sentryClients, logger, opts...) return res, nil } -func (p *TxPool) Start(ctx context.Context) error { +func (p *TxPool) start(ctx context.Context) error { if p.started.Load() { return nil } @@ -1731,7 +1746,7 @@ func (p *TxPool) promote(pendingBaseFee uint64, pendingBlobFee uint64, announcem } } -// MainLoop - does: +// Run - does: // send pending byHash to p2p: // - new byHash // - all pooled byHash to recently connected peers @@ -1739,7 +1754,11 @@ func (p *TxPool) promote(pendingBaseFee uint64, pendingBlobFee uint64, announcem // // promote/demote transactions // reorgs -func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send *Send, newSlotsStreams *NewSlotsStreams, notifyMiningAboutNewSlots func()) { +func (p *TxPool) Run(ctx context.Context) error { + defer p.poolDB.Close() + p.p2pFetcher.ConnectCore() + p.p2pFetcher.ConnectSentries() + syncToNewPeersEvery := time.NewTicker(p.cfg.SyncToNewPeersEvery) defer syncToNewPeersEvery.Stop() processRemoteTxnsEvery := time.NewTicker(p.cfg.ProcessRemoteTxnsEvery) @@ -1749,16 +1768,20 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * logEvery := time.NewTicker(p.cfg.LogEvery) defer logEvery.Stop() - if err := p.Start(ctx); err != nil { + if err := p.start(ctx); err != nil { p.logger.Error("[txpool] Failed to start", "err", err) - return + return err } for { select { case <-ctx.Done(): - _, _ = p.flush(ctx) - return + err := ctx.Err() + _, flushErr := p.flush(context.Background()) // need background ctx since the other one is cancelled + if flushErr != nil { + err = fmt.Errorf("%w: %w", flushErr, err) + } + return err case <-logEvery.C: p.logStats() case <-processRemoteTxnsEvery.C: @@ -1785,11 +1808,11 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * writeToDBBytesCounter.SetUint64(written) p.logger.Debug("[txpool] Commit", "written_kb", written/1024, "in", time.Since(t)) } - case announcements := <-newTxns: + case announcements := <-p.newPendingTxns: go func() { for i := 0; i < 16; i++ { // drain more events from channel, then merge and dedup them select { - case a := <-newTxns: + case a := <-p.newPendingTxns: announcements.AppendOther(a) continue default: @@ -1803,7 +1826,7 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * announcements = announcements.DedupCopy() - notifyMiningAboutNewSlots() + p.builderNotifyNewTxns() if p.cfg.NoGossip { // drain newTxns for emptying newTxn channel @@ -1868,17 +1891,17 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * p.logger.Error("[txpool] collect info to propagate", "err", err) return } - if newSlotsStreams != nil { - newSlotsStreams.Broadcast(&txpoolproto.OnAddReply{RplTxs: slotsRlp}, p.logger) + if p.newSlotsStreams != nil { + p.newSlotsStreams.Broadcast(&txpoolproto.OnAddReply{RplTxs: slotsRlp}, p.logger) } // broadcast local transactions const localTxnsBroadcastMaxPeers uint64 = 10 - txnSentTo := send.BroadcastPooledTxns(localTxnRlps, localTxnsBroadcastMaxPeers) + txnSentTo := p.p2pSender.BroadcastPooledTxns(localTxnRlps, localTxnsBroadcastMaxPeers) for i, peer := range txnSentTo { p.logger.Trace("Local txn broadcast", "txHash", hex.EncodeToString(broadcastHashes.At(i)), "to peer", peer) } - hashSentTo := send.AnnouncePooledTxns(localTxnTypes, localTxnSizes, localTxnHashes, localTxnsBroadcastMaxPeers*2) + hashSentTo := p.p2pSender.AnnouncePooledTxns(localTxnTypes, localTxnSizes, localTxnHashes, localTxnsBroadcastMaxPeers*2) for i := 0; i < localTxnHashes.Len(); i++ { hash := localTxnHashes.At(i) p.logger.Trace("Local txn announced", "txHash", hex.EncodeToString(hash), "to peer", hashSentTo[i], "baseFee", p.pendingBaseFee.Load()) @@ -1886,8 +1909,8 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * // broadcast remote transactions const remoteTxnsBroadcastMaxPeers uint64 = 3 - send.BroadcastPooledTxns(remoteTxnRlps, remoteTxnsBroadcastMaxPeers) - send.AnnouncePooledTxns(remoteTxnTypes, remoteTxnSizes, remoteTxnHashes, remoteTxnsBroadcastMaxPeers*2) + p.p2pSender.BroadcastPooledTxns(remoteTxnRlps, remoteTxnsBroadcastMaxPeers) + p.p2pSender.AnnouncePooledTxns(remoteTxnTypes, remoteTxnSizes, remoteTxnHashes, remoteTxnsBroadcastMaxPeers*2) }() case <-syncToNewPeersEvery.C: // new peer newPeers := p.recentlyConnectedPeers.GetAndClean() @@ -1904,7 +1927,7 @@ func MainLoop(ctx context.Context, p *TxPool, newTxns chan Announcements, send * var types []byte var sizes []uint32 types, sizes, hashes = p.AppendAllAnnouncements(types, sizes, hashes[:0]) - go send.PropagatePooledTxnsToPeersList(newPeers, types, sizes, hashes) + go p.p2pSender.PropagatePooledTxnsToPeersList(newPeers, types, sizes, hashes) propagateToNewPeerTimer.ObserveDuration(t) } } diff --git a/txnprovider/txpool/pool_db.go b/txnprovider/txpool/pool_db.go index 4c81a71faae..f180826d4fc 100644 --- a/txnprovider/txpool/pool_db.go +++ b/txnprovider/txpool/pool_db.go @@ -18,13 +18,17 @@ package txpool import ( "bytes" + "context" "encoding/binary" "encoding/json" + "errors" "fmt" + "time" "github.com/erigontech/erigon-lib/chain" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/log/v3" ) var PoolChainConfigKey = []byte("chain_config") @@ -96,3 +100,73 @@ func PutChainConfig(tx kv.Putter, cc *chain.Config, buf []byte) error { } return nil } + +func SaveChainConfigIfNeed( + ctx context.Context, + coreDB kv.RoDB, + poolDB kv.RwDB, + force bool, + logger log.Logger, +) (cc *chain.Config, blockNum uint64, err error) { + if err = poolDB.View(ctx, func(tx kv.Tx) error { + cc, err = ChainConfig(tx) + if err != nil { + return err + } + blockNum, err = LastSeenBlock(tx) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, 0, err + } + if cc != nil && !force { + if cc.ChainID.Uint64() == 0 { + return nil, 0, errors.New("wrong chain config") + } + return cc, blockNum, nil + } + + for { + if err = coreDB.View(ctx, func(tx kv.Tx) error { + cc, err = chain.GetConfig(tx, nil) + if err != nil { + return err + } + n, err := chain.CurrentBlockNumber(tx) + if err != nil { + return err + } + if n != nil { + blockNum = *n + } + return nil + }); err != nil { + logger.Error("cant read chain config from core db", "err", err) + time.Sleep(5 * time.Second) + continue + } else if cc == nil { + logger.Error("cant read chain config from core db") + time.Sleep(5 * time.Second) + continue + } + break + } + + if err = poolDB.Update(ctx, func(tx kv.RwTx) error { + if err = PutChainConfig(tx, cc, nil); err != nil { + return err + } + if err = PutLastSeenBlock(tx, blockNum, nil); err != nil { + return err + } + return nil + }); err != nil { + return nil, 0, err + } + if cc.ChainID.Uint64() == 0 { + return nil, 0, errors.New("wrong chain config") + } + return cc, blockNum, nil +} diff --git a/txnprovider/txpool/pool_fuzz_test.go b/txnprovider/txpool/pool_fuzz_test.go index 3beb109c7db..72ebf61c204 100644 --- a/txnprovider/txpool/pool_fuzz_test.go +++ b/txnprovider/txpool/pool_fuzz_test.go @@ -325,10 +325,10 @@ func FuzzOnNewBlocks(f *testing.F) { cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) - err = pool.Start(ctx) + err = pool.start(ctx) assert.NoError(err) pool.senders.senderIDs = senderIDs @@ -551,7 +551,7 @@ func FuzzOnNewBlocks(f *testing.F) { check(p2pReceived, TxnSlots{}, "after_flush") checkNotify(p2pReceived, TxnSlots{}, "after_flush") - p2, err := New(ch, db, coreDB, txpoolcfg.DefaultConfig, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + p2, err := New(ctx, ch, db, coreDB, txpoolcfg.DefaultConfig, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) p2.senders = pool.senders // senders are not persisted diff --git a/txnprovider/txpool/pool_test.go b/txnprovider/txpool/pool_test.go index 3d2f15b8db6..fc945fc2822 100644 --- a/txnprovider/txpool/pool_test.go +++ b/txnprovider/txpool/pool_test.go @@ -49,16 +49,15 @@ import ( func TestNonceFromAddress(t *testing.T) { assert, require := assert.New(t), require.New(t) ch := make(chan Announcements, 100) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) - cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 pendingBaseFee := uint64(200000) // start blocks from 0, set empty hash - then kvcache will also work on this @@ -172,13 +171,13 @@ func TestReplaceWithHigherFee(t *testing.T) { ch := make(chan Announcements, 100) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.NotEqual(nil, pool) - ctx := context.Background() var stateVersionID uint64 = 0 pendingBaseFee := uint64(200000) // start blocks from 0, set empty hash - then kvcache will also work on this @@ -289,13 +288,13 @@ func TestReverseNonces(t *testing.T) { ch := make(chan Announcements, 100) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 pendingBaseFee := uint64(1_000_000) // start blocks from 0, set empty hash - then kvcache will also work on this @@ -413,13 +412,13 @@ func TestTxnPoke(t *testing.T) { ch := make(chan Announcements, 100) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 pendingBaseFee := uint64(200000) // start blocks from 0, set empty hash - then kvcache will also work on this @@ -703,10 +702,11 @@ func TestShanghaiValidateTxn(t *testing.T) { shanghaiTime = big.NewInt(0) } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cache := &kvcache.DummyCache{} - pool, err := New(ch, nil, coreDB, cfg, cache, *u256.N1, shanghaiTime, nil /* agraBlock */, nil /* cancunTime */, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, logger) + pool, err := New(ctx, ch, nil, coreDB, cfg, cache, *u256.N1, shanghaiTime, nil /* agraBlock */, nil /* cancunTime */, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, logger, WithFeeCalculator(nil)) asrt.NoError(err) - ctx := context.Background() tx, err := coreDB.BeginRw(ctx) defer tx.Rollback() asrt.NoError(err) @@ -745,17 +745,17 @@ func TestShanghaiValidateTxn(t *testing.T) { func TestSetCodeTxnValidationWithLargeAuthorizationValues(t *testing.T) { maxUint256 := new(uint256.Int).SetAllOne() - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) ch := make(chan Announcements, 1) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) cfg := txpoolcfg.DefaultConfig chainID := *maxUint256 cache := &kvcache.DummyCache{} logger := log.New() - pool, err := New(ch, nil, coreDB, cfg, cache, chainID, common.Big0 /* shanghaiTime */, nil, /* agraBlock */ - common.Big0 /* cancunTime */, common.Big0 /* pragueTime */, fixedgas.DefaultMaxBlobsPerBlock, nil, logger) + pool, err := New(ctx, ch, nil, coreDB, cfg, cache, chainID, common.Big0 /* shanghaiTime */, nil, /* agraBlock */ + common.Big0 /* cancunTime */, common.Big0 /* pragueTime */, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, logger, WithFeeCalculator(nil)) assert.NoError(t, err) - ctx := context.Background() tx, err := coreDB.BeginRw(ctx) defer tx.Rollback() assert.NoError(t, err) @@ -797,13 +797,13 @@ func TestBlobTxnReplacement(t *testing.T) { ch := make(chan Announcements, 5) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0, nil, common.Big0, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0, nil, common.Big0, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 h1 := gointerfaces.ConvertHashToH256([32]byte{}) @@ -977,13 +977,13 @@ func TestDropRemoteAtNoGossip(t *testing.T) { logger := log.New() sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - txnPool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, big.NewInt(0), big.NewInt(0), nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, logger) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + txnPool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, big.NewInt(0), big.NewInt(0), nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, logger, WithFeeCalculator(nil)) assert.NoError(err) require.True(txnPool != nil) - ctx := context.Background() - - err = txnPool.Start(ctx) + err = txnPool.start(ctx) assert.NoError(err) var stateVersionID uint64 = 0 @@ -1078,15 +1078,15 @@ func TestBlobSlots(t *testing.T) { coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) cfg := txpoolcfg.DefaultConfig - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) //Setting limits for blobs in the pool cfg.TotalBlobPoolLimit = 20 sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0, nil, common.Big0, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0, nil, common.Big0, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 h1 := gointerfaces.ConvertHashToH256([32]byte{}) @@ -1152,15 +1152,15 @@ func TestBlobSlots(t *testing.T) { func TestGasLimitChanged(t *testing.T) { assert, require := assert.New(t), require.New(t) ch := make(chan Announcements, 100) - + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) db := memdb.NewTestPoolDB(t) cfg := txpoolcfg.DefaultConfig sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) - pool, err := New(ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, log.New()) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) require.True(pool != nil) - ctx := context.Background() var stateVersionID uint64 = 0 pendingBaseFee := uint64(200000) // start blocks from 0, set empty hash - then kvcache will also work on this diff --git a/txnprovider/txpool/send.go b/txnprovider/txpool/send.go index 38c72d5301b..721269e6c5c 100644 --- a/txnprovider/txpool/send.go +++ b/txnprovider/txpool/send.go @@ -35,25 +35,21 @@ import ( // does not initiate any messages by self type Send struct { ctx context.Context - pool Pool wg *sync.WaitGroup sentryClients []sentryproto.SentryClient // sentry clients that will be used for accessing the network logger log.Logger } -func NewSend(ctx context.Context, sentryClients []sentryproto.SentryClient, pool Pool, logger log.Logger) *Send { +func NewSend(ctx context.Context, sentryClients []sentryproto.SentryClient, logger log.Logger, opts ...Option) *Send { + options := applyOpts(opts...) return &Send{ ctx: ctx, - pool: pool, sentryClients: sentryClients, logger: logger, + wg: options.p2pSenderWg, } } -func (f *Send) SetWaitGroup(wg *sync.WaitGroup) { - f.wg = wg -} - const ( // This is the target size for the packs of transactions or announcements. A // pack can get larger than this if a single transactions exceeds this size. diff --git a/txnprovider/txpool/txpool_grpc_server.go b/txnprovider/txpool/txpool_grpc_server.go index 4b384075444..72fdb977007 100644 --- a/txnprovider/txpool/txpool_grpc_server.go +++ b/txnprovider/txpool/txpool_grpc_server.go @@ -103,14 +103,14 @@ type GrpcServer struct { ctx context.Context txPool txPool db kv.RoDB - NewSlotsStreams *NewSlotsStreams + newSlotsStreams *NewSlotsStreams chainID uint256.Int logger log.Logger } -func NewGrpcServer(ctx context.Context, txPool txPool, db kv.RoDB, chainID uint256.Int, logger log.Logger) *GrpcServer { - return &GrpcServer{ctx: ctx, txPool: txPool, db: db, NewSlotsStreams: &NewSlotsStreams{}, chainID: chainID, logger: logger} +func NewGrpcServer(ctx context.Context, txPool txPool, db kv.RoDB, newSlotsStreams *NewSlotsStreams, chainID uint256.Int, logger log.Logger) *GrpcServer { + return &GrpcServer{ctx: ctx, txPool: txPool, db: db, newSlotsStreams: newSlotsStreams, chainID: chainID, logger: logger} } func (s *GrpcServer) Version(context.Context, *emptypb.Empty) (*typesproto.VersionReply, error) { @@ -248,7 +248,7 @@ func mapDiscardReasonToProto(reason txpoolcfg.DiscardReason) txpool_proto.Import func (s *GrpcServer) OnAdd(req *txpool_proto.OnAddRequest, stream txpool_proto.Txpool_OnAddServer) error { s.logger.Info("New txns subscriber joined") //txpool.Loop does send messages to this streams - remove := s.NewSlotsStreams.Add(stream) + remove := s.newSlotsStreams.Add(stream) defer remove() select { case <-stream.Context().Done(): diff --git a/txnprovider/txpool/txpoolutil/all_components.go b/txnprovider/txpool/txpoolutil/all_components.go deleted file mode 100644 index f4870598354..00000000000 --- a/txnprovider/txpool/txpoolutil/all_components.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2021 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package txpoolutil - -import ( - "context" - "errors" - "math/big" - "time" - - "github.com/c2h5oh/datasize" - "github.com/holiman/uint256" - - "github.com/erigontech/erigon-lib/chain" - "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" - "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon-lib/kv/kvcache" - "github.com/erigontech/erigon-lib/kv/mdbx" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/txnprovider/txpool" - "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" -) - -func SaveChainConfigIfNeed(ctx context.Context, coreDB kv.RoDB, txPoolDB kv.RwDB, force bool, logger log.Logger) (cc *chain.Config, blockNum uint64, err error) { - if err = txPoolDB.View(ctx, func(tx kv.Tx) error { - cc, err = txpool.ChainConfig(tx) - if err != nil { - return err - } - blockNum, err = txpool.LastSeenBlock(tx) - if err != nil { - return err - } - return nil - }); err != nil { - return nil, 0, err - } - if cc != nil && !force { - if cc.ChainID.Uint64() == 0 { - return nil, 0, errors.New("wrong chain config") - } - return cc, blockNum, nil - } - - for { - if err = coreDB.View(ctx, func(tx kv.Tx) error { - cc, err = chain.GetConfig(tx, nil) - if err != nil { - return err - } - n, err := chain.CurrentBlockNumber(tx) - if err != nil { - return err - } - if n != nil { - blockNum = *n - } - return nil - }); err != nil { - logger.Error("cant read chain config from core db", "err", err) - time.Sleep(5 * time.Second) - continue - } else if cc == nil { - logger.Error("cant read chain config from core db") - time.Sleep(5 * time.Second) - continue - } - break - } - - if err = txPoolDB.Update(ctx, func(tx kv.RwTx) error { - if err = txpool.PutChainConfig(tx, cc, nil); err != nil { - return err - } - if err = txpool.PutLastSeenBlock(tx, blockNum, nil); err != nil { - return err - } - return nil - }); err != nil { - return nil, 0, err - } - if cc.ChainID.Uint64() == 0 { - return nil, 0, errors.New("wrong chain config") - } - return cc, blockNum, nil -} - -func AllComponents(ctx context.Context, cfg txpoolcfg.Config, cache kvcache.Cache, newTxns chan txpool.Announcements, chainDB kv.RoDB, - sentryClients []sentryproto.SentryClient, stateChangesClient txpool.StateChangesClient, feeCalculator txpool.FeeCalculator, logger log.Logger) (kv.RwDB, *txpool.TxPool, *txpool.Fetch, *txpool.Send, *txpool.GrpcServer, error) { - opts := mdbx.New(kv.TxPoolDB, logger).Path(cfg.DBDir). - WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }). - WriteMergeThreshold(3 * 8192). - PageSize(16 * datasize.KB). - GrowthStep(16 * datasize.MB). - DirtySpace(uint64(128 * datasize.MB)). - MapSize(1 * datasize.TB). - WriteMap(cfg.MdbxWriteMap) - - if cfg.MdbxPageSize > 0 { - opts = opts.PageSize(cfg.MdbxPageSize) - } - if cfg.MdbxDBSizeLimit > 0 { - opts = opts.MapSize(cfg.MdbxDBSizeLimit) - } - if cfg.MdbxGrowthStep > 0 { - opts = opts.GrowthStep(cfg.MdbxGrowthStep) - } - - txPoolDB, err := opts.Open(ctx) - - if err != nil { - return nil, nil, nil, nil, nil, err - } - - chainConfig, _, err := SaveChainConfigIfNeed(ctx, chainDB, txPoolDB, true, logger) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - chainID, _ := uint256.FromBig(chainConfig.ChainID) - maxBlobsPerBlock := chainConfig.GetMaxBlobsPerBlock() - - shanghaiTime := chainConfig.ShanghaiTime - var agraBlock *big.Int - if chainConfig.Bor != nil { - agraBlock = chainConfig.Bor.GetAgraBlock() - } - cancunTime := chainConfig.CancunTime - pragueTime := chainConfig.PragueTime - if cfg.OverridePragueTime != nil { - pragueTime = cfg.OverridePragueTime - } - - txPool, err := txpool.New( - newTxns, - txPoolDB, - chainDB, - cfg, - cache, - *chainID, - shanghaiTime, - agraBlock, - cancunTime, - pragueTime, - maxBlobsPerBlock, - feeCalculator, - logger, - ) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - fetch := txpool.NewFetch(ctx, sentryClients, txPool, stateChangesClient, chainDB, txPoolDB, *chainID, logger) - //fetch.ConnectCore() - //fetch.ConnectSentries() - - send := txpool.NewSend(ctx, sentryClients, txPool, logger) - txpoolGrpcServer := txpool.NewGrpcServer(ctx, txPool, txPoolDB, *chainID, logger) - return txPoolDB, txPool, fetch, send, txpoolGrpcServer, nil -} From 04799cb621510142ded9df56495f1d58dfa66c5f Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Tue, 7 Jan 2025 19:07:07 +0100 Subject: [PATCH 57/94] Caplin: do not kill caplin when we have non-valid state snapshots (#13313) --- cmd/caplin/caplin1/run.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index ed2bb0730c4..01ec38030a4 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -392,9 +392,6 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi } } stateSnapshots := snapshotsync.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snapshotsync.MakeCaplinStateSnapshotsTypes(indexDB), logger) - if err := stateSnapshots.OpenFolder(); err != nil { - return err - } antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, stateSnapshots, csn, rcsn, syncedDataManager, logger, config.ArchiveStates, config.ArchiveBlocks, config.ArchiveBlobs, config.SnapshotGenerationEnabled, snBuildSema) // Create the antiquary go func() { From e3cb56ec7485eb445a4b5fdff00eb0d54b18c399 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Tue, 7 Jan 2025 21:37:24 +0000 Subject: [PATCH 58/94] downloader: show correct rates after restart (#13324) Set initial values `prevStats.BytesDownload` and `prevStats.BytesCompleted` if they are zero, which can occur if the node was restarted or launched for first time. Fix for https://github.com/erigontech/erigon/issues/13250 and https://github.com/erigontech/erigon/issues/13044 --- erigon-lib/downloader/downloader.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 22e40c13eaa..5b19cb89495 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1973,6 +1973,15 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { stats.BytesDownload = uint64(connStats.BytesReadData.Int64()) stats.BytesCompleted = uint64(connStats.BytesCompleted.Int64()) + // if we have no previous stats, it means that node was restarted and need to set initial values + if prevStats.BytesDownload == 0 { + prevStats.BytesDownload = stats.BytesCompleted + } + + if prevStats.BytesCompleted == 0 { + prevStats.BytesCompleted = stats.BytesCompleted + } + lastMetadataReady := stats.MetadataReady stats.BytesTotal, stats.ConnectionsTotal, stats.MetadataReady = 0, 0, 0 From cc41076ec032a4eaa2b18e5acc056b211f425058 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Wed, 8 Jan 2025 05:41:43 +0100 Subject: [PATCH 59/94] Caplin: Remove cache files completely (#13340) * It was a pre-optimization and it is now not needed * simpler code * less headaches for concurrency --- cl/beacon/handler/block_production.go | 17 +++++++++++------ cl/cltypes/solid/hash_list.go | 6 ++++++ cl/cltypes/solid/uint64slice_byte.go | 5 +++++ cl/cltypes/solid/validator_set.go | 5 +++++ cl/merkle_tree/merkle_tree.go | 6 +++++- cl/phase1/core/state/cache.go | 2 +- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cl/beacon/handler/block_production.go b/cl/beacon/handler/block_production.go index 710fd009c4c..9d9b4bf239c 100644 --- a/cl/beacon/handler/block_production.go +++ b/cl/beacon/handler/block_production.go @@ -300,11 +300,16 @@ func (a *ApiHandler) GetEthV3ValidatorBlock( ) } - // make a simple copy to the current head state - baseState, err := a.forkchoiceStore.GetStateAtBlockRoot( - baseBlockRoot, - true, - ) // we start the block production from this state + var baseState *state.CachingBeaconState + if err := a.syncedData.ViewHeadState(func(headState *state.CachingBeaconState) error { + baseState, err = headState.Copy() + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } if err != nil { return nil, err @@ -1177,7 +1182,7 @@ func (a *ApiHandler) storeBlockAndBlobs( return err } - if err := a.forkchoiceStore.OnBlock(ctx, block, true, false, false); err != nil { + if err := a.forkchoiceStore.OnBlock(ctx, block, true, true, false); err != nil { return err } finalizedBlockRoot := a.forkchoiceStore.FinalizedCheckpoint().Root diff --git a/cl/cltypes/solid/hash_list.go b/cl/cltypes/solid/hash_list.go index 1b8bc09fcd2..cca7e5c435c 100644 --- a/cl/cltypes/solid/hash_list.go +++ b/cl/cltypes/solid/hash_list.go @@ -109,6 +109,12 @@ func (h *hashList) CopyTo(t IterableSSZ[libcommon.Hash]) { tu.MerkleTree = &merkle_tree.MerkleTree{} } h.MerkleTree.CopyInto(tu.MerkleTree) + // make the leaf function on the new buffer + tu.MerkleTree.SetComputeLeafFn(func(idx int, out []byte) { + copy(out, tu.u[idx*length.Hash:]) + }) + } else { + tu.MerkleTree = nil } copy(tu.u, h.u) } diff --git a/cl/cltypes/solid/uint64slice_byte.go b/cl/cltypes/solid/uint64slice_byte.go index a251d683c8f..750e9378ffb 100644 --- a/cl/cltypes/solid/uint64slice_byte.go +++ b/cl/cltypes/solid/uint64slice_byte.go @@ -81,6 +81,11 @@ func (arr *byteBasedUint64Slice) CopyTo(target *byteBasedUint64Slice) { target.MerkleTree = &merkle_tree.MerkleTree{} } arr.MerkleTree.CopyInto(target.MerkleTree) + target.SetComputeLeafFn(func(idx int, out []byte) { + copy(out, target.u[idx*length.Hash:]) + }) + } else { + target.MerkleTree = nil } target.u = target.u[:len(arr.u)] diff --git a/cl/cltypes/solid/validator_set.go b/cl/cltypes/solid/validator_set.go index 6c439c5d53c..79e94d67b4c 100644 --- a/cl/cltypes/solid/validator_set.go +++ b/cl/cltypes/solid/validator_set.go @@ -141,6 +141,11 @@ func (v *ValidatorSet) CopyTo(t *ValidatorSet) { t.MerkleTree = &merkle_tree.MerkleTree{} } v.MerkleTree.CopyInto(t.MerkleTree) + t.MerkleTree.SetComputeLeafFn(func(idx int, out []byte) { + copy(out, t.buffer[idx*validatorSize:]) + }) + } else { + t.MerkleTree = nil } // skip copying (unsupported for phase0) t.phase0Data = make([]Phase0Data, t.l) diff --git a/cl/merkle_tree/merkle_tree.go b/cl/merkle_tree/merkle_tree.go index e7480fb6b4b..819ab224dda 100644 --- a/cl/merkle_tree/merkle_tree.go +++ b/cl/merkle_tree/merkle_tree.go @@ -58,6 +58,10 @@ func (m *MerkleTree) Initialize(leavesCount, maxTreeCacheDepth int, computeLeaf m.dirtyLeaves = make([]atomic.Bool, leavesCount) } +func (m *MerkleTree) SetComputeLeafFn(computeLeaf func(idx int, out []byte)) { + m.computeLeaf = computeLeaf +} + func (m *MerkleTree) MarkLeafAsDirty(idx int) { m.mu.RLock() defer m.mu.RUnlock() @@ -207,7 +211,7 @@ func (m *MerkleTree) CopyInto(other *MerkleTree) { m.mu.RLock() defer m.mu.RUnlock() defer other.mu.Unlock() - other.computeLeaf = m.computeLeaf + //other.computeLeaf = m.computeLeaf if len(other.layers) > len(m.layers) { // reset the internal layers for i := len(m.layers); i < len(other.layers); i++ { diff --git a/cl/phase1/core/state/cache.go b/cl/phase1/core/state/cache.go index a5180932831..e3b76a4916b 100644 --- a/cl/phase1/core/state/cache.go +++ b/cl/phase1/core/state/cache.go @@ -267,7 +267,7 @@ func (b *CachingBeaconState) initCaches() error { func (b *CachingBeaconState) InitBeaconState() error { b.totalActiveBalanceCache = nil b._refreshActiveBalancesIfNeeded() - + b.previousStateRoot = common.Hash{} b.publicKeyIndicies = make(map[[48]byte]uint64) b.ForEachValidator(func(validator solid.Validator, i, total int) bool { b.publicKeyIndicies[validator.PublicKey()] = uint64(i) From ecbe76831b481309147f0cef60e57edcff6cee46 Mon Sep 17 00:00:00 2001 From: sudeep Date: Wed, 8 Jan 2025 13:31:45 +0530 Subject: [PATCH 60/94] move domains/ii registration out of aggregator constructor (#13327) https://github.com/erigontech/erigon/issues/13326 and https://github.com/erigontech/erigon/issues/13325 --- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- cmd/integration/commands/stages.go | 2 +- cmd/rpcdaemon/cli/config.go | 2 +- cmd/state/commands/opcode_tracer.go | 2 +- core/genesis_write.go | 2 +- core/state/intra_block_state_test.go | 2 +- core/state/state_test.go | 4 +- core/test/domains_restart_test.go | 2 +- core/vm/gas_table_test.go | 2 +- core/vm/runtime/runtime.go | 4 +- core/vm/runtime/runtime_test.go | 4 +- .../kv/membatchwithdb/memory_mutation_test.go | 2 +- .../temporaltest/kv_temporal_testdb.go | 2 +- erigon-lib/state/aggregator.go | 165 +--------------- erigon-lib/state/aggregator2.go | 177 ++++++++++++++++++ erigon-lib/state/aggregator_bench_test.go | 2 +- erigon-lib/state/aggregator_fuzz_test.go | 2 +- erigon-lib/state/aggregator_test.go | 6 +- eth/backend.go | 2 +- turbo/app/snapshots_cmd.go | 2 +- turbo/app/squeeze_cmd.go | 4 +- 22 files changed, 206 insertions(+), 188 deletions(-) create mode 100644 erigon-lib/state/aggregator2.go diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 556a0e7cbe1..f35ec298106 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -171,7 +171,7 @@ func runCmd(ctx *cli.Context) error { } else { genesisConfig = new(types.Genesis) } - agg, err := state2.NewAggregator(context.Background(), datadir.New(os.TempDir()), config3.DefaultStepSize, db, log.New()) + agg, err := state2.NewAggregator2(context.Background(), datadir.New(os.TempDir()), config3.DefaultStepSize, db, log.New()) if err != nil { return err } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6b2270207bc..a231ca88576 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -141,7 +141,7 @@ func aggregateResultsFromStateTests( MustOpen() defer _db.Close() - agg, err := libstate.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, _db, log.New()) + agg, err := libstate.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, _db, log.New()) if err != nil { return nil, err } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 6cd8da51232..dd709b75d8e 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1338,7 +1338,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*freezebl blockReader := freezeblocks.NewBlockReader(_allSnapshotsSingleton, _allBorSnapshotsSingleton, _heimdallStoreSingleton, _bridgeStoreSingleton) txNums := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, blockReader)) - _aggSingleton, err = libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + _aggSingleton, err = libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { panic(err) } diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index a55ac8b7af3..66fad4cb9cb 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -424,7 +424,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger blockReader = freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore) txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, blockReader)) - agg, err := libstate.NewAggregator(ctx, cfg.Dirs, config3.DefaultStepSize, rawDB, logger) + agg, err := libstate.NewAggregator2(ctx, cfg.Dirs, config3.DefaultStepSize, rawDB, logger) if err != nil { return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("create aggregator: %w", err) } diff --git a/cmd/state/commands/opcode_tracer.go b/cmd/state/commands/opcode_tracer.go index a5133f2a46c..d0e7319f40c 100644 --- a/cmd/state/commands/opcode_tracer.go +++ b/cmd/state/commands/opcode_tracer.go @@ -436,7 +436,7 @@ func OpcodeTracer(genesis *types.Genesis, blockNum uint64, chaindata string, num rawChainDb := mdbx.MustOpen(dirs.Chaindata) defer rawChainDb.Close() - agg, err := state2.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, rawChainDb, log.New()) + agg, err := state2.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, rawChainDb, log.New()) if err != nil { return err } diff --git a/core/genesis_write.go b/core/genesis_write.go index 4d6b52767ee..ff34fc81614 100644 --- a/core/genesis_write.go +++ b/core/genesis_write.go @@ -505,7 +505,7 @@ func GenesisToBlock(g *types.Genesis, dirs datadir.Dirs, logger log.Logger) (*ty genesisTmpDB := mdbx.New(kv.TemporaryDB, logger).InMem(dirs.DataDir).MapSize(2 * datasize.GB).GrowthStep(1 * datasize.MB).MustOpen() defer genesisTmpDB.Close() - agg, err := state2.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, genesisTmpDB, logger) + agg, err := state2.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, genesisTmpDB, logger) if err != nil { return err } diff --git a/core/state/intra_block_state_test.go b/core/state/intra_block_state_test.go index f7d0767015c..0c463a473c7 100644 --- a/core/state/intra_block_state_test.go +++ b/core/state/intra_block_state_test.go @@ -241,7 +241,7 @@ func (test *snapshotTest) run() bool { db := memdb.NewStateDB("") defer db.Close() - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(""), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(""), 16, db, log.New()) if err != nil { test.err = err return false diff --git a/core/state/state_test.go b/core/state/state_test.go index 7e3ad911053..c1a3e561f08 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -122,7 +122,7 @@ func (s *StateSuite) SetUpTest(c *checker.C) { db := memdb.NewStateDB("") defer db.Close() - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(""), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(""), 16, db, log.New()) if err != nil { panic(err) } @@ -394,7 +394,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.TemporalRwDB, kv.TemporalRwTx, *state. db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := state.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := state.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } diff --git a/core/test/domains_restart_test.go b/core/test/domains_restart_test.go index 952906fbb6b..d5ae4a7f3dd 100644 --- a/core/test/domains_restart_test.go +++ b/core/test/domains_restart_test.go @@ -66,7 +66,7 @@ func testDbAndAggregatorv3(t *testing.T, fpath string, aggStep uint64) (kv.RwDB, db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).MustOpen() t.Cleanup(db.Close) - agg, err := state.NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := state.NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(t, err) t.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 95677431ac5..3a2bd414611 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -104,7 +104,7 @@ func testTemporalDB(t *testing.T) *temporal.DB { t.Cleanup(db.Close) - agg, err := state3.NewAggregator(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) require.NoError(t, err) t.Cleanup(agg.Close) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 77748306170..cdf3d26dd89 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -131,7 +131,7 @@ func Execute(code, input []byte, cfg *Config, tempdir string) ([]byte, *state.In if !externalState { db := memdb.NewStateDB(tempdir) defer db.Close() - agg, err := state3.NewAggregator(context.Background(), datadir.New(tempdir), config3.DefaultStepSize, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(tempdir), config3.DefaultStepSize, db, log.New()) if err != nil { return nil, nil, err } @@ -193,7 +193,7 @@ func Create(input []byte, cfg *Config, blockNr uint64) ([]byte, libcommon.Addres db := memdb.NewStateDB(tmp) defer db.Close() - agg, err := state3.NewAggregator(context.Background(), datadir.New(tmp), config3.DefaultStepSize, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(tmp), config3.DefaultStepSize, db, log.New()) if err != nil { return nil, [20]byte{}, 0, err } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index f3521a1dcf7..9df1c03717b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -56,7 +56,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.RwDB, kv.RwTx, *stateLib.Aggregator) { db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } @@ -177,7 +177,7 @@ func testTemporalDB(t testing.TB) *temporal.DB { t.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) require.NoError(t, err) t.Cleanup(agg.Close) diff --git a/erigon-lib/kv/membatchwithdb/memory_mutation_test.go b/erigon-lib/kv/membatchwithdb/memory_mutation_test.go index cd4ce3cd532..7183bd46fa5 100644 --- a/erigon-lib/kv/membatchwithdb/memory_mutation_test.go +++ b/erigon-lib/kv/membatchwithdb/memory_mutation_test.go @@ -212,7 +212,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.RwDB, kv.RwTx, *stateLib.Aggregator) { db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } diff --git a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go index aba7fec9396..6891112caa8 100644 --- a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go +++ b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go @@ -42,7 +42,7 @@ func NewTestDB(tb testing.TB, dirs datadir.Dirs) (kv.TemporalRwDB, *state.Aggreg rawDB = memdb.New(dirs.DataDir, kv.ChainDB) } - agg, err := state.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, rawDB, log.New()) + agg, err := state.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, rawDB, log.New()) if err != nil { panic(err) } diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 26f147c3958..a589a6e1687 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -126,141 +126,14 @@ func domainIntegrityCheck(name kv.Domain, dirs datadir.Dirs, fromStep, toStep ui } } -var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) - -func init() { - if dbgCommBtIndex { - cfg := Schema[kv.CommitmentDomain] - cfg.IndexList = AccessorBTree | AccessorExistence - Schema[kv.CommitmentDomain] = cfg - } -} - -var Schema = map[kv.Domain]domainCfg{ - kv.AccountsDomain: { - name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, - - IndexList: AccessorBTree | AccessorExistence, - crossDomainIntegrity: domainIntegrityCheck, - compression: seg.CompressNone, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblAccountHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant - - iiCfg: iiCfg{ - keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant - }, - }, - }, - kv.StorageDomain: { - name: kv.StorageDomain, valuesTable: kv.TblStorageVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblStorageHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.StorageDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.StorageDomain.String(), - }, - }, - }, - kv.CodeDomain: { - name: kv.CodeDomain, valuesTable: kv.TblCodeVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet - compressCfg: DomainCompressCfg, - largeValues: true, - - hist: histCfg{ - valuesTable: kv.TblCodeHistoryVals, - compression: seg.CompressKeys | seg.CompressVals, - - historyLargeValues: true, - filenameBase: kv.CodeDomain.String(), - - iiCfg: iiCfg{ - withExistence: false, compressorCfg: seg.DefaultCfg, - keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx, - filenameBase: kv.CodeDomain.String(), - }, - }, - }, - kv.CommitmentDomain: { - name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, - - IndexList: AccessorHashMap, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblCommitmentHistoryVals, - compression: seg.CompressNone, - - snapshotsDisabled: true, - historyLargeValues: false, - filenameBase: kv.CommitmentDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.CommitmentDomain.String(), - }, - }, - }, - kv.ReceiptDomain: { - name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblReceiptHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.ReceiptDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.ReceiptDomain.String(), - }, - }, - }, -} - func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { - tmpdir := dirs.Tmp - salt, err := getStateIndicesSalt(dirs.Snap) - if err != nil { - return nil, err - } - ctx, ctxCancel := context.WithCancel(ctx) - a := &Aggregator{ + return &Aggregator{ ctx: ctx, ctxCancel: ctxCancel, onFreeze: func(frozenFileNames []string) {}, dirs: dirs, - tmpdir: tmpdir, + tmpdir: dirs.Tmp, aggregationStep: aggregationStep, db: db, leakDetector: dbg.NewLeakDetector("agg", dbg.SlowTx()), @@ -272,39 +145,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 commitmentValuesTransform: AggregatorSqueezeCommitmentValues, produce: true, - } - - if err := a.registerDomain(kv.AccountsDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.StorageDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.CodeDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.CommitmentDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.ReceiptDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.TracesToIdxPos, salt, dirs, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { - return nil, err - } - a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history - a.recalcVisibleFiles(a.DirtyFilesEndTxNumMinimax()) - - return a, nil + }, nil } // getStateIndicesSalt - try read salt for all indices from DB. Or fall-back to new salt creation. diff --git a/erigon-lib/state/aggregator2.go b/erigon-lib/state/aggregator2.go new file mode 100644 index 00000000000..f1f51d09121 --- /dev/null +++ b/erigon-lib/state/aggregator2.go @@ -0,0 +1,177 @@ +package state + +import ( + "context" + + "github.com/erigontech/erigon-lib/common/datadir" + "github.com/erigontech/erigon-lib/common/dbg" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/seg" +) + +// this is supposed to register domains/iis + +func NewAggregator2(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { + salt, err := getStateIndicesSalt(dirs.Snap) + if err != nil { + return nil, err + } + + a, err := NewAggregator(ctx, dirs, aggregationStep, db, logger) + if err != nil { + return nil, err + } + if err := a.registerDomain(kv.AccountsDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.StorageDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.CodeDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.CommitmentDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.ReceiptDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesToIdxPos, salt, dirs, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { + return nil, err + } + a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history + a.recalcVisibleFiles(a.DirtyFilesEndTxNumMinimax()) + + return a, nil +} + +var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) + +func init() { + if dbgCommBtIndex { + cfg := Schema[kv.CommitmentDomain] + cfg.IndexList = AccessorBTree | AccessorExistence + Schema[kv.CommitmentDomain] = cfg + } +} + +var Schema = map[kv.Domain]domainCfg{ + kv.AccountsDomain: { + name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, + + IndexList: AccessorBTree | AccessorExistence, + crossDomainIntegrity: domainIntegrityCheck, + compression: seg.CompressNone, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblAccountHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant + + iiCfg: iiCfg{ + keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant + }, + }, + }, + kv.StorageDomain: { + name: kv.StorageDomain, valuesTable: kv.TblStorageVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblStorageHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.StorageDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.StorageDomain.String(), + }, + }, + }, + kv.CodeDomain: { + name: kv.CodeDomain, valuesTable: kv.TblCodeVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet + compressCfg: DomainCompressCfg, + largeValues: true, + + hist: histCfg{ + valuesTable: kv.TblCodeHistoryVals, + compression: seg.CompressKeys | seg.CompressVals, + + historyLargeValues: true, + filenameBase: kv.CodeDomain.String(), + + iiCfg: iiCfg{ + withExistence: false, compressorCfg: seg.DefaultCfg, + keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx, + filenameBase: kv.CodeDomain.String(), + }, + }, + }, + kv.CommitmentDomain: { + name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, + + IndexList: AccessorHashMap, + compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblCommitmentHistoryVals, + compression: seg.CompressNone, + + snapshotsDisabled: true, + historyLargeValues: false, + filenameBase: kv.CommitmentDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.CommitmentDomain.String(), + }, + }, + }, + kv.ReceiptDomain: { + name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblReceiptHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.ReceiptDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.ReceiptDomain.String(), + }, + }, + }, +} diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index 63fa225af81..3ae00f393cb 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -45,7 +45,7 @@ func testDbAndAggregatorBench(b *testing.B, aggStep uint64) (kv.RwDB, *Aggregato dirs := datadir.New(b.TempDir()) db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() b.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(b, err) b.Cleanup(agg.Close) return db, agg diff --git a/erigon-lib/state/aggregator_fuzz_test.go b/erigon-lib/state/aggregator_fuzz_test.go index 559869f66ad..659dbed084d 100644 --- a/erigon-lib/state/aggregator_fuzz_test.go +++ b/erigon-lib/state/aggregator_fuzz_test.go @@ -276,7 +276,7 @@ func testFuzzDbAndAggregatorv3(f *testing.F, aggStep uint64) (kv.RwDB, *Aggregat db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).MustOpen() f.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(err) f.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 816eb25c2e9..759df463408 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -362,7 +362,7 @@ func aggregatorV3_RestartOnDatadir(t *testing.T, rc runCfg) { agg.Close() // Start another aggregator on same datadir - anotherAgg, err := NewAggregator(context.Background(), agg.dirs, aggStep, db, logger) + anotherAgg, err := NewAggregator2(context.Background(), agg.dirs, aggStep, db, logger) require.NoError(t, err) defer anotherAgg.Close() @@ -822,7 +822,7 @@ func TestAggregatorV3_RestartOnFiles(t *testing.T) { newDb := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() t.Cleanup(newDb.Close) - newAgg, err := NewAggregator(context.Background(), agg.dirs, aggStep, newDb, logger) + newAgg, err := NewAggregator2(context.Background(), agg.dirs, aggStep, newDb, logger) require.NoError(t, err) require.NoError(t, newAgg.OpenFolder()) @@ -1091,7 +1091,7 @@ func testDbAndAggregatorv3(tb testing.TB, aggStep uint64) (kv.RwDB, *Aggregator) db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).MustOpen() tb.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(err) tb.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/eth/backend.go b/eth/backend.go index 08da323e42c..6917a1cb0c9 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -1450,7 +1450,7 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf } } blockReader := freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore) - agg, err := libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + agg, err := libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 4835364c440..3aaf44d316d 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -1505,7 +1505,7 @@ func dbCfg(label kv.Label, path string) mdbx.MdbxOpts { Accede(true) // integration tool: open db without creation and without blocking erigon } func openAgg(ctx context.Context, dirs datadir.Dirs, chainDB kv.RwDB, logger log.Logger) *libstate.Aggregator { - agg, err := libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, chainDB, logger) + agg, err := libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, chainDB, logger) if err != nil { panic(err) } diff --git a/turbo/app/squeeze_cmd.go b/turbo/app/squeeze_cmd.go index da096ff2c02..d71d68ed47c 100644 --- a/turbo/app/squeeze_cmd.go +++ b/turbo/app/squeeze_cmd.go @@ -137,7 +137,7 @@ func squeezeStorage(ctx context.Context, dirs datadir.Dirs, logger log.Logger) e ac := agg.BeginFilesRo() defer ac.Close() - aggOld, err := state.NewAggregator(ctx, dirsOld, config3.DefaultStepSize, db, logger) + aggOld, err := state.NewAggregator2(ctx, dirsOld, config3.DefaultStepSize, db, logger) if err != nil { panic(err) } @@ -178,7 +178,7 @@ func squeezeStorage(ctx context.Context, dirs datadir.Dirs, logger log.Logger) e func squeezeCode(ctx context.Context, dirs datadir.Dirs, logger log.Logger) error { db := dbCfg(kv.ChainDB, dirs.Chaindata).MustOpen() defer db.Close() - agg, err := state.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + agg, err := state.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { return err } From 05d72f066ef7436e1e104a8d132c4f4c544c43f1 Mon Sep 17 00:00:00 2001 From: sudeep Date: Wed, 8 Jan 2025 15:40:52 +0530 Subject: [PATCH 61/94] remove prunesmallbatchesdb (#13337) https://github.com/erigontech/erigon/issues/13328 --- erigon-lib/state/aggregator.go | 90 -------------------------- erigon-lib/state/domain_shared_test.go | 9 ++- turbo/app/snapshots_cmd.go | 26 +++++--- 3 files changed, 20 insertions(+), 105 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index a589a6e1687..870cb049934 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -841,96 +841,6 @@ func (ac *AggregatorRoTx) CanUnwindBeforeBlockNum(blockNum uint64, tx kv.Tx) (un return blockNum, true, nil } -func (ac *AggregatorRoTx) PruneSmallBatchesDb(ctx context.Context, timeout time.Duration, db kv.RwDB) (haveMore bool, err error) { - // On tip-of-chain timeout is about `3sec` - // On tip of chain: must be real-time - prune by small batches and prioritize exact-`timeout` - // Not on tip of chain: must be aggressive (prune as much as possible) by bigger batches - - furiousPrune := timeout > 5*time.Hour - aggressivePrune := !furiousPrune && timeout >= 1*time.Minute - - var pruneLimit uint64 = 1_000 - if furiousPrune { - pruneLimit = 10_000_000 - /* disabling this feature for now - seems it doesn't cancel even after prune finished - // start from a bit high limit to give time for warmup - // will disable warmup after first iteration and will adjust pruneLimit based on `time` - withWarmup = true - */ - } - - started := time.Now() - localTimeout := time.NewTicker(timeout) - defer localTimeout.Stop() - logPeriod := 30 * time.Second - logEvery := time.NewTicker(logPeriod) - defer logEvery.Stop() - aggLogEvery := time.NewTicker(600 * time.Second) // to hide specific domain/idx logging - defer aggLogEvery.Stop() - - fullStat := newAggregatorPruneStat() - innerCtx := context.Background() - goExit := false - - for { - err = db.Update(innerCtx, func(tx kv.RwTx) error { - iterationStarted := time.Now() - // `context.Background()` is important here! - // it allows keep DB consistent - prune all keys-related data or noting - // can't interrupt by ctrl+c and leave dirt in DB - stat, err := ac.Prune(innerCtx, tx, pruneLimit, aggLogEvery) - if err != nil { - ac.a.logger.Warn("[snapshots] PruneSmallBatches failed", "err", err) - return err - } - if stat == nil { - if fstat := fullStat.String(); fstat != "" { - ac.a.logger.Info("[snapshots] PruneSmallBatches finished", "took", time.Since(started).String(), "stat", fstat) - } - goExit = true - return nil - } - fullStat.Accumulate(stat) - - if aggressivePrune { - took := time.Since(iterationStarted) - if took < 2*time.Second { - pruneLimit *= 10 - } - if took > logPeriod { - pruneLimit /= 10 - } - } - - select { - case <-logEvery.C: - ac.a.logger.Info("[snapshots] pruning state", - "until commit", time.Until(started.Add(timeout)).String(), - "pruneLimit", pruneLimit, - "aggregatedStep", ac.StepsInFiles(kv.StateDomains...), - "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), - "pruned", fullStat.String(), - ) - default: - } - return nil - }) - if err != nil { - return false, err - } - select { - case <-localTimeout.C: //must be first to improve responsivness - return true, nil - case <-ctx.Done(): - return false, ctx.Err() - default: - } - if goExit { - return false, nil - } - } -} - // PruneSmallBatches is not cancellable, it's over when it's over or failed. // It fills whole timeout with pruning by small batches (of 100 keys) and making some progress func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Duration, tx kv.RwTx) (haveMore bool, err error) { diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index e05c50b192a..29018977b48 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -461,11 +461,10 @@ func TestSharedDomain_StorageIter(t *testing.T) { ac.Close() ac = agg.BeginFilesRo() - //err = db.Update(ctx, func(tx kv.RwTx) error { - // _, err = ac.PruneSmallBatches(ctx, 1*time.Minute, tx) - // return err - //}) - _, err = ac.PruneSmallBatchesDb(ctx, 1*time.Minute, db) + err = db.Update(ctx, func(tx kv.RwTx) error { + _, err = ac.PruneSmallBatches(ctx, 1*time.Minute, tx) + return err + }) require.NoError(t, err) ac.Close() diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 3aaf44d316d..6fa77e759cf 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -1381,15 +1381,18 @@ func doRetireCommand(cliCtx *cli.Context, dirs datadir.Dirs) error { } logger.Info("Prune state history") - ac := agg.BeginFilesRo() - defer ac.Close() for hasMoreToPrune := true; hasMoreToPrune; { - hasMoreToPrune, err = ac.PruneSmallBatchesDb(ctx, 2*time.Minute, db) - if err != nil { + if err := db.Update(ctx, func(tx kv.RwTx) error { + ac := tx.(libstate.HasAggTx).AggTx().(*libstate.AggregatorRoTx) + hasMoreToPrune, err = ac.PruneSmallBatches(ctx, 2*time.Minute, tx) + if err != nil { + return err + } + return nil + }); err != nil { return err } } - ac.Close() logger.Info("Work on state history snapshots") indexWorkers := estimate.IndexSnapshot.Workers() @@ -1435,15 +1438,18 @@ func doRetireCommand(cliCtx *cli.Context, dirs datadir.Dirs) error { return err } - ac = agg.BeginFilesRo() - defer ac.Close() for hasMoreToPrune := true; hasMoreToPrune; { - hasMoreToPrune, err = ac.PruneSmallBatchesDb(context.Background(), 2*time.Minute, db) - if err != nil { + if err := db.Update(ctx, func(tx kv.RwTx) error { + ac := tx.(libstate.HasAggTx).AggTx().(*libstate.AggregatorRoTx) + hasMoreToPrune, err = ac.PruneSmallBatches(ctx, 2*time.Minute, tx) + if err != nil { + return err + } + return nil + }); err != nil { return err } } - ac.Close() if err = agg.MergeLoop(ctx); err != nil { return err From d81cd6efac2c59f9ad05ae0580fa550d0bef5086 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Wed, 8 Jan 2025 12:44:24 +0100 Subject: [PATCH 62/94] Caplin: Avoid republishing in `GossipManager` (#13314) Basically libp2p handles that automatically and I did not know --- cl/beacon/handler/pool.go | 100 +++++++++++------- cl/beacon/handler/utils_test.go | 11 +- cl/cltypes/aggregate.go | 12 --- cl/cltypes/bls_to_execution_change.go | 8 -- cl/cltypes/contribution.go | 14 --- cl/cltypes/validator.go | 8 -- cl/phase1/network/gossip_manager.go | 63 +++++------ .../services/aggregate_and_proof_service.go | 20 +++- .../aggregate_and_proof_service_test.go | 5 +- .../network/services/attestation_service.go | 14 +-- .../services/attestation_service_test.go | 2 +- .../services/batch_signature_verification.go | 32 ++---- .../bls_to_execution_change_service.go | 18 +++- .../bls_to_execution_change_service_test.go | 5 +- cl/phase1/network/services/interface.go | 12 +-- .../aggregate_and_proof_service_mock.go | 15 ++- .../mock_services/attestation_service_mock.go | 13 ++- .../blob_sidecars_service_mock.go | 9 +- .../mock_services/block_service_mock.go | 9 +- .../bls_to_execution_change_service_mock.go | 15 ++- .../proposer_slashing_service_mock.go | 9 +- .../sync_committee_messages_service_mock.go | 15 ++- .../sync_contribution_service_mock.go | 15 ++- .../voluntary_exit_service_mock.go | 15 ++- .../sync_committee_messages_service.go | 17 ++- .../sync_committee_messages_service_test.go | 5 +- .../services/sync_contribution_service.go | 14 ++- .../sync_contribution_service_test.go | 5 +- .../services/voluntary_exit_service.go | 18 +++- .../services/voluntary_exit_service_test.go | 8 +- 30 files changed, 249 insertions(+), 257 deletions(-) diff --git a/cl/beacon/handler/pool.go b/cl/beacon/handler/pool.go index ce8e793fdeb..368bc155f33 100644 --- a/cl/beacon/handler/pool.go +++ b/cl/beacon/handler/pool.go @@ -126,13 +126,8 @@ func (a *ApiHandler) PostEthV1BeaconPoolAttestations(w http.ResponseWriter, r *h beaconhttp.NewEndpointError(http.StatusInternalServerError, err).WriteTo(w) return } - attestationWithGossipData := &services.AttestationWithGossipData{ - Attestation: attestation, - GossipData: &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNamePrefixBeaconAttestation, - SubnetId: &subnet, - }, + attestationWithGossipData := &services.AttestationForGossip{ + Attestation: attestation, ImmediateProcess: true, // we want to process attestation immediately } @@ -144,6 +139,16 @@ func (a *ApiHandler) PostEthV1BeaconPoolAttestations(w http.ResponseWriter, r *h }) continue } + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNamePrefixBeaconAttestation, + SubnetId: &subnet, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } if len(failures) > 0 { errResp := poolingError{ @@ -173,11 +178,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolVoluntaryExits(w http.ResponseWriter, r return } - if err := a.voluntaryExitService.ProcessMessage(r.Context(), nil, &cltypes.SignedVoluntaryExitWithGossipData{ - GossipData: &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNameVoluntaryExit, - }, + if err := a.voluntaryExitService.ProcessMessage(r.Context(), nil, &services.SignedVoluntaryExitForGossip{ SignedVoluntaryExit: &req, ImmediateVerification: true, }); err != nil && !errors.Is(err, services.ErrIgnore) { @@ -185,7 +186,15 @@ func (a *ApiHandler) PostEthV1BeaconPoolVoluntaryExits(w http.ResponseWriter, r return } a.operationsPool.VoluntaryExitsPool.Insert(req.VoluntaryExit.ValidatorIndex, &req) - + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNameVoluntaryExit, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } // Only write 200 w.WriteHeader(http.StatusOK) } @@ -275,16 +284,21 @@ func (a *ApiHandler) PostEthV1BeaconPoolBlsToExecutionChanges(w http.ResponseWri return } - if err := a.blsToExecutionChangeService.ProcessMessage(r.Context(), nil, &cltypes.SignedBLSToExecutionChangeWithGossipData{ + if err := a.blsToExecutionChangeService.ProcessMessage(r.Context(), nil, &services.SignedBLSToExecutionChangeForGossip{ SignedBLSToExecutionChange: v, - GossipData: &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNameBlsToExecutionChange, - }, }); err != nil && !errors.Is(err, services.ErrIgnore) { failures = append(failures, poolingFailure{Index: len(failures), Message: err.Error()}) continue } + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNameBlsToExecutionChange, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } if len(failures) > 0 { @@ -312,21 +326,26 @@ func (a *ApiHandler) PostEthV1ValidatorAggregatesAndProof(w http.ResponseWriter, log.Warn("[Beacon REST] failed to encode aggregate and proof", "err", err) return } - gossipData := &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNameBeaconAggregateAndProof, - } // for this service we are not publishing gossipData as the service does it internally, we just pass that data as a parameter. - if err := a.aggregateAndProofsService.ProcessMessage(r.Context(), nil, &cltypes.SignedAggregateAndProofData{ + if err := a.aggregateAndProofsService.ProcessMessage(r.Context(), nil, &services.SignedAggregateAndProofForGossip{ SignedAggregateAndProof: v, - GossipData: gossipData, ImmediateProcess: true, // we want to process aggregate and proof immediately }); err != nil && !errors.Is(err, services.ErrIgnore) { log.Warn("[Beacon REST] failed to process bls-change", "err", err) failures = append(failures, poolingFailure{Index: len(failures), Message: err.Error()}) continue } + + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNameBeaconAggregateAndProof, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } } @@ -356,7 +375,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolSyncCommittees(w http.ResponseWriter, r for _, subnet := range publishingSubnets { - var syncCommitteeMessageWithGossipData cltypes.SyncCommitteeMessageWithGossipData + var syncCommitteeMessageWithGossipData services.SyncCommitteeMessageForGossip syncCommitteeMessageWithGossipData.SyncCommitteeMessage = v syncCommitteeMessageWithGossipData.ImmediateVerification = true @@ -367,17 +386,22 @@ func (a *ApiHandler) PostEthV1BeaconPoolSyncCommittees(w http.ResponseWriter, r } subnetId := subnet - syncCommitteeMessageWithGossipData.GossipData = &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNamePrefixSyncCommittee, - SubnetId: &subnetId, - } if err = a.syncCommitteeMessagesService.ProcessMessage(r.Context(), &subnet, &syncCommitteeMessageWithGossipData); err != nil && !errors.Is(err, services.ErrIgnore) { log.Warn("[Beacon REST] failed to process attestation in syncCommittee service", "err", err) failures = append(failures, poolingFailure{Index: idx, Message: err.Error()}) break } + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNamePrefixSyncCommittee, + SubnetId: &subnetId, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } } if len(failures) > 0 { @@ -403,7 +427,7 @@ func (a *ApiHandler) PostEthV1ValidatorContributionsAndProofs(w http.ResponseWri continue // skip empty contributions } - var signedContributionAndProofWithGossipData cltypes.SignedContributionAndProofWithGossipData + var signedContributionAndProofWithGossipData services.SignedContributionAndProofForGossip signedContributionAndProofWithGossipData.SignedContributionAndProof = v signedContributionAndProofWithGossipData.ImmediateVerification = true @@ -414,16 +438,20 @@ func (a *ApiHandler) PostEthV1ValidatorContributionsAndProofs(w http.ResponseWri return } - signedContributionAndProofWithGossipData.GossipData = &sentinel.GossipData{ - Data: encodedSSZ, - Name: gossip.TopicNameSyncCommitteeContributionAndProof, - } - if err = a.syncContributionAndProofsService.ProcessMessage(r.Context(), nil, &signedContributionAndProofWithGossipData); err != nil && !errors.Is(err, services.ErrIgnore) { log.Warn("[Beacon REST] failed to process sync contribution", "err", err) failures = append(failures, poolingFailure{Index: idx, Message: err.Error()}) continue } + if a.sentinel != nil { + if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{ + Data: encodedSSZ, + Name: gossip.TopicNameSyncCommitteeContributionAndProof, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } if len(failures) > 0 { diff --git a/cl/beacon/handler/utils_test.go b/cl/beacon/handler/utils_test.go index 16e403388c0..6198ab3cd61 100644 --- a/cl/beacon/handler/utils_test.go +++ b/cl/beacon/handler/utils_test.go @@ -45,6 +45,7 @@ import ( "github.com/erigontech/erigon/cl/persistence/state/historical_states_reader" "github.com/erigontech/erigon/cl/phase1/core/state" mock_services2 "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" + "github.com/erigontech/erigon/cl/phase1/network/services" "github.com/erigontech/erigon/cl/phase1/network/services/mock_services" "github.com/erigontech/erigon/cl/pool" "github.com/erigontech/erigon/cl/utils/eth_clock" @@ -119,22 +120,22 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge mockValidatorMonitor := mockMonitor.NewMockValidatorMonitor(ctrl) // ctx context.Context, subnetID *uint64, msg *cltypes.SyncCommitteeMessage) error - syncCommitteeMessagesService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SyncCommitteeMessageWithGossipData) error { + syncCommitteeMessagesService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *services.SyncCommitteeMessageForGossip) error { return h.syncMessagePool.AddSyncCommitteeMessage(postState, *subnetID, msg.SyncCommitteeMessage) }).AnyTimes() - syncContributionService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedContributionAndProofWithGossipData) error { + syncContributionService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *services.SignedContributionAndProofForGossip) error { return h.syncMessagePool.AddSyncContribution(postState, msg.SignedContributionAndProof.Message.Contribution) }).AnyTimes() - aggregateAndProofsService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedAggregateAndProofData) error { + aggregateAndProofsService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *services.SignedAggregateAndProofForGossip) error { opPool.AttestationsPool.Insert(msg.SignedAggregateAndProof.Message.Aggregate.Signature, msg.SignedAggregateAndProof.Message.Aggregate) return nil }).AnyTimes() - voluntaryExitService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedVoluntaryExitWithGossipData) error { + voluntaryExitService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *services.SignedVoluntaryExitForGossip) error { opPool.VoluntaryExitsPool.Insert(msg.SignedVoluntaryExit.VoluntaryExit.ValidatorIndex, msg.SignedVoluntaryExit) return nil }).AnyTimes() - blsToExecutionChangeService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedBLSToExecutionChangeWithGossipData) error { + blsToExecutionChangeService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *services.SignedBLSToExecutionChangeForGossip) error { opPool.BLSToExecutionChangesPool.Insert(msg.SignedBLSToExecutionChange.Signature, msg.SignedBLSToExecutionChange) return nil }).AnyTimes() diff --git a/cl/cltypes/aggregate.go b/cl/cltypes/aggregate.go index 29920e362a7..2c269cadda9 100644 --- a/cl/cltypes/aggregate.go +++ b/cl/cltypes/aggregate.go @@ -18,7 +18,6 @@ package cltypes import ( libcommon "github.com/erigontech/erigon-lib/common" - sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/merkle_tree" ssz2 "github.com/erigontech/erigon/cl/ssz" @@ -55,17 +54,6 @@ func (a *AggregateAndProof) HashSSZ() ([32]byte, error) { return merkle_tree.HashTreeRoot(a.AggregatorIndex, a.Aggregate, a.SelectionProof[:]) } -// SignedAggregateAndProofData is passed to SignedAggregateAndProof service. The service does the signature verification -// asynchronously. That's why we cannot wait for its ProcessMessage call to finish to check error. The service -// will do re-publishing of the gossip or banning the peer in case of invalid signature by itself. -// that's why we are passing sentinel.SentinelClient and *sentinel.GossipData to enable the service -// to do all of that by itself. -type SignedAggregateAndProofData struct { - SignedAggregateAndProof *SignedAggregateAndProof - GossipData *sentinel.GossipData - ImmediateProcess bool -} - type SignedAggregateAndProof struct { Message *AggregateAndProof `json:"message"` Signature libcommon.Bytes96 `json:"signature"` diff --git a/cl/cltypes/bls_to_execution_change.go b/cl/cltypes/bls_to_execution_change.go index ca6a87f7d9a..00bd1bb0fe4 100644 --- a/cl/cltypes/bls_to_execution_change.go +++ b/cl/cltypes/bls_to_execution_change.go @@ -20,7 +20,6 @@ import ( "fmt" libcommon "github.com/erigontech/erigon-lib/common" - sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon-lib/types/ssz" "github.com/erigontech/erigon/cl/merkle_tree" ssz2 "github.com/erigontech/erigon/cl/ssz" @@ -59,13 +58,6 @@ func (*BLSToExecutionChange) Static() bool { return true } -// SignedBLSToExecutionChangeWithGossipData type represents SignedBLSToExecutionChange with the gossip data where it's coming from. -type SignedBLSToExecutionChangeWithGossipData struct { - SignedBLSToExecutionChange *SignedBLSToExecutionChange - GossipData *sentinel.GossipData - ImmediateVerification bool -} - type SignedBLSToExecutionChange struct { Message *BLSToExecutionChange `json:"message"` Signature libcommon.Bytes96 `json:"signature"` diff --git a/cl/cltypes/contribution.go b/cl/cltypes/contribution.go index 319edc9a427..f1c1244c55d 100644 --- a/cl/cltypes/contribution.go +++ b/cl/cltypes/contribution.go @@ -20,7 +20,6 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/common/length" - sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon-lib/types/clonable" "github.com/erigontech/erigon/cl/merkle_tree" ssz2 "github.com/erigontech/erigon/cl/ssz" @@ -60,13 +59,6 @@ func (a *ContributionAndProof) HashSSZ() ([32]byte, error) { return merkle_tree.HashTreeRoot(a.AggregatorIndex, a.Contribution, a.SelectionProof[:]) } -// SignedContributionAndProofWithGossipData type represents SignedContributionAndProof with the gossip data where it's coming from. -type SignedContributionAndProofWithGossipData struct { - SignedContributionAndProof *SignedContributionAndProof - GossipData *sentinel.GossipData - ImmediateVerification bool -} - type SignedContributionAndProof struct { Message *ContributionAndProof `json:"message"` Signature libcommon.Bytes96 `json:"signature"` @@ -187,12 +179,6 @@ func (agg *SyncContribution) HashSSZ() ([32]byte, error) { } -type SyncCommitteeMessageWithGossipData struct { - SyncCommitteeMessage *SyncCommitteeMessage - GossipData *sentinel.GossipData - ImmediateVerification bool -} - type SyncCommitteeMessage struct { Slot uint64 `json:"slot,string"` BeaconBlockRoot libcommon.Hash `json:"beacon_block_root"` diff --git a/cl/cltypes/validator.go b/cl/cltypes/validator.go index 49c6d6e455a..e4989afd367 100644 --- a/cl/cltypes/validator.go +++ b/cl/cltypes/validator.go @@ -20,7 +20,6 @@ import ( "encoding/json" libcommon "github.com/erigontech/erigon-lib/common" - sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon-lib/types/clonable" "github.com/erigontech/erigon-lib/types/ssz" @@ -129,13 +128,6 @@ func (*VoluntaryExit) EncodingSizeSSZ() int { return 16 } -// SignedVoluntaryExitWithGossipData type represents SignedVoluntaryExit with the gossip data where it's coming from. -type SignedVoluntaryExitWithGossipData struct { - SignedVoluntaryExit *SignedVoluntaryExit - GossipData *sentinel.GossipData - ImmediateVerification bool -} - type SignedVoluntaryExit struct { VoluntaryExit *VoluntaryExit `json:"message"` Signature libcommon.Bytes96 `json:"signature"` diff --git a/cl/phase1/network/gossip_manager.go b/cl/phase1/network/gossip_manager.go index 847d93c5a4b..baae4fb418b 100644 --- a/cl/phase1/network/gossip_manager.go +++ b/cl/phase1/network/gossip_manager.go @@ -135,9 +135,6 @@ func (g *GossipManager) onRecv(ctx context.Context, data *sentinel.GossipData, l g.sentinel.BanPeer(ctx, data.Peer) return err } - if _, err := g.sentinel.PublishGossip(ctx, data); err != nil { - log.Debug("failed to publish gossip", "err", err) - } return nil } @@ -145,24 +142,18 @@ func (g *GossipManager) isReadyToProcessOperations() bool { return g.forkChoice.HighestSeen()+8 >= g.ethClock.GetCurrentSlot() } -func copyOfSentinelData(in *sentinel.GossipData) *sentinel.GossipData { - ret := &sentinel.GossipData{ - Data: common.Copy(in.Data), - Name: in.Name, - } - if in.SubnetId != nil { - ret.SubnetId = new(uint64) - *ret.SubnetId = *in.SubnetId - } - if in.Peer != nil { - ret.Peer = new(sentinel.Peer) - ret.Peer.State = in.Peer.State - ret.Peer.Pid = in.Peer.Pid - ret.Peer.Enr = in.Peer.Enr - ret.Peer.Direction = in.Peer.Direction - ret.Peer.AgentVersion = in.Peer.AgentVersion - ret.Peer.Address = in.Peer.Address +func copyOfPeerData(in *sentinel.GossipData) *sentinel.Peer { + if in == nil || in.Peer == nil { + return nil } + ret := new(sentinel.Peer) + ret.State = in.Peer.State + ret.Pid = in.Peer.Pid + ret.Enr = in.Peer.Enr + ret.Direction = in.Peer.Direction + ret.AgentVersion = in.Peer.AgentVersion + ret.Address = in.Peer.Address + return ret } @@ -183,8 +174,8 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss log.Debug("Received block via gossip", "slot", obj.Block.Slot) return g.blockService.ProcessMessage(ctx, data.SubnetId, obj) case gossip.TopicNameSyncCommitteeContributionAndProof: - obj := &cltypes.SignedContributionAndProofWithGossipData{ - GossipData: copyOfSentinelData(data), + obj := &services.SignedContributionAndProofForGossip{ + Receiver: copyOfPeerData(data), SignedContributionAndProof: &cltypes.SignedContributionAndProof{}, } if err := obj.SignedContributionAndProof.DecodeSSZ(data.Data, int(version)); err != nil { @@ -192,8 +183,8 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss } return g.syncContributionService.ProcessMessage(ctx, data.SubnetId, obj) case gossip.TopicNameVoluntaryExit: - obj := &cltypes.SignedVoluntaryExitWithGossipData{ - GossipData: copyOfSentinelData(data), + obj := &services.SignedVoluntaryExitForGossip{ + Receiver: copyOfPeerData(data), SignedVoluntaryExit: &cltypes.SignedVoluntaryExit{}, } if err := obj.SignedVoluntaryExit.DecodeSSZ(data.Data, int(version)); err != nil { @@ -217,13 +208,10 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss return err } - if _, err := g.sentinel.PublishGossip(ctx, data); err != nil { - log.Debug("failed to publish gossip", "err", err) - } return nil case gossip.TopicNameBlsToExecutionChange: - obj := &cltypes.SignedBLSToExecutionChangeWithGossipData{ - GossipData: copyOfSentinelData(data), + obj := &services.SignedBLSToExecutionChangeForGossip{ + Receiver: copyOfPeerData(data), SignedBLSToExecutionChange: &cltypes.SignedBLSToExecutionChange{}, } if err := obj.SignedBLSToExecutionChange.DecodeSSZ(data.Data, int(version)); err != nil { @@ -231,11 +219,10 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss } return g.blsToExecutionChangeService.ProcessMessage(ctx, data.SubnetId, obj) case gossip.TopicNameBeaconAggregateAndProof: - obj := &cltypes.SignedAggregateAndProofData{ - GossipData: copyOfSentinelData(data), + obj := &services.SignedAggregateAndProofForGossip{ + Receiver: copyOfPeerData(data), SignedAggregateAndProof: &cltypes.SignedAggregateAndProof{}, } - if err := obj.SignedAggregateAndProof.DecodeSSZ(common.CopyBytes(data.Data), int(version)); err != nil { return err } @@ -252,17 +239,17 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss // The background checks above are enough for now. return g.blobService.ProcessMessage(ctx, data.SubnetId, blobSideCar) case gossip.IsTopicSyncCommittee(data.Name): - msg := &cltypes.SyncCommitteeMessageWithGossipData{ - GossipData: copyOfSentinelData(data), + obj := &services.SyncCommitteeMessageForGossip{ + Receiver: copyOfPeerData(data), SyncCommitteeMessage: &cltypes.SyncCommitteeMessage{}, } - if err := msg.SyncCommitteeMessage.DecodeSSZ(common.CopyBytes(data.Data), int(version)); err != nil { + if err := obj.SyncCommitteeMessage.DecodeSSZ(common.CopyBytes(data.Data), int(version)); err != nil { return err } - return g.syncCommitteeMessagesService.ProcessMessage(ctx, data.SubnetId, msg) + return g.syncCommitteeMessagesService.ProcessMessage(ctx, data.SubnetId, obj) case gossip.IsTopicBeaconAttestation(data.Name): - obj := &services.AttestationWithGossipData{ - GossipData: copyOfSentinelData(data), + obj := &services.AttestationForGossip{ + Receiver: copyOfPeerData(data), Attestation: &solid.Attestation{}, SingleAttestation: &solid.SingleAttestation{}, ImmediateProcess: false, diff --git a/cl/phase1/network/services/aggregate_and_proof_service.go b/cl/phase1/network/services/aggregate_and_proof_service.go index 1e80db2a673..6307d554c00 100644 --- a/cl/phase1/network/services/aggregate_and_proof_service.go +++ b/cl/phase1/network/services/aggregate_and_proof_service.go @@ -26,6 +26,7 @@ import ( "github.com/Giulio2002/bls" + sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/cl/beacon/synced_data" @@ -42,8 +43,19 @@ import ( "github.com/erigontech/erigon/cl/utils" ) +// SignedAggregateAndProofData is passed to SignedAggregateAndProof service. The service does the signature verification +// asynchronously. That's why we cannot wait for its ProcessMessage call to finish to check error. The service +// will do re-publishing of the gossip or banning the peer in case of invalid signature by itself. +// that's why we are passing sentinel.SentinelClient and *sentinel.GossipData to enable the service +// to do all of that by itself. +type SignedAggregateAndProofForGossip struct { + SignedAggregateAndProof *cltypes.SignedAggregateAndProof + Receiver *sentinel.Peer + ImmediateProcess bool +} + type aggregateJob struct { - aggregate *cltypes.SignedAggregateAndProofData + aggregate *SignedAggregateAndProofForGossip creationTime time.Time } @@ -96,7 +108,7 @@ func NewAggregateAndProofService( func (a *aggregateAndProofServiceImpl) ProcessMessage( ctx context.Context, subnet *uint64, - aggregateAndProof *cltypes.SignedAggregateAndProofData, + aggregateAndProof *SignedAggregateAndProofForGossip, ) error { selectionProof := aggregateAndProof.SignedAggregateAndProof.Message.SelectionProof aggregateData := aggregateAndProof.SignedAggregateAndProof.Message.Aggregate.Data @@ -232,7 +244,7 @@ func (a *aggregateAndProofServiceImpl) ProcessMessage( a.seenAggreatorIndexes.Add(seenIndex, struct{}{}) } // for this specific request, collect data for potential peer banning or gossip publishing - aggregateVerificationData.GossipData = aggregateAndProof.GossipData + aggregateVerificationData.SendingPeer = aggregateAndProof.Receiver if aggregateAndProof.ImmediateProcess { return a.batchSignatureVerifier.ImmediateVerification(aggregateVerificationData) @@ -361,7 +373,7 @@ func AggregateMessageSignature( } func (a *aggregateAndProofServiceImpl) scheduleAggregateForLaterProcessing( - aggregateAndProof *cltypes.SignedAggregateAndProofData, + aggregateAndProof *SignedAggregateAndProofForGossip, ) { key, err := aggregateAndProof.SignedAggregateAndProof.HashSSZ() if err != nil { diff --git a/cl/phase1/network/services/aggregate_and_proof_service_test.go b/cl/phase1/network/services/aggregate_and_proof_service_test.go index 6745a05a893..82633125207 100644 --- a/cl/phase1/network/services/aggregate_and_proof_service_test.go +++ b/cl/phase1/network/services/aggregate_and_proof_service_test.go @@ -35,11 +35,11 @@ import ( "github.com/erigontech/erigon/cl/pool" ) -func getAggregateAndProofAndState(t *testing.T) (*cltypes.SignedAggregateAndProofData, *state.CachingBeaconState) { +func getAggregateAndProofAndState(t *testing.T) (*SignedAggregateAndProofForGossip, *state.CachingBeaconState) { _, _, s := tests.GetBellatrixRandom() br, _ := s.BlockRoot() checkpoint := s.CurrentJustifiedCheckpoint() - a := &cltypes.SignedAggregateAndProofData{ + a := &SignedAggregateAndProofForGossip{ SignedAggregateAndProof: &cltypes.SignedAggregateAndProof{ Message: &cltypes.AggregateAndProof{ AggregatorIndex: 141, @@ -62,7 +62,6 @@ func getAggregateAndProofAndState(t *testing.T) (*cltypes.SignedAggregateAndProo }, }, }, - GossipData: nil, } a.SignedAggregateAndProof.Message.Aggregate.Data.Target.Epoch = s.Slot() / 32 diff --git a/cl/phase1/network/services/attestation_service.go b/cl/phase1/network/services/attestation_service.go index 6c7adfba1fa..b701a2533cc 100644 --- a/cl/phase1/network/services/attestation_service.go +++ b/cl/phase1/network/services/attestation_service.go @@ -65,10 +65,10 @@ type attestationService struct { } // AttestationWithGossipData type represents attestation with the gossip data where it's coming from. -type AttestationWithGossipData struct { +type AttestationForGossip struct { Attestation *solid.Attestation SingleAttestation *solid.SingleAttestation // New container after Electra - GossipData *sentinel.GossipData + Receiver *sentinel.Peer // ImmediateProcess indicates whether the attestation should be processed immediately or able to be scheduled for later processing. ImmediateProcess bool } @@ -103,7 +103,7 @@ func NewAttestationService( return a } -func (s *attestationService) ProcessMessage(ctx context.Context, subnet *uint64, att *AttestationWithGossipData) error { +func (s *attestationService) ProcessMessage(ctx context.Context, subnet *uint64, att *AttestationForGossip) error { var ( root libcommon.Hash slot uint64 @@ -276,10 +276,10 @@ func (s *attestationService) ProcessMessage(ctx context.Context, subnet *uint64, } aggregateVerificationData := &AggregateVerificationData{ - Signatures: [][]byte{signature[:]}, - SignRoots: [][]byte{signingRoot[:]}, - Pks: [][]byte{pubKey[:]}, - GossipData: att.GossipData, + Signatures: [][]byte{signature[:]}, + SignRoots: [][]byte{signingRoot[:]}, + Pks: [][]byte{pubKey[:]}, + SendingPeer: att.Receiver, F: func() { start := time.Now() defer monitor.ObserveAggregateAttestation(start) diff --git a/cl/phase1/network/services/attestation_service_test.go b/cl/phase1/network/services/attestation_service_test.go index 45d5a008435..b5d61bbf553 100644 --- a/cl/phase1/network/services/attestation_service_test.go +++ b/cl/phase1/network/services/attestation_service_test.go @@ -346,7 +346,7 @@ func (t *attestationTestSuite) TestAttestationProcessMessage() { log.Printf("test case: %s", tt.name) t.SetupTest() tt.mock() - err := t.attService.ProcessMessage(tt.args.ctx, tt.args.subnet, &AttestationWithGossipData{Attestation: tt.args.msg, GossipData: nil, ImmediateProcess: true}) + err := t.attService.ProcessMessage(tt.args.ctx, tt.args.subnet, &AttestationForGossip{Attestation: tt.args.msg, ImmediateProcess: true}) time.Sleep(time.Millisecond * 60) if tt.wantErr { t.Require().Error(err) diff --git a/cl/phase1/network/services/batch_signature_verification.go b/cl/phase1/network/services/batch_signature_verification.go index c7c7cef9ca8..b70db4b9efe 100644 --- a/cl/phase1/network/services/batch_signature_verification.go +++ b/cl/phase1/network/services/batch_signature_verification.go @@ -38,11 +38,11 @@ var ErrInvalidBlsSignature = errors.New("invalid bls signature") // to make sure that we can validate it separately and in case of failure we ban corresponding // GossipData.Peer or simply run F and publish GossipData in case signature verification succeeds. type AggregateVerificationData struct { - Signatures [][]byte - SignRoots [][]byte - Pks [][]byte - F func() - GossipData *sentinel.GossipData + Signatures [][]byte + SignRoots [][]byte + Pks [][]byte + F func() + SendingPeer *sentinel.Peer } func NewBatchSignatureVerifier(ctx context.Context, sentinel sentinel.SentinelClient) *BatchSignatureVerifier { @@ -151,11 +151,6 @@ func (b *BatchSignatureVerifier) processSignatureVerification(aggregateVerificat // Everything went well, run corresponding Fs and send all the gossip data to the network for _, v := range aggregateVerificationData { v.F() - if b.sentinel != nil && v.GossipData != nil { - if _, err := b.sentinel.PublishGossip(b.ctx, v.GossipData); err != nil { - log.Debug("failed to publish gossip", "err", err) - } - } } return nil } @@ -167,19 +162,19 @@ func (b *BatchSignatureVerifier) handleIncorrectSignatures(aggregateVerification valid, err := blsVerifyMultipleSignatures(v.Signatures, v.SignRoots, v.Pks) if err != nil { log.Crit("[BatchVerifier] signature verification failed with the error: " + err.Error()) - if b.sentinel != nil && v.GossipData != nil && v.GossipData.Peer != nil { - b.sentinel.BanPeer(b.ctx, v.GossipData.Peer) + if b.sentinel != nil && v.SendingPeer != nil { + b.sentinel.BanPeer(b.ctx, v.SendingPeer) } continue } if !valid { - if v.GossipData == nil || alreadyBanned { + if v.SendingPeer == nil || alreadyBanned { continue } - log.Debug("[BatchVerifier] received invalid signature on the gossip", "topic", v.GossipData.Name) - if b.sentinel != nil && v.GossipData != nil && v.GossipData.Peer != nil { - b.sentinel.BanPeer(b.ctx, v.GossipData.Peer) + log.Debug("[BatchVerifier] received invalid signature on the gossip", "peer", v.SendingPeer.Pid) + if b.sentinel != nil && v.SendingPeer != nil { + b.sentinel.BanPeer(b.ctx, v.SendingPeer) alreadyBanned = true } continue @@ -187,11 +182,6 @@ func (b *BatchSignatureVerifier) handleIncorrectSignatures(aggregateVerification // run corresponding function and publish the gossip into the network v.F() - if b.sentinel != nil && v.GossipData != nil { - if _, err := b.sentinel.PublishGossip(b.ctx, v.GossipData); err != nil { - log.Debug("failed to publish gossip", "err", err) - } - } } } diff --git a/cl/phase1/network/services/bls_to_execution_change_service.go b/cl/phase1/network/services/bls_to_execution_change_service.go index 2a9a8cfe347..4d8919c15db 100644 --- a/cl/phase1/network/services/bls_to_execution_change_service.go +++ b/cl/phase1/network/services/bls_to_execution_change_service.go @@ -24,6 +24,7 @@ import ( "github.com/Giulio2002/bls" "github.com/erigontech/erigon-lib/common" + sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon/cl/beacon/beaconevents" "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" @@ -38,6 +39,13 @@ var ( blsVerify = bls.Verify ) +// SignedBLSToExecutionChangeForGossip type represents SignedBLSToExecutionChange with the gossip data where it's coming from. +type SignedBLSToExecutionChangeForGossip struct { + SignedBLSToExecutionChange *cltypes.SignedBLSToExecutionChange + Receiver *sentinel.Peer + ImmediateVerification bool +} + type blsToExecutionChangeService struct { operationsPool pool.OperationsPool emitters *beaconevents.EventEmitter @@ -62,7 +70,7 @@ func NewBLSToExecutionChangeService( } } -func (s *blsToExecutionChangeService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedBLSToExecutionChangeWithGossipData) error { +func (s *blsToExecutionChangeService) ProcessMessage(ctx context.Context, subnet *uint64, msg *SignedBLSToExecutionChangeForGossip) error { // https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/p2p-interface.md#bls_to_execution_change // [IGNORE] The signed_bls_to_execution_change is the first valid signed bls to execution change received // for the validator with index signed_bls_to_execution_change.message.validator_index. @@ -117,10 +125,10 @@ func (s *blsToExecutionChangeService) ProcessMessage(ctx context.Context, subnet } aggregateVerificationData := &AggregateVerificationData{ - Signatures: [][]byte{msg.SignedBLSToExecutionChange.Signature[:]}, - SignRoots: [][]byte{signedRoot[:]}, - Pks: [][]byte{change.From[:]}, - GossipData: msg.GossipData, + Signatures: [][]byte{msg.SignedBLSToExecutionChange.Signature[:]}, + SignRoots: [][]byte{signedRoot[:]}, + Pks: [][]byte{change.From[:]}, + SendingPeer: msg.Receiver, F: func() { s.emitters.Operation().SendBlsToExecution(msg.SignedBLSToExecutionChange) s.operationsPool.BLSToExecutionChangesPool.Insert(msg.SignedBLSToExecutionChange.Signature, msg.SignedBLSToExecutionChange) diff --git a/cl/phase1/network/services/bls_to_execution_change_service_test.go b/cl/phase1/network/services/bls_to_execution_change_service_test.go index 6f49fd827e9..1bd041ef3fb 100644 --- a/cl/phase1/network/services/bls_to_execution_change_service_test.go +++ b/cl/phase1/network/services/bls_to_execution_change_service_test.go @@ -78,7 +78,7 @@ func (t *blsToExecutionChangeTestSuite) TearDownTest() { } func (t *blsToExecutionChangeTestSuite) TestProcessMessage() { - mockMsg := &cltypes.SignedBLSToExecutionChangeWithGossipData{ + mockMsg := &SignedBLSToExecutionChangeForGossip{ SignedBLSToExecutionChange: &cltypes.SignedBLSToExecutionChange{ Message: &cltypes.BLSToExecutionChange{ ValidatorIndex: 1, @@ -87,14 +87,13 @@ func (t *blsToExecutionChangeTestSuite) TestProcessMessage() { }, Signature: [96]byte{1, 2, 3}, }, - GossipData: nil, ImmediateVerification: true, } tests := []struct { name string mock func() - msg *cltypes.SignedBLSToExecutionChangeWithGossipData + msg *SignedBLSToExecutionChangeForGossip wantErr bool specificErr error }{ diff --git a/cl/phase1/network/services/interface.go b/cl/phase1/network/services/interface.go index 02a3b26a8d2..fc03bd35af1 100644 --- a/cl/phase1/network/services/interface.go +++ b/cl/phase1/network/services/interface.go @@ -35,22 +35,22 @@ type BlockService Service[*cltypes.SignedBeaconBlock] type BlobSidecarsService Service[*cltypes.BlobSidecar] //go:generate mockgen -typed=true -destination=./mock_services/sync_committee_messages_service_mock.go -package=mock_services . SyncCommitteeMessagesService -type SyncCommitteeMessagesService Service[*cltypes.SyncCommitteeMessageWithGossipData] +type SyncCommitteeMessagesService Service[*SyncCommitteeMessageForGossip] //go:generate mockgen -typed=true -destination=./mock_services/sync_contribution_service_mock.go -package=mock_services . SyncContributionService -type SyncContributionService Service[*cltypes.SignedContributionAndProofWithGossipData] +type SyncContributionService Service[*SignedContributionAndProofForGossip] //go:generate mockgen -typed=true -destination=./mock_services/aggregate_and_proof_service_mock.go -package=mock_services . AggregateAndProofService -type AggregateAndProofService Service[*cltypes.SignedAggregateAndProofData] +type AggregateAndProofService Service[*SignedAggregateAndProofForGossip] //go:generate mockgen -typed=true -destination=./mock_services/attestation_service_mock.go -package=mock_services . AttestationService -type AttestationService Service[*AttestationWithGossipData] +type AttestationService Service[*AttestationForGossip] //go:generate mockgen -typed=true -destination=./mock_services/voluntary_exit_service_mock.go -package=mock_services . VoluntaryExitService -type VoluntaryExitService Service[*cltypes.SignedVoluntaryExitWithGossipData] +type VoluntaryExitService Service[*SignedVoluntaryExitForGossip] //go:generate mockgen -typed=true -destination=./mock_services/bls_to_execution_change_service_mock.go -package=mock_services . BLSToExecutionChangeService -type BLSToExecutionChangeService Service[*cltypes.SignedBLSToExecutionChangeWithGossipData] +type BLSToExecutionChangeService Service[*SignedBLSToExecutionChangeForGossip] //go:generate mockgen -typed=true -destination=./mock_services/proposer_slashing_service_mock.go -package=mock_services . ProposerSlashingService type ProposerSlashingService Service[*cltypes.ProposerSlashing] diff --git a/cl/phase1/network/services/mock_services/aggregate_and_proof_service_mock.go b/cl/phase1/network/services/mock_services/aggregate_and_proof_service_mock.go index 862b6c4043c..7eb06d711f5 100644 --- a/cl/phase1/network/services/mock_services/aggregate_and_proof_service_mock.go +++ b/cl/phase1/network/services/mock_services/aggregate_and_proof_service_mock.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - cltypes "github.com/erigontech/erigon/cl/cltypes" + services "github.com/erigontech/erigon/cl/phase1/network/services" gomock "go.uber.org/mock/gomock" ) @@ -21,7 +21,6 @@ import ( type MockAggregateAndProofService struct { ctrl *gomock.Controller recorder *MockAggregateAndProofServiceMockRecorder - isgomock struct{} } // MockAggregateAndProofServiceMockRecorder is the mock recorder for MockAggregateAndProofService. @@ -42,17 +41,17 @@ func (m *MockAggregateAndProofService) EXPECT() *MockAggregateAndProofServiceMoc } // ProcessMessage mocks base method. -func (m *MockAggregateAndProofService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedAggregateAndProofData) error { +func (m *MockAggregateAndProofService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.SignedAggregateAndProofForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockAggregateAndProofServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockAggregateAndProofServiceProcessMessageCall { +func (mr *MockAggregateAndProofServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockAggregateAndProofServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockAggregateAndProofService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockAggregateAndProofService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockAggregateAndProofServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockAggregateAndProofServiceProcessMessageCall) Return(arg0 error) *Moc } // Do rewrite *gomock.Call.Do -func (c *MockAggregateAndProofServiceProcessMessageCall) Do(f func(context.Context, *uint64, *cltypes.SignedAggregateAndProofData) error) *MockAggregateAndProofServiceProcessMessageCall { +func (c *MockAggregateAndProofServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.SignedAggregateAndProofForGossip) error) *MockAggregateAndProofServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockAggregateAndProofServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *cltypes.SignedAggregateAndProofData) error) *MockAggregateAndProofServiceProcessMessageCall { +func (c *MockAggregateAndProofServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.SignedAggregateAndProofForGossip) error) *MockAggregateAndProofServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/mock_services/attestation_service_mock.go b/cl/phase1/network/services/mock_services/attestation_service_mock.go index 5220a08349a..87b15ca5232 100644 --- a/cl/phase1/network/services/mock_services/attestation_service_mock.go +++ b/cl/phase1/network/services/mock_services/attestation_service_mock.go @@ -21,7 +21,6 @@ import ( type MockAttestationService struct { ctrl *gomock.Controller recorder *MockAttestationServiceMockRecorder - isgomock struct{} } // MockAttestationServiceMockRecorder is the mock recorder for MockAttestationService. @@ -42,17 +41,17 @@ func (m *MockAttestationService) EXPECT() *MockAttestationServiceMockRecorder { } // ProcessMessage mocks base method. -func (m *MockAttestationService) ProcessMessage(ctx context.Context, subnet *uint64, msg *services.AttestationWithGossipData) error { +func (m *MockAttestationService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.AttestationForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockAttestationServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockAttestationServiceProcessMessageCall { +func (mr *MockAttestationServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockAttestationServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockAttestationService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockAttestationService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockAttestationServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockAttestationServiceProcessMessageCall) Return(arg0 error) *MockAttes } // Do rewrite *gomock.Call.Do -func (c *MockAttestationServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.AttestationWithGossipData) error) *MockAttestationServiceProcessMessageCall { +func (c *MockAttestationServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.AttestationForGossip) error) *MockAttestationServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockAttestationServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.AttestationWithGossipData) error) *MockAttestationServiceProcessMessageCall { +func (c *MockAttestationServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.AttestationForGossip) error) *MockAttestationServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/mock_services/blob_sidecars_service_mock.go b/cl/phase1/network/services/mock_services/blob_sidecars_service_mock.go index 1445222c060..ee82d59319a 100644 --- a/cl/phase1/network/services/mock_services/blob_sidecars_service_mock.go +++ b/cl/phase1/network/services/mock_services/blob_sidecars_service_mock.go @@ -21,7 +21,6 @@ import ( type MockBlobSidecarsService struct { ctrl *gomock.Controller recorder *MockBlobSidecarsServiceMockRecorder - isgomock struct{} } // MockBlobSidecarsServiceMockRecorder is the mock recorder for MockBlobSidecarsService. @@ -42,17 +41,17 @@ func (m *MockBlobSidecarsService) EXPECT() *MockBlobSidecarsServiceMockRecorder } // ProcessMessage mocks base method. -func (m *MockBlobSidecarsService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.BlobSidecar) error { +func (m *MockBlobSidecarsService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *cltypes.BlobSidecar) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockBlobSidecarsServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockBlobSidecarsServiceProcessMessageCall { +func (mr *MockBlobSidecarsServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockBlobSidecarsServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBlobSidecarsService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBlobSidecarsService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockBlobSidecarsServiceProcessMessageCall{Call: call} } diff --git a/cl/phase1/network/services/mock_services/block_service_mock.go b/cl/phase1/network/services/mock_services/block_service_mock.go index 52ae2370c10..056061874a9 100644 --- a/cl/phase1/network/services/mock_services/block_service_mock.go +++ b/cl/phase1/network/services/mock_services/block_service_mock.go @@ -21,7 +21,6 @@ import ( type MockBlockService struct { ctrl *gomock.Controller recorder *MockBlockServiceMockRecorder - isgomock struct{} } // MockBlockServiceMockRecorder is the mock recorder for MockBlockService. @@ -42,17 +41,17 @@ func (m *MockBlockService) EXPECT() *MockBlockServiceMockRecorder { } // ProcessMessage mocks base method. -func (m *MockBlockService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedBeaconBlock) error { +func (m *MockBlockService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *cltypes.SignedBeaconBlock) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockBlockServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockBlockServiceProcessMessageCall { +func (mr *MockBlockServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockBlockServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBlockService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBlockService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockBlockServiceProcessMessageCall{Call: call} } diff --git a/cl/phase1/network/services/mock_services/bls_to_execution_change_service_mock.go b/cl/phase1/network/services/mock_services/bls_to_execution_change_service_mock.go index fd4b3da1f37..e8f396b0878 100644 --- a/cl/phase1/network/services/mock_services/bls_to_execution_change_service_mock.go +++ b/cl/phase1/network/services/mock_services/bls_to_execution_change_service_mock.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - cltypes "github.com/erigontech/erigon/cl/cltypes" + services "github.com/erigontech/erigon/cl/phase1/network/services" gomock "go.uber.org/mock/gomock" ) @@ -21,7 +21,6 @@ import ( type MockBLSToExecutionChangeService struct { ctrl *gomock.Controller recorder *MockBLSToExecutionChangeServiceMockRecorder - isgomock struct{} } // MockBLSToExecutionChangeServiceMockRecorder is the mock recorder for MockBLSToExecutionChangeService. @@ -42,17 +41,17 @@ func (m *MockBLSToExecutionChangeService) EXPECT() *MockBLSToExecutionChangeServ } // ProcessMessage mocks base method. -func (m *MockBLSToExecutionChangeService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedBLSToExecutionChangeWithGossipData) error { +func (m *MockBLSToExecutionChangeService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.SignedBLSToExecutionChangeForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockBLSToExecutionChangeServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockBLSToExecutionChangeServiceProcessMessageCall { +func (mr *MockBLSToExecutionChangeServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockBLSToExecutionChangeServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBLSToExecutionChangeService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockBLSToExecutionChangeService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockBLSToExecutionChangeServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockBLSToExecutionChangeServiceProcessMessageCall) Return(arg0 error) * } // Do rewrite *gomock.Call.Do -func (c *MockBLSToExecutionChangeServiceProcessMessageCall) Do(f func(context.Context, *uint64, *cltypes.SignedBLSToExecutionChangeWithGossipData) error) *MockBLSToExecutionChangeServiceProcessMessageCall { +func (c *MockBLSToExecutionChangeServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.SignedBLSToExecutionChangeForGossip) error) *MockBLSToExecutionChangeServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockBLSToExecutionChangeServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *cltypes.SignedBLSToExecutionChangeWithGossipData) error) *MockBLSToExecutionChangeServiceProcessMessageCall { +func (c *MockBLSToExecutionChangeServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.SignedBLSToExecutionChangeForGossip) error) *MockBLSToExecutionChangeServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/mock_services/proposer_slashing_service_mock.go b/cl/phase1/network/services/mock_services/proposer_slashing_service_mock.go index 9be17ca5202..ee9c87ab65b 100644 --- a/cl/phase1/network/services/mock_services/proposer_slashing_service_mock.go +++ b/cl/phase1/network/services/mock_services/proposer_slashing_service_mock.go @@ -21,7 +21,6 @@ import ( type MockProposerSlashingService struct { ctrl *gomock.Controller recorder *MockProposerSlashingServiceMockRecorder - isgomock struct{} } // MockProposerSlashingServiceMockRecorder is the mock recorder for MockProposerSlashingService. @@ -42,17 +41,17 @@ func (m *MockProposerSlashingService) EXPECT() *MockProposerSlashingServiceMockR } // ProcessMessage mocks base method. -func (m *MockProposerSlashingService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.ProposerSlashing) error { +func (m *MockProposerSlashingService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *cltypes.ProposerSlashing) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockProposerSlashingServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockProposerSlashingServiceProcessMessageCall { +func (mr *MockProposerSlashingServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockProposerSlashingServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockProposerSlashingService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockProposerSlashingService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockProposerSlashingServiceProcessMessageCall{Call: call} } diff --git a/cl/phase1/network/services/mock_services/sync_committee_messages_service_mock.go b/cl/phase1/network/services/mock_services/sync_committee_messages_service_mock.go index 02383e81fa8..c2bb9a141a8 100644 --- a/cl/phase1/network/services/mock_services/sync_committee_messages_service_mock.go +++ b/cl/phase1/network/services/mock_services/sync_committee_messages_service_mock.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - cltypes "github.com/erigontech/erigon/cl/cltypes" + services "github.com/erigontech/erigon/cl/phase1/network/services" gomock "go.uber.org/mock/gomock" ) @@ -21,7 +21,6 @@ import ( type MockSyncCommitteeMessagesService struct { ctrl *gomock.Controller recorder *MockSyncCommitteeMessagesServiceMockRecorder - isgomock struct{} } // MockSyncCommitteeMessagesServiceMockRecorder is the mock recorder for MockSyncCommitteeMessagesService. @@ -42,17 +41,17 @@ func (m *MockSyncCommitteeMessagesService) EXPECT() *MockSyncCommitteeMessagesSe } // ProcessMessage mocks base method. -func (m *MockSyncCommitteeMessagesService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SyncCommitteeMessageWithGossipData) error { +func (m *MockSyncCommitteeMessagesService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.SyncCommitteeMessageForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockSyncCommitteeMessagesServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockSyncCommitteeMessagesServiceProcessMessageCall { +func (mr *MockSyncCommitteeMessagesServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockSyncCommitteeMessagesServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockSyncCommitteeMessagesService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockSyncCommitteeMessagesService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockSyncCommitteeMessagesServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockSyncCommitteeMessagesServiceProcessMessageCall) Return(arg0 error) } // Do rewrite *gomock.Call.Do -func (c *MockSyncCommitteeMessagesServiceProcessMessageCall) Do(f func(context.Context, *uint64, *cltypes.SyncCommitteeMessageWithGossipData) error) *MockSyncCommitteeMessagesServiceProcessMessageCall { +func (c *MockSyncCommitteeMessagesServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.SyncCommitteeMessageForGossip) error) *MockSyncCommitteeMessagesServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncCommitteeMessagesServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *cltypes.SyncCommitteeMessageWithGossipData) error) *MockSyncCommitteeMessagesServiceProcessMessageCall { +func (c *MockSyncCommitteeMessagesServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.SyncCommitteeMessageForGossip) error) *MockSyncCommitteeMessagesServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/mock_services/sync_contribution_service_mock.go b/cl/phase1/network/services/mock_services/sync_contribution_service_mock.go index cdc90e8b0c7..a33930a9bb0 100644 --- a/cl/phase1/network/services/mock_services/sync_contribution_service_mock.go +++ b/cl/phase1/network/services/mock_services/sync_contribution_service_mock.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - cltypes "github.com/erigontech/erigon/cl/cltypes" + services "github.com/erigontech/erigon/cl/phase1/network/services" gomock "go.uber.org/mock/gomock" ) @@ -21,7 +21,6 @@ import ( type MockSyncContributionService struct { ctrl *gomock.Controller recorder *MockSyncContributionServiceMockRecorder - isgomock struct{} } // MockSyncContributionServiceMockRecorder is the mock recorder for MockSyncContributionService. @@ -42,17 +41,17 @@ func (m *MockSyncContributionService) EXPECT() *MockSyncContributionServiceMockR } // ProcessMessage mocks base method. -func (m *MockSyncContributionService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedContributionAndProofWithGossipData) error { +func (m *MockSyncContributionService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.SignedContributionAndProofForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockSyncContributionServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockSyncContributionServiceProcessMessageCall { +func (mr *MockSyncContributionServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockSyncContributionServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockSyncContributionService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockSyncContributionService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockSyncContributionServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockSyncContributionServiceProcessMessageCall) Return(arg0 error) *Mock } // Do rewrite *gomock.Call.Do -func (c *MockSyncContributionServiceProcessMessageCall) Do(f func(context.Context, *uint64, *cltypes.SignedContributionAndProofWithGossipData) error) *MockSyncContributionServiceProcessMessageCall { +func (c *MockSyncContributionServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.SignedContributionAndProofForGossip) error) *MockSyncContributionServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncContributionServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *cltypes.SignedContributionAndProofWithGossipData) error) *MockSyncContributionServiceProcessMessageCall { +func (c *MockSyncContributionServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.SignedContributionAndProofForGossip) error) *MockSyncContributionServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/mock_services/voluntary_exit_service_mock.go b/cl/phase1/network/services/mock_services/voluntary_exit_service_mock.go index b2e94717f9a..5d8bc87194e 100644 --- a/cl/phase1/network/services/mock_services/voluntary_exit_service_mock.go +++ b/cl/phase1/network/services/mock_services/voluntary_exit_service_mock.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - cltypes "github.com/erigontech/erigon/cl/cltypes" + services "github.com/erigontech/erigon/cl/phase1/network/services" gomock "go.uber.org/mock/gomock" ) @@ -21,7 +21,6 @@ import ( type MockVoluntaryExitService struct { ctrl *gomock.Controller recorder *MockVoluntaryExitServiceMockRecorder - isgomock struct{} } // MockVoluntaryExitServiceMockRecorder is the mock recorder for MockVoluntaryExitService. @@ -42,17 +41,17 @@ func (m *MockVoluntaryExitService) EXPECT() *MockVoluntaryExitServiceMockRecorde } // ProcessMessage mocks base method. -func (m *MockVoluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedVoluntaryExitWithGossipData) error { +func (m *MockVoluntaryExitService) ProcessMessage(arg0 context.Context, arg1 *uint64, arg2 *services.SignedVoluntaryExitForGossip) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessMessage", ctx, subnet, msg) + ret := m.ctrl.Call(m, "ProcessMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // ProcessMessage indicates an expected call of ProcessMessage. -func (mr *MockVoluntaryExitServiceMockRecorder) ProcessMessage(ctx, subnet, msg any) *MockVoluntaryExitServiceProcessMessageCall { +func (mr *MockVoluntaryExitServiceMockRecorder) ProcessMessage(arg0, arg1, arg2 any) *MockVoluntaryExitServiceProcessMessageCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockVoluntaryExitService)(nil).ProcessMessage), ctx, subnet, msg) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessMessage", reflect.TypeOf((*MockVoluntaryExitService)(nil).ProcessMessage), arg0, arg1, arg2) return &MockVoluntaryExitServiceProcessMessageCall{Call: call} } @@ -68,13 +67,13 @@ func (c *MockVoluntaryExitServiceProcessMessageCall) Return(arg0 error) *MockVol } // Do rewrite *gomock.Call.Do -func (c *MockVoluntaryExitServiceProcessMessageCall) Do(f func(context.Context, *uint64, *cltypes.SignedVoluntaryExitWithGossipData) error) *MockVoluntaryExitServiceProcessMessageCall { +func (c *MockVoluntaryExitServiceProcessMessageCall) Do(f func(context.Context, *uint64, *services.SignedVoluntaryExitForGossip) error) *MockVoluntaryExitServiceProcessMessageCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockVoluntaryExitServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *cltypes.SignedVoluntaryExitWithGossipData) error) *MockVoluntaryExitServiceProcessMessageCall { +func (c *MockVoluntaryExitServiceProcessMessageCall) DoAndReturn(f func(context.Context, *uint64, *services.SignedVoluntaryExitForGossip) error) *MockVoluntaryExitServiceProcessMessageCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/cl/phase1/network/services/sync_committee_messages_service.go b/cl/phase1/network/services/sync_committee_messages_service.go index 83c7c224c57..3ab1c645148 100644 --- a/cl/phase1/network/services/sync_committee_messages_service.go +++ b/cl/phase1/network/services/sync_committee_messages_service.go @@ -22,6 +22,7 @@ import ( "slices" "sync" + sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" @@ -50,6 +51,12 @@ type syncCommitteeMessagesService struct { mu sync.Mutex } +type SyncCommitteeMessageForGossip struct { + SyncCommitteeMessage *cltypes.SyncCommitteeMessage + Receiver *sentinel.Peer + ImmediateVerification bool +} + // NewSyncCommitteeMessagesService creates a new sync committee messages service func NewSyncCommitteeMessagesService( beaconChainCfg *clparams.BeaconChainConfig, @@ -70,7 +77,7 @@ func NewSyncCommitteeMessagesService( } // ProcessMessage processes a sync committee message -func (s *syncCommitteeMessagesService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SyncCommitteeMessageWithGossipData) error { +func (s *syncCommitteeMessagesService) ProcessMessage(ctx context.Context, subnet *uint64, msg *SyncCommitteeMessageForGossip) error { s.mu.Lock() defer s.mu.Unlock() @@ -105,10 +112,10 @@ func (s *syncCommitteeMessagesService) ProcessMessage(ctx context.Context, subne return err } aggregateVerificationData := &AggregateVerificationData{ - Signatures: [][]byte{signature}, - SignRoots: [][]byte{signingRoot}, - Pks: [][]byte{pubKey}, - GossipData: msg.GossipData, + Signatures: [][]byte{signature}, + SignRoots: [][]byte{signingRoot}, + Pks: [][]byte{pubKey}, + SendingPeer: msg.Receiver, F: func() { s.seenSyncCommitteeMessages.Store(seenSyncCommitteeMessageIdentifier, struct{}{}) s.cleanupOldSyncCommitteeMessages() // cleanup old messages diff --git a/cl/phase1/network/services/sync_committee_messages_service_test.go b/cl/phase1/network/services/sync_committee_messages_service_test.go index ca2f0a0fb84..436f7063c79 100644 --- a/cl/phase1/network/services/sync_committee_messages_service_test.go +++ b/cl/phase1/network/services/sync_committee_messages_service_test.go @@ -43,16 +43,15 @@ func setupSyncCommitteesServiceTest(t *testing.T, ctrl *gomock.Controller) (Sync return s, syncedDataManager, ethClock } -func getObjectsForSyncCommitteesServiceTest(t *testing.T, ctrl *gomock.Controller) (*state.CachingBeaconState, *cltypes.SyncCommitteeMessageWithGossipData) { +func getObjectsForSyncCommitteesServiceTest(t *testing.T, ctrl *gomock.Controller) (*state.CachingBeaconState, *SyncCommitteeMessageForGossip) { _, _, state := tests.GetBellatrixRandom() br, _ := state.BlockRoot() - msg := &cltypes.SyncCommitteeMessageWithGossipData{ + msg := &SyncCommitteeMessageForGossip{ SyncCommitteeMessage: &cltypes.SyncCommitteeMessage{ Slot: state.Slot(), BeaconBlockRoot: br, ValidatorIndex: 0, }, - GossipData: nil, ImmediateVerification: true, } return state, msg diff --git a/cl/phase1/network/services/sync_contribution_service.go b/cl/phase1/network/services/sync_contribution_service.go index 27da98964e1..542a533dac2 100644 --- a/cl/phase1/network/services/sync_contribution_service.go +++ b/cl/phase1/network/services/sync_contribution_service.go @@ -28,6 +28,7 @@ import ( "github.com/erigontech/erigon-lib/common" libcommon "github.com/erigontech/erigon-lib/common" + sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon/cl/beacon/beaconevents" "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" @@ -59,6 +60,13 @@ type syncContributionService struct { mu sync.Mutex } +// SignedContributionAndProofWithGossipData type represents SignedContributionAndProof with the gossip data where it's coming from. +type SignedContributionAndProofForGossip struct { + SignedContributionAndProof *cltypes.SignedContributionAndProof + Receiver *sentinel.Peer + ImmediateVerification bool +} + // NewSyncContributionService creates a new sync contribution service func NewSyncContributionService( syncedDataManager *synced_data.SyncedDataManager, @@ -82,7 +90,7 @@ func NewSyncContributionService( } // ProcessMessage processes a sync contribution message -func (s *syncContributionService) ProcessMessage(ctx context.Context, subnet *uint64, signedContribution *cltypes.SignedContributionAndProofWithGossipData) error { +func (s *syncContributionService) ProcessMessage(ctx context.Context, subnet *uint64, signedContribution *SignedContributionAndProofForGossip) error { s.mu.Lock() defer s.mu.Unlock() @@ -138,7 +146,7 @@ func (s *syncContributionService) ProcessMessage(ctx context.Context, subnet *ui return err } - aggregateVerificationData.GossipData = signedContribution.GossipData + aggregateVerificationData.SendingPeer = signedContribution.Receiver // further processing will be done after async signature verification aggregateVerificationData.F = func() { @@ -174,7 +182,7 @@ func (s *syncContributionService) ProcessMessage(ctx context.Context, subnet *ui func (s *syncContributionService) GetSignaturesOnContributionSignatures( headState *state.CachingBeaconState, contributionAndProof *cltypes.ContributionAndProof, - signedContribution *cltypes.SignedContributionAndProofWithGossipData, + signedContribution *SignedContributionAndProofForGossip, subcommiteePubsKeys []libcommon.Bytes48) (*AggregateVerificationData, error) { // [REJECT] The contribution_and_proof.selection_proof is a valid signature of the SyncAggregatorSelectionData derived from the contribution by the validator with index contribution_and_proof.aggregator_index. diff --git a/cl/phase1/network/services/sync_contribution_service_test.go b/cl/phase1/network/services/sync_contribution_service_test.go index 54835758d80..23194bbceb3 100644 --- a/cl/phase1/network/services/sync_contribution_service_test.go +++ b/cl/phase1/network/services/sync_contribution_service_test.go @@ -45,12 +45,12 @@ func setupSyncContributionServiceTest(t *testing.T, ctrl *gomock.Controller) (Sy return s, syncedDataManager, ethClock } -func getObjectsForSyncContributionServiceTest(t *testing.T, ctrl *gomock.Controller) (*state.CachingBeaconState, *cltypes.SignedContributionAndProofWithGossipData) { +func getObjectsForSyncContributionServiceTest(t *testing.T, ctrl *gomock.Controller) (*state.CachingBeaconState, *SignedContributionAndProofForGossip) { _, _, state := tests.GetBellatrixRandom() br, _ := state.BlockRoot() aggBits := make([]byte, 16) aggBits[0] = 1 - msg := &cltypes.SignedContributionAndProofWithGossipData{ + msg := &SignedContributionAndProofForGossip{ SignedContributionAndProof: &cltypes.SignedContributionAndProof{ Message: &cltypes.ContributionAndProof{ AggregatorIndex: 0, @@ -62,7 +62,6 @@ func getObjectsForSyncContributionServiceTest(t *testing.T, ctrl *gomock.Control }, }, }, - GossipData: nil, ImmediateVerification: true, } diff --git a/cl/phase1/network/services/voluntary_exit_service.go b/cl/phase1/network/services/voluntary_exit_service.go index da23aa8b313..bef603d8bdd 100644 --- a/cl/phase1/network/services/voluntary_exit_service.go +++ b/cl/phase1/network/services/voluntary_exit_service.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/erigontech/erigon-lib/common" + sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto" "github.com/erigontech/erigon/cl/beacon/beaconevents" "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" @@ -42,6 +43,13 @@ type voluntaryExitService struct { batchSignatureVerifier *BatchSignatureVerifier } +// SignedVoluntaryExitForGossip type represents SignedVoluntaryExit with the gossip data where it's coming from. +type SignedVoluntaryExitForGossip struct { + SignedVoluntaryExit *cltypes.SignedVoluntaryExit + Receiver *sentinel.Peer + ImmediateVerification bool +} + func NewVoluntaryExitService( operationsPool pool.OperationsPool, emitters *beaconevents.EventEmitter, @@ -60,7 +68,7 @@ func NewVoluntaryExitService( } } -func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedVoluntaryExitWithGossipData) error { +func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint64, msg *SignedVoluntaryExitForGossip) error { // ref: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#voluntary_exit voluntaryExit := msg.SignedVoluntaryExit.VoluntaryExit @@ -133,10 +141,10 @@ func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint6 } aggregateVerificationData := &AggregateVerificationData{ - Signatures: [][]byte{msg.SignedVoluntaryExit.Signature[:]}, - SignRoots: [][]byte{signingRoot[:]}, - Pks: [][]byte{pk[:]}, - GossipData: msg.GossipData, + Signatures: [][]byte{msg.SignedVoluntaryExit.Signature[:]}, + SignRoots: [][]byte{signingRoot[:]}, + Pks: [][]byte{pk[:]}, + SendingPeer: msg.Receiver, F: func() { s.operationsPool.VoluntaryExitsPool.Insert(voluntaryExit.ValidatorIndex, msg.SignedVoluntaryExit) s.emitters.Operation().SendVoluntaryExit(msg.SignedVoluntaryExit) diff --git a/cl/phase1/network/services/voluntary_exit_service_test.go b/cl/phase1/network/services/voluntary_exit_service_test.go index bbb5a3a14a2..fba930fe2fe 100644 --- a/cl/phase1/network/services/voluntary_exit_service_test.go +++ b/cl/phase1/network/services/voluntary_exit_service_test.go @@ -80,7 +80,7 @@ func (t *voluntaryExitTestSuite) TearDownTest() { func (t *voluntaryExitTestSuite) TestProcessMessage() { curEpoch := uint64(100) mockValidatorIndex := uint64(10) - mockMsg := &cltypes.SignedVoluntaryExitWithGossipData{ + mockMsg := &SignedVoluntaryExitForGossip{ SignedVoluntaryExit: &cltypes.SignedVoluntaryExit{ VoluntaryExit: &cltypes.VoluntaryExit{ Epoch: 1, @@ -88,10 +88,9 @@ func (t *voluntaryExitTestSuite) TestProcessMessage() { }, Signature: [96]byte{}, }, - GossipData: nil, ImmediateVerification: true, } - mockMsg2 := &cltypes.SignedVoluntaryExitWithGossipData{ + mockMsg2 := &SignedVoluntaryExitForGossip{ SignedVoluntaryExit: &cltypes.SignedVoluntaryExit{ VoluntaryExit: &cltypes.VoluntaryExit{ Epoch: 1, @@ -99,7 +98,6 @@ func (t *voluntaryExitTestSuite) TestProcessMessage() { }, Signature: [96]byte{}, }, - GossipData: nil, ImmediateVerification: true, } @@ -108,7 +106,7 @@ func (t *voluntaryExitTestSuite) TestProcessMessage() { tests := []struct { name string mock func() - msg *cltypes.SignedVoluntaryExitWithGossipData + msg *SignedVoluntaryExitForGossip wantErr bool err error }{ From 2cea51f65d94ae1c77e17dcd9281f6b6937e58c9 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Wed, 8 Jan 2025 20:22:31 +0800 Subject: [PATCH 63/94] rpcdaemon: Set miner on `eth_getBlockByNumber` on Polygon (#13335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #12690 ```bash ➜ ~ curl --request POST \ -s --url http://127.0.0.1:8545/ \ --header 'Content-Type: application/json' \ --data '{ "method": "eth_getBlockByNumber", "params": [ "0x3F515FC", false ], "id": 1, "jsonrpc": "2.0" }' | jq '.result.miner' "0x83d69448f88bf9c701c1b93f43e1f753d39b2632" ➜ ~ curl --request POST \ -s --url http://127.0.0.1:8545/ \ --header 'Content-Type: application/json' \ --data '{ "method": "eth_getBlockByHash", "params": [ "0xd925f567ee22d63a003f890af94c1e77e6d4d55a3bcef2b35b7b2317948d7f28", false ], "id": 1, "jsonrpc": "2.0" }' | jq '.result.miner' "0x83d69448f88bf9c701c1b93f43e1f753d39b2632" ``` --- turbo/jsonrpc/eth_block.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/turbo/jsonrpc/eth_block.go b/turbo/jsonrpc/eth_block.go index 187e0c95282..9819dd791fb 100644 --- a/turbo/jsonrpc/eth_block.go +++ b/turbo/jsonrpc/eth_block.go @@ -261,6 +261,12 @@ func (api *APIImpl) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber response[field] = nil } } + + if chainConfig.Bor != nil { + borConfig := chainConfig.Bor.(*borcfg.BorConfig) + response["miner"], _ = ecrecover(b.Header(), borConfig) + } + return response, err } @@ -319,18 +325,18 @@ func (api *APIImpl) GetBlockByHash(ctx context.Context, numberOrHash rpc.BlockNu } response, err := ethapi.RPCMarshalBlockEx(block, true, fullTx, borTx, borTxHash, additionalFields) - - if chainConfig.Bor != nil { - borConfig := chainConfig.Bor.(*borcfg.BorConfig) - response["miner"], _ = ecrecover(block.Header(), borConfig) - } - if err == nil && int64(number) == rpc.PendingBlockNumber.Int64() { // Pending blocks need to nil out a few fields for _, field := range []string{"hash", "nonce", "miner"} { response[field] = nil } } + + if chainConfig.Bor != nil { + borConfig := chainConfig.Bor.(*borcfg.BorConfig) + response["miner"], _ = ecrecover(block.Header(), borConfig) + } + return response, err } From 62c8e8193e4bac74ad4a875d4efb131825297f28 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Wed, 8 Jan 2025 22:03:13 +0100 Subject: [PATCH 64/94] Erigon 3: Purified states repressentation (#13227) Purifification of states works by pruning historical states from dangling nodes. The process is to collect all keys into a temporary MDBX with mapping `node -> layer` , then we iterate over each node again and remove all of them whose `node->layer` is not matching in the historical states DB. --------- Co-authored-by: alex.sharov --- RELEASE_INSTRUCTIONS.md | 8 + cmd/integration/commands/state_domains.go | 358 +++++++++++++++++++++- erigon-lib/state/aggregator.go | 6 +- erigon-lib/state/aggregator2.go | 20 +- erigon-lib/state/aggregator_test.go | 2 +- erigon-lib/state/domain.go | 22 +- erigon-lib/state/domain_committed.go | 12 +- erigon-lib/state/domain_shared.go | 4 +- erigon-lib/state/domain_test.go | 18 +- erigon-lib/state/merge.go | 8 +- erigon-lib/state/squeeze.go | 12 +- turbo/app/snapshots_cmd.go | 5 +- 12 files changed, 419 insertions(+), 56 deletions(-) diff --git a/RELEASE_INSTRUCTIONS.md b/RELEASE_INSTRUCTIONS.md index f4f064b12ad..07dbf96f359 100644 --- a/RELEASE_INSTRUCTIONS.md +++ b/RELEASE_INSTRUCTIONS.md @@ -31,3 +31,11 @@ In most cases, it is enough to bump minor version. In the file `ethdb/remote/remotedbserver/server.go` there is variable `KvServiceAPIVersion` that needs to be updated if there are any changes in the remote KV interface, or database schema, leading to data migrations. In most cases, it is enough to bump minor version. It is best to change both DB schema version and remove KV version together. + +## Purify the state domains if a regenration is done + +If a regenration is done, the state domains need to be purified. This can be done by running the following command: +```` +make integration +./build/bin/integration purify_domains --datadir= --replace-in-datadir +```` diff --git a/cmd/integration/commands/state_domains.go b/cmd/integration/commands/state_domains.go index e754165af75..0c673f48a3b 100644 --- a/cmd/integration/commands/state_domains.go +++ b/cmd/integration/commands/state_domains.go @@ -18,14 +18,20 @@ package commands import ( "context" + "encoding/binary" "encoding/hex" "errors" "fmt" + "os" + "path" "path/filepath" + "runtime" + "sort" "strings" + "github.com/erigontech/erigon-lib/etl" + "github.com/erigontech/erigon-lib/seg" state3 "github.com/erigontech/erigon-lib/state" - "github.com/spf13/cobra" "github.com/erigontech/erigon-lib/log/v3" @@ -33,8 +39,11 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/length" + downloadertype "github.com/erigontech/erigon-lib/downloader/snaptype" "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/mdbx" kv2 "github.com/erigontech/erigon-lib/kv/mdbx" + statelib "github.com/erigontech/erigon-lib/state" "github.com/erigontech/erigon/cmd/utils" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/state" @@ -52,12 +61,23 @@ func init() { withStartTx(readDomains) rootCmd.AddCommand(readDomains) + + withDataDir(purifyDomains) + purifyDomains.Flags().StringVar(&outDatadir, "out", "out-purified", "") + purifyDomains.Flags().BoolVar(&purifyOnlyCommitment, "only-commitment", true, "purify only commitment domain") + purifyDomains.Flags().BoolVar(&replaceInDatadir, "replace-in-datadir", false, "replace the purified domains directly in datadir (will remove .kvei and .bt too)") + purifyDomains.Flags().Float64Var(&minSkipRatioL0, "min-skip-ratio-l0", 0.1, "minimum ratio of keys to skip in L0") + rootCmd.AddCommand(purifyDomains) } // if trie variant is not hex, we could not have another rootHash with to verify it var ( - stepSize uint64 - lastStep uint64 + stepSize uint64 + lastStep uint64 + minSkipRatioL0 float64 + outDatadir string + purifyOnlyCommitment bool + replaceInDatadir bool ) // write command to just seek and query state by addr and domain from state db and files (if any) @@ -120,6 +140,322 @@ var readDomains = &cobra.Command{ }, } +var purifyDomains = &cobra.Command{ + Use: "purify_domains", + Short: `Regenerate kv files without repeating keys.`, + Example: "go run ./cmd/integration purify_domains --datadir=... --verbosity=3", + Args: cobra.ArbitraryArgs, + Run: func(cmd *cobra.Command, args []string) { + dirs := datadir.New(datadirCli) + // Iterate over all the files in dirs.SnapDomain and print them + domainDir := dirs.SnapDomain + + // make a temporary dir + tmpDir, err := os.MkdirTemp(dirs.Tmp, "purifyTemp") // make a temporary dir to store the keys + if err != nil { + fmt.Println("Error creating temporary directory: ", err) + return + } + // make a temporary DB to store the keys + + purifyDB := mdbx.MustOpen(tmpDir) + defer purifyDB.Close() + var purificationDomains []string + if purifyOnlyCommitment { + purificationDomains = []string{"commitment"} + } else { + purificationDomains = []string{"account", "storage" /*"code",*/, "commitment", "receipt"} + } + //purificationDomains := []string{"commitment"} + for _, domain := range purificationDomains { + if err := makePurifiableIndexDB(purifyDB, dirs, log.New(), domain); err != nil { + fmt.Println("Error making purifiable index DB: ", err) + return + } + } + for _, domain := range purificationDomains { + if err := makePurifiedDomains(purifyDB, dirs, log.New(), domain); err != nil { + fmt.Println("Error making purifiable index DB: ", err) + return + } + } + if err != nil { + fmt.Printf("error walking the path %q: %v\n", domainDir, err) + } + }, +} + +func makePurifiableIndexDB(db kv.RwDB, dirs datadir.Dirs, logger log.Logger, domain string) error { + var tbl string + switch domain { + case "account": + tbl = kv.MaxTxNum + case "storage": + tbl = kv.HeaderNumber + case "code": + tbl = kv.HeaderCanonical + case "commitment": + tbl = kv.HeaderTD + case "receipt": + tbl = kv.BadHeaderNumber + default: + return fmt.Errorf("invalid domain %s", domain) + } + // Iterate over all the files in dirs.SnapDomain and print them + filesNamesToIndex := []string{} + if err := filepath.Walk(dirs.SnapDomain, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip directories + if info.IsDir() { + return nil + } + if !strings.Contains(info.Name(), domain) { + return nil + } + // Here you can decide if you only want to process certain file extensions + // e.g., .kv files + if filepath.Ext(path) != ".kv" { + // Skip non-kv files if that's your domain’s format + return nil + } + + fmt.Printf("Add file to indexing of %s: %s\n", domain, path) + + filesNamesToIndex = append(filesNamesToIndex, info.Name()) + return nil + }); err != nil { + return fmt.Errorf("failed to walk through the domainDir %s: %w", domain, err) + } + + collector := etl.NewCollector("Purification", dirs.Tmp, etl.NewSortableBuffer(etl.BufferOptimalSize), logger) + defer collector.Close() + // sort the files by name + sort.Slice(filesNamesToIndex, func(i, j int) bool { + res, ok, _ := downloadertype.ParseFileName(dirs.SnapDomain, filesNamesToIndex[i]) + if !ok { + panic("invalid file name") + } + res2, ok, _ := downloadertype.ParseFileName(dirs.SnapDomain, filesNamesToIndex[j]) + if !ok { + panic("invalid file name") + } + return res.From < res2.From + }) + tx, err := db.BeginRw(context.Background()) + if err != nil { + return fmt.Errorf("failed to start transaction: %w", err) + } + defer tx.Rollback() + + // now start the file indexing + for i, fileName := range filesNamesToIndex { + if i == 0 { + continue // we can skip first layer as all the keys are already mapped to 0. + } + layerBytes := make([]byte, 4) + binary.BigEndian.PutUint32(layerBytes, uint32(i)) + count := 0 + + dec, err := seg.NewDecompressor(path.Join(dirs.SnapDomain, fileName)) + if err != nil { + return fmt.Errorf("failed to create decompressor: %w", err) + } + defer dec.Close() + getter := dec.MakeGetter() + fmt.Printf("Indexing file %s\n", fileName) + var buf []byte + for getter.HasNext() { + buf = buf[:0] + buf, _ = getter.Next(buf) + + collector.Collect(buf, layerBytes) + count++ + //fmt.Println("count: ", count, "keyLength: ", len(buf)) + if count%100000 == 0 { + fmt.Printf("Indexed %d keys in file %s\n", count, fileName) + } + // skip values + getter.Skip() + } + fmt.Printf("Indexed %d keys in file %s\n", count, fileName) + } + fmt.Println("Loading the keys to DB") + if err := collector.Load(tx, tbl, etl.IdentityLoadFunc, etl.TransformArgs{}); err != nil { + return fmt.Errorf("failed to load: %w", err) + } + + return tx.Commit() +} + +func makePurifiedDomains(db kv.RwDB, dirs datadir.Dirs, logger log.Logger, domainName string) error { + domain, err := kv.String2Domain(domainName) + if err != nil { + return err + } + + compressionType := statelib.Schema[domain].Compression + compressCfg := statelib.Schema[domain].CompressCfg + compressCfg.Workers = runtime.NumCPU() + + var tbl string + switch domainName { + case "account": + tbl = kv.MaxTxNum + case "storage": + tbl = kv.HeaderNumber + case "code": + tbl = kv.HeaderCanonical + case "commitment": + tbl = kv.HeaderTD + case "receipt": + tbl = kv.BadHeaderNumber + default: + return fmt.Errorf("invalid domainName %s", domainName) + } + // Iterate over all the files in dirs.SnapDomain and print them + filesNamesToPurify := []string{} + if err := filepath.Walk(dirs.SnapDomain, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip directories + if info.IsDir() { + return nil + } + if !strings.Contains(info.Name(), domainName) { + return nil + } + // Here you can decide if you only want to process certain file extensions + // e.g., .kv files + if filepath.Ext(path) != ".kv" { + // Skip non-kv files if that's your domainName’s format + return nil + } + + fmt.Printf("Add file to purification of %s: %s\n", domainName, path) + + filesNamesToPurify = append(filesNamesToPurify, info.Name()) + return nil + }); err != nil { + return fmt.Errorf("failed to walk through the domainDir %s: %w", domainName, err) + } + // sort the files by name + sort.Slice(filesNamesToPurify, func(i, j int) bool { + res, ok, _ := downloadertype.ParseFileName(dirs.SnapDomain, filesNamesToPurify[i]) + if !ok { + panic("invalid file name") + } + res2, ok, _ := downloadertype.ParseFileName(dirs.SnapDomain, filesNamesToPurify[j]) + if !ok { + panic("invalid file name") + } + return res.From < res2.From + }) + + tx, err := db.BeginRo(context.Background()) + if err != nil { + return fmt.Errorf("failed to start transaction: %w", err) + } + defer tx.Rollback() + outD := datadir.New(outDatadir) + + // now start the file indexing + for currentLayer, fileName := range filesNamesToPurify { + count := 0 + skipped := 0 + + dec, err := seg.NewDecompressor(filepath.Join(dirs.SnapDomain, fileName)) + if err != nil { + return fmt.Errorf("failed to create decompressor: %w", err) + } + defer dec.Close() + getter := dec.MakeGetter() + + valuesComp, err := seg.NewCompressor(context.Background(), "Purification", filepath.Join(outD.SnapDomain, fileName), dirs.Tmp, compressCfg, log.LvlTrace, log.New()) + if err != nil { + return fmt.Errorf("create %s values compressor: %w", filepath.Join(outD.SnapDomain, fileName), err) + } + defer valuesComp.Close() + + comp := seg.NewWriter(valuesComp, compressionType) + defer comp.Close() + + fmt.Printf("Indexing file %s\n", fileName) + var ( + bufKey []byte + bufVal []byte + ) + + var layer uint32 + for getter.HasNext() { + // get the key and value for the current entry + bufKey = bufKey[:0] + bufKey, _ = getter.Next(bufKey) + bufVal = bufVal[:0] + bufVal, _ = getter.Next(bufVal) + + layerBytes, err := tx.GetOne(tbl, bufKey) + if err != nil { + return fmt.Errorf("failed to get key %x: %w", bufKey, err) + } + // if the key is not found, then the layer is 0 + layer = 0 + if len(layerBytes) == 4 { + layer = binary.BigEndian.Uint32(layerBytes) + } + if layer != uint32(currentLayer) { + skipped++ + continue + } + if err := comp.AddWord(bufKey); err != nil { + return fmt.Errorf("failed to add key %x: %w", bufKey, err) + } + if err := comp.AddWord(bufVal); err != nil { + return fmt.Errorf("failed to add val %x: %w", bufVal, err) + } + count++ + if count%100000 == 0 { + skipRatio := float64(skipped) / float64(count) + fmt.Printf("Indexed %d keys, skipped %d, in file %s. skip ratio: %.2f\n", count, skipped, fileName, skipRatio) + } + } + + skipRatio := float64(skipped) / float64(count) + if skipRatio < minSkipRatioL0 && currentLayer == 0 { + fmt.Printf("Skip ratio %.2f is less than min-skip-ratio-l0 %.2f, skipping the domainName and file %s\n", skipRatio, minSkipRatioL0, fileName) + return nil + } + fmt.Printf("Loaded %d keys in file %s. now compressing...\n", count, fileName) + if err := comp.Compress(); err != nil { + return fmt.Errorf("failed to compress: %w", err) + } + fmt.Printf("Compressed %d keys in file %s\n", count, fileName) + comp.Close() + if replaceInDatadir { + fmt.Printf("Replacing the file %s in datadir\n", fileName) + if err := os.Rename(filepath.Join(outD.SnapDomain, fileName), filepath.Join(dirs.SnapDomain, fileName)); err != nil { + return fmt.Errorf("failed to replace the file %s: %w", fileName, err) + } + kveiFile := strings.ReplaceAll(fileName, ".kv", ".kvei") + btFile := strings.ReplaceAll(fileName, ".kv", ".bt") + kviFile := strings.ReplaceAll(fileName, ".kv", ".kvi") + removeManyIgnoreError( + filepath.Join(dirs.SnapDomain, fileName+".torrent"), + filepath.Join(dirs.SnapDomain, btFile), + filepath.Join(dirs.SnapDomain, btFile+".torrent"), + filepath.Join(dirs.SnapDomain, kveiFile), + filepath.Join(dirs.SnapDomain, kveiFile+".torrent"), + filepath.Join(dirs.SnapDomain, kviFile), + filepath.Join(dirs.SnapDomain, kviFile+".torrent"), + ) + fmt.Printf("Removed the files %s and %s\n", kveiFile, btFile) + } + } + return nil +} + func requestDomains(chainDb, stateDb kv.RwDB, ctx context.Context, readDomain string, addrs [][]byte, logger log.Logger) error { sn, bsn, agg, _, _, _ := allSnapshots(ctx, chainDb, logger) defer sn.Close() @@ -181,3 +517,19 @@ func requestDomains(chainDb, stateDb kv.RwDB, ctx context.Context, readDomain st } return nil } + +func removeMany(filePaths ...string) error { + for _, filePath := range filePaths { + if err := os.Remove(filePath); err != nil { + _, fileName := filepath.Split(filePath) + return fmt.Errorf("failed to remove the file: %s, %w", fileName, err) + } + } + return nil +} + +func removeManyIgnoreError(filePaths ...string) { + for _, filePath := range filePaths { + os.Remove(filePath) + } +} diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 870cb049934..8f17d44acc9 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -314,7 +314,7 @@ func (a *Aggregator) SetCollateAndBuildWorkers(i int) { a.collateAndBuildWorkers func (a *Aggregator) SetMergeWorkers(i int) { a.mergeWorkers = i } func (a *Aggregator) SetCompressWorkers(i int) { for _, d := range a.d { - d.compressCfg.Workers = i + d.CompressCfg.Workers = i d.History.compressorCfg.Workers = i d.History.InvertedIndex.compressorCfg.Workers = i } @@ -504,7 +504,7 @@ func (sf AggV3StaticFiles) CleanupOnError() { } func (a *Aggregator) buildFiles(ctx context.Context, step uint64) error { - a.logger.Debug("[agg] collate and build", "step", step, "collate_workers", a.collateAndBuildWorkers, "merge_workers", a.mergeWorkers, "compress_workers", a.d[kv.AccountsDomain].compressCfg.Workers) + a.logger.Debug("[agg] collate and build", "step", step, "collate_workers", a.collateAndBuildWorkers, "merge_workers", a.mergeWorkers, "compress_workers", a.d[kv.AccountsDomain].CompressCfg.Workers) var ( logEvery = time.NewTicker(time.Second * 30) @@ -690,7 +690,7 @@ func (a *Aggregator) BuildFiles2(ctx context.Context, fromStep, toStep uint64) e } func (a *Aggregator) mergeLoopStep(ctx context.Context, toTxNum uint64) (somethingDone bool, err error) { - a.logger.Debug("[agg] merge", "collate_workers", a.collateAndBuildWorkers, "merge_workers", a.mergeWorkers, "compress_workers", a.d[kv.AccountsDomain].compressCfg.Workers) + a.logger.Debug("[agg] merge", "collate_workers", a.collateAndBuildWorkers, "merge_workers", a.mergeWorkers, "compress_workers", a.d[kv.AccountsDomain].CompressCfg.Workers) aggTx := a.BeginFilesRo() defer aggTx.Close() diff --git a/erigon-lib/state/aggregator2.go b/erigon-lib/state/aggregator2.go index f1f51d09121..aa8983a7cd6 100644 --- a/erigon-lib/state/aggregator2.go +++ b/erigon-lib/state/aggregator2.go @@ -71,8 +71,8 @@ var Schema = map[kv.Domain]domainCfg{ IndexList: AccessorBTree | AccessorExistence, crossDomainIntegrity: domainIntegrityCheck, - compression: seg.CompressNone, - compressCfg: DomainCompressCfg, + Compression: seg.CompressNone, + CompressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblAccountHistoryVals, @@ -92,8 +92,8 @@ var Schema = map[kv.Domain]domainCfg{ name: kv.StorageDomain, valuesTable: kv.TblStorageVals, IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, + Compression: seg.CompressKeys, + CompressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblStorageHistoryVals, @@ -113,8 +113,8 @@ var Schema = map[kv.Domain]domainCfg{ name: kv.CodeDomain, valuesTable: kv.TblCodeVals, IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet - compressCfg: DomainCompressCfg, + Compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet + CompressCfg: DomainCompressCfg, largeValues: true, hist: histCfg{ @@ -135,8 +135,8 @@ var Schema = map[kv.Domain]domainCfg{ name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, IndexList: AccessorHashMap, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, + Compression: seg.CompressKeys, + CompressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblCommitmentHistoryVals, @@ -157,8 +157,8 @@ var Schema = map[kv.Domain]domainCfg{ name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, - compressCfg: DomainCompressCfg, + Compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, + CompressCfg: DomainCompressCfg, hist: histCfg{ valuesTable: kv.TblReceiptHistoryVals, diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 759df463408..8a89bcb56eb 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1270,7 +1270,7 @@ func TestAggregator_RebuildCommitmentBasedOnFiles(t *testing.T) { roots := make([]common.Hash, 0) // collect latest root from each available file - compression := ac.d[kv.CommitmentDomain].d.compression + compression := ac.d[kv.CommitmentDomain].d.Compression fnames := []string{} for _, f := range ac.d[kv.CommitmentDomain].files { var k, stateVal []byte diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 7d9c2770f0a..58d0e1ffc9e 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -95,8 +95,8 @@ type domainCfg struct { hist histCfg name kv.Domain - compression seg.FileCompression - compressCfg seg.Cfg + Compression seg.FileCompression + CompressCfg seg.Cfg IndexList Accessors // list of indexes for given domain valuesTable string // bucket to store domain values; key -> inverted_step + values (Dupsort) largeValues bool @@ -385,7 +385,7 @@ func (d *Domain) openDirtyFiles() (err error) { if toStep == 0 && d.filenameBase == "commitment" { btM = 128 } - if item.bindex, err = OpenBtreeIndexWithDecompressor(fPath, btM, item.decompressor, d.compression); err != nil { + if item.bindex, err = OpenBtreeIndexWithDecompressor(fPath, btM, item.decompressor, d.Compression); err != nil { _, fName := filepath.Split(fPath) d.logger.Warn("[agg] Domain.openDirtyFiles", "err", err, "f", fName) // don't interrupt on error. other files may be good @@ -774,7 +774,7 @@ func (d *Domain) collateETL(ctx context.Context, stepFrom, stepTo uint64, wal *e }() coll.valuesPath = d.kvFilePath(stepFrom, stepTo) - if coll.valuesComp, err = seg.NewCompressor(ctx, d.filenameBase+".domain.collate", coll.valuesPath, d.dirs.Tmp, d.compressCfg, log.LvlTrace, d.logger); err != nil { + if coll.valuesComp, err = seg.NewCompressor(ctx, d.filenameBase+".domain.collate", coll.valuesPath, d.dirs.Tmp, d.CompressCfg, log.LvlTrace, d.logger); err != nil { return Collation{}, fmt.Errorf("create %s values compressor: %w", d.filenameBase, err) } @@ -783,7 +783,7 @@ func (d *Domain) collateETL(ctx context.Context, stepFrom, stepTo uint64, wal *e //comp := seg.NewWriter(coll.valuesComp, seg.CompressNone) // compress := seg.CompressNone if stepTo-stepFrom > DomainMinStepsToCompress { - compress = d.compression + compress = d.Compression } comp := seg.NewWriter(coll.valuesComp, compress) @@ -886,7 +886,7 @@ func (d *Domain) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv }() coll.valuesPath = d.kvFilePath(step, step+1) - if coll.valuesComp, err = seg.NewCompressor(ctx, d.filenameBase+".domain.collate", coll.valuesPath, d.dirs.Tmp, d.compressCfg, log.LvlTrace, d.logger); err != nil { + if coll.valuesComp, err = seg.NewCompressor(ctx, d.filenameBase+".domain.collate", coll.valuesPath, d.dirs.Tmp, d.CompressCfg, log.LvlTrace, d.logger); err != nil { return Collation{}, fmt.Errorf("create %s values compressor: %w", d.filenameBase, err) } @@ -1068,7 +1068,7 @@ func (d *Domain) buildFileRange(ctx context.Context, stepFrom, stepTo uint64, co btM = 128 } - bt, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesDecomp, d.compression, *d.salt, ps, d.dirs.Tmp, d.logger, d.noFsync) + bt, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesDecomp, d.Compression, *d.salt, ps, d.dirs.Tmp, d.logger, d.noFsync) if err != nil { return StaticFiles{}, fmt.Errorf("build %s .bt idx: %w", d.filenameBase, err) } @@ -1170,7 +1170,7 @@ func (d *Domain) buildFiles(ctx context.Context, step uint64, collation Collatio if step == 0 && d.filenameBase == "commitment" { btM = 128 } - bt, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesDecomp, d.compression, *d.salt, ps, d.dirs.Tmp, d.logger, d.noFsync) + bt, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesDecomp, d.Compression, *d.salt, ps, d.dirs.Tmp, d.logger, d.noFsync) if err != nil { return StaticFiles{}, fmt.Errorf("build %s .bt idx: %w", d.filenameBase, err) } @@ -1211,7 +1211,7 @@ func (d *Domain) buildAccessor(ctx context.Context, fromStep, toStep uint64, dat Salt: d.salt, NoFsync: d.noFsync, } - return buildAccessor(ctx, data, d.compression, idxPath, false, cfg, ps, d.logger) + return buildAccessor(ctx, data, d.Compression, idxPath, false, cfg, ps, d.logger) } func (d *Domain) missedBtreeAccessors() (l []*filesItem) { @@ -1274,7 +1274,7 @@ func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps g.Go(func() error { fromStep, toStep := item.startTxNum/d.aggregationStep, item.endTxNum/d.aggregationStep idxPath := d.kvBtFilePath(fromStep, toStep) - if err := BuildBtreeIndexWithDecompressor(idxPath, item.decompressor, d.compression, ps, d.dirs.Tmp, *d.salt, d.logger, d.noFsync); err != nil { + if err := BuildBtreeIndexWithDecompressor(idxPath, item.decompressor, d.Compression, ps, d.dirs.Tmp, *d.salt, d.logger, d.noFsync); err != nil { return fmt.Errorf("failed to build btree index for %s: %w", item.decompressor.FileName(), err) } return nil @@ -1613,7 +1613,7 @@ func (dt *DomainRoTx) statelessGetter(i int) *seg.Reader { } r := dt.getters[i] if r == nil { - r = seg.NewReader(dt.files[i].src.decompressor.MakeGetter(), dt.d.compression) + r = seg.NewReader(dt.files[i].src.decompressor.MakeGetter(), dt.d.Compression) dt.getters[i] = r } return r diff --git a/erigon-lib/state/domain_committed.go b/erigon-lib/state/domain_committed.go index 8daef3f6b26..b9456dc231d 100644 --- a/erigon-lib/state/domain_committed.go +++ b/erigon-lib/state/domain_committed.go @@ -255,7 +255,7 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto if _, ok := accountFileMap[f.startTxNum]; !ok { accountFileMap[f.startTxNum] = make(map[uint64]*seg.Reader) } - accountFileMap[f.startTxNum][f.endTxNum] = seg.NewReader(f.decompressor.MakeGetter(), accounts.d.compression) + accountFileMap[f.startTxNum][f.endTxNum] = seg.NewReader(f.decompressor.MakeGetter(), accounts.d.Compression) } } storageFileMap := make(map[uint64]map[uint64]*seg.Reader) @@ -264,12 +264,12 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto if _, ok := storageFileMap[f.startTxNum]; !ok { storageFileMap[f.startTxNum] = make(map[uint64]*seg.Reader) } - storageFileMap[f.startTxNum][f.endTxNum] = seg.NewReader(f.decompressor.MakeGetter(), storage.d.compression) + storageFileMap[f.startTxNum][f.endTxNum] = seg.NewReader(f.decompressor.MakeGetter(), storage.d.Compression) } } - ms := seg.NewReader(mergedStorage.decompressor.MakeGetter(), storage.d.compression) - ma := seg.NewReader(mergedAccount.decompressor.MakeGetter(), accounts.d.compression) + ms := seg.NewReader(mergedStorage.decompressor.MakeGetter(), storage.d.Compression) + ma := seg.NewReader(mergedAccount.decompressor.MakeGetter(), accounts.d.Compression) dt.d.logger.Debug("prepare commitmentValTransformDomain", "merge", rng.String("range", dt.d.aggregationStep), "Mstorage", hadToLookupStorage, "Maccount", hadToLookupAccount) vt := func(valBuf []byte, keyFromTxNum, keyEndTxNum uint64) (transValBuf []byte, err error) { @@ -285,7 +285,7 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto if dirty == nil { return nil, fmt.Errorf("dirty storage file not found %d-%d", keyFromTxNum/dt.d.aggregationStep, keyEndTxNum/dt.d.aggregationStep) } - sig = seg.NewReader(dirty.decompressor.MakeGetter(), storage.d.compression) + sig = seg.NewReader(dirty.decompressor.MakeGetter(), storage.d.Compression) storageFileMap[keyFromTxNum][keyEndTxNum] = sig } @@ -298,7 +298,7 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto if dirty == nil { return nil, fmt.Errorf("dirty account file not found %d-%d", keyFromTxNum/dt.d.aggregationStep, keyEndTxNum/dt.d.aggregationStep) } - aig = seg.NewReader(dirty.decompressor.MakeGetter(), accounts.d.compression) + aig = seg.NewReader(dirty.decompressor.MakeGetter(), accounts.d.Compression) accountFileMap[keyFromTxNum][keyEndTxNum] = aig } diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index e570c8c7263..495caa8462e 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -491,8 +491,8 @@ func (sd *SharedDomains) replaceShortenedKeysInBranch(prefix []byte, branch comm sd.logger.Crit("dereference key during commitment read", "failed", err.Error()) return nil, err } - storageGetter := seg.NewReader(storageItem.decompressor.MakeGetter(), sto.d.compression) - accountGetter := seg.NewReader(accountItem.decompressor.MakeGetter(), acc.d.compression) + storageGetter := seg.NewReader(storageItem.decompressor.MakeGetter(), sto.d.Compression) + accountGetter := seg.NewReader(accountItem.decompressor.MakeGetter(), acc.d.Compression) metricI := 0 for i, f := range sd.aggTx.d[kv.CommitmentDomain].files { if i > 5 { diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 18451c4bc6b..cf125bd5728 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -135,7 +135,7 @@ func testCollationBuild(t *testing.T, compressDomainVals bool) { ctx := context.Background() if compressDomainVals { - d.compression = seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressKeys | seg.CompressVals } tx, err := db.BeginRw(ctx) @@ -206,7 +206,7 @@ func testCollationBuild(t *testing.T, compressDomainVals bool) { defer sf.CleanupOnError() c.Close() - g := seg.NewReader(sf.valuesDecomp.MakeGetter(), d.compression) + g := seg.NewReader(sf.valuesDecomp.MakeGetter(), d.Compression) g.Reset(0) var words []string for g.HasNext() { @@ -1122,7 +1122,7 @@ func TestDomain_CollationBuildInMem(t *testing.T) { defer sf.CleanupOnError() c.Close() - g := seg.NewReader(sf.valuesDecomp.MakeGetter(), d.compression) + g := seg.NewReader(sf.valuesDecomp.MakeGetter(), d.Compression) g.Reset(0) var words []string for g.HasNext() { @@ -1459,7 +1459,7 @@ func TestDomain_GetAfterAggregation(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals d.filenameBase = kv.FileCommitmentDomain dc := d.BeginFilesRo() @@ -1529,7 +1529,7 @@ func TestDomainRange(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals d.filenameBase = kv.FileAccountDomain dc := d.BeginFilesRo() @@ -1624,7 +1624,7 @@ func TestDomain_CanPruneAfterAggregation(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressKeys | seg.CompressVals d.filenameBase = kv.FileCommitmentDomain dc := d.BeginFilesRo() @@ -1720,7 +1720,7 @@ func TestDomain_PruneAfterAggregation(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals dc := d.BeginFilesRo() defer dc.Close() @@ -1863,7 +1863,7 @@ func TestDomain_PruneProgress(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressKeys | seg.CompressVals + d.Compression = seg.CompressKeys | seg.CompressVals dc := d.BeginFilesRo() defer dc.Close() @@ -2479,7 +2479,7 @@ func TestDomainContext_findShortenedKey(t *testing.T) { lastFile := findFile(st, en) require.NotNilf(t, lastFile, "%d-%d", st/dc.d.aggregationStep, en/dc.d.aggregationStep) - lf := seg.NewReader(lastFile.decompressor.MakeGetter(), d.compression) + lf := seg.NewReader(lastFile.decompressor.MakeGetter(), d.Compression) shortenedKey, found := dc.findShortenedKey([]byte(key), lf, lastFile) require.Truef(t, found, "key %d/%d %x file %d %d %s", ki, len(data), []byte(key), lastFile.startTxNum, lastFile.endTxNum, lastFile.decompressor.FileName()) diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go index 2c22e540a30..bfa69b354a3 100644 --- a/erigon-lib/state/merge.go +++ b/erigon-lib/state/merge.go @@ -406,12 +406,12 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h fromStep, toStep := r.values.from/r.aggStep, r.values.to/r.aggStep kvFilePath := dt.d.kvFilePath(fromStep, toStep) - kvFile, err := seg.NewCompressor(ctx, "merge domain "+dt.d.filenameBase, kvFilePath, dt.d.dirs.Tmp, dt.d.compressCfg, log.LvlTrace, dt.d.logger) + kvFile, err := seg.NewCompressor(ctx, "merge domain "+dt.d.filenameBase, kvFilePath, dt.d.dirs.Tmp, dt.d.CompressCfg, log.LvlTrace, dt.d.logger) if err != nil { return nil, nil, nil, fmt.Errorf("merge %s compressor: %w", dt.d.filenameBase, err) } - compression := dt.d.compression + compression := dt.d.Compression if toStep-fromStep < DomainMinStepsToCompress { compression = seg.CompressNone } @@ -425,7 +425,7 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h var cp CursorHeap heap.Init(&cp) for _, item := range domainFiles { - g := seg.NewReader(item.decompressor.MakeGetter(), dt.d.compression) + g := seg.NewReader(item.decompressor.MakeGetter(), dt.d.Compression) g.Reset(0) if g.HasNext() { key, _ := g.Next(nil) @@ -521,7 +521,7 @@ func (dt *DomainRoTx) mergeFiles(ctx context.Context, domainFiles, indexFiles, h if toStep == 0 && dt.d.filenameBase == "commitment" { btM = 128 } - valuesIn.bindex, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesIn.decompressor, dt.d.compression, *dt.d.salt, ps, dt.d.dirs.Tmp, dt.d.logger, dt.d.noFsync) + valuesIn.bindex, err = CreateBtreeIndexWithDecompressor(btPath, btM, valuesIn.decompressor, dt.d.Compression, *dt.d.salt, ps, dt.d.dirs.Tmp, dt.d.logger, dt.d.noFsync) if err != nil { return nil, nil, nil, fmt.Errorf("merge %s btindex [%d-%d]: %w", dt.d.filenameBase, r.values.from, r.values.to, err) } diff --git a/erigon-lib/state/squeeze.go b/erigon-lib/state/squeeze.go index ad929203d53..4a611627ed5 100644 --- a/erigon-lib/state/squeeze.go +++ b/erigon-lib/state/squeeze.go @@ -75,8 +75,8 @@ func (a *Aggregator) sqeezeDomainFile(ctx context.Context, domain kv.Domain, fro panic("please use SqueezeCommitmentFiles func") } - compression := a.d[domain].compression - compressCfg := a.d[domain].compressCfg + compression := a.d[domain].Compression + compressCfg := a.d[domain].CompressCfg a.logger.Info("[sqeeze] file", "f", to, "cfg", compressCfg, "c", compression) decompressor, err := seg.NewDecompressor(from) @@ -200,24 +200,24 @@ func (ac *AggregatorRoTx) SqueezeCommitmentFiles() error { err = func() error { steps := cf.endTxNum/ac.a.aggregationStep - cf.startTxNum/ac.a.aggregationStep - compression := commitment.d.compression + compression := commitment.d.Compression if steps < DomainMinStepsToCompress { compression = seg.CompressNone } ac.a.logger.Info("[squeeze_migration] file start", "original", cf.decompressor.FileName(), - "progress", fmt.Sprintf("%d/%d", ri+1, len(ranges)), "compress_cfg", commitment.d.compressCfg, "compress", compression) + "progress", fmt.Sprintf("%d/%d", ri+1, len(ranges)), "compress_cfg", commitment.d.CompressCfg, "compress", compression) originalPath := cf.decompressor.FilePath() squeezedTmpPath := originalPath + sqExt + ".tmp" squeezedCompr, err := seg.NewCompressor(context.Background(), "squeeze", squeezedTmpPath, ac.a.dirs.Tmp, - commitment.d.compressCfg, log.LvlInfo, commitment.d.logger) + commitment.d.CompressCfg, log.LvlInfo, commitment.d.logger) if err != nil { return err } defer squeezedCompr.Close() - writer := seg.NewWriter(squeezedCompr, commitment.d.compression) + writer := seg.NewWriter(squeezedCompr, commitment.d.Compression) reader := seg.NewReader(cf.decompressor.MakeGetter(), compression) reader.Reset(0) diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 6fa77e759cf..3ea6d4366b1 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -951,7 +951,7 @@ func doMeta(cliCtx *cli.Context) error { panic(err) } defer src.Close() - log.Info("meta", "count", src.Count(), "size", datasize.ByteSize(src.Size()).HumanReadable(), "serialized_dict", datasize.ByteSize(src.SerializedDictSize()).HumanReadable(), "dict_words", src.DictWords(), "name", src.FileName()) + log.Info("meta", "count", src.Count(), "size", datasize.ByteSize(src.Size()).HumanReadable(), "serialized_dict", datasize.ByteSize(src.SerializedDictSize()).HumanReadable(), "dict_words", src.DictWords(), "name", src.FileName(), "detected_compression_type", seg.DetectCompressType(src.MakeGetter())) } else if strings.HasSuffix(fname, ".bt") { kvFPath := strings.TrimSuffix(fname, ".bt") + ".kv" src, err := seg.NewDecompressor(kvFPath) @@ -1262,6 +1262,9 @@ func doCompress(cliCtx *cli.Context) error { if dbg.EnvBool("OnlyVals", false) { compression = seg.CompressVals } + if dbg.EnvBool("NoCompress", false) { + compression = seg.CompressNone + } logger.Info("[compress] file", "datadir", dirs.DataDir, "f", f, "cfg", compressCfg) c, err := seg.NewCompressor(ctx, "compress", f, dirs.Tmp, compressCfg, log.LvlInfo, logger) From a469e91c30b2507110ffb56efc8d7880873e0e5b Mon Sep 17 00:00:00 2001 From: Ilya Mikheev <54912776+JkLondon@users.noreply.github.com> Date: Thu, 9 Jan 2025 05:44:23 +0100 Subject: [PATCH 65/94] p2p receipts by default (#13350) Co-authored-by: JkLondon --- p2p/sentry/sentry_multi_client/sentry_multi_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client.go b/p2p/sentry/sentry_multi_client/sentry_multi_client.go index 6fce17787a0..7db44bdd60f 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client.go @@ -568,7 +568,7 @@ func (cs *MultiClient) getBlockBodies66(ctx context.Context, inreq *proto_sentry } var ( - EnableP2PReceipts = dbg.EnvBool("P2P_RECEIPTS", false) + EnableP2PReceipts = dbg.EnvBool("P2P_RECEIPTS", true) ) func (cs *MultiClient) getReceipts66(ctx context.Context, inreq *proto_sentry.InboundMessage, sentryClient proto_sentry.SentryClient) error { From e00c814534f8160d1f132e9c69faf4ea1edffc1f Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:08:40 +0000 Subject: [PATCH 66/94] polygon: astrid fix --no-downloader issue due to bridge snapshot store bug (#13351) Running Amoy with --no-downloader failed with trie root mismatch because the bridge didn't return correct events mapping for the first block with events which is 35072. This was because the snapshot store incorrectly checked in snapshots instead of in the DB store when Events was called with start=0 --- polygon/bridge/snapshot_store.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polygon/bridge/snapshot_store.go b/polygon/bridge/snapshot_store.go index 30d66fd0256..2b0be23f310 100644 --- a/polygon/bridge/snapshot_store.go +++ b/polygon/bridge/snapshot_store.go @@ -266,7 +266,8 @@ func (s *SnapshotStore) BlockEventIdsRange(ctx context.Context, blockNum uint64) } func (s *SnapshotStore) Events(ctx context.Context, start, end uint64) ([][]byte, error) { - if start > s.LastFrozenEventId() { + lastFrozenEventId := s.LastFrozenEventId() + if start > lastFrozenEventId || lastFrozenEventId == 0 { return s.Store.Events(ctx, start, end) } From 91bda3a305478a6794c6432f79126a012d5b1d23 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Thu, 9 Jan 2025 12:19:01 +0100 Subject: [PATCH 67/94] Added gas limit cap to txs in txpool (#13352) fixes a potential vulnerability --- txnprovider/txpool/pool.go | 8 +++ txnprovider/txpool/pool_test.go | 82 +++++++++++++++++++---- txnprovider/txpool/txpoolcfg/txpoolcfg.go | 1 + 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/txnprovider/txpool/pool.go b/txnprovider/txpool/pool.go index 6a6b8bbadf5..04126223a0c 100644 --- a/txnprovider/txpool/pool.go +++ b/txnprovider/txpool/pool.go @@ -901,6 +901,7 @@ func (p *TxPool) validateTx(txn *TxnSlot, isLocal bool, stateCache kvcache.Cache } return txpoolcfg.UnderPriced } + gas, reason := txpoolcfg.CalcIntrinsicGas(uint64(txn.DataLen), uint64(txn.DataNonZeroLen), uint64(authorizationLen), nil, txn.Creation, true, true, isShanghai) if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx intrinsic gas idHash=%x gas=%d", txn.IDHash, gas)) @@ -917,6 +918,13 @@ func (p *TxPool) validateTx(txn *TxnSlot, isLocal bool, stateCache kvcache.Cache } return txpoolcfg.IntrinsicGas } + if txn.Gas > p.blockGasLimit.Load() { + if txn.Traced { + p.logger.Info(fmt.Sprintf("TX TRACING: validateTx txn.gas > block gas limit idHash=%x gas=%d, block gas limit=%d", txn.IDHash, txn.Gas, p.blockGasLimit.Load())) + } + return txpoolcfg.GasLimitTooHigh + } + if !isLocal && uint64(p.all.count(txn.SenderID)) > p.cfg.AccountSlots { if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx marked as spamming idHash=%x slots=%d, limit=%d", txn.IDHash, p.all.count(txn.SenderID), p.cfg.AccountSlots)) diff --git a/txnprovider/txpool/pool_test.go b/txnprovider/txpool/pool_test.go index fc945fc2822..4c1cc236847 100644 --- a/txnprovider/txpool/pool_test.go +++ b/txnprovider/txpool/pool_test.go @@ -733,7 +733,7 @@ func TestShanghaiValidateTxn(t *testing.T) { asrt.NoError(err) view, err := cache.View(ctx, tx) asrt.NoError(err) - + pool.blockGasLimit.Store(30_000_000) reason := pool.validateTx(txn, false, view) if reason != test.expected { @@ -743,6 +743,64 @@ func TestShanghaiValidateTxn(t *testing.T) { } } +func TestTooHighGasLimitTxnValidation(t *testing.T) { + assert, require := assert.New(t), require.New(t) + ch := make(chan Announcements, 100) + coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + db := memdb.NewTestPoolDB(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cfg := txpoolcfg.DefaultConfig + sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, nil, nil, nil, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) + assert.NoError(err) + require.True(pool != nil) + var stateVersionID uint64 = 0 + pendingBaseFee := uint64(200000) + // start blocks from 0, set empty hash - then kvcache will also work on this + h1 := gointerfaces.ConvertHashToH256([32]byte{}) + change := &remote.StateChangeBatch{ + StateVersionId: stateVersionID, + PendingBlockBaseFee: pendingBaseFee, + BlockGasLimit: 1000000, + ChangeBatch: []*remote.StateChange{ + {BlockHeight: 0, BlockHash: h1}, + }, + } + var addr [20]byte + addr[0] = 1 + v := types2.EncodeAccountBytesV3(2, uint256.NewInt(1*common.Ether), make([]byte, 32), 1) + change.ChangeBatch[0].Changes = append(change.ChangeBatch[0].Changes, &remote.AccountChange{ + Action: remote.Action_UPSERT, + Address: gointerfaces.ConvertAddressToH160(addr), + Data: v, + }) + + tx, err := db.BeginRw(ctx) + require.NoError(err) + defer tx.Rollback() + + err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{}) + assert.NoError(err) + + { + var txnSlots TxnSlots + txnSlot := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 1000001, + Nonce: 2, + } + txnSlot.IDHash[0] = 1 + txnSlots.Append(txnSlot, addr[:], true) + + reasons, err := pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(err) + assert.Len(reasons, 1) + assert.Equal(reasons[0], txpoolcfg.GasLimitTooHigh) + } +} + func TestSetCodeTxnValidationWithLargeAuthorizationValues(t *testing.T) { maxUint256 := new(uint256.Int).SetAllOne() ctx, cancel := context.WithCancel(context.Background()) @@ -756,6 +814,7 @@ func TestSetCodeTxnValidationWithLargeAuthorizationValues(t *testing.T) { pool, err := New(ctx, ch, nil, coreDB, cfg, cache, chainID, common.Big0 /* shanghaiTime */, nil, /* agraBlock */ common.Big0 /* cancunTime */, common.Big0 /* pragueTime */, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, logger, WithFeeCalculator(nil)) assert.NoError(t, err) + pool.blockGasLimit.Store(30_000_000) tx, err := coreDB.BeginRw(ctx) defer tx.Rollback() assert.NoError(t, err) @@ -803,6 +862,7 @@ func TestBlobTxnReplacement(t *testing.T) { sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0, nil, common.Big0, nil, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) assert.NoError(err) + require.True(pool != nil) var stateVersionID uint64 = 0 @@ -810,7 +870,7 @@ func TestBlobTxnReplacement(t *testing.T) { change := &remote.StateChangeBatch{ StateVersionId: stateVersionID, PendingBlockBaseFee: 200_000, - BlockGasLimit: 1000000, + BlockGasLimit: math.MaxUint64, PendingBlobFeePerGas: 100_000, ChangeBatch: []*remote.StateChange{ {BlockHeight: 0, BlockHash: h1}, @@ -958,7 +1018,6 @@ func makeBlobTxn() TxnSlot { blobTxn.BlobHashes = make([]common.Hash, 2) blobTxn.BlobHashes[0] = common.Hash(kzg.KZGToVersionedHash(commitment0)) blobTxn.BlobHashes[1] = common.Hash(kzg.KZGToVersionedHash(commitment1)) - blobTxn.Tip = *tip blobTxn.FeeCap = *feeCap blobTxn.BlobFeeCap = *blobFeeCap @@ -1093,7 +1152,7 @@ func TestBlobSlots(t *testing.T) { change := &remote.StateChangeBatch{ StateVersionId: stateVersionID, PendingBlockBaseFee: 200_000, - BlockGasLimit: 1000000, + BlockGasLimit: math.MaxUint64, PendingBlobFeePerGas: 100_000, ChangeBatch: []*remote.StateChange{ {BlockHeight: 0, BlockHash: h1}, @@ -1201,25 +1260,20 @@ func TestGasLimitChanged(t *testing.T) { reasons, err := pool.AddLocalTxns(ctx, txnSlots) assert.NoError(err) for _, reason := range reasons { - assert.Equal(txpoolcfg.Success, reason, reason.String()) + assert.Equal(reason, txpoolcfg.GasLimitTooHigh) } - mtx, ok := pool.byHash[string(txnSlot1.IDHash[:])] - assert.True(ok) - assert.Zero(mtx.subPool&NotTooMuchGas, "Should be insufficient block space for the tx") - change.ChangeBatch[0].Changes = nil change.BlockGasLimit = 150_000 err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{}) assert.NoError(err) - assert.NotZero(mtx.subPool&NotTooMuchGas, "Should now have block space for the tx") - - change.BlockGasLimit = 50_000 - err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{}) + reasons, err = pool.AddLocalTxns(ctx, txnSlots) assert.NoError(err) - assert.Zero(mtx.subPool&NotTooMuchGas, "Should now have block space (again) for the tx") + for _, reason := range reasons { + assert.Equal(txpoolcfg.Success, reason, reason.String()) + } } // sender - immutable structure which stores only nonce and balance of account diff --git a/txnprovider/txpool/txpoolcfg/txpoolcfg.go b/txnprovider/txpool/txpoolcfg/txpoolcfg.go index 9924a9ba0bb..876f4a88d2d 100644 --- a/txnprovider/txpool/txpoolcfg/txpoolcfg.go +++ b/txnprovider/txpool/txpoolcfg/txpoolcfg.go @@ -120,6 +120,7 @@ const ( BlobTxReplace DiscardReason = 30 // Cannot replace type-3 blob txn with another type of txn BlobPoolOverflow DiscardReason = 31 // The total number of blobs (through blob txns) in the pool has reached its limit NoAuthorizations DiscardReason = 32 // EIP-7702 transactions with an empty authorization list are invalid + GasLimitTooHigh DiscardReason = 33 // Gas limit is too high ) func (r DiscardReason) String() string { From 18d5754fa078e6ebe553b6441ef0a59206f7b58a Mon Sep 17 00:00:00 2001 From: Somnath Date: Thu, 9 Jan 2025 15:59:06 +0400 Subject: [PATCH 68/94] Add pectra devnet 5 tests (eest disabled) (#13225) Issue board: https://github.com/erigontech/erigon/issues/12401 --- tests/exec_spec_test.go | 1 + tests/execution-spec-tests | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exec_spec_test.go b/tests/exec_spec_test.go index e93e660f97b..eba6534150e 100644 --- a/tests/exec_spec_test.go +++ b/tests/exec_spec_test.go @@ -30,6 +30,7 @@ func TestExecutionSpec(t *testing.T) { log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StderrHandler)) bt := new(testMatcher) + bt.skipLoad(`^`) dir := filepath.Join(".", "execution-spec-tests") checkStateRoot := true diff --git a/tests/execution-spec-tests b/tests/execution-spec-tests index c53c43d7efe..fe2fb053a53 160000 --- a/tests/execution-spec-tests +++ b/tests/execution-spec-tests @@ -1 +1 @@ -Subproject commit c53c43d7efe94c5e972252533b1aee3669323fe1 +Subproject commit fe2fb053a53bb32d061043e90f7dddfd354e6bcf From 808b44ee9916e5cecd1a132cadeee9258ca91d9f Mon Sep 17 00:00:00 2001 From: Somnath Date: Thu, 9 Jan 2025 16:43:29 +0400 Subject: [PATCH 69/94] BlockHash EIP-2935 changes for pectra-devnet-5 (#13349) See: https://github.com/ethereum/EIPs/pull/9144 Issue board: https://github.com/erigontech/erigon/issues/12401 --- params/protocol_params.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/protocol_params.go b/params/protocol_params.go index fc3c878f181..0827aed2d8c 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -177,7 +177,7 @@ const ( P256VerifyGas uint64 = 3450 // EIP-2935: Historical block hashes in state - BlockHashHistoryServeWindow uint64 = 8192 + BlockHashHistoryServeWindow uint64 = 8191 BlockHashOldWindow uint64 = 256 // EIP-7702 @@ -190,7 +190,7 @@ var DelegatedDesignationPrefix = []byte{0xef, 0x01, 0x00} var BeaconRootsAddress = common.HexToAddress("0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02") // EIP-2935: Historical block hashes in state -var HistoryStorageAddress = common.HexToAddress("0x0aae40965e6800cd9b1f4b05ff21581047e3f91e") +var HistoryStorageAddress = common.HexToAddress("0x0F792be4B0c0cb4DAE440Ef133E90C0eCD48CCCC") // EIP-7002: Execution layer triggerable withdrawals var WithdrawalRequestAddress = common.HexToAddress("0x09Fc772D0857550724b07B850a4323f39112aAaA") From f812087e11e06e68bf322476518e04a32fcd31c1 Mon Sep 17 00:00:00 2001 From: Somnath Date: Thu, 9 Jan 2025 19:59:44 +0400 Subject: [PATCH 70/94] engineapi: Fix `engine_getClientVersionV1` (#13358) For specs: https://github.com/ethereum/execution-apis/blob/main/src/engine/identification.md#engine_getclientversionv1 The issue earlier was in json marshalling of strings returned by CL clients. Since Git commits are already hex, no need to further hexlify the string. The commit string here won't have "0x" prefix - that's contentious, but, going by the example given in the spec. The version string doesn't have size limits, hence extending it to what we would be using otherwise. --------- Co-authored-by: Giulio --- turbo/engineapi/engine_api_methods.go | 13 +++++++------ turbo/engineapi/engine_types/jsonrpc.go | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/turbo/engineapi/engine_api_methods.go b/turbo/engineapi/engine_api_methods.go index 43e9b98bce2..d8fa979e41c 100644 --- a/turbo/engineapi/engine_api_methods.go +++ b/turbo/engineapi/engine_api_methods.go @@ -140,17 +140,18 @@ func (e *EngineServer) GetClientVersionV1(ctx context.Context, callerVersion *en if callerVersion != nil { e.logger.Info("[GetClientVersionV1] Received request from" + callerVersion.String()) } - commitBytes := [4]byte{} - c := []byte(params.GitCommit) - if len(c) >= 4 { - copy(commitBytes[:], c[0:4]) + commitString := params.GitCommit + if len(commitString) >= 8 { + commitString = commitString[:8] + } else { + commitString = "00000000" // shouldn't be triggered } result := make([]engine_types.ClientVersionV1, 1) result[0] = engine_types.ClientVersionV1{ Code: params.ClientCode, Name: params.ClientName, - Version: params.Version, - Commit: commitBytes, + Version: params.VersionWithCommit(params.GitCommit), + Commit: "0x" + commitString, } return result, nil } diff --git a/turbo/engineapi/engine_types/jsonrpc.go b/turbo/engineapi/engine_types/jsonrpc.go index 021a4a96125..64e97e3bc44 100644 --- a/turbo/engineapi/engine_types/jsonrpc.go +++ b/turbo/engineapi/engine_types/jsonrpc.go @@ -109,10 +109,10 @@ type GetPayloadResponse struct { } type ClientVersionV1 struct { - Code string `json:"code" gencodec:"required"` - Name string `json:"name" gencodec:"required"` - Version string `json:"version" gencodec:"required"` - Commit [4]byte `json:"commit" gencodec:"required"` + Code string `json:"code" gencodec:"required"` + Name string `json:"name" gencodec:"required"` + Version string `json:"version" gencodec:"required"` + Commit string `json:"commit" gencodec:"required"` } func (c ClientVersionV1) String() string { From 8cf8b1f76073d015d41c45e8566807a6bbe7ed01 Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:26:15 +0000 Subject: [PATCH 71/94] polygon: keep devp2p max peers default to 100 as before (#13361) Follow up after https://github.com/erigontech/erigon/pull/13003 changed default devp2p peers from 100 to 32 Keep the previous default of 100 pers for polygon since the recommendation for it is to keep a higher peer count as per recent changes to their peering strategy - https://forum.polygon.technology/t/introducing-our-new-dns-discovery-for-polygon-pos-faster-smarter-more-connected/19871 --- cmd/utils/flags.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 97ae92d9205..de1539344f0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1420,8 +1420,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config, nodeName, datadir string, l cfg.MetricsEnabled = ctx.Bool(MetricsEnabledFlag.Name) } - ethPeers := cfg.MaxPeers - logger.Info("Maximum peer count", "ETH", ethPeers, "total", cfg.MaxPeers) + logger.Info("Maximum peer count", "total", cfg.MaxPeers) if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) @@ -1679,15 +1678,23 @@ func setClique(ctx *cli.Context, cfg *params.ConsensusSnapshotConfig, datadir st } } -func setBorConfig(ctx *cli.Context, cfg *ethconfig.Config) { +func setBorConfig(ctx *cli.Context, cfg *ethconfig.Config, nodeConfig *nodecfg.Config, logger log.Logger) { cfg.HeimdallURL = ctx.String(HeimdallURLFlag.Name) cfg.WithoutHeimdall = ctx.Bool(WithoutHeimdallFlag.Name) cfg.WithHeimdallMilestones = ctx.Bool(WithHeimdallMilestones.Name) cfg.WithHeimdallWaypointRecording = ctx.Bool(WithHeimdallWaypoints.Name) cfg.PolygonSync = ctx.Bool(PolygonSyncFlag.Name) cfg.PolygonSyncStage = ctx.Bool(PolygonSyncStageFlag.Name) - heimdall.RecordWayPoints( - cfg.WithHeimdallWaypointRecording || cfg.PolygonSync || cfg.PolygonSyncStage) + heimdall.RecordWayPoints(cfg.WithHeimdallWaypointRecording || cfg.PolygonSync || cfg.PolygonSyncStage) + + chainConfig := params.ChainConfigByChainName(ctx.String(ChainFlag.Name)) + if chainConfig.Bor != nil && !ctx.IsSet(MaxPeersFlag.Name) { + // override default max devp2p peers for polygon as per + // https://forum.polygon.technology/t/introducing-our-new-dns-discovery-for-polygon-pos-faster-smarter-more-connected/19871 + // which encourages high peer count + nodeConfig.P2P.MaxPeers = 100 + logger.Info("Maximum peer count default sanitizing for bor", "total", nodeConfig.P2P.MaxPeers) + } } func setMiner(ctx *cli.Context, cfg *params.MiningConfig) { @@ -1928,7 +1935,7 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C setClique(ctx, &cfg.Clique, nodeConfig.Dirs.DataDir) setMiner(ctx, &cfg.Miner) setWhitelist(ctx, cfg) - setBorConfig(ctx, cfg) + setBorConfig(ctx, cfg, nodeConfig, logger) setSilkworm(ctx, cfg) if err := setBeaconAPI(ctx, cfg); err != nil { log.Error("Failed to set beacon API", "err", err) From c5fe93682c16258e91089f2158edcf95867ae2fe Mon Sep 17 00:00:00 2001 From: dashangcun <907225865@qq.com> Date: Fri, 10 Jan 2025 10:22:25 +0100 Subject: [PATCH 72/94] refactor: use slices.contains (#13356) Using standard library methods to simplify code, inspired by https://github.com/erigontech/erigon/pull/11380/files Signed-off-by: dashangcun <907225865@qq.com> --- erigon-lib/bptree/node.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/erigon-lib/bptree/node.go b/erigon-lib/bptree/node.go index 0d675c56a1a..de64d71dd96 100644 --- a/erigon-lib/bptree/node.go +++ b/erigon-lib/bptree/node.go @@ -18,6 +18,7 @@ package bptree import ( "fmt" + "slices" "strings" "unsafe" ) @@ -31,12 +32,7 @@ func (keys Keys) Less(i, j int) bool { return keys[i] < keys[j] } func (keys Keys) Swap(i, j int) { keys[i], keys[j] = keys[j], keys[i] } func (keys Keys) Contains(key Felt) bool { - for _, k := range keys { - if k == key { - return true - } - } - return false + return slices.Contains(keys, key) } func (keys Keys) String() string { From f5c815dd3e95bead126f1f013ca3d439f3abe1d4 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Fri, 10 Jan 2025 12:09:06 +0100 Subject: [PATCH 73/94] Caplin: fixed small bugs (#13362) * Fixed inconsistent dependet_root * Fixed panics when Erigon is not synced in the Beacon API * Also do not error on failing to publish --- cl/beacon/beaconhttp/api.go | 14 +++++++++++--- cl/beacon/handler/block_production.go | 6 ++---- cl/beacon/handler/duties_attester.go | 4 ++++ cl/beacon/handler/pool.go | 24 ++++++++---------------- cl/rpc/rpc.go | 12 ------------ 5 files changed, 25 insertions(+), 35 deletions(-) diff --git a/cl/beacon/beaconhttp/api.go b/cl/beacon/beaconhttp/api.go index 1aef8a84a39..85c69abf2d5 100644 --- a/cl/beacon/beaconhttp/api.go +++ b/cl/beacon/beaconhttp/api.go @@ -103,13 +103,21 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc { ans, err := h.Handle(w, r) if err != nil { var endpointError *EndpointError - var e *EndpointError - if errors.As(err, &e) { + if e, ok := err.(*EndpointError); ok { + // Directly use the error if it's already an *EndpointError endpointError = e } else { + // Wrap the error in an EndpointError otherwise endpointError = WrapEndpointError(err) } - endpointError.WriteTo(w) + + // Write the endpoint error to the response writer + if endpointError != nil { + endpointError.WriteTo(w) + } else { + // Failsafe: If the error is nil, write a generic 500 error + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + } return } // TODO: potentially add a context option to buffer these diff --git a/cl/beacon/handler/block_production.go b/cl/beacon/handler/block_production.go index 9d9b4bf239c..a1a43c00bd8 100644 --- a/cl/beacon/handler/block_production.go +++ b/cl/beacon/handler/block_production.go @@ -1144,8 +1144,7 @@ func (a *ApiHandler) broadcastBlock(ctx context.Context, blk *cltypes.SignedBeac Name: gossip.TopicNameBeaconBlock, Data: blkSSZ, }); err != nil { - log.Error("Failed to publish block", "err", err) - return err + a.logger.Error("Failed to publish block", "err", err) } for idx, blob := range blobsSidecarsBytes { idx64 := uint64(idx) @@ -1154,8 +1153,7 @@ func (a *ApiHandler) broadcastBlock(ctx context.Context, blk *cltypes.SignedBeac Data: blob, SubnetId: &idx64, }); err != nil { - log.Error("Failed to publish blob sidecar", "err", err) - return err + a.logger.Error("Failed to publish blob sidecar", "err", err) } } return nil diff --git a/cl/beacon/handler/duties_attester.go b/cl/beacon/handler/duties_attester.go index 46731805819..84a47232a84 100644 --- a/cl/beacon/handler/duties_attester.go +++ b/cl/beacon/handler/duties_attester.go @@ -48,6 +48,10 @@ func (a *ApiHandler) getDependentRoot(epoch uint64, attester bool) (libcommon.Ha if attester { dependentRootSlot = ((epoch - 1) * a.beaconChainCfg.SlotsPerEpoch) - 1 } + if !a.syncedData.Syncing() && dependentRootSlot == a.syncedData.HeadSlot() { + dependentRoot = a.syncedData.HeadRoot() + return nil + } maxIterations := 2048 for i := 0; i < maxIterations; i++ { if dependentRootSlot > epoch*a.beaconChainCfg.SlotsPerEpoch { diff --git a/cl/beacon/handler/pool.go b/cl/beacon/handler/pool.go index 368bc155f33..2b00af03713 100644 --- a/cl/beacon/handler/pool.go +++ b/cl/beacon/handler/pool.go @@ -145,8 +145,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolAttestations(w http.ResponseWriter, r *h Name: gossip.TopicNamePrefixBeaconAttestation, SubnetId: &subnet, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish attestation to gossip", "err", err) } } } @@ -191,8 +190,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolVoluntaryExits(w http.ResponseWriter, r Data: encodedSSZ, Name: gossip.TopicNameVoluntaryExit, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish voluntary exit to gossip", "err", err) } } // Only write 200 @@ -222,8 +220,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolAttesterSlashings(w http.ResponseWriter, Data: encodedSSZ, Name: gossip.TopicNameAttesterSlashing, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish attester slashing to gossip", "err", err) } } // Only write 200 @@ -251,8 +248,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolProposerSlashings(w http.ResponseWriter, Data: encodedSSZ, Name: gossip.TopicNameProposerSlashing, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish proposer slashing to gossip", "err", err) } } // Only write 200 @@ -295,8 +291,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolBlsToExecutionChanges(w http.ResponseWri Data: encodedSSZ, Name: gossip.TopicNameBlsToExecutionChange, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish bls-to-execution-change to gossip", "err", err) } } } @@ -342,8 +337,7 @@ func (a *ApiHandler) PostEthV1ValidatorAggregatesAndProof(w http.ResponseWriter, Data: encodedSSZ, Name: gossip.TopicNameBeaconAggregateAndProof, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish aggregate and proof to gossip", "err", err) } } } @@ -398,8 +392,7 @@ func (a *ApiHandler) PostEthV1BeaconPoolSyncCommittees(w http.ResponseWriter, r Name: gossip.TopicNamePrefixSyncCommittee, SubnetId: &subnetId, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish sync committee message to gossip", "err", err) } } } @@ -448,8 +441,7 @@ func (a *ApiHandler) PostEthV1ValidatorContributionsAndProofs(w http.ResponseWri Data: encodedSSZ, Name: gossip.TopicNameSyncCommitteeContributionAndProof, }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + a.logger.Debug("[Beacon REST] failed to publish sync committee contribution to gossip", "err", err) } } } diff --git a/cl/rpc/rpc.go b/cl/rpc/rpc.go index 7f71b382a07..2fec27767c2 100644 --- a/cl/rpc/rpc.go +++ b/cl/rpc/rpc.go @@ -296,18 +296,6 @@ func (b *BeaconRpcP2P) SetStatus(finalizedRoot libcommon.Hash, finalizedEpoch ui return err } -func (b *BeaconRpcP2P) PropagateBlock(block *cltypes.SignedBeaconBlock) error { - encoded, err := block.EncodeSSZ(nil) - if err != nil { - return err - } - _, err = b.sentinel.PublishGossip(b.ctx, &sentinel.GossipData{ - Data: encoded, - Name: "beacon_block", - }) - return err -} - func (b *BeaconRpcP2P) BanPeer(pid string) { b.sentinel.BanPeer(b.ctx, &sentinel.Peer{Pid: pid}) } From ea136cbb9e5e1889e9525cd1180a99aae7569d3e Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Fri, 10 Jan 2025 12:53:21 +0000 Subject: [PATCH 74/94] downloader: small changes (#13334) Removed the rate calculation logic from the `ReCalcStats` function and replaced it with a call to the new `calculateRates` function. --------- Co-authored-by: awskii --- erigon-lib/downloader/downloader.go | 69 ++++++++++------------------- 1 file changed, 23 insertions(+), 46 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 5b19cb89495..bc64a3b10a3 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -2230,52 +2230,11 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } } - decay := func(prev uint64) uint64 { - switch { - case prev < 1000: - return prev / 16 - case stats.FlushRate < 10000: - return prev / 8 - case stats.FlushRate < 100000: - return prev / 4 - default: - return prev / 2 - } - } - - if stats.BytesDownload > prevStats.BytesDownload { - stats.DownloadRate = (stats.BytesDownload - prevStats.BytesDownload) / uint64(interval.Seconds()) - } else { - stats.DownloadRate = decay(prevStats.DownloadRate) - } - - if stats.BytesHashed > prevStats.BytesHashed { - stats.HashRate = (stats.BytesHashed - prevStats.BytesHashed) / uint64(interval.Seconds()) - } else { - stats.HashRate = decay(prevStats.HashRate) - } - - if stats.BytesCompleted > stats.BytesTotal { - stats.BytesCompleted = stats.BytesTotal - } - - if stats.BytesCompleted > prevStats.BytesCompleted { - stats.CompletionRate = (stats.BytesCompleted - prevStats.BytesCompleted) / uint64(interval.Seconds()) - } else { - stats.CompletionRate = decay(prevStats.CompletionRate) - } - - if stats.BytesFlushed > prevStats.BytesFlushed { - stats.FlushRate = (stats.BytesFlushed - prevStats.BytesFlushed) / uint64(interval.Seconds()) - } else { - stats.FlushRate = decay(prevStats.FlushRate) - } - - if stats.BytesUpload > prevStats.BytesUpload { - stats.UploadRate = (stats.BytesUpload - prevStats.BytesUpload) / uint64(interval.Seconds()) - } else { - stats.UploadRate = decay(prevStats.UploadRate) - } + stats.DownloadRate = calculateRate(stats.BytesDownload, prevStats.BytesDownload, prevStats.DownloadRate, interval) + stats.HashRate = calculateRate(stats.BytesHashed, prevStats.BytesHashed, prevStats.HashRate, interval) + stats.FlushRate = calculateRate(stats.BytesFlushed, prevStats.BytesFlushed, prevStats.FlushRate, interval) + stats.UploadRate = calculateRate(stats.BytesUpload, prevStats.BytesUpload, prevStats.UploadRate, interval) + stats.CompletionRate = calculateRate(stats.BytesCompleted, prevStats.BytesCompleted, prevStats.CompletionRate, interval) if stats.BytesTotal == 0 { stats.Progress = 0 @@ -2324,6 +2283,24 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } } +// Calculating rate with decay in order to avoid rate spikes +func calculateRate(current, previous uint64, prevRate uint64, interval time.Duration) uint64 { + if current > previous { + return (current - previous) / uint64(interval.Seconds()) + } + + switch { + case prevRate < 1000: + return prevRate / 16 + case prevRate < 10000: + return prevRate / 8 + case prevRate < 100000: + return prevRate / 4 + default: + return prevRate / 2 + } +} + type filterWriter struct { files map[string][]byte remainder []byte From 334e628e9c649d6485ed8f4af4a820a359cc4700 Mon Sep 17 00:00:00 2001 From: luchenhan <168071714+luchenhan@users.noreply.github.com> Date: Fri, 10 Jan 2025 22:01:39 +0900 Subject: [PATCH 75/94] refactor: using slices.Contains to simplify the code (#13315) Signed-off-by: luchenhan --- node/node.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/node/node.go b/node/node.go index 65b65ad96fe..f43ae11ad22 100644 --- a/node/node.go +++ b/node/node.go @@ -25,6 +25,7 @@ import ( "fmt" "path/filepath" "reflect" + "slices" "strings" "sync" "time" @@ -203,12 +204,7 @@ func (n *Node) doClose(errs []error) error { // containsLifecycle checks if 'lfs' contains 'l'. func containsLifecycle(lfs []Lifecycle, l Lifecycle) bool { - for _, obj := range lfs { - if obj == l { - return true - } - } - return false + return slices.Contains(lfs, l) } // stopServices terminates running services, RPC and p2p networking. From b031b3566e0a8380ae8601ec54c970df763e2672 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Fri, 10 Jan 2025 20:03:12 +0700 Subject: [PATCH 76/94] trace to set txnIndex and blockHash (#13294) --- polygon/tracer/trace_bor_state_sync_txn.go | 3 ++- turbo/jsonrpc/tracing.go | 16 +++++++++------- turbo/transactions/tracing.go | 8 ++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/polygon/tracer/trace_bor_state_sync_txn.go b/polygon/tracer/trace_bor_state_sync_txn.go index b87b481cab2..a032b28153c 100644 --- a/polygon/tracer/trace_bor_state_sync_txn.go +++ b/polygon/tracer/trace_bor_state_sync_txn.go @@ -48,9 +48,10 @@ func TraceBorStateSyncTxnDebugAPI( stream *jsoniter.Stream, callTimeout time.Duration, msgs []*types.Message, + txIndex int, ) (usedGas uint64, err error) { txCtx := initStateSyncTxContext(blockNum, blockHash) - tracer, streaming, cancel, err := transactions.AssembleTracer(ctx, traceConfig, txCtx.TxHash, stream, callTimeout) + tracer, streaming, cancel, err := transactions.AssembleTracer(ctx, traceConfig, txCtx.TxHash, blockHash, txIndex, stream, callTimeout) if err != nil { stream.WriteNil() return usedGas, err diff --git a/turbo/jsonrpc/tracing.go b/turbo/jsonrpc/tracing.go index d4361c07e75..b9b53c563c2 100644 --- a/turbo/jsonrpc/tracing.go +++ b/turbo/jsonrpc/tracing.go @@ -137,7 +137,7 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp } var usedGas uint64 - for idx, txn := range txns { + for txnIndex, txn := range txns { isBorStateSyncTxn := borStateSyncTxn == txn var txnHash common.Hash if isBorStateSyncTxn { @@ -159,7 +159,7 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp stream.WriteArrayEnd() return ctx.Err() } - ibs.SetTxContext(idx) + ibs.SetTxContext(txnIndex) msg, _ := txn.AsMessage(*signer, block.BaseFee(), rules) if msg.FeeCap().IsZero() && engine != nil { @@ -196,11 +196,12 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp stream, api.evmCallTimeout, stateSyncEvents, + txnIndex, ) usedGas += _usedGas } else { var _usedGas uint64 - _usedGas, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, ibs, config, chainConfig, stream, api.evmCallTimeout) + _usedGas, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, block.Hash(), txnIndex, ibs, config, chainConfig, stream, api.evmCallTimeout) usedGas += _usedGas } if err == nil { @@ -214,7 +215,7 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp } stream.WriteObjectEnd() - if idx != len(txns)-1 { + if txnIndex != len(txns)-1 { stream.WriteMore() } @@ -352,6 +353,7 @@ func (api *PrivateDebugAPIImpl) TraceTransaction(ctx context.Context, hash commo stream, api.evmCallTimeout, stateSyncEvents, + txnIndex, ) return err } @@ -363,7 +365,7 @@ func (api *PrivateDebugAPIImpl) TraceTransaction(ctx context.Context, hash commo } // Trace the transaction and return - _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, ibs, config, chainConfig, stream, api.evmCallTimeout) + _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, block.Hash(), txnIndex, ibs, config, chainConfig, stream, api.evmCallTimeout) return err } @@ -432,7 +434,7 @@ func (api *PrivateDebugAPIImpl) TraceCall(ctx context.Context, args ethapi.CallA blockCtx := transactions.NewEVMBlockContext(engine, header, blockNrOrHash.RequireCanonical, dbtx, api._blockReader, chainConfig) txCtx := core.NewEVMTxContext(msg) // Trace the transaction and return - _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, ibs, config, chainConfig, stream, api.evmCallTimeout) + _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, hash, 0, ibs, config, chainConfig, stream, api.evmCallTimeout) return err } @@ -570,7 +572,7 @@ func (api *PrivateDebugAPIImpl) TraceCallMany(ctx context.Context, bundles []Bun } txCtx = core.NewEVMTxContext(msg) ibs.SetTxContext(txnIndex) - _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, evm.IntraBlockState(), config, chainConfig, stream, api.evmCallTimeout) + _, err = transactions.TraceTx(ctx, msg, blockCtx, txCtx, block.Hash(), txnIndex, evm.IntraBlockState(), config, chainConfig, stream, api.evmCallTimeout) if err != nil { stream.WriteArrayEnd() stream.WriteArrayEnd() diff --git a/turbo/transactions/tracing.go b/turbo/transactions/tracing.go index a36d0ff3aa3..ffd7db24f59 100644 --- a/turbo/transactions/tracing.go +++ b/turbo/transactions/tracing.go @@ -102,13 +102,15 @@ func TraceTx( message core.Message, blockCtx evmtypes.BlockContext, txCtx evmtypes.TxContext, + blockHash libcommon.Hash, + txnIndex int, ibs evmtypes.IntraBlockState, config *tracersConfig.TraceConfig, chainConfig *chain.Config, stream *jsoniter.Stream, callTimeout time.Duration, ) (usedGas uint64, err error) { - tracer, streaming, cancel, err := AssembleTracer(ctx, config, txCtx.TxHash, stream, callTimeout) + tracer, streaming, cancel, err := AssembleTracer(ctx, config, txCtx.TxHash, blockHash, txnIndex, stream, callTimeout) if err != nil { stream.WriteNil() return 0, err @@ -134,6 +136,8 @@ func AssembleTracer( ctx context.Context, config *tracersConfig.TraceConfig, txHash libcommon.Hash, + blockHash libcommon.Hash, + txnIndex int, stream *jsoniter.Stream, callTimeout time.Duration, ) (vm.EVMLogger, bool, context.CancelFunc, error) { @@ -155,7 +159,7 @@ func AssembleTracer( if config != nil && config.TracerConfig != nil { cfg = *config.TracerConfig } - tracer, err := tracers.New(*config.Tracer, &tracers.Context{TxHash: txHash}, cfg) + tracer, err := tracers.New(*config.Tracer, &tracers.Context{TxHash: txHash, TxIndex: txnIndex, BlockHash: blockHash}, cfg) if err != nil { return nil, false, func() {}, err } From dbd36e0a74a613c71b576f5b1816ff7e6f8048d5 Mon Sep 17 00:00:00 2001 From: Michelangelo Riccobene Date: Fri, 10 Jan 2025 14:06:52 +0100 Subject: [PATCH 77/94] qa-tests: upload metrics based plots generated in the tip-tracking test (#13194) Upload plots created during test --- .github/workflows/qa-tip-tracking-polygon.yml | 18 +++++++++++++++--- .github/workflows/qa-tip-tracking.yml | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/qa-tip-tracking-polygon.yml b/.github/workflows/qa-tip-tracking-polygon.yml index 0fc9d9f4a18..dbb1bb95514 100644 --- a/.github/workflows/qa-tip-tracking-polygon.yml +++ b/.github/workflows/qa-tip-tracking-polygon.yml @@ -97,9 +97,21 @@ jobs: uses: actions/upload-artifact@v4 with: name: test-results - path: | - ${{ github.workspace }}/result-${{ env.CHAIN }}.json - ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log + path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json + + - name: Upload erigon log + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: erigon-log + path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log + + - name: Upload metric plots + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: metric-plots + path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots* - name: Delete Erigon Testbed Data Directory if: always() diff --git a/.github/workflows/qa-tip-tracking.yml b/.github/workflows/qa-tip-tracking.yml index aba0c4a5026..3e338cf5603 100644 --- a/.github/workflows/qa-tip-tracking.yml +++ b/.github/workflows/qa-tip-tracking.yml @@ -97,9 +97,21 @@ jobs: uses: actions/upload-artifact@v4 with: name: test-results - path: | - ${{ github.workspace }}/result-${{ env.CHAIN }}.json - ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log + path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json + + - name: Upload erigon log + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: erigon-log + path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log + + - name: Upload metric plots + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: metric-plots + path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots* - name: Delete Erigon Testbed Data Directory if: always() From 5aa205511f1309ed6e9a57c6e2190bac00ab06ef Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Fri, 10 Jan 2025 13:14:41 +0000 Subject: [PATCH 78/94] diagnostics: additional peer info (#12488) Collecting additional peer info in order to simplify debugging torrent for p2p enable: - pieces count allocated for each peer - upload rate --- erigon-lib/diagnostics/entities.go | 9 ++++-- erigon-lib/downloader/downloader.go | 48 +++++++++++++++++++---------- erigon-lib/go.mod | 6 ++-- erigon-lib/go.sum | 12 ++++---- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 06a68ee0d98..a84f4c23fdb 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -116,8 +116,13 @@ type FileDownloadedStatisticsUpdate struct { } type SegmentPeer struct { - Url string `json:"url"` - DownloadRate uint64 `json:"downloadRate"` + Url string `json:"url"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + PiecesCount uint64 `json:"piecesCount"` + RemoteAddr string `json:"remoteAddr"` + PeerId [20]byte `json:"peerId"` + TorrentName string `json:"torrentName"` } type SnapshotIndexingStatistics struct { diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index bc64a3b10a3..ed1cac77c20 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1569,7 +1569,7 @@ func (d *Downloader) torrentDownload(t *torrent.Torrent, statusChan chan downloa return case <-t.Complete.On(): downloadTime := time.Since(downloadStarted) - downloaded := t.Stats().BytesReadUsefulData + downloaded := t.Stats().BytesCompleted diagnostics.Send(diagnostics.FileDownloadedStatisticsUpdate{ FileName: t.Name(), @@ -2261,7 +2261,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { d.lock.Unlock() if !stats.Completed { - logger.Debug("[snapshots] downloading", + log.Debug("[snapshots] downloading", "len", len(torrents), "webTransfers", webTransfers, "torrent", torrentInfo, @@ -2273,6 +2273,8 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { "completion-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.CompletionRate)), "flushed", common.ByteCount(stats.BytesFlushed), "flush-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.FlushRate)), + "downloaded", common.ByteCount(stats.BytesDownload), + "download-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.DownloadRate)), "webseed-trips", stats.WebseedTripCount.Load(), "webseed-active", stats.WebseedActiveTrips.Load(), "webseed-max-active", stats.WebseedMaxActiveTrips.Load(), @@ -2360,10 +2362,16 @@ func getWebseedsRatesForlogs(weebseedPeersOfThisFile []*torrent.Peer, fName stri if peerUrl, err := webPeerUrl(peer); err == nil { if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil { rate := uint64(peer.DownloadRate()) + upRate := uint64(peer.UploadRate()) if !finished { seed := diagnostics.SegmentPeer{ Url: peerUrl.Host, DownloadRate: rate, + UploadRate: upRate, + RemoteAddr: peer.RemoteAddr.String(), + PeerId: [20]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + PiecesCount: 0, + TorrentName: fName, } seeds = append(seeds, seed) } @@ -2387,11 +2395,17 @@ func getPeersRatesForlogs(peersOfThisFile []*torrent.PeerConn, fName string) ([] for _, peer := range peersOfThisFile { dr := uint64(peer.DownloadRate()) + ur := uint64(peer.UploadRate()) url := fmt.Sprintf("%v", peer.PeerClientName.Load()) segPeer := diagnostics.SegmentPeer{ Url: url, DownloadRate: dr, + UploadRate: ur, + PiecesCount: peer.PeerPieces().GetCardinality(), + RemoteAddr: peer.RemoteAddr.String(), + PeerId: peer.PeerID, + TorrentName: fName, } peers = append(peers, segPeer) rates = append(rates, url, fmt.Sprintf("%s/s", common.ByteCount(dr))) @@ -2890,22 +2904,22 @@ func (d *Downloader) logProgress() { "completion-rate", fmt.Sprintf("%s/s", common.ByteCount(d.stats.CompletionRate)), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys)) - - diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ - Downloaded: bytesDone, - Total: d.stats.BytesTotal, - TotalTime: time.Since(d.startTime).Round(time.Second).Seconds(), - DownloadRate: d.stats.DownloadRate, - UploadRate: d.stats.UploadRate, - Peers: d.stats.PeersUnique, - Files: d.stats.FilesTotal, - Connections: d.stats.ConnectionsTotal, - Alloc: m.Alloc, - Sys: m.Sys, - DownloadFinished: d.stats.Completed, - TorrentMetadataReady: d.stats.MetadataReady, - }) } + + diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ + Downloaded: bytesDone, + Total: d.stats.BytesTotal, + TotalTime: time.Since(d.startTime).Round(time.Second).Seconds(), + DownloadRate: d.stats.DownloadRate, + UploadRate: d.stats.UploadRate, + Peers: d.stats.PeersUnique, + Files: d.stats.FilesTotal, + Connections: d.stats.ConnectionsTotal, + Alloc: m.Alloc, + Sys: m.Sys, + DownloadFinished: d.stats.Completed, + TorrentMetadataReady: d.stats.MetadataReady, + }) } func calculateTime(amountLeft, rate uint64) string { diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 9b374ea6817..4d34c948767 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -82,10 +82,10 @@ require ( github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/anacrolix/chansync v0.3.0 // indirect github.com/anacrolix/envpprof v1.3.0 // indirect - github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 // indirect + github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect - github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect + github.com/anacrolix/missinggo/v2 v2.7.3 // indirect github.com/anacrolix/mmsg v1.0.0 // indirect github.com/anacrolix/multiless v0.3.1-0.20221221005021-2d12701f83f7 // indirect github.com/anacrolix/stm v0.4.1-0.20221221005312-96d17df0e496 // indirect @@ -107,7 +107,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect - github.com/go-llsqlite/crawshaw v0.4.0 // indirect + github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 4d9209e71d9..892cec6d07f 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -39,8 +39,8 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tcJDk= github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 h1:Kmcl3I9K2+5AdnnR7hvrnVT0TLeFWWMa9bxnm55aVIg= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab h1:MvuAC/UJtcohN6xWc8zYXSZfllh1LVNepQ0R3BCX5I4= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0= github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= @@ -62,8 +62,8 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 h1:W/oGeHhYwxueeiDjQfmK9G+X9M2xJgfTtow62v0TWAs= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= +github.com/anacrolix/missinggo/v2 v2.7.3 h1:Ee//CmZBMadeNiYB/hHo9ly2PFOEZ4Fhsbnug3rDAIE= +github.com/anacrolix/missinggo/v2 v2.7.3/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw= github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= @@ -184,8 +184,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= -github.com/go-llsqlite/crawshaw v0.4.0 h1:L02s2jZBBJj80xm1VkkdyB/JlQ/Fi0kLbNHfXA8yrec= -github.com/go-llsqlite/crawshaw v0.4.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 h1:3EpZo8LxIzF4q3BT+vttQQlRfA6uTtTb/cxVisWa5HM= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= From 92fb52c5aca8eefa9c16ebbe3fe1102042964577 Mon Sep 17 00:00:00 2001 From: Mark Holt <135143369+mh0lt@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:10:23 +0000 Subject: [PATCH 79/94] Move downloader config after bor (#13374) This fixes `no metadata` in the downloader becuase checkpoints and milestones are only optionally included at the moment. If the downloader config is initiated before the bor flags have run the optional files will be exluded from the downloading whitelist and therefore ignore. --- cmd/utils/flags.go | 64 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index de1539344f0..42da9f80cd5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1888,37 +1888,6 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C cfg.Snapshot.Verify = ctx.Bool(DownloaderVerifyFlag.Name) cfg.Snapshot.DownloaderAddr = strings.TrimSpace(ctx.String(DownloaderAddrFlag.Name)) cfg.Snapshot.ChainName = chain - if cfg.Snapshot.DownloaderAddr == "" { - downloadRateStr := ctx.String(TorrentDownloadRateFlag.Name) - uploadRateStr := ctx.String(TorrentUploadRateFlag.Name) - var downloadRate, uploadRate datasize.ByteSize - if err := downloadRate.UnmarshalText([]byte(downloadRateStr)); err != nil { - panic(err) - } - if err := uploadRate.UnmarshalText([]byte(uploadRateStr)); err != nil { - panic(err) - } - lvl, _, err := downloadercfg2.Int2LogLevel(ctx.Int(TorrentVerbosityFlag.Name)) - if err != nil { - panic(err) - } - logger.Info("torrent verbosity", "level", lvl.LogString()) - version := "erigon: " + params.VersionWithCommit(params.GitCommit) - webseedsList := libcommon.CliString2Array(ctx.String(WebSeedsFlag.Name)) - if known, ok := snapcfg.KnownWebseeds[chain]; ok { - webseedsList = append(webseedsList, known...) - } - cfg.Downloader, err = downloadercfg2.New(ctx.Context, cfg.Dirs, version, lvl, downloadRate, uploadRate, - ctx.Int(TorrentPortFlag.Name), ctx.Int(TorrentConnsPerFileFlag.Name), ctx.Int(TorrentDownloadSlotsFlag.Name), - libcommon.CliString2Array(ctx.String(TorrentStaticPeersFlag.Name)), - webseedsList, chain, true, ctx.Bool(DbWriteMapFlag.Name), - ) - if err != nil { - panic(err) - } - downloadernat.DoNat(nodeConfig.P2P.NAT, cfg.Downloader.ClientConfig, logger) - } - nodeConfig.Http.Snap = cfg.Snapshot if ctx.Command.Name == "import" { @@ -2009,6 +1978,39 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C if ctx.IsSet(TrustedSetupFile.Name) { libkzg.SetTrustedSetupFilePath(ctx.String(TrustedSetupFile.Name)) } + + // Do this after chain config as there are chain type registration + // dependencies for know config which need to be set-up + if cfg.Snapshot.DownloaderAddr == "" { + downloadRateStr := ctx.String(TorrentDownloadRateFlag.Name) + uploadRateStr := ctx.String(TorrentUploadRateFlag.Name) + var downloadRate, uploadRate datasize.ByteSize + if err := downloadRate.UnmarshalText([]byte(downloadRateStr)); err != nil { + panic(err) + } + if err := uploadRate.UnmarshalText([]byte(uploadRateStr)); err != nil { + panic(err) + } + lvl, _, err := downloadercfg2.Int2LogLevel(ctx.Int(TorrentVerbosityFlag.Name)) + if err != nil { + panic(err) + } + logger.Info("torrent verbosity", "level", lvl.LogString()) + version := "erigon: " + params.VersionWithCommit(params.GitCommit) + webseedsList := libcommon.CliString2Array(ctx.String(WebSeedsFlag.Name)) + if known, ok := snapcfg.KnownWebseeds[chain]; ok { + webseedsList = append(webseedsList, known...) + } + cfg.Downloader, err = downloadercfg2.New(ctx.Context, cfg.Dirs, version, lvl, downloadRate, uploadRate, + ctx.Int(TorrentPortFlag.Name), ctx.Int(TorrentConnsPerFileFlag.Name), ctx.Int(TorrentDownloadSlotsFlag.Name), + libcommon.CliString2Array(ctx.String(TorrentStaticPeersFlag.Name)), + webseedsList, chain, true, ctx.Bool(DbWriteMapFlag.Name), + ) + if err != nil { + panic(err) + } + downloadernat.DoNat(nodeConfig.P2P.NAT, cfg.Downloader.ClientConfig, logger) + } } // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if From bec753d7558feeacdec7d9f5d7165eb6724ede96 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Sat, 11 Jan 2025 01:57:34 +0100 Subject: [PATCH 80/94] Fixed `cannot alloc memory` (#13375) Seems it happens with many allocs in ETL buffers --- cl/antiquary/beacon_states_collector.go | 71 ++++++++++++++++++------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/cl/antiquary/beacon_states_collector.go b/cl/antiquary/beacon_states_collector.go index d78f4047ceb..2f602e14755 100644 --- a/cl/antiquary/beacon_states_collector.go +++ b/cl/antiquary/beacon_states_collector.go @@ -20,7 +20,9 @@ import ( "bytes" "context" "io" + "sync" + "github.com/c2h5oh/datasize" "github.com/klauspost/compress/zstd" libcommon "github.com/erigontech/erigon-lib/common" @@ -36,7 +38,20 @@ import ( "github.com/erigontech/erigon/cl/transition/impl/eth2" ) -var stateAntiquaryBufSz = etl.BufferOptimalSize / 8 // 18 collectors * 256mb / 8 = 512mb in worst case +var stateAntiquaryBufSz = etl.BufferOptimalSize / 16 // 18 collectors * 256mb / 16 = 256mb in worst case + +const EnabledPreAllocate = true + +var etlBufferPool = &sync.Pool{ + New: func() interface{} { + buf := etl.NewSortableBuffer(stateAntiquaryBufSz) + // preallocate 20_000 items with a 2MB overflow buffer + if EnabledPreAllocate { + buf.Prealloc(20_000, int(stateAntiquaryBufSz+2*datasize.MB)) + } + return buf + }, +} // RATIONALE: MDBX locks the entire database when writing to it, so we need to minimize the time spent in the write lock. // so instead of writing the historical states on write transactions, we accumulate them in memory and write them in a single write transaction. @@ -63,6 +78,8 @@ type beaconStatesCollector struct { balancesDumpsCollector *etl.Collector effectiveBalancesDumpCollector *etl.Collector + buffers []etl.Buffer + buf *bytes.Buffer compressor *zstd.Encoder @@ -76,30 +93,40 @@ func newBeaconStatesCollector(beaconCfg *clparams.BeaconChainConfig, tmpdir stri if err != nil { panic(err) } + + var buffers []etl.Buffer + makeETLBuffer := func() etl.Buffer { + b := etlBufferPool.Get().(etl.Buffer) + b.Reset() + buffers = append(buffers, b) + return b + } + return &beaconStatesCollector{ - effectiveBalanceCollector: etl.NewCollector(kv.ValidatorEffectiveBalance, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - balancesCollector: etl.NewCollector(kv.ValidatorBalance, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - randaoMixesCollector: etl.NewCollector(kv.RandaoMixes, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - intraRandaoMixesCollector: etl.NewCollector(kv.IntraRandaoMixes, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - proposersCollector: etl.NewCollector(kv.Proposers, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - slashingsCollector: etl.NewCollector(kv.ValidatorSlashings, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - blockRootsCollector: etl.NewCollector(kv.BlockRoot, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - stateRootsCollector: etl.NewCollector(kv.StateRoot, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - slotDataCollector: etl.NewCollector(kv.SlotData, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - epochDataCollector: etl.NewCollector(kv.EpochData, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - inactivityScoresCollector: etl.NewCollector(kv.InactivityScores, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - nextSyncCommitteeCollector: etl.NewCollector(kv.NextSyncCommittee, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - currentSyncCommitteeCollector: etl.NewCollector(kv.CurrentSyncCommittee, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - eth1DataVotesCollector: etl.NewCollector(kv.Eth1DataVotes, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - stateEventsCollector: etl.NewCollector(kv.StateEvents, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - activeValidatorIndiciesCollector: etl.NewCollector(kv.ActiveValidatorIndicies, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - balancesDumpsCollector: etl.NewCollector(kv.BalancesDump, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), - effectiveBalancesDumpCollector: etl.NewCollector(kv.EffectiveBalancesDump, tmpdir, etl.NewSortableBuffer(stateAntiquaryBufSz), logger).LogLvl(log.LvlTrace), + effectiveBalanceCollector: etl.NewCollector(kv.ValidatorEffectiveBalance, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + balancesCollector: etl.NewCollector(kv.ValidatorBalance, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + randaoMixesCollector: etl.NewCollector(kv.RandaoMixes, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + intraRandaoMixesCollector: etl.NewCollector(kv.IntraRandaoMixes, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + proposersCollector: etl.NewCollector(kv.Proposers, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + slashingsCollector: etl.NewCollector(kv.ValidatorSlashings, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + blockRootsCollector: etl.NewCollector(kv.BlockRoot, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + stateRootsCollector: etl.NewCollector(kv.StateRoot, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + slotDataCollector: etl.NewCollector(kv.SlotData, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + epochDataCollector: etl.NewCollector(kv.EpochData, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + inactivityScoresCollector: etl.NewCollector(kv.InactivityScores, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + nextSyncCommitteeCollector: etl.NewCollector(kv.NextSyncCommittee, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + currentSyncCommitteeCollector: etl.NewCollector(kv.CurrentSyncCommittee, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + eth1DataVotesCollector: etl.NewCollector(kv.Eth1DataVotes, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + stateEventsCollector: etl.NewCollector(kv.StateEvents, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + activeValidatorIndiciesCollector: etl.NewCollector(kv.ActiveValidatorIndicies, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + balancesDumpsCollector: etl.NewCollector(kv.BalancesDump, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), + effectiveBalancesDumpCollector: etl.NewCollector(kv.EffectiveBalancesDump, tmpdir, makeETLBuffer(), logger).LogLvl(log.LvlTrace), logger: logger, beaconCfg: beaconCfg, buf: buf, compressor: compressor, + buffers: buffers, } } @@ -361,6 +388,12 @@ func (i *beaconStatesCollector) close() { i.activeValidatorIndiciesCollector.Close() i.balancesDumpsCollector.Close() i.effectiveBalancesDumpCollector.Close() + for _, b := range i.buffers { + b.Reset() + } + for _, b := range i.buffers { + etlBufferPool.Put(b) + } } // antiquateFullUint64List goes on mdbx as it is full of common repeated patter always and thus fits with 16KB pages. From 20e61f11db97fb4561a78236e084fb3efaf3cc86 Mon Sep 17 00:00:00 2001 From: Somnath Date: Sat, 11 Jan 2025 14:08:39 +0400 Subject: [PATCH 81/94] Implement calldata cost increase per EIP-7623 (#13080) See https://github.com/ethereum/EIPs/blob/master/EIPS/eip-7623.md Issue board - https://github.com/erigontech/erigon/issues/12401 --- core/state_transition.go | 19 +++++--- erigon-lib/common/fixedgas/protocol.go | 1 + tests/transaction_test_util.go | 5 ++- txnprovider/txpool/pool.go | 12 +++++- txnprovider/txpool/pool_test.go | 3 +- txnprovider/txpool/txpoolcfg/txpoolcfg.go | 43 ++++++++++++------- .../txpool/txpoolcfg/txpoolcfg_test.go | 32 ++++++++++++++ 7 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 txnprovider/txpool/txpoolcfg/txpoolcfg_test.go diff --git a/core/state_transition.go b/core/state_transition.go index 52141f37dbe..0af8c519fab 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -108,7 +108,7 @@ type Message interface { // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. // TODO: convert the input to a struct -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool, authorizationsLen uint64) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860, isPrague bool, authorizationsLen uint64) (uint64, uint64, error) { // Zero and non-zero bytes are priced differently dataLen := uint64(len(data)) dataNonZeroLen := uint64(0) @@ -118,11 +118,11 @@ func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation b } } - gas, status := txpoolcfg.CalcIntrinsicGas(dataLen, dataNonZeroLen, authorizationsLen, accessList, isContractCreation, isHomestead, isEIP2028, isEIP3860) + gas, floorGas7623, status := txpoolcfg.CalcIntrinsicGas(dataLen, dataNonZeroLen, authorizationsLen, accessList, isContractCreation, isHomestead, isEIP2028, isEIP3860, isPrague) if status != txpoolcfg.Success { - return 0, ErrGasUintOverflow + return 0, 0, ErrGasUintOverflow } - return gas, nil + return gas, floorGas7623, nil } // NewStateTransition initialises and returns a new state transition object. @@ -460,12 +460,12 @@ func (st *StateTransition) TransitionDb(refunds bool, gasBailout bool) (*evmtype } // Check clauses 4-5, subtract intrinsic gas if everything is correct - gas, err := IntrinsicGas(st.data, accessTuples, contractCreation, rules.IsHomestead, rules.IsIstanbul, isEIP3860, uint64(len(auths))) + gas, floorGas7623, err := IntrinsicGas(st.data, accessTuples, contractCreation, rules.IsHomestead, rules.IsIstanbul, isEIP3860, rules.IsPrague, uint64(len(auths))) if err != nil { return nil, err } - if st.gasRemaining < gas { - return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gasRemaining, gas) + if st.gasRemaining < gas || st.gasRemaining < floorGas7623 { + return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gasRemaining, max(gas, floorGas7623)) } st.gasRemaining -= gas @@ -505,6 +505,11 @@ func (st *StateTransition) TransitionDb(refunds bool, gasBailout bool) (*evmtype } else { ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), st.data, st.gasRemaining, st.value, bailout) } + gasUsed := st.gasUsed() + if gasUsed < floorGas7623 && rules.IsPrague { + gasUsed = floorGas7623 + st.gasRemaining = st.initialGas - gasUsed + } if refunds && !gasBailout { if rules.IsLondon { // After EIP-3529: refunds are capped to gasUsed / 5 diff --git a/erigon-lib/common/fixedgas/protocol.go b/erigon-lib/common/fixedgas/protocol.go index cc73737a733..c002d070cf3 100644 --- a/erigon-lib/common/fixedgas/protocol.go +++ b/erigon-lib/common/fixedgas/protocol.go @@ -24,6 +24,7 @@ const ( TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list + TxTotalCostFloorPerToken uint64 = 10 // Per token of calldata in a transaction, as a minimum the txn must pay (EIP-7623) MaxCodeSize = 24576 // Maximum bytecode to permit for a contract diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index a78135aedce..e83b04382ff 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -78,7 +78,10 @@ func (tt *TransactionTest) Run(chainID *big.Int) error { if stx, ok := tx.(*types.SetCodeTransaction); ok { authorizationsLen = uint64(len(stx.GetAuthorizations())) } - requiredGas, err := core.IntrinsicGas(msg.Data(), msg.AccessList(), msg.To() == nil, rules.IsHomestead, rules.IsIstanbul, rules.IsShanghai, authorizationsLen) + requiredGas, floorGas, err := core.IntrinsicGas(msg.Data(), msg.AccessList(), msg.To() == nil, rules.IsHomestead, rules.IsIstanbul, rules.IsShanghai, rules.IsPrague, authorizationsLen) + if rules.IsPrague && floorGas > requiredGas { + requiredGas = floorGas + } if err != nil { return nil, nil, 0, err } diff --git a/txnprovider/txpool/pool.go b/txnprovider/txpool/pool.go index 04126223a0c..d79d8ccabf3 100644 --- a/txnprovider/txpool/pool.go +++ b/txnprovider/txpool/pool.go @@ -671,6 +671,7 @@ func (p *TxPool) best(ctx context.Context, n int, txns *TxnsRlp, onTopOf, availa best := p.pending.best isShanghai := p.isShanghai() || p.isAgra() + isPrague := p.isPrague() txns.Resize(uint(min(n, len(best.ms)))) var toRemove []*metaTxn @@ -724,7 +725,10 @@ func (p *TxPool) best(ctx context.Context, n int, txns *TxnsRlp, onTopOf, availa // not an exact science using intrinsic gas but as close as we could hope for at // this stage authorizationLen := uint64(len(mt.TxnSlot.Authorizations)) - intrinsicGas, _ := txpoolcfg.CalcIntrinsicGas(uint64(mt.TxnSlot.DataLen), uint64(mt.TxnSlot.DataNonZeroLen), authorizationLen, nil, mt.TxnSlot.Creation, true, true, isShanghai) + intrinsicGas, floorGas, _ := txpoolcfg.CalcIntrinsicGas(uint64(mt.TxnSlot.DataLen), uint64(mt.TxnSlot.DataNonZeroLen), authorizationLen, nil, mt.TxnSlot.Creation, true, true, isShanghai, isPrague) + if isPrague && floorGas > intrinsicGas { + intrinsicGas = floorGas + } if intrinsicGas > availableGas { // we might find another txn with a low enough intrinsic gas to include so carry on continue @@ -902,7 +906,11 @@ func (p *TxPool) validateTx(txn *TxnSlot, isLocal bool, stateCache kvcache.Cache return txpoolcfg.UnderPriced } - gas, reason := txpoolcfg.CalcIntrinsicGas(uint64(txn.DataLen), uint64(txn.DataNonZeroLen), uint64(authorizationLen), nil, txn.Creation, true, true, isShanghai) + gas, floorGas, reason := txpoolcfg.CalcIntrinsicGas(uint64(txn.DataLen), uint64(txn.DataNonZeroLen), uint64(authorizationLen), nil, txn.Creation, true, true, isShanghai, p.isPrague()) + if p.isPrague() && floorGas > gas { + gas = floorGas + } + if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx intrinsic gas idHash=%x gas=%d", txn.IDHash, gas)) } diff --git a/txnprovider/txpool/pool_test.go b/txnprovider/txpool/pool_test.go index 4c1cc236847..11fcc6a4306 100644 --- a/txnprovider/txpool/pool_test.go +++ b/txnprovider/txpool/pool_test.go @@ -631,7 +631,8 @@ func TestShanghaiIntrinsicGas(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { - gas, reason := txpoolcfg.CalcIntrinsicGas(c.dataLen, c.dataNonZeroLen, c.authorizationsLen, nil, c.creation, true, true, c.isShanghai) + // Todo (@somnathb1) - Factor in EIP-7623 + gas, _, reason := txpoolcfg.CalcIntrinsicGas(c.dataLen, c.dataNonZeroLen, c.authorizationsLen, nil, c.creation, true, true, c.isShanghai, false) if reason != txpoolcfg.Success { t.Errorf("expected success but got reason %v", reason) } diff --git a/txnprovider/txpool/txpoolcfg/txpoolcfg.go b/txnprovider/txpool/txpoolcfg/txpoolcfg.go index 876f4a88d2d..24321d5e507 100644 --- a/txnprovider/txpool/txpoolcfg/txpoolcfg.go +++ b/txnprovider/txpool/txpoolcfg/txpoolcfg.go @@ -192,13 +192,13 @@ func (r DiscardReason) String() string { // CalcIntrinsicGas computes the 'intrinsic gas' for a message with the given data. // TODO: move input data to a struct -func CalcIntrinsicGas(dataLen, dataNonZeroLen, authorizationsLen uint64, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028, isShanghai bool) (uint64, DiscardReason) { +func CalcIntrinsicGas(dataLen, dataNonZeroLen, authorizationsLen uint64, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028, isShanghai, isPrague bool) (gas uint64, floorGas7623 uint64, d DiscardReason) { // Set the starting gas for the raw transaction - var gas uint64 if isContractCreation && isHomestead { gas = fixedgas.TxGasContractCreation } else { gas = fixedgas.TxGas + floorGas7623 = fixedgas.TxGas } // Bump the required gas by the amount of transactional data if dataLen > 0 { @@ -212,68 +212,81 @@ func CalcIntrinsicGas(dataLen, dataNonZeroLen, authorizationsLen uint64, accessL product, overflow := emath.SafeMul(nz, nonZeroGas) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } z := dataLen - nz product, overflow = emath.SafeMul(z, fixedgas.TxDataZeroGas) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } if isContractCreation && isShanghai { numWords := toWordSize(dataLen) product, overflow = emath.SafeMul(numWords, fixedgas.InitCodeWordGas) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow + } + } + + // EIP-7623 + if isPrague { + tokenLen := dataLen + 3*nz + dataGas, overflow := emath.SafeMul(tokenLen, fixedgas.TxTotalCostFloorPerToken) + if overflow { + return 0, 0, GasUintOverflow + } + floorGas7623, overflow = emath.SafeAdd(floorGas7623, dataGas) + if overflow { + return 0, 0, GasUintOverflow } } } if accessList != nil { product, overflow := emath.SafeMul(uint64(len(accessList)), fixedgas.TxAccessListAddressGas) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } product, overflow = emath.SafeMul(uint64(accessList.StorageKeys()), fixedgas.TxAccessListStorageKeyGas) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } } // Add the cost of authorizations product, overflow := emath.SafeMul(authorizationsLen, fixedgas.PerEmptyAccountCost) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } gas, overflow = emath.SafeAdd(gas, product) if overflow { - return 0, GasUintOverflow + return 0, 0, GasUintOverflow } - return gas, Success + return gas, floorGas7623, Success } // toWordSize returns the ceiled word size required for memory expansion. diff --git a/txnprovider/txpool/txpoolcfg/txpoolcfg_test.go b/txnprovider/txpool/txpoolcfg/txpoolcfg_test.go new file mode 100644 index 00000000000..c1159b8949e --- /dev/null +++ b/txnprovider/txpool/txpoolcfg/txpoolcfg_test.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package txpoolcfg + +import ( + "testing" + + "github.com/erigontech/erigon-lib/common/fixedgas" + "github.com/stretchr/testify/assert" +) + +func TestZeroDataIntrinsicGas(t *testing.T) { + assert := assert.New(t) + gas, floorGas7623, discardReason := CalcIntrinsicGas(0, 0, 0, nil, false, true, true, true, true) + assert.Equal(discardReason, Success) + assert.Equal(gas, fixedgas.TxGas) + assert.Equal(floorGas7623, fixedgas.TxGas) +} From 81f0ab91e0bfd08826f4e251bbe26be9909e8989 Mon Sep 17 00:00:00 2001 From: Somnath Date: Sat, 11 Jan 2025 19:57:11 +0400 Subject: [PATCH 82/94] Update addresses for EIPs 7002 and 7251 (#13069) See https://github.com/ethereum/EIPs/pull/9119 and https://github.com/ethereum/EIPs/pull/9118 Issue board: https://github.com/erigontech/erigon/issues/12401 --- .github/workflows/ci.yml | 1 - .github/workflows/test-integration-caplin.yml | 1 - .github/workflows/test-integration-erigon.yml | 1 - params/protocol_params.go | 4 ++-- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d480c032cb..257bb698f3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,6 @@ jobs: - name: Install dependencies run: | choco upgrade mingw -y --no-progress --version 13.2.0 - choco install cmake -y --no-progress --version 3.27.8 - name: Build run: .\wmake.ps1 all diff --git a/.github/workflows/test-integration-caplin.yml b/.github/workflows/test-integration-caplin.yml index 2c9b745d7ac..086d7a652c8 100644 --- a/.github/workflows/test-integration-caplin.yml +++ b/.github/workflows/test-integration-caplin.yml @@ -66,7 +66,6 @@ jobs: - name: Install dependencies run: | choco upgrade mingw -y --no-progress --version 13.2.0 - choco install cmake -y --no-progress --version 3.27.8 - name: test-integration-caplin run: cd ./cl/spectest/ && .\wmake.ps1 Tests Mainnet diff --git a/.github/workflows/test-integration-erigon.yml b/.github/workflows/test-integration-erigon.yml index cab34a13929..1108579b38f 100644 --- a/.github/workflows/test-integration-erigon.yml +++ b/.github/workflows/test-integration-erigon.yml @@ -66,7 +66,6 @@ jobs: - name: Install dependencies run: | choco upgrade mingw -y --no-progress --version 13.2.0 - choco install cmake -y --no-progress --version 3.27.8 - name: test-integration run: .\wmake.ps1 test-integration diff --git a/params/protocol_params.go b/params/protocol_params.go index 0827aed2d8c..d348ad8c1d3 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -193,10 +193,10 @@ var BeaconRootsAddress = common.HexToAddress("0x000F3df6D732807Ef1319fB7B8bB8522 var HistoryStorageAddress = common.HexToAddress("0x0F792be4B0c0cb4DAE440Ef133E90C0eCD48CCCC") // EIP-7002: Execution layer triggerable withdrawals -var WithdrawalRequestAddress = common.HexToAddress("0x09Fc772D0857550724b07B850a4323f39112aAaA") +var WithdrawalRequestAddress = common.HexToAddress("0x0c15F14308530b7CDB8460094BbB9cC28b9AaaAA") // EIP-7251 -var ConsolidationRequestAddress = common.HexToAddress("0x01aBEa29659e5e97C95107F20bb753cD3e09bBBb") +var ConsolidationRequestAddress = common.HexToAddress("0x00431F263cE400f4455c2dCf564e53007Ca4bbBb") // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations var Bls12381MultiExpDiscountTable = [128]uint64{1200, 888, 764, 641, 594, 547, 500, 453, 438, 423, 408, 394, 379, 364, 349, 334, 330, 326, 322, 318, 314, 310, 306, 302, 298, 294, 289, 285, 281, 277, 273, 269, 268, 266, 265, 263, 262, 260, 259, 257, 256, 254, 253, 251, 250, 248, 247, 245, 244, 242, 241, 239, 238, 236, 235, 233, 232, 231, 229, 228, 226, 225, 223, 222, 221, 220, 219, 219, 218, 217, 216, 216, 215, 214, 213, 213, 212, 211, 211, 210, 209, 208, 208, 207, 206, 205, 205, 204, 203, 202, 202, 201, 200, 199, 199, 198, 197, 196, 196, 195, 194, 193, 193, 192, 191, 191, 190, 189, 188, 188, 187, 186, 185, 185, 184, 183, 182, 182, 181, 180, 179, 179, 178, 177, 176, 176, 175, 174} From 0af73b723d22e7b777b7a04896a0765146b0feed Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Sun, 12 Jan 2025 01:19:18 +0100 Subject: [PATCH 83/94] Erigon3: Remove legacy receipts reading (#13389) --- core/rawdb/accessors_chain.go | 29 ---------- core/rawdb/accessors_chain_test.go | 93 ------------------------------ 2 files changed, 122 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index dcff36247d3..210553c65d9 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -34,7 +34,6 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/common/dbg" "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/kv" @@ -836,34 +835,6 @@ func ReadRawReceipts(db kv.Tx, blockNum uint64) types.Receipts { return receipts } -// ReadReceipts retrieves all the transaction receipts belonging to a block, including -// its corresponding metadata fields. If it is unable to populate these metadata -// fields then nil is returned. -// -// The current implementation populates these metadata fields by reading the receipts' -// corresponding block body, so if the block body is not found it will return nil even -// if the receipt itself is stored. -func ReadReceipts(db kv.Tx, block *types.Block, senders []common.Address) types.Receipts { - if block == nil { - return nil - } - // We're deriving many fields from the block body, retrieve beside the receipt - receipts := ReadRawReceipts(db, block.NumberU64()) - if receipts == nil { - return nil - } - if len(senders) > 0 { - block.SendersToTxs(senders) - } else { - senders = block.Body().SendersFromTxs() - } - if err := receipts.DeriveFields(block.Hash(), block.NumberU64(), block.Transactions(), senders); err != nil { - log.Error("Failed to derive block receipts fields", "hash", block.Hash(), "number", block.NumberU64(), "err", err, "stack", dbg.Stack()) - return nil - } - return receipts -} - // WriteReceipts stores all the transaction receipts belonging to a block. func WriteReceipts(tx kv.Putter, number uint64, receipts types.Receipts) error { buf := bytes.NewBuffer(make([]byte, 0, 1024)) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index cfa44a1e01a..73df7b067c0 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -460,99 +460,6 @@ func TestHeadStorage(t *testing.T) { } } -// Tests that receipts associated with a single block can be stored and retrieved. -func TestBlockReceiptStorage(t *testing.T) { - t.Parallel() - m := mock.Mock(t) - tx, err := m.DB.BeginRw(m.Ctx) - require.NoError(t, err) - defer tx.Rollback() - br := m.BlockReader - require := require.New(t) - ctx := m.Ctx - - // Create a live block since we need metadata to reconstruct the receipt - tx1 := types.NewTransaction(1, libcommon.HexToAddress("0x1"), u256.Num1, 1, u256.Num1, nil) - tx2 := types.NewTransaction(2, libcommon.HexToAddress("0x2"), u256.Num2, 2, u256.Num2, nil) - - body := &types.Body{Transactions: types.Transactions{tx1, tx2}} - - // Create the two receipts to manage afterwards - receipt1 := &types.Receipt{ - Status: types.ReceiptStatusFailed, - CumulativeGasUsed: 1, - Logs: []*types.Log{ - {Address: libcommon.BytesToAddress([]byte{0x11})}, - {Address: libcommon.BytesToAddress([]byte{0x01, 0x11})}, - }, - TxHash: tx1.Hash(), - ContractAddress: libcommon.BytesToAddress([]byte{0x01, 0x11, 0x11}), - GasUsed: 111111, - } - //receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1}) - - receipt2 := &types.Receipt{ - PostState: libcommon.Hash{2}.Bytes(), - CumulativeGasUsed: 2, - Logs: []*types.Log{ - {Address: libcommon.BytesToAddress([]byte{0x22})}, - {Address: libcommon.BytesToAddress([]byte{0x02, 0x22})}, - }, - TxHash: tx2.Hash(), - ContractAddress: libcommon.BytesToAddress([]byte{0x02, 0x22, 0x22}), - GasUsed: 222222, - } - //receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2}) - receipts := []*types.Receipt{receipt1, receipt2} - header := &types.Header{Number: big.NewInt(1)} - - // Check that no receipt entries are in a pristine database - hash := header.Hash() //libcommon.BytesToHash([]byte{0x03, 0x14}) - - rawdb.WriteCanonicalHash(tx, header.Hash(), header.Number.Uint64()) - rawdb.WriteHeader(tx, header) - // Insert the body that corresponds to the receipts - require.NoError(rawdb.WriteBody(tx, hash, 1, body)) - require.NoError(rawdb.WriteSenders(tx, hash, 1, body.SendersFromTxs())) - - // Insert the receipt slice into the database and check presence - require.NoError(rawdb.WriteReceipts(tx, 1, receipts)) - - b, senders, err := br.BlockWithSenders(ctx, tx, hash, 1) - require.NoError(err) - require.NotNil(b) - if rs := rawdb.ReadReceipts(tx, b, senders); len(rs) == 0 { - t.Fatalf("no receipts returned") - } else { - if err := checkReceiptsRLP(rs, receipts); err != nil { - t.Fatal(err.Error()) - } - } - // Delete the body and ensure that the receipts are no longer returned (metadata can't be recomputed) - rawdb.DeleteHeader(tx, hash, 1) - rawdb.DeleteBody(tx, hash, 1) - b, senders, err = br.BlockWithSenders(ctx, tx, hash, 1) - require.NoError(err) - require.Nil(b) - if rs := rawdb.ReadReceipts(tx, b, senders); rs != nil { - t.Fatalf("receipts returned when body was deleted: %v", rs) - } - // Ensure that receipts without metadata can be returned without the block body too - if err := checkReceiptsRLP(rawdb.ReadRawReceipts(tx, 1), receipts); err != nil { - t.Fatal(err) - } - rawdb.WriteHeader(tx, header) - // Sanity check that body alone without the receipt is a full purge - require.NoError(rawdb.WriteBody(tx, hash, 1, body)) - require.NoError(rawdb.TruncateReceipts(tx, 1)) - b, senders, err = br.BlockWithSenders(ctx, tx, hash, 1) - require.NoError(err) - require.NotNil(b) - if rs := rawdb.ReadReceipts(tx, b, senders); len(rs) != 0 { - t.Fatalf("deleted receipts returned: %v", rs) - } -} - // Tests block storage and retrieval operations with withdrawals. func TestBlockWithdrawalsStorage(t *testing.T) { t.Parallel() From be1c034dddc68700f840da63178c7b1ff27afd25 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:58:48 +0100 Subject: [PATCH 84/94] Use libcommon.CopyBytes instead of bytesCopy (#13406) --- .../kv/remotedbserver/remotedbserver.go | 21 ++++++------------- p2p/discover/v5wire/encoding.go | 13 ++++-------- turbo/jsonrpc/eth_call.go | 15 ++++++------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/erigon-lib/kv/remotedbserver/remotedbserver.go b/erigon-lib/kv/remotedbserver/remotedbserver.go index b7e46c598f0..7316e92fbf4 100644 --- a/erigon-lib/kv/remotedbserver/remotedbserver.go +++ b/erigon-lib/kv/remotedbserver/remotedbserver.go @@ -260,8 +260,8 @@ func (s *KvServer) Tx(stream remote.KV_TxServer) error { if err != nil { return fmt.Errorf("kvserver: %w", err) } - c.k = bytesCopy(k) - c.v = bytesCopy(v) + c.k = common.CopyBytes(k) + c.v = common.CopyBytes(v) } if err := s.renew(stream.Context(), id); err != nil { @@ -421,15 +421,6 @@ func handleOp(c kv.Cursor, stream remote.KV_TxServer, in *remote.Cursor) error { return nil } -func bytesCopy(b []byte) []byte { - if b == nil { - return nil - } - copiedBytes := make([]byte, len(b)) - copy(copiedBytes, b) - return copiedBytes -} - func (s *KvServer) StateChanges(_ *remote.StateChangeRequest, server remote.KV_StateChangesServer) error { ch, remove := s.stateChangeStreams.Sub() defer remove() @@ -647,8 +638,8 @@ func (s *KvServer) HistoryRange(_ context.Context, req *remote.HistoryRangeReq) if err != nil { return err } - key := bytesCopy(k) - value := bytesCopy(v) + key := common.CopyBytes(k) + value := common.CopyBytes(v) reply.Keys = append(reply.Keys, key) reply.Values = append(reply.Values, value) } @@ -692,8 +683,8 @@ func (s *KvServer) RangeAsOf(_ context.Context, req *remote.RangeAsOfReq) (*remo if err != nil { return err } - key := bytesCopy(k) - value := bytesCopy(v) + key := common.CopyBytes(k) + value := common.CopyBytes(v) reply.Keys = append(reply.Keys, key) reply.Values = append(reply.Values, value) limit-- diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 5db2fa4d5d3..af5170729e4 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -31,6 +31,7 @@ import ( "fmt" "hash" + libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/mclock" "github.com/erigontech/erigon-lib/rlp" "github.com/erigontech/erigon/p2p/enode" @@ -199,7 +200,7 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar // Store sent WHOAREYOU challenges. if challenge, ok := packet.(*Whoareyou); ok { - challenge.ChallengeData = bytesCopy(&c.buf) + challenge.ChallengeData = libcommon.CopyBytes(c.buf.Bytes()) c.sc.storeSentHandshake(id, addr, challenge) } else if msgData == nil { headerData := c.buf.Bytes() @@ -293,7 +294,7 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error // Create header. head := c.makeHeader(toID, flagWhoareyou, 0) - head.AuthData = bytesCopy(&c.buf) + head.AuthData = libcommon.CopyBytes(c.buf.Bytes()) head.Nonce = packet.Nonce // Encode auth data. @@ -400,7 +401,7 @@ func (c *Codec) encodeMessageHeader(toID enode.ID, s *session) (Header, error) { auth := messageAuthData{SrcID: c.localnode.ID()} c.buf.Reset() binary.Write(&c.buf, binary.BigEndian, &auth) //nolint:errcheck - head.AuthData = bytesCopy(&c.buf) + head.AuthData = libcommon.CopyBytes(c.buf.Bytes()) head.Nonce = nonce return head, err } @@ -649,9 +650,3 @@ func (h *Header) mask(destID enode.ID) cipher.Stream { } return cipher.NewCTR(block, h.IV[:]) } - -func bytesCopy(r *bytes.Buffer) []byte { - b := make([]byte, r.Len()) - copy(b, r.Bytes()) - return b -} diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index eb4c18780be..1b8f9a7f46d 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -23,16 +23,10 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon-lib/kv/dbutils" - "github.com/erigontech/erigon-lib/trie" - - "github.com/erigontech/erigon-lib/commitment" - libstate "github.com/erigontech/erigon-lib/state" "github.com/holiman/uint256" "google.golang.org/grpc" - "github.com/erigontech/erigon-lib/kv/membatchwithdb" - + "github.com/erigontech/erigon-lib/commitment" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutil" "github.com/erigontech/erigon-lib/common/hexutility" @@ -40,8 +34,12 @@ import ( "github.com/erigontech/erigon-lib/gointerfaces" txpool_proto "github.com/erigontech/erigon-lib/gointerfaces/txpoolproto" "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/kv/dbutils" + "github.com/erigontech/erigon-lib/kv/membatchwithdb" "github.com/erigontech/erigon-lib/kv/rawdbv3" "github.com/erigontech/erigon-lib/log/v3" + libstate "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/core" @@ -636,8 +634,7 @@ func (api *BaseAPI) getWitness(ctx context.Context, db kv.RoDB, blockNrOrHash rp fmt.Printf("state root mismatch after stateless execution actual(%x) != expected(%x)\n", newStateRoot.Bytes(), block.Root().Bytes()) } witnessBufBytes := witnessBuffer.Bytes() - witnessBufBytesCopy := make([]byte, len(witnessBufBytes)) - copy(witnessBufBytesCopy, witnessBufBytes) + witnessBufBytesCopy := libcommon.CopyBytes(witnessBufBytes) return witnessBufBytesCopy, nil } From e2563f8a6705f62b7b3b2fac56012008dc9c2627 Mon Sep 17 00:00:00 2001 From: awskii Date: Mon, 13 Jan 2025 13:54:12 +0000 Subject: [PATCH 85/94] commitment: clarify update kind based on `afterMap` (#13348) Closes https://github.com/erigontech/erigon/issues/13235 --- erigon-lib/commitment/hex_patricia_hashed.go | 53 +++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 46dd0251fb5..3d692dca755 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -1566,6 +1566,42 @@ func (hph *HexPatriciaHashed) createCellGetter(b []byte, updateKey []byte, row, } } +// updateKind is a type of update that is being applied to the trie structure. +type updateKind uint8 + +const ( + // updateKindDelete means after we processed longest common prefix, row ended up empty. + updateKindDelete updateKind = 0b0 + + // updateKindPropagate is an update operation ended up with a single nibble which is leaf or extension node. + // We do not store keys with only one cell as a value in db, instead we copy them upwards to the parent branch. + // + // In case current prefix existed before and node is fused to upper level, this causes deletion for current prefix + // and update of branch value on upper level. + // e.g.: leaf was at prefix 0xbeef, but we fuse it in level above, so + // - delete 0xbeef + // - update 0xbee + updateKindPropagate updateKind = 0b01 + + // updateKindBranch is an update operation ended up as a branch of 2+ cells. + // That does not necessarily means that branch is NEW, it could be an existing branch that was updated. + updateKindBranch updateKind = 0b10 +) + +// Kind defines how exactly given update should be folded upwards to the parent branch or root. +// It also returns number of nibbles that left in branch after the operation. +func afterMapUpdateKind(afterMap uint16) (kind updateKind, nibblesAfterUpdate int) { + nibblesAfterUpdate = bits.OnesCount16(afterMap) + switch nibblesAfterUpdate { + case 0: + return updateKindDelete, nibblesAfterUpdate + case 1: + return updateKindPropagate, nibblesAfterUpdate + default: + return updateKindBranch, nibblesAfterUpdate + } +} + // The purpose of fold is to reduce hph.currentKey[:hph.currentKeyLen]. It should be invoked // until that current key becomes a prefix of hashedKey that we will process next // (in other words until the needFolding function returns 0) @@ -1597,7 +1633,6 @@ func (hph *HexPatriciaHashed) fold() (err error) { depth := hph.depths[row] updateKey := hexToCompact(hph.currentKey[:updateKeyLen]) - partsCount := bits.OnesCount16(hph.afterMap[row]) defer func() { hph.depthsToTxNum[depth] = 0 }() if hph.trace { @@ -1605,8 +1640,9 @@ func (hph *HexPatriciaHashed) fold() (err error) { row, updatedNibs(hph.touchMap[row]&hph.afterMap[row]), depth, hph.currentKey[:hph.currentKeyLen], hph.touchMap[row], hph.afterMap[row]) } - switch partsCount { - case 0: // Everything deleted + updateKind, nibblesLeftAfterUpdate := afterMapUpdateKind(hph.afterMap[row]) + switch updateKind { + case updateKindDelete: // Everything deleted if hph.touchMap[row] != 0 { if row == 0 { // Root is deleted because the tree is empty @@ -1636,7 +1672,7 @@ func (hph *HexPatriciaHashed) fold() (err error) { } else { hph.currentKeyLen = 0 } - case 1: // Leaf or extension node + case updateKindPropagate: // Leaf or extension node if hph.touchMap[row] != 0 { // any modifications if row == 0 { @@ -1650,9 +1686,10 @@ func (hph *HexPatriciaHashed) fold() (err error) { cell := &hph.grid[row][nibble] upCell.extLen = 0 upCell.stateHashLen = 0 + // propagate cell into parent row upCell.fillFromLowerCell(cell, depth, hph.currentKey[upDepth:hph.currentKeyLen], nibble) - // Delete if it existed - if hph.branchBefore[row] { + + if hph.branchBefore[row] { // encode Delete if prefix existed before //fmt.Printf("delete existed row %d prefix %x\n", row, updateKey) _, err := hph.branchEncoder.CollectUpdate(hph.ctx, updateKey, 0, hph.touchMap[row], 0, RetrieveCellNoop) if err != nil { @@ -1664,7 +1701,7 @@ func (hph *HexPatriciaHashed) fold() (err error) { if hph.trace { fmt.Printf("formed leaf (%d %x, depth=%d) [%x] %s\n", row, nibble, depth, updateKey, cell.FullString()) } - default: // Branch node + case updateKindBranch: if hph.touchMap[row] != 0 { // any modifications if row == 0 { hph.rootTouched = true @@ -1682,7 +1719,7 @@ func (hph *HexPatriciaHashed) fold() (err error) { } // Calculate total length of all hashes - totalBranchLen := 17 - partsCount // For every empty cell, one byte + totalBranchLen := 17 - nibblesLeftAfterUpdate // For every empty cell, one byte for bitset, j := hph.afterMap[row], 0; bitset != 0; j++ { bit := bitset & -bitset nibble := bits.TrailingZeros16(bit) From cc9388b689e1262b85447e226d29ffadddd57304 Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:05:44 +0000 Subject: [PATCH 86/94] shutter: listen for decryption keys (#13306) closes https://github.com/erigontech/erigon/issues/13191 --- cmd/utils/flags.go | 28 +- eth/backend.go | 16 +- turbo/cli/default_flags.go | 5 +- txnprovider/shutter/.gitignore | 1 + txnprovider/shutter/Makefile | 37 ++ txnprovider/shutter/config.go | 75 ++- .../shutter/decryption_keys_listener.go | 294 ++++++++++ .../shutter/decryption_keys_processor.go | 62 ++ .../shutter/decryption_keys_validator.go | 113 ++++ .../shutter/decryption_keys_validator_test.go | 200 +++++++ .../testhelpers/collecting_log_handler.go | 66 +++ .../testhelpers/decryption_keys_mock_data.go | 122 ++++ txnprovider/shutter/pool.go | 37 +- txnprovider/shutter/proto/serialization.go | 71 +++ .../shutter/proto/serialization_test.go | 68 +++ txnprovider/shutter/proto/shutter.pb.go | 536 ++++++++++++++++++ txnprovider/shutter/proto/shutter.proto | 43 ++ 17 files changed, 1733 insertions(+), 41 deletions(-) create mode 100644 txnprovider/shutter/.gitignore create mode 100644 txnprovider/shutter/Makefile create mode 100644 txnprovider/shutter/decryption_keys_listener.go create mode 100644 txnprovider/shutter/decryption_keys_processor.go create mode 100644 txnprovider/shutter/decryption_keys_validator.go create mode 100644 txnprovider/shutter/decryption_keys_validator_test.go create mode 100644 txnprovider/shutter/internal/testhelpers/collecting_log_handler.go create mode 100644 txnprovider/shutter/internal/testhelpers/decryption_keys_mock_data.go create mode 100644 txnprovider/shutter/proto/serialization.go create mode 100644 txnprovider/shutter/proto/serialization_test.go create mode 100644 txnprovider/shutter/proto/shutter.pb.go create mode 100644 txnprovider/shutter/proto/shutter.proto diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 42da9f80cd5..8dbce0b6789 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1092,13 +1092,17 @@ var ( Usage: "Enable 'chaos monkey' to generate spontaneous network/consensus/etc failures. Use ONLY for testing", Value: false, } - ShutterEnabled = cli.BoolFlag{ + ShutterEnabledFlag = cli.BoolFlag{ Name: "shutter", Usage: "Enable the Shutter encrypted transactions mempool (defaults to false)", } - ShutterKeyperBootnodes = cli.StringSliceFlag{ - Name: "shutter.keyper.bootnodes", - Usage: "Use to override the default keyper bootnodes (defaults to using the bootnodes from the embedded config)", + ShutterP2pBootstrapNodesFlag = cli.StringSliceFlag{ + Name: "shutter.p2p.bootstrap.nodes", + Usage: "Use to override the default p2p bootstrap nodes (defaults to using the values in the embedded config)", + } + ShutterP2pListenPortFlag = cli.UintFlag{ + Name: "shutter.p2p.listen.port", + Usage: "Use to override the default p2p listen port (defaults to 23102)", } ) @@ -1582,18 +1586,22 @@ func setTxPool(ctx *cli.Context, dbDir string, fullCfg *ethconfig.Config) { fullCfg.TxPool = cfg } -func setShutter(ctx *cli.Context, chainName string, cfg *ethconfig.Config) { - if enabled := ctx.Bool(ShutterEnabled.Name); !enabled { +func setShutter(ctx *cli.Context, chainName string, nodeConfig *nodecfg.Config, ethConfig *ethconfig.Config) { + if enabled := ctx.Bool(ShutterEnabledFlag.Name); !enabled { return } config := shutter.ConfigByChainName(chainName) + config.PrivateKey = nodeConfig.P2P.PrivateKey // check for cli overrides - if ctx.IsSet(ShutterKeyperBootnodes.Name) { - config.KeyperBootnodes = ctx.StringSlice(ShutterKeyperBootnodes.Name) + if ctx.IsSet(ShutterP2pBootstrapNodesFlag.Name) { + config.BootstrapNodes = ctx.StringSlice(ShutterP2pBootstrapNodesFlag.Name) + } + if ctx.IsSet(ShutterP2pListenPortFlag.Name) { + config.ListenPort = ctx.Uint64(ShutterP2pListenPortFlag.Name) } - cfg.Shutter = config + ethConfig.Shutter = config } func setEthash(ctx *cli.Context, datadir string, cfg *ethconfig.Config) { @@ -1898,7 +1906,7 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C setGPO(ctx, &cfg.GPO) setTxPool(ctx, nodeConfig.Dirs.TxPool, cfg) - setShutter(ctx, chain, cfg) + setShutter(ctx, chain, nodeConfig, cfg) setEthash(ctx, nodeConfig.Dirs.DataDir, cfg) setClique(ctx, &cfg.Clique, nodeConfig.Dirs.DataDir) diff --git a/eth/backend.go b/eth/backend.go index 6917a1cb0c9..d16f0b0cf98 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -1589,11 +1589,23 @@ func (s *Ethereum) Start() error { // 1) Hive tests requires us to do so and starting it from eth_sendRawTransaction is not viable as we have not enough data // to initialize it properly. // 2) we cannot propose for block 1 regardless. - s.bgComponentsEg.Go(func() error { return s.txPool.Run(s.sentryCtx) }) + s.bgComponentsEg.Go(func() error { + err := s.txPool.Run(s.sentryCtx) + if err != nil { + s.logger.Error("txPool.Run error", "err", err) + } + return err + }) } if s.shutterPool != nil { - s.bgComponentsEg.Go(func() error { return s.shutterPool.Run(s.sentryCtx) }) + s.bgComponentsEg.Go(func() error { + err := s.shutterPool.Run(s.sentryCtx) + if err != nil { + s.logger.Error("shutterPool.Run error", "err", err) + } + return err + }) } return nil diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index e05e469cf24..e5c54cb0c3a 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -236,6 +236,7 @@ var DefaultFlags = []cli.Flag{ &utils.ChaosMonkeyFlag, - &utils.ShutterEnabled, - &utils.ShutterKeyperBootnodes, + &utils.ShutterEnabledFlag, + &utils.ShutterP2pBootstrapNodesFlag, + &utils.ShutterP2pListenPortFlag, } diff --git a/txnprovider/shutter/.gitignore b/txnprovider/shutter/.gitignore new file mode 100644 index 00000000000..567609b1234 --- /dev/null +++ b/txnprovider/shutter/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/txnprovider/shutter/Makefile b/txnprovider/shutter/Makefile new file mode 100644 index 00000000000..24929202268 --- /dev/null +++ b/txnprovider/shutter/Makefile @@ -0,0 +1,37 @@ +GOBINREL = build/bin +GOBIN = $(CURDIR)/$(GOBINREL) + +OS = $(shell uname -s) +ARCH = $(shell uname -m) + +ifeq ($(OS),Darwin) +PROTOC_OS := osx +ifeq ($(ARCH),arm64) +ARCH = aarch_64 +endif +endif +ifeq ($(OS),Linux) +PROTOC_OS = linux +endif + +PROTOC_INCLUDE = build/include/google +PROTO_DIR = $(CURDIR)/proto + +$(GOBINREL): + mkdir -p "$(GOBIN)" + +$(GOBINREL)/protoc: $(GOBINREL) + $(eval PROTOC_TMP := $(shell mktemp -d)) + curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-$(PROTOC_OS)-$(ARCH).zip -o "$(PROTOC_TMP)/protoc.zip" + cd "$(PROTOC_TMP)" && unzip protoc.zip + cp "$(PROTOC_TMP)/bin/protoc" "$(GOBIN)" + mkdir -p "$(PROTOC_INCLUDE)" + cp -R "$(PROTOC_TMP)/include/google/" "$(PROTOC_INCLUDE)" + rm -rf "$(PROTOC_TMP)" + +$(GOBINREL)/protoc-gen-go: $(GOBINREL) + go build -o "$(GOBIN)/protoc-gen-go" google.golang.org/protobuf/cmd/protoc-gen-go + +.PHONY: proto +proto: $(GOBINREL)/protoc $(GOBINREL)/protoc-gen-go + PATH="$(GOBIN):$(PATH)" protoc -I=$(PROTO_DIR) --go_out=$(PROTO_DIR) $(PROTO_DIR)/shutter.proto diff --git a/txnprovider/shutter/config.go b/txnprovider/shutter/config.go index bd5a3e6c873..9d41941d0a4 100644 --- a/txnprovider/shutter/config.go +++ b/txnprovider/shutter/config.go @@ -17,18 +17,48 @@ package shutter import ( + "crypto/ecdsa" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/erigontech/erigon-lib/chain/networkname" - "github.com/erigontech/erigon/params" ) type Config struct { - Enabled bool `json:"-"` - InstanceId uint64 `json:"instanceId"` - SequencerContractAddress string `json:"sequencerContractAddress"` - ValidatorRegistryContractAddress string `json:"validatorRegistryContractAddress"` - KeyBroadcastContractAddress string `json:"keyBroadcastContractAddress"` - KeyperSetManagerContractAddress string `json:"keyperSetManagerContractAddress"` - KeyperBootnodes []string `json:"keyperBootnodes"` + P2pConfig + Enabled bool + InstanceId uint64 + SequencerContractAddress string + ValidatorRegistryContractAddress string + KeyBroadcastContractAddress string + KeyperSetManagerContractAddress string + MaxNumKeysPerMessage uint64 +} + +type P2pConfig struct { + PrivateKey *ecdsa.PrivateKey + ListenPort uint64 + BootstrapNodes []string +} + +func (c P2pConfig) BootstrapNodesAddrInfo() ([]peer.AddrInfo, error) { + addrInfos := make([]peer.AddrInfo, len(c.BootstrapNodes)) + for i, node := range c.BootstrapNodes { + ma, err := multiaddr.NewMultiaddr(node) + if err != nil { + return nil, err + } + + ai, err := peer.AddrInfoFromP2pAddr(ma) + if err != nil { + return nil, err + } + + addrInfos[i] = *ai + } + + return addrInfos, nil } func ConfigByChainName(chainName string) Config { @@ -45,27 +75,40 @@ func ConfigByChainName(chainName string) Config { var ( chiadoConfig = Config{ Enabled: true, - InstanceId: params.ChiadoChainConfig.ChainID.Uint64(), + InstanceId: 102_000, SequencerContractAddress: "0x2aD8E2feB0ED5b2EC8e700edB725f120576994ed", ValidatorRegistryContractAddress: "0xa9289A3Dd14FEBe10611119bE81E5d35eAaC3084", KeyBroadcastContractAddress: "0x9D31865BEffcE842FBd36CDA587aDDA8bef804B7", KeyperSetManagerContractAddress: "0xC4DE9FAf4ec882b33dA0162CBE628B0D8205D0c0", - KeyperBootnodes: []string{ - "/ip4/167.99.177.227/tcp/23005/p2p/12D3KooWSdm5guPBdn8DSaBphVBzUUgPLg9sZLnazEUrcbtLy254", - "/ip4/159.89.15.119/tcp/23005/p2p/12D3KooWPP6bp2PJQR8rUvG1SD4qNH4WFrKve6DMgWThyKxwNbbH", + MaxNumKeysPerMessage: defaultMaxNumKeysPerMessage, + P2pConfig: P2pConfig{ + ListenPort: defaultP2PListenPort, + BootstrapNodes: []string{ + "/ip4/167.99.177.227/tcp/23005/p2p/12D3KooWSdm5guPBdn8DSaBphVBzUUgPLg9sZLnazEUrcbtLy254", + "/ip4/159.89.15.119/tcp/23005/p2p/12D3KooWPP6bp2PJQR8rUvG1SD4qNH4WFrKve6DMgWThyKxwNbbH", + }, }, } gnosisConfig = Config{ Enabled: true, - InstanceId: params.GnosisChainConfig.ChainID.Uint64(), + InstanceId: 1_000, SequencerContractAddress: "0xc5C4b277277A1A8401E0F039dfC49151bA64DC2E", ValidatorRegistryContractAddress: "0xefCC23E71f6bA9B22C4D28F7588141d44496A6D6", KeyBroadcastContractAddress: "0x626dB87f9a9aC47070016A50e802dd5974341301", KeyperSetManagerContractAddress: "0x7C2337f9bFce19d8970661DA50dE8DD7d3D34abb", - KeyperBootnodes: []string{ - "/ip4/167.99.177.227/tcp/23003/p2p/12D3KooWD35AESYCttDEi3J5WnQdTFuM5JNtmuXEb1x4eQ28gb1s", - "/ip4/159.89.15.119/tcp/23003/p2p/12D3KooWRzAhgPA16DiBQhiuYoasYzJaQSAbtc5i5FvgTi9ZDQtS", + MaxNumKeysPerMessage: defaultMaxNumKeysPerMessage, + P2pConfig: P2pConfig{ + ListenPort: defaultP2PListenPort, + BootstrapNodes: []string{ + "/ip4/167.99.177.227/tcp/23003/p2p/12D3KooWD35AESYCttDEi3J5WnQdTFuM5JNtmuXEb1x4eQ28gb1s", + "/ip4/159.89.15.119/tcp/23003/p2p/12D3KooWRzAhgPA16DiBQhiuYoasYzJaQSAbtc5i5FvgTi9ZDQtS", + }, }, } ) + +const ( + defaultP2PListenPort = 23_102 + defaultMaxNumKeysPerMessage = 500 +) diff --git a/txnprovider/shutter/decryption_keys_listener.go b/txnprovider/shutter/decryption_keys_listener.go new file mode 100644 index 00000000000..3da6be015a8 --- /dev/null +++ b/txnprovider/shutter/decryption_keys_listener.go @@ -0,0 +1,294 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package shutter + +import ( + "context" + "strconv" + "time" + + "github.com/libp2p/go-libp2p" + pubsub "github.com/libp2p/go-libp2p-pubsub" + libp2pcrypto "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "golang.org/x/sync/errgroup" + + "github.com/erigontech/erigon-lib/event" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/params" + "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +const ( + ProtocolVersion = "/shutter/0.1.0" + DecryptionKeysTopic = "decryptionKeys" +) + +type DecryptionKeysListener struct { + logger log.Logger + config Config + observers *event.Observers[*proto.DecryptionKeys] +} + +func NewDecryptionKeysListener(logger log.Logger, config Config) DecryptionKeysListener { + return DecryptionKeysListener{ + logger: logger, + config: config, + observers: event.NewObservers[*proto.DecryptionKeys](), + } +} + +func (dkl DecryptionKeysListener) RegisterObserver(observer event.Observer[*proto.DecryptionKeys]) event.UnregisterFunc { + return dkl.observers.Register(observer) +} + +func (dkl DecryptionKeysListener) Run(ctx context.Context) error { + dkl.logger.Info("running decryption keys listener") + + p2pHost, err := dkl.initP2pHost() + if err != nil { + return err + } + + pubSub, err := dkl.initGossipSub(ctx, p2pHost) + if err != nil { + return err + } + + err = dkl.connectBootstrapNodes(ctx, p2pHost) + if err != nil { + return err + } + + // + // TODO play around with go-libp2p-kad-dht for routing and discovery analogous to rolling-shutter + // check if it improves number of peers for topic + // + + // + // TODO persist connected nodes to be able to re-use on restart + // + + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { return dkl.listenLoop(ctx, pubSub) }) + eg.Go(func() error { return dkl.peerInfoLoop(ctx, pubSub) }) + return eg.Wait() +} + +func (dkl DecryptionKeysListener) initP2pHost() (host.Host, error) { + listenAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + strconv.FormatUint(dkl.config.ListenPort, 10)) + if err != nil { + return nil, err + } + + privKey, err := libp2pcrypto.UnmarshalSecp256k1PrivateKey(dkl.config.PrivateKey.D.Bytes()) + if err != nil { + return nil, err + } + + p2pHost, err := libp2p.New( + libp2p.Identity(privKey), + libp2p.ListenAddrs(listenAddr), + libp2p.UserAgent("erigon/shutter/"+params.VersionWithCommit(params.GitCommit)), + libp2p.ProtocolVersion(ProtocolVersion), + ) + if err != nil { + return nil, err + } + + dkl.logger.Info("shutter p2p host initialised", "addr", listenAddr, "id", p2pHost.ID()) + return p2pHost, nil +} + +func (dkl DecryptionKeysListener) initGossipSub(ctx context.Context, host host.Host) (*pubsub.PubSub, error) { + // NOTE: gossipSubParams, peerScoreParams, peerScoreThresholds are taken from + // https://github.com/shutter-network/rolling-shutter/blob/main/rolling-shutter/p2p/params.go#L16 + gossipSubParams := pubsub.DefaultGossipSubParams() + gossipSubParams.HeartbeatInterval = 700 * time.Millisecond + gossipSubParams.HistoryLength = 6 + + bootstrapNodes, err := dkl.config.BootstrapNodesAddrInfo() + if err != nil { + return nil, err + } + + bootstrapNodesSet := make(map[peer.ID]bool, len(dkl.config.BootstrapNodes)) + for _, node := range bootstrapNodes { + bootstrapNodesSet[node.ID] = true + } + + // NOTE: loosely from the gossipsub spec: + // Only the bootstrappers / highly trusted PX'ing nodes + // should reach the AcceptPXThreshold thus they need + // to be treated differently in the scoring function. + appSpecificScoringFn := func(p peer.ID) float64 { + _, ok := bootstrapNodesSet[p] + if !ok { + return 0 + } + // In order to be able to participate in the gossipsub, + // a peer has to be PX'ed by a bootstrap node - this is only + // possible if the AcceptPXThreshold peer-score is reached. + + // NOTE: we have yet to determine a value that is + // sufficient to reach the AcceptPXThreshold most of the time, + // but don't overshoot and trust the bootstrap peers + // unconditionally - they should still be punishable + // for malicous behavior + return 200 + } + peerScoreParams := &pubsub.PeerScoreParams{ + // Topics score-map will be filled later while subscribing to topics. + Topics: make(map[string]*pubsub.TopicScoreParams), + TopicScoreCap: 32.72, + AppSpecificScore: appSpecificScoringFn, + AppSpecificWeight: 1, + IPColocationFactorWeight: -35.11, + IPColocationFactorThreshold: 10, + IPColocationFactorWhitelist: nil, + BehaviourPenaltyWeight: -15.92, + BehaviourPenaltyThreshold: 6, + BehaviourPenaltyDecay: 0.928, + DecayInterval: 12 * time.Second, + DecayToZero: 0.01, + RetainScore: 12 * time.Hour, + } + + peerScoreThresholds := &pubsub.PeerScoreThresholds{ + GossipThreshold: -4000, + PublishThreshold: -8000, + GraylistThreshold: -16000, + AcceptPXThreshold: 100, + OpportunisticGraftThreshold: 5, + } + + return pubsub.NewGossipSub( + ctx, + host, + pubsub.WithGossipSubParams(gossipSubParams), + pubsub.WithPeerScore(peerScoreParams, peerScoreThresholds), + ) +} + +func (dkl DecryptionKeysListener) connectBootstrapNodes(ctx context.Context, host host.Host) error { + nodes, err := dkl.config.BootstrapNodesAddrInfo() + if err != nil { + return err + } + + wg, ctx := errgroup.WithContext(ctx) + for _, node := range nodes { + wg.Go(func() error { + err := host.Connect(ctx, node) + if err != nil { + dkl.logger.Error("failed to connect to bootstrap node", "node", node, "err", err) + } + return nil + }) + } + + return wg.Wait() +} + +func (dkl DecryptionKeysListener) listenLoop(ctx context.Context, pubSub *pubsub.PubSub) error { + topicValidator := NewDecryptionKeysP2pValidatorEx(dkl.logger, dkl.config) + err := pubSub.RegisterTopicValidator(DecryptionKeysTopic, topicValidator) + if err != nil { + return err + } + + topic, err := pubSub.Join(DecryptionKeysTopic) + if err != nil { + return err + } + defer func() { + if err := topic.Close(); err != nil { + dkl.logger.Error("failed to close decryption keys topic", "err", err) + } + }() + + err = topic.SetScoreParams(decryptionKeysTopicScoreParams()) + if err != nil { + return err + } + + sub, err := topic.Subscribe() + if err != nil { + return err + } + defer sub.Cancel() + + for { + msg, err := sub.Next(ctx) + if err != nil { + return err + } + + decryptionKeys, err := proto.UnmarshallDecryptionKeys(msg.Data) + if err != nil { + dkl.logger.Debug("failed to unmarshal decryption keys, skipping message", "err", err) + continue + } + + dkl.observers.Notify(decryptionKeys) + } +} + +func (dkl DecryptionKeysListener) peerInfoLoop(ctx context.Context, pubSub *pubsub.PubSub) error { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + peers := pubSub.ListPeers(DecryptionKeysTopic) + dkl.logger.Info("decryption keys peer count", "peers", len(peers)) + } + } +} + +func decryptionKeysTopicScoreParams() *pubsub.TopicScoreParams { + // NOTE: this is taken from + // https://github.com/shutter-network/rolling-shutter/blob/main/rolling-shutter/p2p/params.go#L100 + // + // Based on attestation topic in beacon chain network. The formula uses the number of + // validators which we set to a fixed number which could be the number of keypers. + n := float64(200) + return &pubsub.TopicScoreParams{ + TopicWeight: 1, + TimeInMeshWeight: 0.0324, + TimeInMeshQuantum: 12 * time.Second, + TimeInMeshCap: 300, + FirstMessageDeliveriesWeight: 0.05, + FirstMessageDeliveriesDecay: 0.631, + FirstMessageDeliveriesCap: n / 755.712, + MeshMessageDeliveriesWeight: -0.026, + MeshMessageDeliveriesDecay: 0.631, + MeshMessageDeliveriesCap: n / 94.464, + MeshMessageDeliveriesThreshold: n / 377.856, + MeshMessageDeliveriesWindow: 200 * time.Millisecond, + MeshMessageDeliveriesActivation: 4 * 12 * time.Second, + MeshFailurePenaltyWeight: -0.0026, + MeshFailurePenaltyDecay: 0.631, + InvalidMessageDeliveriesWeight: -99, + InvalidMessageDeliveriesDecay: 0.9994, + } +} diff --git a/txnprovider/shutter/decryption_keys_processor.go b/txnprovider/shutter/decryption_keys_processor.go new file mode 100644 index 00000000000..50c1743352c --- /dev/null +++ b/txnprovider/shutter/decryption_keys_processor.go @@ -0,0 +1,62 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package shutter + +import ( + "context" + + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +type DecryptionKeysProcessor struct { + logger log.Logger + queue chan *proto.DecryptionKeys +} + +func NewDecryptionKeysProcessor(logger log.Logger) DecryptionKeysProcessor { + return DecryptionKeysProcessor{ + logger: logger, + queue: make(chan *proto.DecryptionKeys), + } +} + +func (dkp DecryptionKeysProcessor) Enqueue(msg *proto.DecryptionKeys) { + dkp.queue <- msg +} + +func (dkp DecryptionKeysProcessor) Run(ctx context.Context) error { + dkp.logger.Info("running decryption keys processor") + + for { + select { + case msg := <-dkp.queue: + dkp.logger.Debug( + "processing decryption keys message", + "instanceId", msg.InstanceId, + "eon", msg.Eon, + "slot", msg.GetGnosis().Slot, + "txPointer", msg.GetGnosis().TxPointer, + ) + // + // TODO process msg + // + case <-ctx.Done(): + return ctx.Err() + } + } +} diff --git a/txnprovider/shutter/decryption_keys_validator.go b/txnprovider/shutter/decryption_keys_validator.go new file mode 100644 index 00000000000..7bcd21e6427 --- /dev/null +++ b/txnprovider/shutter/decryption_keys_validator.go @@ -0,0 +1,113 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package shutter + +import ( + "context" + "errors" + "fmt" + "math" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +var ( + ErrInstanceIdMismatch = errors.New("instance id mismatch") + ErrMissingGnosisExtraData = errors.New("missing gnosis extra data") + ErrSlotTooLarge = errors.New("slot too large") + ErrTxPointerTooLarge = errors.New("tx pointer too large") + ErrEonTooLarge = errors.New("eon too large") + ErrEmptyKeys = errors.New("empty keys") + ErrTooManyKeys = errors.New("too many keys") +) + +type DecryptionKeysValidator struct { + config Config +} + +func NewDecryptionKeysValidator(config Config) DecryptionKeysValidator { + return DecryptionKeysValidator{ + config: config, + } +} + +func (v DecryptionKeysValidator) Validate(msg *proto.DecryptionKeys) error { + if msg.InstanceId != v.config.InstanceId { + return fmt.Errorf("%w: %d", ErrInstanceIdMismatch, msg.InstanceId) + } + + gnosisExtraData := msg.GetGnosis() + if gnosisExtraData == nil { + return ErrMissingGnosisExtraData + } + + if gnosisExtraData.Slot > math.MaxInt64 { + return fmt.Errorf("%w: %d", ErrSlotTooLarge, gnosisExtraData.Slot) + } + + if gnosisExtraData.TxPointer > math.MaxInt32 { + return fmt.Errorf("%w: %d", ErrTxPointerTooLarge, gnosisExtraData.TxPointer) + } + + if msg.Eon > math.MaxInt64 { + return fmt.Errorf("%w: %d", ErrEonTooLarge, msg.Eon) + } + + if len(msg.Keys) == 0 { + return ErrEmptyKeys + } + + if len(msg.Keys) > int(v.config.MaxNumKeysPerMessage) { + return fmt.Errorf("%w: %d", ErrTooManyKeys, len(msg.Keys)) + } + + // + // TODO when we add the shcrypto library and smart contract state accessors: + // - add validation for signers and signatures based on keyper set and eon infos + // - add DecryptionKeys.Validate() equivalent which checks the Key unmarshalling into shcrypto.EpochSecretKey + // - add validation forVerifyEpochSecretKey: check if we should be doing this validation + // + return nil +} + +func NewDecryptionKeysP2pValidatorEx(logger log.Logger, config Config) pubsub.ValidatorEx { + dkv := NewDecryptionKeysValidator(config) + return func(ctx context.Context, id peer.ID, msg *pubsub.Message) pubsub.ValidationResult { + if topic := msg.GetTopic(); topic != DecryptionKeysTopic { + logger.Debug("rejecting decryption keys msg due to topic mismatch", "topic", topic, "peer", id) + return pubsub.ValidationReject + } + + decryptionKeys, err := proto.UnmarshallDecryptionKeys(msg.GetData()) + if err != nil { + logger.Debug("rejecting decryption keys msg due to unmarshalling error", "err", err, "peer", id) + return pubsub.ValidationReject + } + + err = dkv.Validate(decryptionKeys) + if err != nil { + logger.Debug("rejecting decryption keys msg due to data validation error", "err", err, "peer", id) + return pubsub.ValidationReject + } + + return pubsub.ValidationAccept + } +} diff --git a/txnprovider/shutter/decryption_keys_validator_test.go b/txnprovider/shutter/decryption_keys_validator_test.go new file mode 100644 index 00000000000..224313c5b53 --- /dev/null +++ b/txnprovider/shutter/decryption_keys_validator_test.go @@ -0,0 +1,200 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package shutter_test + +import ( + "context" + "math" + "testing" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/turbo/testlog" + "github.com/erigontech/erigon/txnprovider/shutter" + "github.com/erigontech/erigon/txnprovider/shutter/internal/testhelpers" + shutterproto "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +func TestDecryptionKeysValidator(t *testing.T) { + t.Parallel() + for _, tc := range decryptionKeysValidatorTestCases(t) { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + msg, err := shutterproto.UnmarshallDecryptionKeys(tc.msg.Data) + require.NoError(t, err) + + validator := shutter.NewDecryptionKeysValidator(shutter.Config{ + InstanceId: testhelpers.TestInstanceId, + MaxNumKeysPerMessage: testhelpers.TestMaxNumKeysPerMessage, + }) + haveErr := validator.Validate(msg) + require.ErrorIs(t, haveErr, tc.wantErr) + }) + } +} + +func TestDecryptionKeysP2pValidatorEx(t *testing.T) { + t.Parallel() + for _, tc := range decryptionKeysP2pValidatorExTestCases(t) { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := testlog.Logger(t, log.LvlCrit) + logHandler := testhelpers.NewCollectingLogHandler(logger.GetHandler()) + logger.SetHandler(logHandler) + + validator := shutter.NewDecryptionKeysP2pValidatorEx(logger, shutter.Config{ + InstanceId: testhelpers.TestInstanceId, + MaxNumKeysPerMessage: testhelpers.TestMaxNumKeysPerMessage, + }) + haveValidationResult := validator(ctx, "peer1", tc.msg) + require.Equal(t, tc.wantValidationResult, haveValidationResult) + require.True( + t, + logHandler.ContainsAll(tc.wantValidationLogMsgs), + "%v vs %v", + tc.wantValidationLogMsgs, + logHandler.FormattedRecords(), + ) + }) + } +} + +type decryptionKeysValidationTestCase struct { + name string + msg *pubsub.Message + wantErr error + wantValidationResult pubsub.ValidationResult + wantValidationLogMsgs []string +} + +func decryptionKeysValidatorTestCases(t *testing.T) []decryptionKeysValidationTestCase { + return []decryptionKeysValidationTestCase{ + { + name: "instance id mismatch", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + InstanceIdOverride: 999999, + }), + wantErr: shutter.ErrInstanceIdMismatch, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "instance id mismatch: 999999", + }, + }, + { + name: "missing gnosis extra data", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + NilExtra: true, + }), + wantErr: shutter.ErrMissingGnosisExtraData, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "missing gnosis extra data", + }, + }, + { + name: "slot too large", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + Slot: math.MaxInt64 + 1, + }), + wantErr: shutter.ErrSlotTooLarge, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "slot too large: 9223372036854775808", + }, + }, + { + name: "tx pointer too large", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + TxPointer: math.MaxInt32 + 1, + }), + wantErr: shutter.ErrTxPointerTooLarge, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "tx pointer too large: 2147483648", + }, + }, + { + name: "eon too large", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + Eon: math.MaxInt64 + 1, + }), + wantErr: shutter.ErrEonTooLarge, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "eon too large: 9223372036854775808", + }, + }, + { + name: "empty keys", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + Keys: [][]byte{}, + IdentityPreimages: [][]byte{}, + }), + wantErr: shutter.ErrEmptyKeys, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "empty keys", + }, + }, + { + name: "too many keys", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + Keys: [][]byte{[]byte("key1"), []byte("key2"), []byte("key3"), []byte("key4")}, + IdentityPreimages: [][]byte{[]byte("id1"), []byte("id2"), []byte("id3"), []byte("id4")}, + }), + wantErr: shutter.ErrTooManyKeys, + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to data validation error", + "too many keys: 4", + }, + }, + } +} + +func decryptionKeysP2pValidatorExTestCases(t *testing.T) []decryptionKeysValidationTestCase { + return append( + []decryptionKeysValidationTestCase{ + { + name: "unmarshalling error", + msg: testhelpers.MockDecryptionKeysMsg(t, testhelpers.MockDecryptionKeysMsgOptions{ + EnvelopeDataOverride: []byte("invalid"), + }), + wantValidationResult: pubsub.ValidationReject, + wantValidationLogMsgs: []string{ + "rejecting decryption keys msg due to unmarshalling error", + "cannot parse invalid wire-format data", + }, + }, + }, + decryptionKeysValidatorTestCases(t)..., + ) +} diff --git a/txnprovider/shutter/internal/testhelpers/collecting_log_handler.go b/txnprovider/shutter/internal/testhelpers/collecting_log_handler.go new file mode 100644 index 00000000000..bfca777ddd1 --- /dev/null +++ b/txnprovider/shutter/internal/testhelpers/collecting_log_handler.go @@ -0,0 +1,66 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package testhelpers + +import ( + "strings" + + "github.com/erigontech/erigon-lib/log/v3" +) + +type CollectingLogHandler struct { + records []*log.Record + handler log.Handler +} + +func NewCollectingLogHandler(handler log.Handler) *CollectingLogHandler { + return &CollectingLogHandler{ + handler: handler, + } +} + +func (clh *CollectingLogHandler) Log(r *log.Record) error { + clh.records = append(clh.records, r) + return clh.handler.Log(r) +} + +func (clh *CollectingLogHandler) ContainsAll(subStrs []string) bool { + for _, subStr := range subStrs { + if !clh.Contains(subStr) { + return false + } + } + return true +} + +func (clh *CollectingLogHandler) Contains(subStr string) bool { + for _, r := range clh.records { + msg := string(log.TerminalFormatNoColor().Format(r)) + if strings.Contains(msg, subStr) { + return true + } + } + return false +} + +func (clh *CollectingLogHandler) FormattedRecords() []string { + formattedRecords := make([]string, len(clh.records)) + for i, record := range clh.records { + formattedRecords[i] = strings.TrimSuffix(string(log.TerminalFormatNoColor().Format(record)), "\n") + } + return formattedRecords +} diff --git a/txnprovider/shutter/internal/testhelpers/decryption_keys_mock_data.go b/txnprovider/shutter/internal/testhelpers/decryption_keys_mock_data.go new file mode 100644 index 00000000000..346f5e95ecb --- /dev/null +++ b/txnprovider/shutter/internal/testhelpers/decryption_keys_mock_data.go @@ -0,0 +1,122 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package testhelpers + +import ( + "testing" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/erigontech/erigon/txnprovider/shutter" + shutterproto "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +const ( + TestInstanceId = 123 + TestMaxNumKeysPerMessage = 3 +) + +type MockDecryptionKeysMsgOptions struct { + Eon uint64 + Keys [][]byte + IdentityPreimages [][]byte + NilExtra bool + Slot uint64 + TxPointer uint64 + SignerIndices []uint64 + Signatures [][]byte + InstanceIdOverride uint64 + VersionOverride string + TopicOverride string + EnvelopeDataOverride []byte +} + +func MockDecryptionKeysMsg(t *testing.T, opts MockDecryptionKeysMsgOptions) *pubsub.Message { + var data []byte + if opts.EnvelopeDataOverride != nil { + data = opts.EnvelopeDataOverride + } else { + data = MockDecryptionKeysEnvelopeData(t, opts) + } + + var topic string + if opts.TopicOverride != "" { + topic = opts.TopicOverride + } else { + topic = shutter.DecryptionKeysTopic + } + + return &pubsub.Message{ + Message: &pb.Message{ + Data: data, + Topic: &topic, + }, + } +} + +func MockDecryptionKeysEnvelopeData(t *testing.T, opts MockDecryptionKeysMsgOptions) []byte { + var instanceId uint64 + if opts.InstanceIdOverride != 0 { + instanceId = opts.InstanceIdOverride + } else { + instanceId = TestInstanceId + } + + decryptionKeys := &shutterproto.DecryptionKeys{ + InstanceId: instanceId, + Eon: opts.Eon, + } + + require.Equal(t, len(opts.Keys), len(opts.IdentityPreimages)) + for i := range opts.Keys { + decryptionKeys.Keys = append(decryptionKeys.Keys, &shutterproto.Key{ + Key: opts.Keys[i], + IdentityPreimage: opts.IdentityPreimages[i], + }) + } + + if !opts.NilExtra { + decryptionKeys.Extra = &shutterproto.DecryptionKeys_Gnosis{ + Gnosis: &shutterproto.GnosisDecryptionKeysExtra{ + Slot: opts.Slot, + TxPointer: opts.TxPointer, + SignerIndices: opts.SignerIndices, + Signatures: opts.Signatures, + }, + } + } + + var version string + if opts.VersionOverride != "" { + version = opts.VersionOverride + } else { + version = shutterproto.EnvelopeVersion + } + + decryptionKeysMsg, err := anypb.New(decryptionKeys) + require.NoError(t, err) + envelopeData, err := proto.Marshal(&shutterproto.Envelope{ + Version: version, + Message: decryptionKeysMsg, + }) + require.NoError(t, err) + return envelopeData +} diff --git a/txnprovider/shutter/pool.go b/txnprovider/shutter/pool.go index cce91c1d309..7fcc73a8b76 100644 --- a/txnprovider/shutter/pool.go +++ b/txnprovider/shutter/pool.go @@ -19,34 +19,49 @@ package shutter import ( "context" + "golang.org/x/sync/errgroup" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/txnprovider" + "github.com/erigontech/erigon/txnprovider/shutter/proto" ) var _ txnprovider.TxnProvider = (*Pool)(nil) type Pool struct { - logger log.Logger - config Config - secondaryTxnProvider txnprovider.TxnProvider + logger log.Logger + config Config + secondaryTxnProvider txnprovider.TxnProvider + decryptionKeysListener DecryptionKeysListener + decryptionKeysProcessor DecryptionKeysProcessor } func NewPool(logger log.Logger, config Config, secondaryTxnProvider txnprovider.TxnProvider) *Pool { + logger = logger.New("component", "shutter") + decryptionKeysListener := NewDecryptionKeysListener(logger, config) + decryptionKeysProcessor := NewDecryptionKeysProcessor(logger) return &Pool{ - logger: logger.New("component", "shutter"), - config: config, - secondaryTxnProvider: secondaryTxnProvider, + logger: logger, + config: config, + secondaryTxnProvider: secondaryTxnProvider, + decryptionKeysListener: decryptionKeysListener, + decryptionKeysProcessor: decryptionKeysProcessor, } } func (p Pool) Run(ctx context.Context) error { p.logger.Info("running pool") - // - // TODO - start pool, sentinel listeners for keyper decryption keys and other necessary background goroutines - // blocks until all sub-components have shutdown or have error-ed - // - return nil + + unregisterDkpObserver := p.decryptionKeysListener.RegisterObserver(func(msg *proto.DecryptionKeys) { + p.decryptionKeysProcessor.Enqueue(msg) + }) + defer unregisterDkpObserver() + + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { return p.decryptionKeysListener.Run(ctx) }) + eg.Go(func() error { return p.decryptionKeysProcessor.Run(ctx) }) + return eg.Wait() } func (p Pool) ProvideTxns(ctx context.Context, opts ...txnprovider.ProvideOption) ([]types.Transaction, error) { diff --git a/txnprovider/shutter/proto/serialization.go b/txnprovider/shutter/proto/serialization.go new file mode 100644 index 00000000000..fb8bf1dbf9a --- /dev/null +++ b/txnprovider/shutter/proto/serialization.go @@ -0,0 +1,71 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package proto + +import ( + "errors" + "fmt" + "strings" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + EnvelopeVersion = "0.0.1" +) + +var ( + ErrEmptyEnvelope = errors.New("empty envelope") + ErrEnveloperVersionMismatch = errors.New("envelope version mismatch") + ErrProtoUnmarshall = errors.New("issue unmarshalling proto bytes") +) + +func UnmarshallDecryptionKeys(envelopeBytes []byte) (*DecryptionKeys, error) { + envelope, err := UnmarshallEnvelope(envelopeBytes) + if err != nil { + return nil, err + } + + if envelope.Message == nil { + return nil, fmt.Errorf("%w", ErrEmptyEnvelope) + } + + // needed to avoid marshalling error, gist is that keypers use p2pmsg package name instead of proto + envelope.Message.TypeUrl = strings.Replace(envelope.Message.TypeUrl, "p2pmsg", "proto", 1) + decryptionKeys := DecryptionKeys{} + err = anypb.UnmarshalTo(envelope.Message, &decryptionKeys, proto.UnmarshalOptions{}) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrProtoUnmarshall, err) + } + + return &decryptionKeys, nil +} + +func UnmarshallEnvelope(envelopeBytes []byte) (*Envelope, error) { + envelope := Envelope{} + err := proto.Unmarshal(envelopeBytes, &envelope) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrProtoUnmarshall, err) + } + + if envelope.Version != EnvelopeVersion { + return nil, fmt.Errorf("%w: %s", ErrEnveloperVersionMismatch, envelope.Version) + } + + return &envelope, nil +} diff --git a/txnprovider/shutter/proto/serialization_test.go b/txnprovider/shutter/proto/serialization_test.go new file mode 100644 index 00000000000..cfcb8aa8f19 --- /dev/null +++ b/txnprovider/shutter/proto/serialization_test.go @@ -0,0 +1,68 @@ +// Copyright 2025 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package proto_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/erigontech/erigon/txnprovider/shutter/internal/testhelpers" + shutterproto "github.com/erigontech/erigon/txnprovider/shutter/proto" +) + +func TestUnmarshallDecryptionKeys(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + data []byte + wantErr error + }{ + { + name: "invalid envelope version", + data: testhelpers.MockDecryptionKeysEnvelopeData(t, testhelpers.MockDecryptionKeysMsgOptions{ + VersionOverride: "XXX", + }), + wantErr: shutterproto.ErrEnveloperVersionMismatch, + }, + { + name: "empty envelope message", + data: func(t *testing.T) []byte { + data, err := proto.Marshal(&shutterproto.Envelope{ + Version: shutterproto.EnvelopeVersion, + }) + require.NoError(t, err) + return data + }(t), + wantErr: shutterproto.ErrEmptyEnvelope, + }, + { + name: "invalid data bytes", + data: []byte("invalid"), + wantErr: shutterproto.ErrProtoUnmarshall, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, haveErr := shutterproto.UnmarshallDecryptionKeys(tc.data) + require.ErrorIs(t, haveErr, tc.wantErr) + }) + } +} diff --git a/txnprovider/shutter/proto/shutter.pb.go b/txnprovider/shutter/proto/shutter.pb.go new file mode 100644 index 00000000000..7e98db5e835 --- /dev/null +++ b/txnprovider/shutter/proto/shutter.pb.go @@ -0,0 +1,536 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.1 +// protoc v5.27.1 +// source: shutter.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Envelope struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + Message *anypb.Any `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + Trace *TraceContext `protobuf:"bytes,3,opt,name=trace,proto3,oneof" json:"trace,omitempty"` +} + +func (x *Envelope) Reset() { + *x = Envelope{} + mi := &file_shutter_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Envelope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Envelope) ProtoMessage() {} + +func (x *Envelope) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Envelope.ProtoReflect.Descriptor instead. +func (*Envelope) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{0} +} + +func (x *Envelope) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *Envelope) GetMessage() *anypb.Any { + if x != nil { + return x.Message + } + return nil +} + +func (x *Envelope) GetTrace() *TraceContext { + if x != nil { + return x.Trace + } + return nil +} + +type TraceContext struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TraceId []byte `protobuf:"bytes,1,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanId []byte `protobuf:"bytes,2,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + TraceFlags []byte `protobuf:"bytes,3,opt,name=trace_flags,json=traceFlags,proto3" json:"trace_flags,omitempty"` + TraceState string `protobuf:"bytes,4,opt,name=trace_state,json=traceState,proto3" json:"trace_state,omitempty"` +} + +func (x *TraceContext) Reset() { + *x = TraceContext{} + mi := &file_shutter_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceContext) ProtoMessage() {} + +func (x *TraceContext) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceContext.ProtoReflect.Descriptor instead. +func (*TraceContext) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{1} +} + +func (x *TraceContext) GetTraceId() []byte { + if x != nil { + return x.TraceId + } + return nil +} + +func (x *TraceContext) GetSpanId() []byte { + if x != nil { + return x.SpanId + } + return nil +} + +func (x *TraceContext) GetTraceFlags() []byte { + if x != nil { + return x.TraceFlags + } + return nil +} + +func (x *TraceContext) GetTraceState() string { + if x != nil { + return x.TraceState + } + return "" +} + +type DecryptionKeys struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + InstanceId uint64 `protobuf:"varint,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + Eon uint64 `protobuf:"varint,2,opt,name=eon,proto3" json:"eon,omitempty"` + Keys []*Key `protobuf:"bytes,3,rep,name=keys,proto3" json:"keys,omitempty"` + // Types that are assignable to Extra: + // + // *DecryptionKeys_Gnosis + // *DecryptionKeys_Optimism + Extra isDecryptionKeys_Extra `protobuf_oneof:"extra"` +} + +func (x *DecryptionKeys) Reset() { + *x = DecryptionKeys{} + mi := &file_shutter_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DecryptionKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DecryptionKeys) ProtoMessage() {} + +func (x *DecryptionKeys) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DecryptionKeys.ProtoReflect.Descriptor instead. +func (*DecryptionKeys) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{2} +} + +func (x *DecryptionKeys) GetInstanceId() uint64 { + if x != nil { + return x.InstanceId + } + return 0 +} + +func (x *DecryptionKeys) GetEon() uint64 { + if x != nil { + return x.Eon + } + return 0 +} + +func (x *DecryptionKeys) GetKeys() []*Key { + if x != nil { + return x.Keys + } + return nil +} + +func (m *DecryptionKeys) GetExtra() isDecryptionKeys_Extra { + if m != nil { + return m.Extra + } + return nil +} + +func (x *DecryptionKeys) GetGnosis() *GnosisDecryptionKeysExtra { + if x, ok := x.GetExtra().(*DecryptionKeys_Gnosis); ok { + return x.Gnosis + } + return nil +} + +func (x *DecryptionKeys) GetOptimism() *OptimismDecryptionKeysExtra { + if x, ok := x.GetExtra().(*DecryptionKeys_Optimism); ok { + return x.Optimism + } + return nil +} + +type isDecryptionKeys_Extra interface { + isDecryptionKeys_Extra() +} + +type DecryptionKeys_Gnosis struct { + Gnosis *GnosisDecryptionKeysExtra `protobuf:"bytes,4,opt,name=gnosis,proto3,oneof"` +} + +type DecryptionKeys_Optimism struct { + Optimism *OptimismDecryptionKeysExtra `protobuf:"bytes,5,opt,name=optimism,proto3,oneof"` +} + +func (*DecryptionKeys_Gnosis) isDecryptionKeys_Extra() {} + +func (*DecryptionKeys_Optimism) isDecryptionKeys_Extra() {} + +type Key struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IdentityPreimage []byte `protobuf:"bytes,1,opt,name=identity_preimage,json=identityPreimage,proto3" json:"identity_preimage,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *Key) Reset() { + *x = Key{} + mi := &file_shutter_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Key) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Key) ProtoMessage() {} + +func (x *Key) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Key.ProtoReflect.Descriptor instead. +func (*Key) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{3} +} + +func (x *Key) GetIdentityPreimage() []byte { + if x != nil { + return x.IdentityPreimage + } + return nil +} + +func (x *Key) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +type GnosisDecryptionKeysExtra struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Slot uint64 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` + TxPointer uint64 `protobuf:"varint,2,opt,name=tx_pointer,json=txPointer,proto3" json:"tx_pointer,omitempty"` + SignerIndices []uint64 `protobuf:"varint,3,rep,packed,name=signer_indices,json=signerIndices,proto3" json:"signer_indices,omitempty"` + Signatures [][]byte `protobuf:"bytes,4,rep,name=signatures,proto3" json:"signatures,omitempty"` +} + +func (x *GnosisDecryptionKeysExtra) Reset() { + *x = GnosisDecryptionKeysExtra{} + mi := &file_shutter_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GnosisDecryptionKeysExtra) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GnosisDecryptionKeysExtra) ProtoMessage() {} + +func (x *GnosisDecryptionKeysExtra) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GnosisDecryptionKeysExtra.ProtoReflect.Descriptor instead. +func (*GnosisDecryptionKeysExtra) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{4} +} + +func (x *GnosisDecryptionKeysExtra) GetSlot() uint64 { + if x != nil { + return x.Slot + } + return 0 +} + +func (x *GnosisDecryptionKeysExtra) GetTxPointer() uint64 { + if x != nil { + return x.TxPointer + } + return 0 +} + +func (x *GnosisDecryptionKeysExtra) GetSignerIndices() []uint64 { + if x != nil { + return x.SignerIndices + } + return nil +} + +func (x *GnosisDecryptionKeysExtra) GetSignatures() [][]byte { + if x != nil { + return x.Signatures + } + return nil +} + +type OptimismDecryptionKeysExtra struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *OptimismDecryptionKeysExtra) Reset() { + *x = OptimismDecryptionKeysExtra{} + mi := &file_shutter_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OptimismDecryptionKeysExtra) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OptimismDecryptionKeysExtra) ProtoMessage() {} + +func (x *OptimismDecryptionKeysExtra) ProtoReflect() protoreflect.Message { + mi := &file_shutter_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OptimismDecryptionKeysExtra.ProtoReflect.Descriptor instead. +func (*OptimismDecryptionKeysExtra) Descriptor() ([]byte, []int) { + return file_shutter_proto_rawDescGZIP(), []int{5} +} + +var File_shutter_proto protoreflect.FileDescriptor + +var file_shutter_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x73, 0x68, 0x75, 0x74, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x8e, 0x01, 0x0a, 0x08, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x48, 0x00, 0x52, 0x05, + 0x74, 0x72, 0x61, 0x63, 0x65, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x22, 0x84, 0x01, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x74, 0x72, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x17, + 0x0a, 0x07, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x73, 0x70, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x74, 0x72, + 0x61, 0x63, 0x65, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x63, + 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x0e, 0x44, 0x65, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x1f, 0x0a, 0x0b, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x10, 0x0a, + 0x03, 0x65, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x65, 0x6f, 0x6e, 0x12, + 0x1e, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, + 0x3a, 0x0a, 0x06, 0x67, 0x6e, 0x6f, 0x73, 0x69, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x6e, 0x6f, 0x73, 0x69, 0x73, 0x44, 0x65, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x78, 0x74, 0x72, + 0x61, 0x48, 0x00, 0x52, 0x06, 0x67, 0x6e, 0x6f, 0x73, 0x69, 0x73, 0x12, 0x40, 0x0a, 0x08, 0x6f, + 0x70, 0x74, 0x69, 0x6d, 0x69, 0x73, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x73, 0x6d, 0x44, 0x65, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x78, 0x74, 0x72, + 0x61, 0x48, 0x00, 0x52, 0x08, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x73, 0x6d, 0x42, 0x07, 0x0a, + 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x22, 0x44, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x2b, 0x0a, + 0x11, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x50, 0x72, 0x65, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x95, 0x01, 0x0a, + 0x19, 0x47, 0x6e, 0x6f, 0x73, 0x69, 0x73, 0x44, 0x65, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x78, 0x74, 0x72, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x6c, + 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x6c, 0x6f, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x74, 0x78, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x09, 0x74, 0x78, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x25, 0x0a, + 0x0e, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x04, 0x52, 0x0d, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x64, + 0x69, 0x63, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x73, 0x6d, + 0x44, 0x65, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x78, + 0x74, 0x72, 0x61, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x2f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_shutter_proto_rawDescOnce sync.Once + file_shutter_proto_rawDescData = file_shutter_proto_rawDesc +) + +func file_shutter_proto_rawDescGZIP() []byte { + file_shutter_proto_rawDescOnce.Do(func() { + file_shutter_proto_rawDescData = protoimpl.X.CompressGZIP(file_shutter_proto_rawDescData) + }) + return file_shutter_proto_rawDescData +} + +var file_shutter_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_shutter_proto_goTypes = []any{ + (*Envelope)(nil), // 0: proto.Envelope + (*TraceContext)(nil), // 1: proto.TraceContext + (*DecryptionKeys)(nil), // 2: proto.DecryptionKeys + (*Key)(nil), // 3: proto.Key + (*GnosisDecryptionKeysExtra)(nil), // 4: proto.GnosisDecryptionKeysExtra + (*OptimismDecryptionKeysExtra)(nil), // 5: proto.OptimismDecryptionKeysExtra + (*anypb.Any)(nil), // 6: google.protobuf.Any +} +var file_shutter_proto_depIdxs = []int32{ + 6, // 0: proto.Envelope.message:type_name -> google.protobuf.Any + 1, // 1: proto.Envelope.trace:type_name -> proto.TraceContext + 3, // 2: proto.DecryptionKeys.keys:type_name -> proto.Key + 4, // 3: proto.DecryptionKeys.gnosis:type_name -> proto.GnosisDecryptionKeysExtra + 5, // 4: proto.DecryptionKeys.optimism:type_name -> proto.OptimismDecryptionKeysExtra + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_shutter_proto_init() } +func file_shutter_proto_init() { + if File_shutter_proto != nil { + return + } + file_shutter_proto_msgTypes[0].OneofWrappers = []any{} + file_shutter_proto_msgTypes[2].OneofWrappers = []any{ + (*DecryptionKeys_Gnosis)(nil), + (*DecryptionKeys_Optimism)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_shutter_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_shutter_proto_goTypes, + DependencyIndexes: file_shutter_proto_depIdxs, + MessageInfos: file_shutter_proto_msgTypes, + }.Build() + File_shutter_proto = out.File + file_shutter_proto_rawDesc = nil + file_shutter_proto_goTypes = nil + file_shutter_proto_depIdxs = nil +} diff --git a/txnprovider/shutter/proto/shutter.proto b/txnprovider/shutter/proto/shutter.proto new file mode 100644 index 00000000000..bc891c64b98 --- /dev/null +++ b/txnprovider/shutter/proto/shutter.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; +package proto; + +import "google/protobuf/any.proto"; + +option go_package = "./;proto"; + +message Envelope { + string version = 1; + google.protobuf.Any message = 2; + optional TraceContext trace = 3; +} + +message TraceContext { + bytes trace_id = 1; + bytes span_id = 2; + bytes trace_flags = 3; + string trace_state = 4; +} + +message DecryptionKeys { + uint64 instance_id = 1; + uint64 eon = 2; + repeated Key keys = 3; + oneof extra { + GnosisDecryptionKeysExtra gnosis = 4; + OptimismDecryptionKeysExtra optimism = 5; + } +} + +message Key { + bytes identity_preimage = 1; + bytes key = 2; +} + +message GnosisDecryptionKeysExtra { + uint64 slot = 1; + uint64 tx_pointer = 2; + repeated uint64 signer_indices = 3; + repeated bytes signatures = 4; +} + +message OptimismDecryptionKeysExtra {} From 4f3befef208d058d7332a6961c64fbc87860405f Mon Sep 17 00:00:00 2001 From: antonis19 Date: Mon, 13 Jan 2025 16:44:57 +0100 Subject: [PATCH 87/94] Add sync times table in readme (#13410) Current source: https://erigon.tech/erigon-3-alpha-2-introducing-blazingly-fast-sync-on-archive-nodes-with-ottersync-and-other-improvements/ In the future we might want to update this based on automatically collected QA testing data. --------- Co-authored-by: antonis19 --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 627a2c2dd4c..194bc7e6d8a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ frontier. - [Erigon](#erigon) - [System Requirements](#system-requirements) +- [Sync Times](#sync-times) - [Usage](#usage) - [Getting Started](#getting-started) - [Datadir structure](#datadir-structure) @@ -97,6 +98,18 @@ on [cloud-network-drives](https://github.com/erigontech/erigon?tab=readme-ov-fil 🔬 More details on what type of data stored [here](https://ledgerwatch.github.io/turbo_geth_release.html#Disk-space) +Sync Times +========== + +These are the approximate sync times syncing from scratch to the tip of the chain (results may vary depending on hardware and bandwidth). + + +| Chain | Archive | Full | Minimal | +|------------|-----------------|----------------|----------------| +| Ethereum | 7 Hours, 55 Minutes | 4 Hours, 23 Minutes | 1 Hour, 41 Minutes | +| Gnosis | 2 Hours, 10 Minutes | 1 Hour, 5 Minutes | 33 Minutes | +| Polygon | 1 Day, 21 Hours | 21 Hours, 41 Minutes | 11 Hours, 54 Minutes | + Usage ===== From 2b7119ae77ddfef36760b1d881f7b7e01e42b0a9 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Tue, 14 Jan 2025 00:09:23 +0800 Subject: [PATCH 88/94] txpool: Don't propagate transactions with duplicate authorizations (#13137) Fixes #11763 Tests adapted from: https://github.com/lightclient/go-ethereum/commit/fe828ff59ecaf16ed67db1735e481aaa78dfb472 Futher work: use (remote/local) block reader to get latest account nonce as set code txns may increment nonce more than once. --- core/types/authorization.go | 41 ++-- params/protocol_params.go | 1 + txnprovider/txpool/pool.go | 67 +++++- txnprovider/txpool/pool_test.go | 266 ++++++++++++++++++++- txnprovider/txpool/pool_txn_parser.go | 4 + txnprovider/txpool/pool_txn_parser_test.go | 69 ++++++ txnprovider/txpool/senders.go | 5 + txnprovider/txpool/txpoolcfg/txpoolcfg.go | 69 +++--- 8 files changed, 464 insertions(+), 58 deletions(-) diff --git a/core/types/authorization.go b/core/types/authorization.go index 602df7bb805..fa723465ad3 100644 --- a/core/types/authorization.go +++ b/core/types/authorization.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math" "github.com/holiman/uint256" @@ -36,8 +37,12 @@ func (ath *Authorization) copy() *Authorization { } func (ath *Authorization) RecoverSigner(data *bytes.Buffer, b []byte) (*libcommon.Address, error) { + if ath.Nonce == math.MaxUint64 { + return nil, errors.New("failed assertion: auth.nonce < 2**64 - 1") + } + authLen := rlp.U64Len(ath.ChainID) - authLen += (1 + length.Addr) + authLen += 1 + length.Addr authLen += rlp.U64Len(ath.Nonce) if err := rlp.EncodeStructSizePrefix(authLen, data, b); err != nil { @@ -57,46 +62,46 @@ func (ath *Authorization) RecoverSigner(data *bytes.Buffer, b []byte) (*libcommo return nil, err } + return RecoverSignerFromRLP(data.Bytes(), ath.YParity, ath.R, ath.S) +} + +func RecoverSignerFromRLP(rlp []byte, yParity uint8, r uint256.Int, s uint256.Int) (*libcommon.Address, error) { hashData := []byte{params.SetCodeMagicPrefix} - hashData = append(hashData, data.Bytes()...) + hashData = append(hashData, rlp...) hash := crypto.Keccak256Hash(hashData) var sig [65]byte - r := ath.R.Bytes() - s := ath.S.Bytes() - copy(sig[32-len(r):32], r) - copy(sig[64-len(s):64], s) + rBytes := r.Bytes() + sBytes := s.Bytes() + copy(sig[32-len(rBytes):32], rBytes) + copy(sig[64-len(sBytes):64], sBytes) - if ath.Nonce == 1<<64-1 { - return nil, errors.New("failed assertion: auth.nonce < 2**64 - 1") - } - if ath.YParity == 0 || ath.YParity == 1 { - sig[64] = ath.YParity - } else { - return nil, fmt.Errorf("invalid y parity value: %d", ath.YParity) + if yParity > 1 { + return nil, fmt.Errorf("invalid y parity value: %d", yParity) } + sig[64] = yParity - if !crypto.TransactionSignatureIsValid(sig[64], &ath.R, &ath.S, false /* allowPreEip2s */) { + if !crypto.TransactionSignatureIsValid(sig[64], &r, &s, false /* allowPreEip2s */) { return nil, errors.New("invalid signature") } - pubkey, err := crypto.Ecrecover(hash.Bytes(), sig[:]) + pubKey, err := crypto.Ecrecover(hash.Bytes(), sig[:]) if err != nil { return nil, err } - if len(pubkey) == 0 || pubkey[0] != 4 { + if len(pubKey) == 0 || pubKey[0] != 4 { return nil, errors.New("invalid public key") } var authority libcommon.Address - copy(authority[:], crypto.Keccak256(pubkey[1:])[12:]) + copy(authority[:], crypto.Keccak256(pubKey[1:])[12:]) return &authority, nil } func authorizationSize(auth Authorization) (authLen int) { authLen = rlp.U64Len(auth.ChainID) authLen += rlp.U64Len(auth.Nonce) - authLen += (1 + length.Addr) + authLen += 1 + length.Addr authLen += rlp.U64Len(uint64(auth.YParity)) + (1 + rlp.Uint256LenExcludingHead(&auth.R)) + (1 + rlp.Uint256LenExcludingHead(&auth.S)) diff --git a/params/protocol_params.go b/params/protocol_params.go index d348ad8c1d3..bd136ad4ec6 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -184,6 +184,7 @@ const ( SetCodeMagicPrefix = byte(0x05) ) +// EIP-7702: Set EOA account code var DelegatedDesignationPrefix = []byte{0xef, 0x01, 0x00} // EIP-4788: Beacon block root in the EVM diff --git a/txnprovider/txpool/pool.go b/txnprovider/txpool/pool.go index d79d8ccabf3..514c5ca7c86 100644 --- a/txnprovider/txpool/pool.go +++ b/txnprovider/txpool/pool.go @@ -150,6 +150,7 @@ type TxPool struct { newSlotsStreams *NewSlotsStreams builderNotifyNewTxns func() logger log.Logger + auths map[common.Address]*metaTxn // All accounts with a pooled authorization } type FeeCalculator interface { @@ -226,6 +227,7 @@ func New( builderNotifyNewTxns: builderNotifyNewTxns, newSlotsStreams: newSlotsStreams, logger: logger, + auths: map[common.Address]*metaTxn{}, } if shanghaiTime != nil { @@ -420,6 +422,22 @@ func (p *TxPool) OnNewBlock(ctx context.Context, stateChanges *remote.StateChang return err } + // remove auths from pool map + for _, mt := range minedTxns.Txns { + if mt.Type == SetCodeTxnType { + numAuths := len(mt.AuthRaw) + for i := range numAuths { + signature := mt.Authorizations[i] + signer, err := types.RecoverSignerFromRLP(mt.AuthRaw[i], uint8(signature.V.Uint64()), signature.R, signature.S) + if err != nil { + continue + } + + delete(p.auths, *signer) + } + } + } + var announcements Announcements announcements, err = p.addTxnsOnNewBlock(block, cacheView, stateChanges, p.senders, unwindTxns, /* newTxns */ @@ -1272,6 +1290,7 @@ func (p *TxPool) addTxns(blockNum uint64, cacheView kvcache.CacheView, senders * continue } mt := newMetaTxn(txn, newTxns.IsLocal[i], blockNum) + if reason := p.addLocked(mt, &announcements); reason != txpoolcfg.NotSet { discardReasons[i] = reason continue @@ -1436,6 +1455,40 @@ func (p *TxPool) addLocked(mt *metaTxn, announcements *Announcements) txpoolcfg. return txpoolcfg.FeeTooLow } + // Check if we have txn with same authorization in the pool + if mt.TxnSlot.Type == SetCodeTxnType { + numAuths := len(mt.TxnSlot.AuthRaw) + foundDuplicate := false + for i := range numAuths { + signature := mt.TxnSlot.Authorizations[i] + signer, err := types.RecoverSignerFromRLP(mt.TxnSlot.AuthRaw[i], uint8(signature.V.Uint64()), signature.R, signature.S) + if err != nil { + continue + } + + if _, ok := p.auths[*signer]; ok { + foundDuplicate = true + break + } + + p.auths[*signer] = mt + } + + if foundDuplicate { + return txpoolcfg.ErrAuthorityReserved + } + } + + // Do not allow transaction from reserved authority + addr, ok := p.senders.getAddr(mt.TxnSlot.SenderID) + if !ok { + p.logger.Info("senderID not registered, discarding transaction for safety") + return txpoolcfg.InvalidSender + } + if _, ok := p.auths[addr]; ok { + return txpoolcfg.ErrAuthorityReserved + } + hashStr := string(mt.TxnSlot.IDHash[:]) p.byHash[hashStr] = mt @@ -1472,6 +1525,18 @@ func (p *TxPool) discardLocked(mt *metaTxn, reason txpoolcfg.DiscardReason) { t := p.totalBlobsInPool.Load() p.totalBlobsInPool.Store(t - uint64(len(mt.TxnSlot.BlobHashes))) } + if mt.TxnSlot.Type == SetCodeTxnType { + numAuths := len(mt.TxnSlot.AuthRaw) + for i := range numAuths { + signature := mt.TxnSlot.Authorizations[i] + signer, err := types.RecoverSignerFromRLP(mt.TxnSlot.AuthRaw[i], uint8(signature.V.Uint64()), signature.R, signature.S) + if err != nil { + continue + } + + delete(p.auths, *signer) + } + } } // Cache recently mined blobs in anticipation of reorg, delete finalized ones @@ -1539,7 +1604,7 @@ func (p *TxPool) removeMined(byNonce *BySenderAndNonce, minedTxns []*TxnSlot) er for _, txn := range minedTxns { nonce, ok := noncesToRemove[txn.SenderID] if !ok || txn.Nonce > nonce { - noncesToRemove[txn.SenderID] = txn.Nonce + noncesToRemove[txn.SenderID] = txn.Nonce // TODO: after 7702 nonce can be incremented more than once, may affect this } } diff --git a/txnprovider/txpool/pool_test.go b/txnprovider/txpool/pool_test.go index 11fcc6a4306..b5785d415ed 100644 --- a/txnprovider/txpool/pool_test.go +++ b/txnprovider/txpool/pool_test.go @@ -28,21 +28,25 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/erigontech/erigon-lib/common/datadir" - "github.com/erigontech/erigon-lib/kv/temporal/temporaltest" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/core/types/typestest" - "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/fixedgas" + "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/common/u256" + "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/crypto/kzg" "github.com/erigontech/erigon-lib/gointerfaces" remote "github.com/erigontech/erigon-lib/gointerfaces/remoteproto" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" "github.com/erigontech/erigon-lib/kv/memdb" + "github.com/erigontech/erigon-lib/kv/temporal/temporaltest" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/rlp" types2 "github.com/erigontech/erigon-lib/types" + "github.com/erigontech/erigon/core/types" + "github.com/erigontech/erigon/core/types/typestest" + "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/txnprovider/txpool/txpoolcfg" ) @@ -166,6 +170,258 @@ func TestNonceFromAddress(t *testing.T) { } } +func TestMultipleAuthorizations(t *testing.T) { + ch := make(chan Announcements, 100) + coreDB, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + db := memdb.NewTestPoolDB(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + cfg := txpoolcfg.DefaultConfig + sendersCache := kvcache.New(kvcache.DefaultCoherentConfig) + pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, *u256.N1, common.Big0 /* shanghaiTime */, nil /* agraBlock */, common.Big0 /* cancunTime */, common.Big0 /* pragueTime */, fixedgas.DefaultMaxBlobsPerBlock, nil, nil, func() {}, nil, log.New(), WithFeeCalculator(nil)) + assert.NoError(t, err) + require.True(t, pool != nil) + + var stateVersionID uint64 = 0 + pendingBaseFee := uint64(200000) + // start blocks from 0, set empty hash - then kvcache will also work on this + h1 := gointerfaces.ConvertHashToH256([32]byte{}) + change := &remote.StateChangeBatch{ + StateVersionId: stateVersionID, + PendingBlockBaseFee: pendingBaseFee, + BlockGasLimit: 1000000, + ChangeBatch: []*remote.StateChange{ + {BlockHeight: 0, BlockHash: h1}, + }, + } + + chainID := uint64(7078815900) + privateKey, err := crypto.GenerateKey() + assert.NoError(t, err) + authAddress := crypto.PubkeyToAddress(privateKey.PublicKey) + + var addr1, addr2 [20]byte + addr2[0] = 1 + v := types2.EncodeAccountBytesV3(0, uint256.NewInt(1*common.Ether), make([]byte, 32), 1) + change.ChangeBatch[0].Changes = append(change.ChangeBatch[0].Changes, &remote.AccountChange{ + Action: remote.Action_UPSERT, + Address: gointerfaces.ConvertAddressToH160(addr1), + Data: v, + }) + change.ChangeBatch[0].Changes = append(change.ChangeBatch[0].Changes, &remote.AccountChange{ + Action: remote.Action_UPSERT, + Address: gointerfaces.ConvertAddressToH160(addr2), + Data: v, + }) + change.ChangeBatch[0].Changes = append(change.ChangeBatch[0].Changes, &remote.AccountChange{ + Action: remote.Action_UPSERT, + Address: gointerfaces.ConvertAddressToH160(authAddress), + Data: v, + }) + tx, err := db.BeginRw(ctx) + require.NoError(t, err) + defer tx.Rollback() + err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{}) + assert.NoError(t, err) + + // Generate auth data for transactions + var b [33]byte + data := bytes.NewBuffer(b[:]) + data.Reset() + + authLen := rlp.U64Len(chainID) + authLen += 1 + length.Addr + authLen += rlp.U64Len(0) + assert.NoError(t, rlp.EncodeStructSizePrefix(authLen, data, b[:])) + assert.NoError(t, rlp.EncodeInt(chainID, data, b[:])) + assert.NoError(t, rlp.EncodeOptionalAddress(&authAddress, data, b[:])) + assert.NoError(t, rlp.EncodeInt(0, data, b[:])) + + hashData := []byte{params.SetCodeMagicPrefix} + hashData = append(hashData, data.Bytes()...) + hash := crypto.Keccak256Hash(hashData) + + sig, err := crypto.Sign(hash.Bytes(), privateKey) + assert.NoError(t, err) + + r := uint256.NewInt(0).SetBytes(sig[:32]) + s := uint256.NewInt(0).SetBytes(sig[32:64]) + yParity := sig[64] + + var auth Signature + auth.ChainID.Set(uint256.NewInt(chainID)) + auth.V.Set(uint256.NewInt(uint64(yParity))) + auth.R.Set(r) + auth.S.Set(s) + + logger := log.New() + + // txn with existing authorization should not be accepted + { + var txnSlots TxnSlots + txnSlot1 := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 100000, + Nonce: 0, + Authorizations: []Signature{auth}, + AuthRaw: [][]byte{data.Bytes()}, + Type: SetCodeTxnType, + } + txnSlot1.IDHash[0] = 1 + txnSlots.Append(txnSlot1, addr1[:], true) + + txnSlot2 := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 100000, + Nonce: 0, + Authorizations: []Signature{auth}, + AuthRaw: [][]byte{data.Bytes()}, + Type: SetCodeTxnType, + } + txnSlot2.IDHash[0] = 2 + txnSlots.Append(txnSlot2, addr2[:], true) + + assert.NoError(t, pool.senders.registerNewSenders(&txnSlots, logger)) + + reasons, err := pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(t, err) + assert.Equal(t, reasons, []txpoolcfg.DiscardReason{txpoolcfg.Success, txpoolcfg.ErrAuthorityReserved}) + + assert.Len(t, pool.auths, 1) // auth address should be in pool auth + _, ok := pool.auths[authAddress] + assert.True(t, ok) + + err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{[]*TxnSlot{txnSlot1}, Addresses{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, []bool{true}}) + assert.NoError(t, err) + + assert.Len(t, pool.auths, 0) // auth address should not be there after block has been mined + } + + // fee bump + { + var txnSlots TxnSlots + txnSlot1 := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 100000, + Nonce: 1, + Authorizations: []Signature{auth}, + AuthRaw: [][]byte{data.Bytes()}, + Type: SetCodeTxnType, + } + txnSlot1.IDHash[0] = 3 + txnSlots.Append(txnSlot1, addr1[:], true) + + assert.NoError(t, pool.senders.registerNewSenders(&txnSlots, logger)) + reasons, err := pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(t, err) + assert.Equal(t, reasons, []txpoolcfg.DiscardReason{txpoolcfg.Success}) + + txnSlots = TxnSlots{} + txnSlot2 := &TxnSlot{ + Tip: *uint256.NewInt(900000), + FeeCap: *uint256.NewInt(900000), + Gas: 100000, + Nonce: 1, + Authorizations: []Signature{auth}, + AuthRaw: [][]byte{data.Bytes()}, + Type: SetCodeTxnType, + } + txnSlot2.IDHash[0] = 4 + txnSlots.Append(txnSlot2, addr1[:], true) + + assert.NoError(t, pool.senders.registerNewSenders(&txnSlots, logger)) + reasons, err = pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(t, err) + assert.Equal(t, reasons, []txpoolcfg.DiscardReason{txpoolcfg.Success}) + assert.Equal(t, pool.queued.Best().TxnSlot, txnSlot2) + + err = pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{[]*TxnSlot{txnSlot1}, Addresses{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, []bool{true}}) + assert.NoError(t, err) + } + + // do not allow transactions from delegated addresses + { + var txnSlots TxnSlots + txnSlot1 := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 100000, + Nonce: 1, + Authorizations: []Signature{auth}, + AuthRaw: [][]byte{data.Bytes()}, + Type: SetCodeTxnType, + } + txnSlot1.IDHash[0] = 5 + txnSlots.Append(txnSlot1, addr1[:], true) + + assert.NoError(t, pool.senders.registerNewSenders(&txnSlots, logger)) + reasons, err := pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(t, err) + assert.Equal(t, reasons, []txpoolcfg.DiscardReason{txpoolcfg.Success}) + + txnSlots = TxnSlots{} + txnSlot2 := &TxnSlot{ + Tip: *uint256.NewInt(300000), + FeeCap: *uint256.NewInt(300000), + Gas: 100000, + Nonce: 1, + Type: DynamicFeeTxnType, + } + txnSlot2.IDHash[0] = 6 + + txnSlots.Append(txnSlot2, authAddress.Bytes(), true) + reasons, err = pool.AddLocalTxns(ctx, txnSlots) + assert.NoError(t, err) + assert.Equal(t, reasons, []txpoolcfg.DiscardReason{txpoolcfg.ErrAuthorityReserved}) + } +} + +func TestRecoverSignerFromRLP_ValidData(t *testing.T) { + privateKey, err := crypto.GenerateKey() + assert.NoError(t, err) + pubKey := crypto.PubkeyToAddress(privateKey.PublicKey) + chainID := uint64(7078815900) + + var b [33]byte + data := bytes.NewBuffer(b[:]) + data.Reset() + + // Encode RLP data exactly as in the previous implementation + authLen := rlp.U64Len(chainID) + authLen += 1 + length.Addr + authLen += rlp.U64Len(0) // nonce + assert.NoError(t, rlp.EncodeStructSizePrefix(authLen, data, b[:])) + assert.NoError(t, rlp.EncodeInt(chainID, data, b[:])) + assert.NoError(t, rlp.EncodeOptionalAddress(&pubKey, data, b[:])) + assert.NoError(t, rlp.EncodeInt(0, data, b[:])) + + // Prepare hash data exactly as before + hashData := []byte{params.SetCodeMagicPrefix} + hashData = append(hashData, data.Bytes()...) + hash := crypto.Keccak256Hash(hashData) + + // Sign the hash + sig, err := crypto.Sign(hash.Bytes(), privateKey) + assert.NoError(t, err) + + // Separate signature components + r := uint256.NewInt(0).SetBytes(sig[:32]) + s := uint256.NewInt(0).SetBytes(sig[32:64]) + yParity := sig[64] + + // Recover signer using the explicit RecoverSignerFromRLP function + recoveredAddress, err := types.RecoverSignerFromRLP(data.Bytes(), yParity, *r, *s) + assert.NoError(t, err) + assert.NotNil(t, recoveredAddress) + + // Verify the recovered address matches the original public key address + assert.Equal(t, pubKey, *recoveredAddress) +} + func TestReplaceWithHigherFee(t *testing.T) { assert, require := assert.New(t), require.New(t) ch := make(chan Announcements, 100) diff --git a/txnprovider/txpool/pool_txn_parser.go b/txnprovider/txpool/pool_txn_parser.go index 654bfd96b02..4b7a301d2b0 100644 --- a/txnprovider/txpool/pool_txn_parser.go +++ b/txnprovider/txpool/pool_txn_parser.go @@ -476,6 +476,7 @@ func (ctx *TxnParseContext) parseTransactionBody(payload []byte, pos, p0 int, sl } var sig Signature p2 := authPos + rawStart := p2 p2, err = rlp.ParseU256(payload, p2, &sig.ChainID) if err != nil { return 0, fmt.Errorf("%w: authorization chainId: %s", ErrParseTxn, err) //nolint @@ -493,11 +494,13 @@ func (ctx *TxnParseContext) parseTransactionBody(payload []byte, pos, p0 int, sl if err != nil { return 0, fmt.Errorf("%w: authorization nonce: %s", ErrParseTxn, err) //nolint } + rawEnd := p2 p2, _, err = parseSignature(payload, p2, false /* legacy */, nil /* cfgChainId */, &sig) if err != nil { return 0, fmt.Errorf("%w: authorization signature: %s", ErrParseTxn, err) //nolint } slot.Authorizations = append(slot.Authorizations, sig) + slot.AuthRaw = append(slot.AuthRaw, common.CopyBytes(payload[rawStart:rawEnd])) authPos += authLen if authPos != p2 { return 0, fmt.Errorf("%w: authorization: unexpected list items", ErrParseTxn) @@ -685,6 +688,7 @@ type TxnSlot struct { // EIP-7702: set code tx Authorizations []Signature + AuthRaw [][]byte // rlp encoded chainID+address+nonce, used to recover authorization address in txpool } // nolint diff --git a/txnprovider/txpool/pool_txn_parser_test.go b/txnprovider/txpool/pool_txn_parser_test.go index aeca9ab91d3..eb5fa2f3827 100644 --- a/txnprovider/txpool/pool_txn_parser_test.go +++ b/txnprovider/txpool/pool_txn_parser_test.go @@ -27,8 +27,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/fixedgas" "github.com/erigontech/erigon-lib/common/hexutility" + "github.com/erigontech/erigon-lib/rlp" ) func TestParseTransactionRLP(t *testing.T) { @@ -293,6 +295,73 @@ func TestBlobTxnParsing(t *testing.T) { assert.Equal(t, proof1, fatTxn.Proofs[1]) } +func TestSetCodeAuthRawParsing(t *testing.T) { + // generated using this in core/types/encdec_test.go + /* + func TestGenerateSetCodeTxnRlp(t *testing.T) { + tr := NewTRand() + var txn Transaction + requiredAuthLen := 1 + for txn = tr.RandTransaction(SetCodeTxType); txn.Type() != SetCodeTxType || len(txn.(*SetCodeTransaction).GetAuthorizations()) != requiredAuthLen; txn = tr.RandTransaction(SetCodeTxType) { + } + v, _, _ := txn.RawSignatureValues() + v.SetUint64(uint64(randIntInRange(0, 2))) + + txn.GetChainID().SetUint64(1) + + auths := txn.(*SetCodeTransaction).GetAuthorizations() + auths[0].ChainID = 1 + auths[0].Nonce = 10 + auths[0].Address = libcommon.HexToAddress("0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe") + w := bytes.NewBuffer(nil) + if err := txn.MarshalBinary(w); err != nil { + t.Error(err) + } + + hex := hexutility.Bytes(w.Bytes()).String() + authj, err := json.Marshal(txn.(*SetCodeTransaction).GetAuthorizations()) + if err != nil { + t.Error(err) + } + fmt.Println("txn", hex, len(txn.(*SetCodeTransaction).GetAuthorizations()), string(authj)) + } + */ + bodyRlxHex := "0x04f9049e018889ee3d6b9ebf6ba1888b1a5f65f3cdca7d88f68834f2b4d0715588dd8a498e12c308cb945fd038e869663db8dd239fe1a9147c2a99ddcb0188c901aea8e00d085cb902cd7be4e3a63e15d2dec92a4de7787cd3bf97c422b9824ce02e032d987beb6518e8020b1ab8d4cb290b509b9dc7833f006da4979cf7cfb697edd7222778ced565d1a999a44e31f080638f918e0eec79b4b2dd46f0e964f3e405bfe235f1fd27bffeb90cf393711147799321dee0d3ae22eacb60c4024bae203b82968111ef579d0a3e68a14c31c12abdddfae54a9c488bfde0b1f3f5635b275d67a49530501fafedf9e29a4a04b8720c3d77ccc00d8772816d4b16292a995b4a44840f2e149c35bc62bdbbd64a47ffcf506d41e3b72da2387e8daf2e96131fa74a91172bf14350773a85628b3a2213d37cf8624d55bfab63133f4da0da5559431dbf2178c8a69f0549c100871abfbcec9099092f0027f2a146ad8e73d705dcee036385b2ba8fcc6bb43a98298e4149689d62d20e2b7aa229ea4ddabc8cc17243faacbc1b405c4e0d66babb42761ac0109b852d6f3486de201a39d639a685cf35bb122b500f6533970beaa003a124b12dff03dadd7de82f5f8487e3deb6c0debd3675b1d9bfbb568a4ad3eeea8d76f11ab76c1403e3b094ab6af4bcf89971ca8ee2821960a0a8892c0160bbfd92f69891aeec50c6a1f01b411cc7d1902e101acb5672821192e8ba80be74937a59b497d4bb6f766612b8f487e299f76e505bd0a10af95494bee0fbd1ba95aa52fd739f896d13b09bb605f9ba8cfb2b413a5d590c5e351ecd99ae4737443493a2d3fbfd165b5669118f82c4a6e6fc184a1880203fc6e1fd917abfd77110ba01b1867a38bd1286c6557efc630feb385c533cd84194a9c95def3a0a132116c6d75107db7ad114329425817e152c30069b6556f34559ed678699fddcd35a05fee0144f7964995f09faa4cde325cb5c2da4224f8252b55b987fc54296d0316a9e40b7aae0cb2f58d33d3418bf29fb21e6cabea12563c83cdfe4e2d848823d450f43eee919d5591f719006d9545b6773f14a1ff076fb5774e4de4eed61efd44d105b43a1768aaded355355c0f9010ff794a1d26391185070ad08c5b4bf1df855ffc0e897c5e1a026452ffaac6a96832bf62b550b30b4825a5dbd3c2f46419b6a857eaec43e2764f859946867fce4bcb4339a4fda05f556336499ebf0182df842a021824f29b1b799cf739d944ec397754facbd8d0b16b2afe56d26616466b1a760a01e979136ab5dcc20835eed318eb62413dfd5f3f69787c0f07e1a079371b71a2bf87a946416515c2d28ac7c7e9afa09fc30b550e4996677f863a0aa40e5bfbb7f445acc791c24c6db15ef7e9167cd8fb5dd2940d96135ff4d6a4fa01869b7699d771b48e973b84e916b4bc2a06b0dec665214f5ec43a2a4fb5b4ec1a05c371dc0735b30ef12037f33c1239e0e1211173f1e59cf6c04561a94a7867f3ff84df84b0194de0b295669a9fd93d5f28d9ec85e40f4cb697bae0a579309053d89140f8cee2816d5b671d40401b111d19e37a34b21c05c0f22be62b95a6a019d8395bbeeb2ebaa12e60b652e8f95000194297aef1a6b3c8065bdb0c233ef83e529307a19ad93ef44f0dd4adddc83ad8b0dad25d1072d034400" + bodyRlx := hexutility.MustDecodeHex(bodyRlxHex) + + ctx := NewTxnParseContext(*uint256.NewInt(1)) + ctx.withSender = false + var txn TxnSlot + + _, err := ctx.ParseTransaction(bodyRlx, 0, &txn, nil, false, false, nil) + require.NoError(t, err) + require.Len(t, txn.AuthRaw, 1) + + payload := txn.AuthRaw[0] + p := 0 + + var chainID uint256.Int + var nonce uint64 + var address common.Address + actualAddress := common.HexToAddress("0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe") + + p, err = rlp.ParseU256(payload, p, &chainID) + require.NoError(t, err) + require.Equal(t, chainID.Uint64(), uint64(1)) + + p, datalen, err := rlp.ParseString(payload, p) + require.NoError(t, err) + require.Equal(t, datalen, 20) + + address = common.BytesToAddress(payload[p : p+datalen]) + require.Equal(t, address, actualAddress) + + p += 20 + _, nonce, err = rlp.ParseU64(payload, p) + require.NoError(t, err) + require.Equal(t, nonce, uint64(10)) +} + func TestSetCodeTxnParsing(t *testing.T) { bodyRlxHex := "0x04f9041701880124ec419e9796da8868b499f209983df888bb35ca86e3d9ea47882486c24309101b0e94e4ec12c49d6bcf7cc1325aa50afff92a561229fe880c716dca0e3e3d28b902b6779e563691f1ca8a86a02efdd93db261215047dad430a475d0e191f66b580d6e759a7c7a739532455e65160acf92dc1e1cc11970e7851277278e9d5d2549e451de8c8dd98ebdd3c55e73cd0b465875b72ea6d54917474f7ddfbd1f66d1a929694becc69bc3064c79c32b2db2a094844b400133724e046d9a96f2b6c7888fe008e6a667a970068487ce9a8e6c1260973956b26c1b78235f3452e21c5ed6d47507023ec4072b9ebea8ea9bde77ea64352ef7a6a8efb2ca61fbd0cf7c31491a4c38e3081dfc7b5e8066fca60d8f57b641032f23119a67a37ad0514529df22ba73b4028dc4a6aef0b26161371d731a81d8ac20ea90515b924f2534e32c240d0b75b5d1683e1bc7ecf8b82b73fb4c40d7cfc38e8c32f2c4d3424a86ba8c6e867f13328be201dd8d5e8ee47e03c1d9096968b71228b068cc21514f6bab7867a0d0a2651f40e927079b008c3ef11d571eb5f71d729ee9cfb3d2a99d258c10371fa1df271f4588e031498b155244295490fd842b3055e240ea89843a188b7f15be53252367761b9a8d21818d2c756822c0383246e167dd645722aefe4ecc5e78608bcc851dc5a51255a3f91e908bb5fa53063596458f45c6e25a712de4b2a5b36eea57f5b772c84f1d0f2f2ae103445fb7f2d38493041ca452f1e846c34331bea7b5b350d02306fa3a15b50e978b4efebccce8a3479479d51c95a08e0cab0732fc4f8095337d7502c6a962199342ed127701a6f5b0e54cbdd88f23556aab406a3a7ef49f848c3efbf4cf62052999bde1940abf4944158aefc5472f4ec9e23308cfb63deedc79e9a4f39d8b353c7e6f15d36f4c63987ae6f32701c6579e68f05f9ae86b6fbbc8d57bc17e5c2f3e5389ea75d102017767205c10d6bf5cf6e33a94ad9e6cfac5accf56d61dcee39f2e954ea89b7241e480e6021fa099a81bc9d28d6ca58a11d36f406b212be70c721bd8a4d1d643fa2bf30ebd59a4f838f794fbba2afaae8cabd778b6e151b0431e3fef0a033ce1a07081820b2a08cc2ed4355811644547f23597f7ebe516538baac51d97cbccee97f8ccf201941d994a07f0b3e925d332d4eae10c9ba474da3d8a8806320d2ae09c60e880887dbf8422d2f6549088321947f20ebcbfeff20194327d773bdc6c27cd28a533e81074372dc33a8afd884ef63dce09c5e56c8088cb702ac89cff765f88d26fe11c3d471949f20194f61ffc773a97207c8124c29526a59e6fa0b34a52880e563a787da952ab808884f2a19b171abfb2882d473907f3ada086f20194c1d608bb39e078a99086e7564e89a7625ed86dca88e8a0ab45821912e88088df6c3d43080350518895a828c35680a0278088e2487fd89ca40b3488689accdbeb8d4d2e" bodyRlx := hexutility.MustDecodeHex(bodyRlxHex) diff --git a/txnprovider/txpool/senders.go b/txnprovider/txpool/senders.go index 6ee17d8e1ad..4b0ad0a4170 100644 --- a/txnprovider/txpool/senders.go +++ b/txnprovider/txpool/senders.go @@ -194,6 +194,11 @@ func (sc *sendersBatch) getID(addr common.Address) (uint64, bool) { return id, ok } +func (sc *sendersBatch) getAddr(id uint64) (common.Address, bool) { + addr, ok := sc.senderID2Addr[id] + return addr, ok +} + var traceAllSenders = false func (sc *sendersBatch) getOrCreateID(addr common.Address, logger log.Logger) (uint64, bool) { diff --git a/txnprovider/txpool/txpoolcfg/txpoolcfg.go b/txnprovider/txpool/txpoolcfg/txpoolcfg.go index 24321d5e507..e9949ecbdca 100644 --- a/txnprovider/txpool/txpoolcfg/txpoolcfg.go +++ b/txnprovider/txpool/txpoolcfg/txpoolcfg.go @@ -87,40 +87,41 @@ var DefaultConfig = Config{ type DiscardReason uint8 const ( - NotSet DiscardReason = 0 // analog of "nil-value", means it will be set in future - Success DiscardReason = 1 - AlreadyKnown DiscardReason = 2 - Mined DiscardReason = 3 - ReplacedByHigherTip DiscardReason = 4 - UnderPriced DiscardReason = 5 - ReplaceUnderpriced DiscardReason = 6 // if a transaction is attempted to be replaced with a different one without the required price bump. - FeeTooLow DiscardReason = 7 - OversizedData DiscardReason = 8 - InvalidSender DiscardReason = 9 - NegativeValue DiscardReason = 10 // ensure no one is able to specify a transaction with a negative value. - Spammer DiscardReason = 11 - PendingPoolOverflow DiscardReason = 12 - BaseFeePoolOverflow DiscardReason = 13 - QueuedPoolOverflow DiscardReason = 14 - GasUintOverflow DiscardReason = 15 - IntrinsicGas DiscardReason = 16 - RLPTooLong DiscardReason = 17 - NonceTooLow DiscardReason = 18 - InsufficientFunds DiscardReason = 19 - NotReplaced DiscardReason = 20 // There was an existing transaction with the same sender and nonce, not enough price bump to replace - DuplicateHash DiscardReason = 21 // There was an existing transaction with the same hash - InitCodeTooLarge DiscardReason = 22 // EIP-3860 - transaction init code is too large - TypeNotActivated DiscardReason = 23 // For example, an EIP-4844 transaction is submitted before Cancun activation - InvalidCreateTxn DiscardReason = 24 // EIP-4844 & 7702 transactions cannot have the form of a create transaction - NoBlobs DiscardReason = 25 // Blob transactions must have at least one blob - TooManyBlobs DiscardReason = 26 // There's a limit on how many blobs a block (and thus any transaction) may have - UnequalBlobTxExt DiscardReason = 27 // blob_versioned_hashes, blobs, commitments and proofs must have equal number - BlobHashCheckFail DiscardReason = 28 // KZGcommitment's versioned hash has to be equal to blob_versioned_hash at the same index - UnmatchedBlobTxExt DiscardReason = 29 // KZGcommitments must match the corresponding blobs and proofs - BlobTxReplace DiscardReason = 30 // Cannot replace type-3 blob txn with another type of txn - BlobPoolOverflow DiscardReason = 31 // The total number of blobs (through blob txns) in the pool has reached its limit - NoAuthorizations DiscardReason = 32 // EIP-7702 transactions with an empty authorization list are invalid - GasLimitTooHigh DiscardReason = 33 // Gas limit is too high + NotSet DiscardReason = 0 // analog of "nil-value", means it will be set in future + Success DiscardReason = 1 + AlreadyKnown DiscardReason = 2 + Mined DiscardReason = 3 + ReplacedByHigherTip DiscardReason = 4 + UnderPriced DiscardReason = 5 + ReplaceUnderpriced DiscardReason = 6 // if a transaction is attempted to be replaced with a different one without the required price bump. + FeeTooLow DiscardReason = 7 + OversizedData DiscardReason = 8 + InvalidSender DiscardReason = 9 + NegativeValue DiscardReason = 10 // ensure no one is able to specify a transaction with a negative value. + Spammer DiscardReason = 11 + PendingPoolOverflow DiscardReason = 12 + BaseFeePoolOverflow DiscardReason = 13 + QueuedPoolOverflow DiscardReason = 14 + GasUintOverflow DiscardReason = 15 + IntrinsicGas DiscardReason = 16 + RLPTooLong DiscardReason = 17 + NonceTooLow DiscardReason = 18 + InsufficientFunds DiscardReason = 19 + NotReplaced DiscardReason = 20 // There was an existing transaction with the same sender and nonce, not enough price bump to replace + DuplicateHash DiscardReason = 21 // There was an existing transaction with the same hash + InitCodeTooLarge DiscardReason = 22 // EIP-3860 - transaction init code is too large + TypeNotActivated DiscardReason = 23 // For example, an EIP-4844 transaction is submitted before Cancun activation + InvalidCreateTxn DiscardReason = 24 // EIP-4844 & 7702 transactions cannot have the form of a create transaction + NoBlobs DiscardReason = 25 // Blob transactions must have at least one blob + TooManyBlobs DiscardReason = 26 // There's a limit on how many blobs a block (and thus any transaction) may have + UnequalBlobTxExt DiscardReason = 27 // blob_versioned_hashes, blobs, commitments and proofs must have equal number + BlobHashCheckFail DiscardReason = 28 // KZGcommitment's versioned hash has to be equal to blob_versioned_hash at the same index + UnmatchedBlobTxExt DiscardReason = 29 // KZGcommitments must match the corresponding blobs and proofs + BlobTxReplace DiscardReason = 30 // Cannot replace type-3 blob txn with another type of txn + BlobPoolOverflow DiscardReason = 31 // The total number of blobs (through blob txns) in the pool has reached its limit + NoAuthorizations DiscardReason = 32 // EIP-7702 transactions with an empty authorization list are invalid + GasLimitTooHigh DiscardReason = 33 // Gas limit is too high + ErrAuthorityReserved DiscardReason = 34 // EIP-7702 transaction with authority already reserved ) func (r DiscardReason) String() string { From 40539304e9f330a748be2fc50a7fb863e75c432b Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:17:48 +0000 Subject: [PATCH 89/94] integrity: fix no gaps in bor events integrity check (#13413) logic is incorrect, prevEventId should be 0 not 1 since events start at 1 so when we check the first event the prevEventId should be 0 --- eth/integrity/no_gaps_bor_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/integrity/no_gaps_bor_events.go b/eth/integrity/no_gaps_bor_events.go index 5b70f69adee..c9bf0566ebb 100644 --- a/eth/integrity/no_gaps_bor_events.go +++ b/eth/integrity/no_gaps_bor_events.go @@ -57,7 +57,7 @@ func NoGapsInBorEvents(ctx context.Context, db kv.RoDB, blockReader services.Ful snapshots := blockReader.BorSnapshots().(*heimdall.RoSnapshots) - var prevEventId uint64 = 1 + var prevEventId uint64 var maxBlockNum uint64 if to > 0 { From 9eb039411dda0671188e98c4d15bcdf96116139e Mon Sep 17 00:00:00 2001 From: Shota Date: Tue, 14 Jan 2025 01:07:27 +0400 Subject: [PATCH 90/94] fix comments (#13404) Co-authored-by: shota.silagadze --- erigon-lib/kv/kv_interface.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/erigon-lib/kv/kv_interface.go b/erigon-lib/kv/kv_interface.go index f8832cbbb02..8b7f8879137 100644 --- a/erigon-lib/kv/kv_interface.go +++ b/erigon-lib/kv/kv_interface.go @@ -240,19 +240,7 @@ type RoDB interface { ReadOnly() bool View(ctx context.Context, f func(tx Tx) error) error - // BeginRo - creates transaction - // tx may be discarded by .Rollback() method - // - // A transaction and its cursors must only be used by a single - // thread (not goroutine), and a thread may only have a single transaction at a time. - // It happen automatically by - because this method calls runtime.LockOSThread() inside (Rollback/Commit releases it) - // By this reason application code can't call runtime.UnlockOSThread() - it leads to undefined behavior. - // - // If this `parent` is non-NULL, the new transaction - // will be a nested transaction, with the transaction indicated by parent - // as its parent. Transactions may be nested to any level. A parent - // transaction and its cursors may not issue any other operations than - // Commit and Rollback while it has active child transactions. + // BeginRo - creates transaction, must not be moved between gorotines BeginRo(ctx context.Context) (Tx, error) AllTables() TableCfg PageSize() datasize.ByteSize @@ -291,6 +279,19 @@ type RwDB interface { Update(ctx context.Context, f func(tx RwTx) error) error UpdateNosync(ctx context.Context, f func(tx RwTx) error) error + // BeginRw - creates transaction + // tx may be discarded by .Rollback() method + // + // A transaction and its cursors must only be used by a single + // thread (not goroutine), and a thread may only have a single transaction at a time. + // It happen automatically by - because this method calls runtime.LockOSThread() inside (Rollback/Commit releases it) + // By this reason application code can't call runtime.UnlockOSThread() - it leads to undefined behavior. + // + // If this `parent` is non-NULL, the new transaction + // will be a nested transaction, with the transaction indicated by parent + // as its parent. Transactions may be nested to any level. A parent + // transaction and its cursors may not issue any other operations than + // Commit and Rollback while it has active child transactions. BeginRw(ctx context.Context) (RwTx, error) BeginRwNosync(ctx context.Context) (RwTx, error) } From 85844e6bcfe1ff593c81ec7fcbe4a36f81e38b4d Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Tue, 14 Jan 2025 02:14:36 +0000 Subject: [PATCH 91/94] jsonrpc: fix state sync event txn for bor with astrid (#13416) found while running https://github.com/erigontech/erigon/pull/13415 few apis are missing calls to the new flow with the bridge reader when run with astrid hence not returning bor state sync txn --- eth/backend.go | 9 +++++---- turbo/jsonrpc/eth_block.go | 36 ++++++++++++++++++++++++++++++------ turbo/jsonrpc/eth_txs.go | 28 ++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index d16f0b0cf98..af8ff662077 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -103,6 +103,7 @@ import ( "github.com/erigontech/erigon/ethdb/prune" "github.com/erigontech/erigon/ethstats" "github.com/erigontech/erigon/node" + "github.com/erigontech/erigon/node/nodecfg" "github.com/erigontech/erigon/p2p" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/sentry" @@ -334,7 +335,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, logger, segmentsBuildLimiter) + blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, stack.Config(), logger, segmentsBuildLimiter) if err != nil { return nil, err } @@ -1423,7 +1424,7 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl return err } -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, logger log.Logger, blockSnapBuildSema *semaphore.Weighted) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *libstate.Aggregator, error) { +func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, nodeConfig *nodecfg.Config, logger log.Logger, blockSnapBuildSema *semaphore.Weighted) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *libstate.Aggregator, error) { var minFrozenBlock uint64 if frozenLimit := snConfig.Sync.FrozenBlockLimit; frozenLimit != 0 { @@ -1442,8 +1443,8 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf allBorSnapshots = heimdall.NewRoSnapshots(snConfig.Snapshot, dirs.Snap, minFrozenBlock, logger) if snConfig.PolygonSync { - bridgeStore = bridge.NewSnapshotStore(bridge.NewMdbxStore(dirs.DataDir, logger, false, 0), allBorSnapshots, chainConfig.Bor) - heimdallStore = heimdall.NewSnapshotStore(heimdall.NewMdbxStore(logger, dirs.DataDir, 0), allBorSnapshots) + bridgeStore = bridge.NewSnapshotStore(bridge.NewMdbxStore(dirs.DataDir, logger, false, int64(nodeConfig.Http.DBReadConcurrency)), allBorSnapshots, chainConfig.Bor) + heimdallStore = heimdall.NewSnapshotStore(heimdall.NewMdbxStore(logger, dirs.DataDir, int64(nodeConfig.Http.DBReadConcurrency)), allBorSnapshots) } else { bridgeStore = bridge.NewSnapshotStore(bridge.NewDbStore(db), allBorSnapshots, chainConfig.Bor) heimdallStore = heimdall.NewSnapshotStore(heimdall.NewDbStore(db), allBorSnapshots) diff --git a/turbo/jsonrpc/eth_block.go b/turbo/jsonrpc/eth_block.go index 9819dd791fb..46952cd7dc4 100644 --- a/turbo/jsonrpc/eth_block.go +++ b/turbo/jsonrpc/eth_block.go @@ -248,9 +248,21 @@ func (api *APIImpl) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber var borTx types.Transaction var borTxHash common.Hash if chainConfig.Bor != nil { - borTx = rawdb.ReadBorTransactionForBlock(tx, b.NumberU64()) - if borTx != nil { - borTxHash = bortypes.ComputeBorTxHash(b.NumberU64(), b.Hash()) + if api.useBridgeReader { + possibleBorTxnHash := bortypes.ComputeBorTxHash(b.NumberU64(), b.Hash()) + _, ok, err := api.bridgeReader.EventTxnLookup(ctx, possibleBorTxnHash) + if err != nil { + return nil, err + } + if ok { + borTx = bortypes.NewBorTransaction() + borTxHash = possibleBorTxnHash + } + } else { + borTx = rawdb.ReadBorTransactionForBlock(tx, b.NumberU64()) + if borTx != nil { + borTxHash = bortypes.ComputeBorTxHash(b.NumberU64(), b.Hash()) + } } } @@ -318,9 +330,21 @@ func (api *APIImpl) GetBlockByHash(ctx context.Context, numberOrHash rpc.BlockNu var borTx types.Transaction var borTxHash common.Hash if chainConfig.Bor != nil { - borTx = rawdb.ReadBorTransactionForBlock(tx, number) - if borTx != nil { - borTxHash = bortypes.ComputeBorTxHash(number, block.Hash()) + if api.useBridgeReader { + possibleBorTxnHash := bortypes.ComputeBorTxHash(block.NumberU64(), block.Hash()) + _, ok, err := api.bridgeReader.EventTxnLookup(ctx, possibleBorTxnHash) + if err != nil { + return nil, err + } + if ok { + borTx = bortypes.NewBorTransaction() + borTxHash = possibleBorTxnHash + } + } else { + borTx = rawdb.ReadBorTransactionForBlock(tx, number) + if borTx != nil { + borTxHash = bortypes.ComputeBorTxHash(number, block.Hash()) + } } } diff --git a/turbo/jsonrpc/eth_txs.go b/turbo/jsonrpc/eth_txs.go index 980a9da3a98..e08f9bf43dd 100644 --- a/turbo/jsonrpc/eth_txs.go +++ b/turbo/jsonrpc/eth_txs.go @@ -225,7 +225,19 @@ func (api *APIImpl) GetTransactionByBlockHashAndIndex(ctx context.Context, block if chainConfig.Bor == nil { return nil, nil // not error } - borTx := rawdb.ReadBorTransactionForBlock(tx, block.NumberU64()) + var borTx types2.Transaction + if api.useBridgeReader { + possibleBorTxnHash := bortypes.ComputeBorTxHash(block.NumberU64(), block.Hash()) + _, ok, err := api.bridgeReader.EventTxnLookup(ctx, possibleBorTxnHash) + if err != nil { + return nil, err + } + if ok { + borTx = bortypes.NewBorTransaction() + } + } else { + borTx = rawdb.ReadBorTransactionForBlock(tx, block.NumberU64()) + } if borTx == nil { return nil, nil // not error } @@ -289,7 +301,19 @@ func (api *APIImpl) GetTransactionByBlockNumberAndIndex(ctx context.Context, blo if chainConfig.Bor == nil { return nil, nil // not error } - borTx := rawdb.ReadBorTransactionForBlock(tx, blockNum) + var borTx types2.Transaction + if api.useBridgeReader { + possibleBorTxnHash := bortypes.ComputeBorTxHash(blockNum, hash) + _, ok, err := api.bridgeReader.EventTxnLookup(ctx, possibleBorTxnHash) + if err != nil { + return nil, err + } + if ok { + borTx = bortypes.NewBorTransaction() + } + } else { + borTx = rawdb.ReadBorTransactionForBlock(tx, blockNum) + } if borTx == nil { return nil, nil } From 75e1e0db8383fe556c98bf1f5a7a1869a96042f2 Mon Sep 17 00:00:00 2001 From: blxdyx Date: Tue, 14 Jan 2025 14:13:05 +0800 Subject: [PATCH 92/94] upstream --- eth/backend.go | 11 +++++------ eth/stagedsync/stage_snapshots.go | 2 +- turbo/stages/stageloop.go | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index fbbc91ffc43..074bcdf8262 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -342,7 +342,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, allBscSnapshots, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, logger, segmentsBuildLimiter) + blockReader, blockWriter, allSnapshots, allBorSnapshots, bridgeStore, heimdallStore, allBscSnapshots, agg, err := setUpBlockReader(ctx, rawChainDB, config.Dirs, config, chainConfig, stack.Config(), logger, segmentsBuildLimiter) if err != nil { return nil, err } @@ -810,7 +810,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger ethBackendRPC := privateapi.NewEthBackendServer(ctx, backend, backend.chainDB, backend.notifications, blockReader, logger, latestBlockBuiltStore) // initialize engine backend - blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, heimdallStore, bridgeStore, backend.chainConfig, config, backend.notifications.Events, segmentsBuildLimiter, logger) + blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, blobStore, heimdallStore, bridgeStore, backend.chainConfig, config, backend.notifications.Events, segmentsBuildLimiter, logger) var miningRPC txpoolproto.MiningServer = privateapi.NewMiningServer(ctx, backend, ethashApi, logger) @@ -1014,7 +1014,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.polygonDownloadSync = stagedsync.New(backend.config.Sync, stagedsync.DownloadSyncStages( backend.sentryCtx, stagedsync.StageSnapshotsCfg( backend.chainDB, *backend.sentriesClient.ChainConfig, config.Sync, dirs, blockRetire, backend.downloaderClient, - blockReader, backend.notifications, backend.engine, false, false, false, backend.silkworm, config.Prune, + blockReader, backend.notifications, backend.engine, false, false, false, backend.silkworm, config.Prune, )), nil, nil, backend.logger, stages.ModeApplyingBlocks) // these range extractors set the db to the local db instead of the chain db @@ -1457,8 +1457,7 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl return err } -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, logger log.Logger) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *freezeblocks.BscRoSnapshots, *libstate.Aggregator, error) { -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, nodeConfig *nodecfg.Config, logger log.Logger, blockSnapBuildSema *semaphore.Weighted) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *libstate.Aggregator, error) { +func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig *ethconfig.Config, chainConfig *chain.Config, nodeConfig *nodecfg.Config, logger log.Logger, blockSnapBuildSema *semaphore.Weighted) (*freezeblocks.BlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *heimdall.RoSnapshots, bridge.Store, heimdall.Store, *freezeblocks.BscRoSnapshots, *libstate.Aggregator, error) { var minFrozenBlock uint64 if frozenLimit := snConfig.Sync.FrozenBlockLimit; frozenLimit != 0 { @@ -1490,7 +1489,7 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf allBscSnapshots = freezeblocks.NewBscRoSnapshots(snConfig.Snapshot, dirs.Snap, minFrozenBlock, logger) } - blockReader := freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore) + blockReader := freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore, allBscSnapshots) agg, err := libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { return nil, nil, nil, nil, nil, nil, nil, nil, err diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 9bf8f82b825..632c07766ff 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -335,7 +335,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R } diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Fill DB"}) - if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, agg, cfg.engine, logger); err != nil { + if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, agg, cfg.chainConfig, cfg.engine, logger); err != nil { return err } diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 4754c396cd8..6ff0a287dca 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -692,7 +692,7 @@ func NewDefaultStages(ctx context.Context, runInTestMode := cfg.ImportMode return stagedsync.DefaultStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling || cfg.DisableBlobPrune, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs || cfg.DisableBlobPrune, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, heimdallStore, bridgeStore, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures, cfg.WithHeimdallWaypointRecording, nil), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), @@ -734,7 +734,7 @@ func NewPipelineStages(ctx context.Context, if len(cfg.Sync.UploadLocation) == 0 { return stagedsync.PipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, agg, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling || cfg.DisableBlobPrune, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs || cfg.DisableBlobPrune, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg(db, cfg.Prune, cfg.BatchSize, controlServer.ChainConfig, controlServer.Engine, &vm.Config{}, notifications, cfg.StateStream, false, dirs, blockReader, controlServer.Hd, cfg.Genesis, cfg.Sync, SilkwormForExecutionStage(silkworm, cfg)), @@ -743,7 +743,7 @@ func NewPipelineStages(ctx context.Context, } return stagedsync.UploaderPipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, agg, cfg.InternalCL && cfg.CaplinConfig.Backfilling, cfg.CaplinConfig.BlobBackfilling || cfg.DisableBlobPrune, cfg.CaplinConfig.Archive, silkworm, cfg.Prune), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, blockReader, notifications, engine, cfg.InternalCL && cfg.CaplinConfig.ArchiveBlocks, cfg.CaplinConfig.ArchiveBlobs || cfg.DisableBlobPrune, cfg.CaplinConfig.ArchiveStates, silkworm, cfg.Prune), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, cfg.Sync, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, cfg.Sync, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), From c65e18eb2054b05aae3374c9136081cfe1db7706 Mon Sep 17 00:00:00 2001 From: blxdyx Date: Tue, 14 Jan 2025 14:57:22 +0800 Subject: [PATCH 93/94] don't remove salt --- turbo/app/snapshots_cmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index cc089e97a07..ba063e67e91 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -886,8 +886,8 @@ func doClearIndexing(cliCtx *cli.Context) error { } // remove salt-state.txt and salt-block.txt - os.Remove(filepath.Join(snapDir, "salt-state.txt")) - os.Remove(filepath.Join(snapDir, "salt-blocks.txt")) + //os.Remove(filepath.Join(snapDir, "salt-state.txt")) + //os.Remove(filepath.Join(snapDir, "salt-blocks.txt")) return nil } From b13159b687acabb92018d3419a529c679865a9f0 Mon Sep 17 00:00:00 2001 From: blxdyx Date: Tue, 14 Jan 2025 15:19:25 +0800 Subject: [PATCH 94/94] go fmt --- turbo/stages/mock/mock_sentry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index e771a427a3f..ca6a0c04ef2 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -526,7 +526,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK mock.agg.SetProduceMod(mock.BlockReader.FreezingCfg().ProduceE3) mock.Sync = stagedsync.New( cfg.Sync, - stagedsync.DefaultStages(mock.Ctx, stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, mock.BlockReader, mock.Notifications, mock.Engine, false, false, false, nil, prune), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, cfg.Sync, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications), stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil, nil, nil, mock.BlockReader, nil, nil, recents, signatures, false, nil), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(mock.DB, nil, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, blockWriter), stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, cfg.Sync, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), stagedsync.StageExecuteBlocksCfg( + stagedsync.DefaultStages(mock.Ctx, stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, cfg.Sync, dirs, blockRetire, snapDownloader, mock.BlockReader, mock.Notifications, mock.Engine, false, false, false, nil, prune), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, cfg.Sync, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications), stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil, nil, nil, mock.BlockReader, nil, nil, recents, signatures, false, nil), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(mock.DB, nil, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, blockWriter), stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, cfg.Sync, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), stagedsync.StageExecuteBlocksCfg( mock.DB, prune, cfg.BatchSize,