From f3dfb376934e288aac99b5014b78af5b92178fac Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 14 Mar 2022 12:11:38 +0100 Subject: [PATCH 01/39] Update config templates --- Cargo.lock | 7 +++++++ bee-ledger/Cargo.toml | 1 + .../bee-node/config.chrysalis-devnet.json | 19 ++++++++++++++----- .../bee-node/config.chrysalis-devnet.toml | 13 ++++++++++--- .../bee-node/config.chrysalis-mainnet.json | 19 ++++++++++++++----- .../bee-node/config.chrysalis-mainnet.toml | 13 ++++++++++--- 6 files changed, 56 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c4a0e414b..d3891a715e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,6 +430,7 @@ dependencies = [ "futures 0.3.21", "hashbrown", "hex", + "humanize-rs", "iota-crypto", "log", "ref-cast", @@ -1943,6 +1944,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humanize-rs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016b02deb8b0c415d8d56a6f0ab265e50c22df61194e37f9be75ed3a722de8a6" + [[package]] name = "humantime" version = "2.1.0" diff --git a/bee-ledger/Cargo.toml b/bee-ledger/Cargo.toml index 286744bd13..def84fdab9 100644 --- a/bee-ledger/Cargo.toml +++ b/bee-ledger/Cargo.toml @@ -23,6 +23,7 @@ digest = { version = "0.9.0", default-features = false, optional = true } futures = { version = "0.3.17", default-features = false, optional = true } hashbrown = { version = "0.11.2", default-features = false, optional = true } hex = { version = "0.4.3", default-features = false, optional = true } +humanize-rs = "0.1.5" iota-crypto = { version = "0.10.0", default-features = false, features = [ "blake2b" ], optional = true } log = { version = "0.4.14", default-features = false, optional = true } ref-cast = { version = "1.0.6", default-features = false, optional = true } diff --git a/bee-node/bee-node/config.chrysalis-devnet.json b/bee-node/bee-node/config.chrysalis-devnet.json index 568730c5ee..07c8464750 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.json +++ b/bee-node/bee-node/config.chrysalis-devnet.json @@ -24,8 +24,7 @@ "reconnectIntervalSecs": 30, "maxUnknownPeers": 4, "maxDiscoveredPeers": 8, - "peering": { - } + "peering": {} }, "autopeering": { "enabled": false, @@ -94,9 +93,19 @@ ] }, "pruning": { - "enabled": true, - "delay": 60480, - "pruneReceipts": false + "milestones": { + "enabled": false, + "maxMilestonesToKeep": 60480 + }, + "size": { + "enabled": true, + "targetSize": "30GB", + "thresholdPercentage": 10.0, + "cooldownTime": "5m" + }, + "receipts": { + "enabled": false + } }, "storage": { "path": "./storage/devnet/tangle" diff --git a/bee-node/bee-node/config.chrysalis-devnet.toml b/bee-node/bee-node/config.chrysalis-devnet.toml index 96b4977c83..3ab5607cb3 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.toml +++ b/bee-node/bee-node/config.chrysalis-devnet.toml @@ -86,9 +86,16 @@ full = "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest delta = "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest-delta_snapshot.bin" [pruning] -enabled = true -delay = 60480 -prune_receipts = false +[pruning.milestones] +enabled = true +max_milestones_to_keep = 60480 +[pruning.receipts] +enabled = false +[pruning.size] +enabled = true +target_size = 30GB +threshold_percentage = 10.0 +cooldown_time = 5m [storage] path = "./storage/devnet/tangle" diff --git a/bee-node/bee-node/config.chrysalis-mainnet.json b/bee-node/bee-node/config.chrysalis-mainnet.json index edf72c2733..93a4d416ca 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.json +++ b/bee-node/bee-node/config.chrysalis-mainnet.json @@ -24,8 +24,7 @@ "reconnectIntervalSecs": 30, "maxUnknownPeers": 4, "maxDiscoveredPeers": 8, - "peering": { - } + "peering": {} }, "autopeering": { "enabled": false, @@ -133,9 +132,19 @@ ] }, "pruning": { - "enabled": true, - "delay": 60480, - "pruneReceipts": false + "milestones": { + "enabled": false, + "maxMilestonesToKeep": 60480 + }, + "size": { + "enabled": true, + "targetSize": "30GB", + "thresholdPercentage": 10.0, + "cooldownTime": "5m" + }, + "receipts": { + "enabled": false + } }, "storage": { "path": "./storage/mainnet/tangle" diff --git a/bee-node/bee-node/config.chrysalis-mainnet.toml b/bee-node/bee-node/config.chrysalis-mainnet.toml index 7441bae064..edc75e0a96 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.toml +++ b/bee-node/bee-node/config.chrysalis-mainnet.toml @@ -119,9 +119,16 @@ full = "https://cdn.tanglebay.com/snapshots/mainnet/full_snapshot.bin" delta = "https://cdn.tanglebay.com/snapshots/mainnet/delta_snapshot.bin" [pruning] -enabled = true -delay = 60480 -prune_receipts = false +[pruning.milestones] +enabled = true +max_milestones_to_keep = 60480 +[pruning.receipts] +enabled = false +[pruning.size] +enabled = true +target_size = 30GB +threshold_percentage = 10.0 +cooldown_time = 5m [storage] path = "./storage/mainnet/tangle" From a485519fde03452151bd2d8c56e78f46509d91d6 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 14 Mar 2022 14:19:57 +0100 Subject: [PATCH 02/39] Refactor pruning config builder --- bee-ledger/src/workers/consensus/worker.rs | 6 +- bee-ledger/src/workers/pruning/condition.rs | 2 +- bee-ledger/src/workers/pruning/config.rs | 240 +++++++++++++++--- bee-ledger/src/workers/pruning/prune.rs | 3 +- .../bee-node/config.chrysalis-devnet.json | 8 +- .../bee-node/config.chrysalis-devnet.toml | 2 +- .../bee-node/config.chrysalis-mainnet.json | 8 +- .../bee-node/config.chrysalis-mainnet.toml | 2 +- 8 files changed, 224 insertions(+), 47 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 10d57b6ce3..e17ff0567f 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -292,15 +292,15 @@ where let snapshot_pruning_delta = bmd + EXTRA_PRUNING_DEPTH; let pruning_delay_min = snapshot_depth + snapshot_pruning_delta; - let pruning_delay = if pruning_config.delay() < pruning_delay_min { + let pruning_delay = if pruning_config.pruning_milestones().max_milestones_to_keep() < pruning_delay_min { warn!( "Configuration value for \"pruning.delay\" is too low ({}), value changed to {}.", - pruning_config.delay(), + pruning_config.pruning_milestones().max_milestones_to_keep(), pruning_delay_min ); pruning_delay_min } else { - pruning_config.delay() + pruning_config.pruning_milestones().max_milestones_to_keep() }; // Unwrap is fine because ledger index was already in storage or just added by the snapshot worker. diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index bae415e9b8..e5c80b7a7c 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -23,7 +23,7 @@ pub(crate) fn should_prune( pruning_delay: u32, config: &PruningConfig, ) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> { - if config.disabled() { + if !config.pruning_milestones().enabled() { return Err(PruningSkipReason::Disabled); } diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 5aff3a7419..1e2d0c3e00 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -3,20 +3,27 @@ //! Module containing pruning configuration. +use std::time::Duration; + +use humanize_rs::{bytes, duration}; use serde::Deserialize; -const DEFAULT_ENABLED: bool = true; -const DEFAULT_DELAY: u32 = 60480; -const DEFAULT_PRUNE_RECEIPTS: bool = false; +const PRUNING_MILESTONES_ENABLED_DEFAULT: bool = true; +const PRUNING_RECEIPTS_ENABLED_DEFAULT: bool = false; +const PRUNING_BY_SIZE_ENABLED_DEFAULT: bool = true; +const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 60480; +const THRESHOLD_PERCENTAGE_DEFAULT: f32 = 10.0; +const COOLDOWN_TIME_DEFAULT: &str = "5m"; +const TARGET_SIZE_DEFAULT: &str = "30Gb"; /// Builder for a [`PruningConfig`]. #[derive(Default, Deserialize, PartialEq)] #[must_use] pub struct PruningConfigBuilder { - enabled: Option, - delay: Option, - #[serde(alias = "pruneReceipts")] - prune_receipts: Option, + milestones: Option, + receipts: Option, + #[serde(alias = "bySize")] + by_size: Option, } impl PruningConfigBuilder { @@ -25,21 +32,21 @@ impl PruningConfigBuilder { Self::default() } - /// Enables pruning. - pub fn enabled(mut self, enabled: bool) -> Self { - self.enabled.replace(enabled); + /// TODO + pub fn milestones(mut self, builder: PruningMilestonesConfigBuilder) -> Self { + self.milestones.replace(builder); self } - /// Sets the pruning delay. - pub fn delay(mut self, delay: u32) -> Self { - self.delay.replace(delay); + /// TODO + pub fn receipts(mut self, builder: PruningReceiptsConfigBuilder) -> Self { + self.receipts.replace(builder); self } - /// Sets whether receipts should be pruned as well. - pub fn prune_receipts(mut self, prune_receipts: bool) -> Self { - self.prune_receipts.replace(prune_receipts); + /// TODO + pub fn by_size(mut self, builder: PruningBySizeConfigBuilder) -> Self { + self.by_size.replace(builder); self } @@ -47,9 +54,9 @@ impl PruningConfigBuilder { #[must_use] pub fn finish(self) -> PruningConfig { PruningConfig { - enabled: self.enabled.unwrap_or(DEFAULT_ENABLED), - delay: self.delay.unwrap_or(DEFAULT_DELAY), - prune_receipts: self.prune_receipts.unwrap_or(DEFAULT_PRUNE_RECEIPTS), + pruning_milestones: self.milestones.unwrap_or_default().finish(), + pruning_receipts: self.receipts.unwrap_or_default().finish(), + pruning_by_size: self.by_size.unwrap_or_default().finish(), } } } @@ -57,9 +64,9 @@ impl PruningConfigBuilder { /// The pruning configuration. #[derive(Clone)] pub struct PruningConfig { - enabled: bool, - delay: u32, - prune_receipts: bool, + pruning_milestones: PruningMilestonesConfig, + pruning_receipts: PruningReceiptsConfig, + pruning_by_size: PruningBySizeConfig, } impl PruningConfig { @@ -68,23 +75,192 @@ impl PruningConfig { PruningConfigBuilder::new() } - /// Returns whether pruning is enabled. + /// TODO + pub fn pruning_milestones(&self) -> &PruningMilestonesConfig { + &self.pruning_milestones + } + + /// TODO + pub fn pruning_receipts(&self) -> &PruningReceiptsConfig { + &self.pruning_receipts + } + + /// TODO + pub fn pruning_by_size(&self) -> &PruningBySizeConfig { + &self.pruning_by_size + } +} + +/// TODO +#[derive(Default, Deserialize, PartialEq)] +#[must_use] +pub struct PruningMilestonesConfigBuilder { + enabled: Option, + max_milestones_to_keep: Option, +} + +impl PruningMilestonesConfigBuilder { + /// TODO + pub fn enabled(mut self, enabled: bool) -> Self { + self.enabled.replace(enabled); + self + } + + /// TODO + pub fn max_milestones_to_keep(mut self, max_milestones_to_keep: u32) -> Self { + self.max_milestones_to_keep.replace(max_milestones_to_keep); + self + } + + /// TODO + #[must_use] + pub fn finish(self) -> PruningMilestonesConfig { + PruningMilestonesConfig { + enabled: self.enabled.unwrap_or(PRUNING_MILESTONES_ENABLED_DEFAULT), + max_milestones_to_keep: self.max_milestones_to_keep.unwrap_or(MAX_MILESTONES_TO_KEEP_DEFAULT), + } + } +} + +/// TODO +#[derive(Default, Deserialize, PartialEq)] +#[must_use] +pub struct PruningReceiptsConfigBuilder { + enabled: Option, +} + +impl PruningReceiptsConfigBuilder { + /// TODO + pub fn enabled(mut self, enabled: bool) -> Self { + self.enabled.replace(enabled); + self + } + + /// TODO + #[must_use] + pub fn finish(self) -> PruningReceiptsConfig { + PruningReceiptsConfig { + enabled: self.enabled.unwrap_or(PRUNING_RECEIPTS_ENABLED_DEFAULT), + } + } +} + +/// TODO +#[derive(Default, Deserialize, PartialEq)] +#[must_use] +pub struct PruningBySizeConfigBuilder { + enabled: Option, + target_size: Option, + threshold_percentage: Option, + cooldown_time: Option, +} + +impl PruningBySizeConfigBuilder { + /// TODO + pub fn enabled(mut self, enabled: bool) -> Self { + self.enabled.replace(enabled); + self + } + + /// TODO + pub fn target_size(mut self, target_size: String) -> Self { + self.target_size.replace(target_size); + self + } + + /// TODO + pub fn threshold_percentage(mut self, threshold_percentage: f32) -> Self { + self.threshold_percentage.replace(threshold_percentage); + self + } + + /// TODO + pub fn cooldown_time(mut self, cooldown_time: String) -> Self { + self.cooldown_time.replace(cooldown_time); + self + } + + /// TODO + #[must_use] + pub fn finish(self) -> PruningBySizeConfig { + let target_size = self.target_size.unwrap_or_else(|| TARGET_SIZE_DEFAULT.to_string()); + let target_size = target_size + .parse::() + .expect("parse human-readable pruning target size") + .size(); + + let cooldown_time = self.cooldown_time.unwrap_or_else(|| COOLDOWN_TIME_DEFAULT.to_string()); + let cooldown_time = + duration::parse(cooldown_time.as_ref()).expect("parse human-readable pruning cooldown time"); + + PruningBySizeConfig { + enabled: self.enabled.unwrap_or(PRUNING_BY_SIZE_ENABLED_DEFAULT), + target_size, + threshold_percentage: self.threshold_percentage.unwrap_or(THRESHOLD_PERCENTAGE_DEFAULT), + cooldown_time, + } + } +} + +/// TODO +#[derive(Clone)] +pub struct PruningMilestonesConfig { + enabled: bool, + max_milestones_to_keep: u32, +} + +impl PruningMilestonesConfig { + /// TODO + pub fn enabled(&self) -> bool { + self.enabled + } + + /// TODO + pub fn max_milestones_to_keep(&self) -> u32 { + self.max_milestones_to_keep + } +} + +/// TODO +#[derive(Clone)] +pub struct PruningReceiptsConfig { + enabled: bool, +} + +impl PruningReceiptsConfig { + /// TODO + pub fn enabled(&self) -> bool { + self.enabled + } +} + +/// TODO +#[derive(Clone)] +pub struct PruningBySizeConfig { + enabled: bool, + target_size: usize, + threshold_percentage: f32, + cooldown_time: Duration, +} + +impl PruningBySizeConfig { + /// TODO pub fn enabled(&self) -> bool { self.enabled } - /// Returns whether pruning is disabled. - pub fn disabled(&self) -> bool { - !self.enabled + /// TODO + pub fn target_size(&self) -> usize { + self.target_size } - /// Returns the pruning delay. - pub fn delay(&self) -> u32 { - self.delay + /// TODO + pub fn threshold_percentage(&self) -> f32 { + self.threshold_percentage } - /// Returns whether [`Receipt`](crate::types::Receipt)s are pruned. - pub fn prune_receipts(&self) -> bool { - self.prune_receipts + /// TODO + pub fn cooldown_time(&self) -> Duration { + self.cooldown_time } } diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 9df7da2d46..00c046bc45 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -129,7 +129,8 @@ pub async fn prune( tangle.update_entry_point_index(index); let batch_milestones = Instant::now(); - let milestone_data_metrics = batch::prune_milestone_data(storage, &mut batch, index, config.prune_receipts())?; + let milestone_data_metrics = + batch::prune_milestone_data(storage, &mut batch, index, config.pruning_receipts().enabled())?; timings.batch_milestone_data = batch_milestones.elapsed(); metrics.receipts = milestone_data_metrics.receipts; diff --git a/bee-node/bee-node/config.chrysalis-devnet.json b/bee-node/bee-node/config.chrysalis-devnet.json index 07c8464750..57086cac3f 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.json +++ b/bee-node/bee-node/config.chrysalis-devnet.json @@ -97,14 +97,14 @@ "enabled": false, "maxMilestonesToKeep": 60480 }, - "size": { + "receipts": { + "enabled": false + }, + "bySize": { "enabled": true, "targetSize": "30GB", "thresholdPercentage": 10.0, "cooldownTime": "5m" - }, - "receipts": { - "enabled": false } }, "storage": { diff --git a/bee-node/bee-node/config.chrysalis-devnet.toml b/bee-node/bee-node/config.chrysalis-devnet.toml index 3ab5607cb3..0fa387349f 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.toml +++ b/bee-node/bee-node/config.chrysalis-devnet.toml @@ -91,7 +91,7 @@ enabled = true max_milestones_to_keep = 60480 [pruning.receipts] enabled = false -[pruning.size] +[pruning.by_size] enabled = true target_size = 30GB threshold_percentage = 10.0 diff --git a/bee-node/bee-node/config.chrysalis-mainnet.json b/bee-node/bee-node/config.chrysalis-mainnet.json index 93a4d416ca..08e5270429 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.json +++ b/bee-node/bee-node/config.chrysalis-mainnet.json @@ -136,14 +136,14 @@ "enabled": false, "maxMilestonesToKeep": 60480 }, - "size": { + "receipts": { + "enabled": false + }, + "bySize": { "enabled": true, "targetSize": "30GB", "thresholdPercentage": 10.0, "cooldownTime": "5m" - }, - "receipts": { - "enabled": false } }, "storage": { diff --git a/bee-node/bee-node/config.chrysalis-mainnet.toml b/bee-node/bee-node/config.chrysalis-mainnet.toml index edc75e0a96..1ac3f82e7e 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.toml +++ b/bee-node/bee-node/config.chrysalis-mainnet.toml @@ -124,7 +124,7 @@ enabled = true max_milestones_to_keep = 60480 [pruning.receipts] enabled = false -[pruning.size] +[pruning.by_size] enabled = true target_size = 30GB threshold_percentage = 10.0 From 1ffc253c04a71fa603ab40510199e4e7e2746e68 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 21 Mar 2022 11:27:38 +0100 Subject: [PATCH 03/39] Debug log actual and target storage size --- bee-ledger/src/workers/pruning/config.rs | 2 +- bee-ledger/src/workers/pruning/prune.rs | 3 +-- bee-storage/bee-storage-rocksdb/src/storage.rs | 4 +--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 1e2d0c3e00..1987f677b6 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -14,7 +14,7 @@ const PRUNING_BY_SIZE_ENABLED_DEFAULT: bool = true; const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 60480; const THRESHOLD_PERCENTAGE_DEFAULT: f32 = 10.0; const COOLDOWN_TIME_DEFAULT: &str = "5m"; -const TARGET_SIZE_DEFAULT: &str = "30Gb"; +const TARGET_SIZE_DEFAULT: &str = "30Gb"; /// Builder for a [`PruningConfig`]. #[derive(Default, Deserialize, PartialEq)] diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 00c046bc45..9a82c9be44 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -58,8 +58,7 @@ pub async fn prune( let index = MilestoneIndex(index); debug!("Pruning milestone {}...", index); - - // + debug!("Storage size: actual {} limit {}", storage.size().expect("ok storage size").expect("some storage size"), config.pruning_by_size().target_size()); // Measurement of the full pruning step. let full_prune = Instant::now(); diff --git a/bee-storage/bee-storage-rocksdb/src/storage.rs b/bee-storage/bee-storage-rocksdb/src/storage.rs index 15805606a1..73b6d2946b 100644 --- a/bee-storage/bee-storage-rocksdb/src/storage.rs +++ b/bee-storage/bee-storage-rocksdb/src/storage.rs @@ -212,9 +212,7 @@ impl StorageBackend for Storage { } fn size(&self) -> Result, Self::Error> { - Ok(Some( - self.inner.live_files()?.iter().fold(0, |acc, file| acc + file.size), - )) + Ok(Some(self.inner.live_files()?.iter().map(|file| file.size).sum())) } fn get_health(&self) -> Result, Self::Error> { From fd466074afe083424bff1657bdbac9fac54c4759 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 21 Mar 2022 13:43:56 +0100 Subject: [PATCH 04/39] Put human-friendly sizes/durations in quotation marks in configs --- bee-ledger/src/workers/pruning/config.rs | 4 ++++ bee-node/bee-node/config.chrysalis-devnet.toml | 4 ++-- bee-node/bee-node/config.chrysalis-mainnet.toml | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 1987f677b6..bf1b4b77ae 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -96,6 +96,7 @@ impl PruningConfig { #[must_use] pub struct PruningMilestonesConfigBuilder { enabled: Option, + #[serde(alias = "maxMilestonesToKeep")] max_milestones_to_keep: Option, } @@ -150,8 +151,11 @@ impl PruningReceiptsConfigBuilder { #[must_use] pub struct PruningBySizeConfigBuilder { enabled: Option, + #[serde(alias = "targetSize")] target_size: Option, + #[serde(alias = "thresholdPercentage")] threshold_percentage: Option, + #[serde(alias = "cooldownTime")] cooldown_time: Option, } diff --git a/bee-node/bee-node/config.chrysalis-devnet.toml b/bee-node/bee-node/config.chrysalis-devnet.toml index 0fa387349f..a5914f29bb 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.toml +++ b/bee-node/bee-node/config.chrysalis-devnet.toml @@ -93,9 +93,9 @@ max_milestones_to_keep = 60480 enabled = false [pruning.by_size] enabled = true -target_size = 30GB +target_size = "30GB" threshold_percentage = 10.0 -cooldown_time = 5m +cooldown_time = "5m" [storage] path = "./storage/devnet/tangle" diff --git a/bee-node/bee-node/config.chrysalis-mainnet.toml b/bee-node/bee-node/config.chrysalis-mainnet.toml index 1ac3f82e7e..2bc6c593ab 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.toml +++ b/bee-node/bee-node/config.chrysalis-mainnet.toml @@ -126,9 +126,9 @@ max_milestones_to_keep = 60480 enabled = false [pruning.by_size] enabled = true -target_size = 30GB +target_size = "30GB" threshold_percentage = 10.0 -cooldown_time = 5m +cooldown_time = "5m" [storage] path = "./storage/mainnet/tangle" From 5e54892804de03af2b4c4b83d6e1f2a037512c73 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 21 Mar 2022 14:05:49 +0100 Subject: [PATCH 05/39] Log actual and target storage size in 'should_prune' --- bee-ledger/src/workers/consensus/worker.rs | 2 +- bee-ledger/src/workers/pruning/condition.rs | 22 ++++++++++++++------- bee-ledger/src/workers/pruning/prune.rs | 1 - 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index e17ff0567f..57afa4f4ee 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -345,7 +345,7 @@ where } } - match should_prune(&tangle, ledger_index, pruning_delay, &pruning_config) { + match should_prune(&tangle, &storage, ledger_index, pruning_delay, &pruning_config) { Ok((start_index, target_index)) => { if let Err(e) = prune::prune(&tangle, &storage, &bus, start_index, target_index, &pruning_config) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index e5c80b7a7c..d13221f473 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -17,25 +17,33 @@ pub enum PruningSkipReason { BelowThreshold { reached_in: u32 }, } -pub(crate) fn should_prune( - tangle: &Tangle, +pub(crate) fn should_prune( + tangle: &Tangle, + storage: &S, ledger_index: LedgerIndex, - pruning_delay: u32, + max_milestones_to_keep: u32, config: &PruningConfig, -) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> { - if !config.pruning_milestones().enabled() { +) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> +{ + log::debug!( + "Storage size: actual {} limit {}", + storage.size().expect("ok storage size").expect("some storage size"), + config.pruning_by_size().target_size() + ); + + if !config.pruning_milestones().enabled() && !config.pruning_by_size().enabled() { return Err(PruningSkipReason::Disabled); } let pruning_index = *tangle.get_pruning_index() + 1; - let pruning_threshold = pruning_index + pruning_delay; + let pruning_threshold = pruning_index + max_milestones_to_keep; if *ledger_index < pruning_threshold { Err(PruningSkipReason::BelowThreshold { reached_in: pruning_threshold - *ledger_index, }) } else { - let target_pruning_index = *ledger_index - pruning_delay; + let target_pruning_index = *ledger_index - max_milestones_to_keep; Ok(( pruning_index.into(), diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 9a82c9be44..26a24dcf08 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -58,7 +58,6 @@ pub async fn prune( let index = MilestoneIndex(index); debug!("Pruning milestone {}...", index); - debug!("Storage size: actual {} limit {}", storage.size().expect("ok storage size").expect("some storage size"), config.pruning_by_size().target_size()); // Measurement of the full pruning step. let full_prune = Instant::now(); From 317be5e647493d531cb989a73d08e7abe3a03f6b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 30 Mar 2022 15:53:16 +0200 Subject: [PATCH 06/39] Add pruning config unit test --- Cargo.lock | 2 + bee-ledger/Cargo.toml | 4 + bee-ledger/src/workers/consensus/worker.rs | 6 +- bee-ledger/src/workers/pruning/condition.rs | 7 +- bee-ledger/src/workers/pruning/config.rs | 310 ++++++++++++------ bee-ledger/src/workers/pruning/prune.rs | 2 +- .../bee-node/config.chrysalis-devnet.json | 10 +- .../bee-node/config.chrysalis-devnet.toml | 6 +- .../bee-node/config.chrysalis-mainnet.json | 10 +- .../bee-node/config.chrysalis-mainnet.toml | 6 +- 10 files changed, 234 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3891a715e..7bac9a65e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -436,10 +436,12 @@ dependencies = [ "ref-cast", "reqwest", "serde", + "serde_json", "thiserror", "time-helper", "tokio", "tokio-stream", + "toml", "trace-tools", "tracing", "url", diff --git a/bee-ledger/Cargo.toml b/bee-ledger/Cargo.toml index def84fdab9..6388c03ae1 100644 --- a/bee-ledger/Cargo.toml +++ b/bee-ledger/Cargo.toml @@ -37,6 +37,10 @@ trace-tools = { version = "0.3.0", default-features = false, optional = true } tracing = { version = "0.1.29", default-features = false, optional = true } url = { version = "2.2.2", default-features = false, optional = true } +[dev-dependencies] +serde_json = { version = "1.0.68", default-features = false, features = [ "std" ] } +toml = { version = "0.5.8", default-features = false } + [features] workers = [ "bee-runtime", diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 57afa4f4ee..58e7671699 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -292,15 +292,15 @@ where let snapshot_pruning_delta = bmd + EXTRA_PRUNING_DEPTH; let pruning_delay_min = snapshot_depth + snapshot_pruning_delta; - let pruning_delay = if pruning_config.pruning_milestones().max_milestones_to_keep() < pruning_delay_min { + let pruning_delay = if pruning_config.milestones().max_milestones_to_keep() < pruning_delay_min { warn!( "Configuration value for \"pruning.delay\" is too low ({}), value changed to {}.", - pruning_config.pruning_milestones().max_milestones_to_keep(), + pruning_config.milestones().max_milestones_to_keep(), pruning_delay_min ); pruning_delay_min } else { - pruning_config.pruning_milestones().max_milestones_to_keep() + pruning_config.milestones().max_milestones_to_keep() }; // Unwrap is fine because ledger index was already in storage or just added by the snapshot worker. diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index d13221f473..19768fbca7 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -23,15 +23,14 @@ pub(crate) fn should_prune( ledger_index: LedgerIndex, max_milestones_to_keep: u32, config: &PruningConfig, -) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> -{ +) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> { log::debug!( "Storage size: actual {} limit {}", storage.size().expect("ok storage size").expect("some storage size"), - config.pruning_by_size().target_size() + config.size().target_size() ); - if !config.pruning_milestones().enabled() && !config.pruning_by_size().enabled() { + if !config.milestones().enabled() && !config.size().enabled() { return Err(PruningSkipReason::Disabled); } diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index bf1b4b77ae..9a48641634 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -9,21 +9,20 @@ use humanize_rs::{bytes, duration}; use serde::Deserialize; const PRUNING_MILESTONES_ENABLED_DEFAULT: bool = true; +const PRUNING_SIZE_ENABLED_DEFAULT: bool = true; const PRUNING_RECEIPTS_ENABLED_DEFAULT: bool = false; -const PRUNING_BY_SIZE_ENABLED_DEFAULT: bool = true; -const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 60480; +const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 99999; //60480; const THRESHOLD_PERCENTAGE_DEFAULT: f32 = 10.0; const COOLDOWN_TIME_DEFAULT: &str = "5m"; const TARGET_SIZE_DEFAULT: &str = "30Gb"; /// Builder for a [`PruningConfig`]. -#[derive(Default, Deserialize, PartialEq)] +#[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] pub struct PruningConfigBuilder { milestones: Option, + size: Option, receipts: Option, - #[serde(alias = "bySize")] - by_size: Option, } impl PruningConfigBuilder { @@ -32,21 +31,21 @@ impl PruningConfigBuilder { Self::default() } - /// TODO + /// Sets the [`PruningMilestonesConfigBuilder`]. pub fn milestones(mut self, builder: PruningMilestonesConfigBuilder) -> Self { self.milestones.replace(builder); self } - /// TODO - pub fn receipts(mut self, builder: PruningReceiptsConfigBuilder) -> Self { - self.receipts.replace(builder); + /// Sets the [`PruningSizeConfigBuilder`]. + pub fn size(mut self, builder: PruningSizeConfigBuilder) -> Self { + self.size.replace(builder); self } - /// TODO - pub fn by_size(mut self, builder: PruningBySizeConfigBuilder) -> Self { - self.by_size.replace(builder); + /// Sets the [`PruningReceiptsConfigBuilder`]. + pub fn receipts(mut self, builder: PruningReceiptsConfigBuilder) -> Self { + self.receipts.replace(builder); self } @@ -54,45 +53,15 @@ impl PruningConfigBuilder { #[must_use] pub fn finish(self) -> PruningConfig { PruningConfig { - pruning_milestones: self.milestones.unwrap_or_default().finish(), - pruning_receipts: self.receipts.unwrap_or_default().finish(), - pruning_by_size: self.by_size.unwrap_or_default().finish(), + milestones: self.milestones.unwrap_or_default().finish(), + receipts: self.receipts.unwrap_or_default().finish(), + size: self.size.unwrap_or_default().finish(), } } } -/// The pruning configuration. -#[derive(Clone)] -pub struct PruningConfig { - pruning_milestones: PruningMilestonesConfig, - pruning_receipts: PruningReceiptsConfig, - pruning_by_size: PruningBySizeConfig, -} - -impl PruningConfig { - /// Returns a builder to create a [`PruningConfig`]. - pub fn build() -> PruningConfigBuilder { - PruningConfigBuilder::new() - } - - /// TODO - pub fn pruning_milestones(&self) -> &PruningMilestonesConfig { - &self.pruning_milestones - } - - /// TODO - pub fn pruning_receipts(&self) -> &PruningReceiptsConfig { - &self.pruning_receipts - } - - /// TODO - pub fn pruning_by_size(&self) -> &PruningBySizeConfig { - &self.pruning_by_size - } -} - -/// TODO -#[derive(Default, Deserialize, PartialEq)] +/// Builder for a [`PruningMilestonesConfig`]. +#[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] pub struct PruningMilestonesConfigBuilder { enabled: Option, @@ -101,19 +70,19 @@ pub struct PruningMilestonesConfigBuilder { } impl PruningMilestonesConfigBuilder { - /// TODO + /// Sets whether pruning based on milestone indexes is enabled. pub fn enabled(mut self, enabled: bool) -> Self { self.enabled.replace(enabled); self } - /// TODO + /// Sets how many milestones to hold available in the storage. pub fn max_milestones_to_keep(mut self, max_milestones_to_keep: u32) -> Self { self.max_milestones_to_keep.replace(max_milestones_to_keep); self } - /// TODO + /// Finishes this builder into a [`PruningMilestonesConfig`]. #[must_use] pub fn finish(self) -> PruningMilestonesConfig { PruningMilestonesConfig { @@ -123,33 +92,10 @@ impl PruningMilestonesConfigBuilder { } } -/// TODO -#[derive(Default, Deserialize, PartialEq)] -#[must_use] -pub struct PruningReceiptsConfigBuilder { - enabled: Option, -} - -impl PruningReceiptsConfigBuilder { - /// TODO - pub fn enabled(mut self, enabled: bool) -> Self { - self.enabled.replace(enabled); - self - } - - /// TODO - #[must_use] - pub fn finish(self) -> PruningReceiptsConfig { - PruningReceiptsConfig { - enabled: self.enabled.unwrap_or(PRUNING_RECEIPTS_ENABLED_DEFAULT), - } - } -} - -/// TODO -#[derive(Default, Deserialize, PartialEq)] +/// Builder for a [`PruningSizeConfig`]. +#[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] -pub struct PruningBySizeConfigBuilder { +pub struct PruningSizeConfigBuilder { enabled: Option, #[serde(alias = "targetSize")] target_size: Option, @@ -159,8 +105,8 @@ pub struct PruningBySizeConfigBuilder { cooldown_time: Option, } -impl PruningBySizeConfigBuilder { - /// TODO +impl PruningSizeConfigBuilder { + /// Sets whether pruning based on storage size is enabled. pub fn enabled(mut self, enabled: bool) -> Self { self.enabled.replace(enabled); self @@ -184,9 +130,9 @@ impl PruningBySizeConfigBuilder { self } - /// TODO + /// Finishes this builder into a [`PruningSizeConfig`]. #[must_use] - pub fn finish(self) -> PruningBySizeConfig { + pub fn finish(self) -> PruningSizeConfig { let target_size = self.target_size.unwrap_or_else(|| TARGET_SIZE_DEFAULT.to_string()); let target_size = target_size .parse::() @@ -197,8 +143,8 @@ impl PruningBySizeConfigBuilder { let cooldown_time = duration::parse(cooldown_time.as_ref()).expect("parse human-readable pruning cooldown time"); - PruningBySizeConfig { - enabled: self.enabled.unwrap_or(PRUNING_BY_SIZE_ENABLED_DEFAULT), + PruningSizeConfig { + enabled: self.enabled.unwrap_or(PRUNING_SIZE_ENABLED_DEFAULT), target_size, threshold_percentage: self.threshold_percentage.unwrap_or(THRESHOLD_PERCENTAGE_DEFAULT), cooldown_time, @@ -206,54 +152,97 @@ impl PruningBySizeConfigBuilder { } } -/// TODO -#[derive(Clone)] -pub struct PruningMilestonesConfig { - enabled: bool, - max_milestones_to_keep: u32, +/// Builder for a [`PruningReceiptsConfig`]. +#[derive(Default, Debug, Deserialize, PartialEq)] +#[must_use] +pub struct PruningReceiptsConfigBuilder { + enabled: Option, } -impl PruningMilestonesConfig { - /// TODO - pub fn enabled(&self) -> bool { - self.enabled +impl PruningReceiptsConfigBuilder { + /// Sets whether receipts will be pruned. + pub fn enabled(mut self, enabled: bool) -> Self { + self.enabled.replace(enabled); + self } - /// TODO - pub fn max_milestones_to_keep(&self) -> u32 { - self.max_milestones_to_keep + /// Finishes this builder into a [`PruningReceiptsConfig`]. + #[must_use] + pub fn finish(self) -> PruningReceiptsConfig { + PruningReceiptsConfig { + enabled: self.enabled.unwrap_or(PRUNING_RECEIPTS_ENABLED_DEFAULT), + } } } -/// TODO -#[derive(Clone)] -pub struct PruningReceiptsConfig { +/// The pruning configuration. +#[derive(Clone, Debug)] +pub struct PruningConfig { + milestones: PruningMilestonesConfig, + size: PruningSizeConfig, + receipts: PruningReceiptsConfig, +} + +impl PruningConfig { + /// Returns a builder to create a [`PruningConfig`]. + pub fn build() -> PruningConfigBuilder { + PruningConfigBuilder::new() + } + + /// Returns the `[PruningMilestonesConfig`]. + #[inline(always)] + pub fn milestones(&self) -> &PruningMilestonesConfig { + &self.milestones + } + + /// Returns the `[PruningSizeConfig`]. + #[inline(always)] + pub fn size(&self) -> &PruningSizeConfig { + &self.size + } + + /// Returns the `[PruningReceiptsConfig`]. + #[inline(always)] + pub fn receipts(&self) -> &PruningReceiptsConfig { + &self.receipts + } +} + +/// The config associated with milestone index based pruning. +#[derive(Clone, Debug)] +pub struct PruningMilestonesConfig { enabled: bool, + max_milestones_to_keep: u32, } -impl PruningReceiptsConfig { - /// TODO +impl PruningMilestonesConfig { + /// Returns whether pruning based on milestone indexes is enabled. pub fn enabled(&self) -> bool { self.enabled } + + /// Returns the maximum number of milestones to hold available in the storage. + pub fn max_milestones_to_keep(&self) -> u32 { + self.max_milestones_to_keep + } } -/// TODO -#[derive(Clone)] -pub struct PruningBySizeConfig { +/// The config associated with storage size based pruning. +#[derive(Clone, Debug)] +pub struct PruningSizeConfig { enabled: bool, target_size: usize, threshold_percentage: f32, cooldown_time: Duration, } -impl PruningBySizeConfig { - /// TODO +impl PruningSizeConfig { + /// Returns whether pruning based on a target storage size is enabled. pub fn enabled(&self) -> bool { self.enabled } - /// TODO + /// Returns the target size. pub fn target_size(&self) -> usize { self.target_size } @@ -268,3 +257,114 @@ impl PruningBySizeConfig { self.cooldown_time } } + +/// The config associated with pruning receipts. +#[cfg_attr(test, derive(Eq, PartialEq))] +#[derive(Clone, Debug)] +pub struct PruningReceiptsConfig { + enabled: bool, +} + +impl PruningReceiptsConfig { + /// Returns whether pruning receipts is enabled. + pub fn enabled(&self) -> bool { + self.enabled + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug)] + struct NodeConfig { + pruning: PruningConfig, + } + + #[derive(Default, Debug, Deserialize)] + #[must_use] + struct NodeConfigBuilder { + pruning: Option, + } + + impl NodeConfigBuilder { + fn finish(self) -> NodeConfig { + NodeConfig { pruning: self.pruning.unwrap().finish()} + } + } + + fn create_config_from_json_str() -> PruningConfig { + let config_json_str = r#" + { + "pruning": { + "milestones": { + "enabled": false, + "maxMilestonesToKeep": 200 + }, + "size": { + "enabled": false, + "targetSize": "500MB", + "thresholdPercentage": 20.0, + "cooldownTime": "1m" + }, + "receipts": { + "enabled": true + } + } + }"#; + + let node_config = serde_json::from_str::(config_json_str) + .expect("error deserializing json config str") + .finish(); + + node_config.pruning + } + + fn create_config_from_toml_str() -> PruningConfig { + let config_toml_str = r#" + [pruning] + [pruning.milestones] + enabled = false + max_milestones_to_keep = 200 + [pruning.size] + enabled = false + target_size = "500MB" + threshold_percentage = 20.0 + cooldown_time = "1m" + [pruning.receipts] + enabled = true + "#; + + let node_config_builder = toml::from_str::(config_toml_str) + .expect("error deserializing toml config str"); + + println!("{:?}", node_config_builder); + let node_config = node_config_builder.finish(); + + node_config.pruning + } + + #[test] + fn deserialize_json_and_toml_repr_into_same_config() { + let json_config = create_config_from_json_str(); + let toml_config = create_config_from_toml_str(); + + assert!(!json_config.milestones().enabled()); + assert_eq!(json_config.milestones().max_milestones_to_keep(), 200); + assert_eq!(json_config.size().target_size(), 500000000); + assert_eq!(json_config.size().threshold_percentage(), 20.0); + assert_eq!(json_config.size().cooldown_time(), Duration::from_secs(60)); + assert!(!json_config.size().enabled()); + assert!(json_config.receipts().enabled()); + + assert!(!toml_config.milestones().enabled()); + assert_eq!(toml_config.milestones().max_milestones_to_keep(), 200); + assert_eq!(toml_config.size().target_size(), 500000000); + assert_eq!(toml_config.size().threshold_percentage(), 20.0); + assert_eq!(toml_config.size().cooldown_time(), Duration::from_secs(60)); + assert!(!toml_config.size().enabled()); + assert!(toml_config.receipts().enabled()); + + } + +} diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 26a24dcf08..c164420483 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -128,7 +128,7 @@ pub async fn prune( let batch_milestones = Instant::now(); let milestone_data_metrics = - batch::prune_milestone_data(storage, &mut batch, index, config.pruning_receipts().enabled())?; + batch::prune_milestone_data(storage, &mut batch, index, config.receipts().enabled())?; timings.batch_milestone_data = batch_milestones.elapsed(); metrics.receipts = milestone_data_metrics.receipts; diff --git a/bee-node/bee-node/config.chrysalis-devnet.json b/bee-node/bee-node/config.chrysalis-devnet.json index 57086cac3f..14b3f74218 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.json +++ b/bee-node/bee-node/config.chrysalis-devnet.json @@ -94,17 +94,17 @@ }, "pruning": { "milestones": { - "enabled": false, + "enabled": true, "maxMilestonesToKeep": 60480 }, - "receipts": { - "enabled": false - }, - "bySize": { + "size": { "enabled": true, "targetSize": "30GB", "thresholdPercentage": 10.0, "cooldownTime": "5m" + }, + "receipts": { + "enabled": false } }, "storage": { diff --git a/bee-node/bee-node/config.chrysalis-devnet.toml b/bee-node/bee-node/config.chrysalis-devnet.toml index a5914f29bb..9b3fec5440 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.toml +++ b/bee-node/bee-node/config.chrysalis-devnet.toml @@ -89,13 +89,13 @@ delta = "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest [pruning.milestones] enabled = true max_milestones_to_keep = 60480 -[pruning.receipts] -enabled = false -[pruning.by_size] +[pruning.size] enabled = true target_size = "30GB" threshold_percentage = 10.0 cooldown_time = "5m" +[pruning.receipts] +enabled = false [storage] path = "./storage/devnet/tangle" diff --git a/bee-node/bee-node/config.chrysalis-mainnet.json b/bee-node/bee-node/config.chrysalis-mainnet.json index 08e5270429..b9e8ca8df8 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.json +++ b/bee-node/bee-node/config.chrysalis-mainnet.json @@ -133,17 +133,17 @@ }, "pruning": { "milestones": { - "enabled": false, + "enabled": true, "maxMilestonesToKeep": 60480 }, - "receipts": { - "enabled": false - }, - "bySize": { + "size": { "enabled": true, "targetSize": "30GB", "thresholdPercentage": 10.0, "cooldownTime": "5m" + }, + "receipts": { + "enabled": false } }, "storage": { diff --git a/bee-node/bee-node/config.chrysalis-mainnet.toml b/bee-node/bee-node/config.chrysalis-mainnet.toml index 2bc6c593ab..56fcee02cb 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.toml +++ b/bee-node/bee-node/config.chrysalis-mainnet.toml @@ -122,13 +122,13 @@ delta = "https://cdn.tanglebay.com/snapshots/mainnet/delta_snapshot.bin" [pruning.milestones] enabled = true max_milestones_to_keep = 60480 -[pruning.receipts] -enabled = false -[pruning.by_size] +[pruning.size] enabled = true target_size = "30GB" threshold_percentage = 10.0 cooldown_time = "5m" +[pruning.receipts] +enabled = false [storage] path = "./storage/mainnet/tangle" From 553292ffd807050b13d9db8ff1860906bbb7679d Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 13 Apr 2022 14:53:59 +0200 Subject: [PATCH 07/39] Update docs in PruningSizeConfig --- bee-ledger/src/workers/pruning/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 9a48641634..88fc86ee3f 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -242,17 +242,17 @@ impl PruningSizeConfig { self.enabled } - /// Returns the target size. + /// Returns the target size of the database. pub fn target_size(&self) -> usize { self.target_size } - /// TODO + /// Returns the percentage the database gets reduced if the target size is reached. pub fn threshold_percentage(&self) -> f32 { self.threshold_percentage } - /// TODO + /// Returns the cooldown time between two pruning-by-database size events. pub fn cooldown_time(&self) -> Duration { self.cooldown_time } From 193a02820b5b4c67b63caeb11c2095349a9cf5e5 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 13 Apr 2022 16:02:49 +0200 Subject: [PATCH 08/39] Prune milestones until final size is reached --- bee-ledger/src/workers/consensus/worker.rs | 49 ++-- bee-ledger/src/workers/pruning/condition.rs | 89 ++++-- bee-ledger/src/workers/pruning/config.rs | 10 +- bee-ledger/src/workers/pruning/prune.rs | 300 +++++++++++--------- 4 files changed, 266 insertions(+), 182 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 58e7671699..ba63fbc6f8 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -24,7 +24,11 @@ use crate::{ consensus::{metadata::WhiteFlagMetadata, state::validate_ledger_state, white_flag}, error::Error, event::{MessageReferenced, MilestoneConfirmed, OutputConsumed, OutputCreated}, - pruning::{condition::should_prune, config::PruningConfig, prune}, + pruning::{ + condition::{should_prune, PruningTask}, + config::PruningConfig, + prune, + }, snapshot::{condition::should_snapshot, config::SnapshotConfig, worker::SnapshotWorker}, storage::{self, StorageBackend}, }, @@ -291,14 +295,14 @@ where }; let snapshot_pruning_delta = bmd + EXTRA_PRUNING_DEPTH; - let pruning_delay_min = snapshot_depth + snapshot_pruning_delta; - let pruning_delay = if pruning_config.milestones().max_milestones_to_keep() < pruning_delay_min { + let milestones_to_keep_min = snapshot_depth + snapshot_pruning_delta; + let milestones_to_keep = if pruning_config.milestones().max_milestones_to_keep() < milestones_to_keep_min { warn!( - "Configuration value for \"pruning.delay\" is too low ({}), value changed to {}.", + "Configuration value for \"max_milestones_to_keep\" is too low ({}), value changed to {}.", pruning_config.milestones().max_milestones_to_keep(), - pruning_delay_min + milestones_to_keep_min ); - pruning_delay_min + milestones_to_keep_min } else { pruning_config.milestones().max_milestones_to_keep() }; @@ -345,18 +349,29 @@ where } } - match should_prune(&tangle, &storage, ledger_index, pruning_delay, &pruning_config) { - Ok((start_index, target_index)) => { - if let Err(e) = - prune::prune(&tangle, &storage, &bus, start_index, target_index, &pruning_config) - .await - { - error!("Pruning failed: {:?}.", e); - // TODO: consider initiating an emergency (but graceful) shutdown instead of just - // panicking. - panic!("Aborting due to an unexpected pruning error."); + match should_prune(&tangle, &storage, ledger_index, milestones_to_keep, &pruning_config) { + Ok(pruning_task) => match pruning_task { + PruningTask::ByRange(start_index, target_index) => { + if let Err(e) = prune::prune_by_range( + &tangle, + &storage, + &bus, + start_index, + target_index, + &pruning_config, + ) + .await + { + error!("Pruning failed: {e:?}."); + // TODO: consider initiating an emergency (but graceful) shutdown instead of just + // panicking. + panic!("Aborting due to an unexpected pruning error."); + } } - } + PruningTask::BySize(reduced_size) => { + todo!("pruning by size"); + } + }, Err(reason) => { debug!("Pruning skipped: {:?}", reason); } diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 19768fbca7..6cb2cffdae 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -1,6 +1,8 @@ // Copyright 2020-2021 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +use std::time::Duration; + use bee_message::milestone::MilestoneIndex; use bee_tangle::{storage::StorageBackend, Tangle}; @@ -10,47 +12,74 @@ const PRUNING_BATCH_SIZE_MAX: u32 = 200; /// Reasons for skipping pruning. #[derive(Debug)] -pub enum PruningSkipReason { +pub(crate) enum PruningSkipReason { /// Pruning is disabled in the config. Disabled, - /// Not enough data yet to be pruned. - BelowThreshold { reached_in: u32 }, + /// The storage in use doesn't support this pruning task. + PruningTaskNotSupported, + /// Not enough history yet to be pruned. + BelowMaxMilestonesToKeepThreshold { reached_in: u32 }, + /// The database has not yet reached its target size. + BelowTargetSize { reached_in: usize }, + /// The cooldown time has not yet been passed. + BelowCooldownTimeThreshold { reached_in: Duration }, +} + +pub(crate) enum PruningTask { + // TODO: consider using named structs: start index, target index + ByRange(MilestoneIndex, MilestoneIndex), + // TODO: same: reduced size + BySize(usize), } pub(crate) fn should_prune( tangle: &Tangle, storage: &S, ledger_index: LedgerIndex, - max_milestones_to_keep: u32, + milestones_to_keep: u32, config: &PruningConfig, -) -> Result<(MilestoneIndex, MilestoneIndex), PruningSkipReason> { - log::debug!( - "Storage size: actual {} limit {}", - storage.size().expect("ok storage size").expect("some storage size"), - config.size().target_size() - ); - - if !config.milestones().enabled() && !config.size().enabled() { - return Err(PruningSkipReason::Disabled); - } +) -> Result { + if config.milestones().enabled() { + let pruning_index = *tangle.get_pruning_index() + 1; + let pruning_threshold = pruning_index + milestones_to_keep; + + if *ledger_index < pruning_threshold { + Err(PruningSkipReason::BelowMaxMilestonesToKeepThreshold { + reached_in: pruning_threshold - *ledger_index, + }) + } else { + let target_pruning_index = *ledger_index - milestones_to_keep; + + Ok(PruningTask::ByRange( + pruning_index.into(), + if target_pruning_index > pruning_index + PRUNING_BATCH_SIZE_MAX { + (pruning_index + PRUNING_BATCH_SIZE_MAX).into() + } else { + target_pruning_index.into() + }, + )) + } + } else if config.size().enabled() { + // TODO: return `PruningTAskNotSupported` if we can't get the size of the storage. + let actual_size = storage.size().expect("ok storage size").expect("some storage size"); + let target_size = config.size().target_size(); + + log::debug!("Storage size: actual {actual_size} target {target_size}"); + + if actual_size < target_size { + // Panic: cannot underflow. + Err(PruningSkipReason::BelowTargetSize { + reached_in: target_size - actual_size, + }) + } else { + let reduced_size = + actual_size - (config.size().threshold_percentage() as f64 / 100.0 * target_size as f64) as usize; - let pruning_index = *tangle.get_pruning_index() + 1; - let pruning_threshold = pruning_index + max_milestones_to_keep; + log::debug!("Reduced size: {reduced_size}"); - if *ledger_index < pruning_threshold { - Err(PruningSkipReason::BelowThreshold { - reached_in: pruning_threshold - *ledger_index, - }) + Ok(PruningTask::BySize(reduced_size)) + } } else { - let target_pruning_index = *ledger_index - max_milestones_to_keep; - - Ok(( - pruning_index.into(), - if target_pruning_index > pruning_index + PRUNING_BATCH_SIZE_MAX { - (pruning_index + PRUNING_BATCH_SIZE_MAX).into() - } else { - target_pruning_index.into() - }, - )) + Err(PruningSkipReason::Disabled) } } diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 88fc86ee3f..7a8834185f 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -289,7 +289,9 @@ mod tests { impl NodeConfigBuilder { fn finish(self) -> NodeConfig { - NodeConfig { pruning: self.pruning.unwrap().finish()} + NodeConfig { + pruning: self.pruning.unwrap().finish(), + } } } @@ -335,8 +337,8 @@ mod tests { enabled = true "#; - let node_config_builder = toml::from_str::(config_toml_str) - .expect("error deserializing toml config str"); + let node_config_builder = + toml::from_str::(config_toml_str).expect("error deserializing toml config str"); println!("{:?}", node_config_builder); let node_config = node_config_builder.finish(); @@ -364,7 +366,5 @@ mod tests { assert_eq!(toml_config.size().cooldown_time(), Duration::from_secs(60)); assert!(!toml_config.size().enabled()); assert!(toml_config.receipts().enabled()); - } - } diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index c164420483..c290dbeaeb 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -29,7 +29,7 @@ static NUM_PRUNINGS: AtomicUsize = AtomicUsize::new(0); /// Performs pruning of data from `start_index` to `target_index`. #[cfg_attr(feature = "trace", trace_tools::observe)] -pub async fn prune( +pub async fn prune_by_range( tangle: &Tangle, storage: &S, bus: &Bus<'_>, @@ -55,145 +55,185 @@ pub async fn prune( } for index in *start_index..=*target_index { - let index = MilestoneIndex(index); - - debug!("Pruning milestone {}...", index); - - // Measurement of the full pruning step. - let full_prune = Instant::now(); - - // Get the current set of SEPs. - let get_curr_seps = Instant::now(); - let mut curr_seps = tangle.get_solid_entry_points().await; - timings.get_curr_seps = get_curr_seps.elapsed(); - - metrics.curr_seps = curr_seps.len(); - - // Start a batch to make changes to the storage in a single atomic step. - let mut batch = S::batch_begin(); - - // Add confirmed data to the delete batch. - // NOTE: This is the most costly thing during pruning, because it has to perform a past-cone traversal. - let batch_confirmed_data = Instant::now(); - let (mut new_seps, confirmed_data_metrics) = - batch::batch_prunable_confirmed_data(storage, &mut batch, index, &curr_seps)?; - timings.batch_confirmed_data = batch_confirmed_data.elapsed(); - - metrics.new_seps = new_seps.len(); - metrics.messages = confirmed_data_metrics.prunable_messages; - metrics.edges = confirmed_data_metrics.prunable_edges; - metrics.indexations = confirmed_data_metrics.prunable_indexations; - - // Keep still relevant SEPs. - // - // Note: - // Currently Bee is reliant on the snapshot file generated by Hornet, which stores the confirmation index - // of an SEP along with it. It then keeps it long enough to be (pretty) sure the coordinator would reject a - // message directly referencing it. In Bee, however, we wanted to try a different approach, which doesn't - // trust the Coordinator's tip selection, and stores the highest confirmation index of any of its direct - // approvers instead. - // - // For the first X milestones we keep the initial SEP set (from the snapshot file) around, after that, we keep - // only the necessary SEPs (the ones that will be referenced in future prunings). - let filter_curr_seps = Instant::now(); - if NUM_PRUNINGS.fetch_add(1, Ordering::Relaxed) >= KEEP_INITIAL_SNAPSHOT_SEPS { - curr_seps.retain(|_, v| **v > *index); - } - timings.filter_curr_seps = filter_curr_seps.elapsed(); + prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + } - metrics.kept_seps = curr_seps.len(); + if start_index == target_index { + info!("Pruned milestone {}.", start_index); + } else { + info!("Pruned from milestone {} to milestone {}.", start_index, target_index); + } - // Create the union of both sets: - new_seps.extend(curr_seps); + Ok(()) +} - let num_next_seps = new_seps.len(); +/// Performs pruning of data until a certain size is reached. +#[cfg_attr(feature = "trace", trace_tools::observe)] +pub async fn prune_by_size( + tangle: &Tangle, + storage: &S, + bus: &Bus<'_>, + reduced_size: usize, + config: &PruningConfig, +) -> Result<(), Error> { + let mut timings = Timings::default(); + let mut metrics = PruningMetrics::default(); - metrics.next_seps = num_next_seps; + loop { + let actual_size = storage.size().expect("ok storage size").expect("some storage size"); - // Write the new set of SEPs to the storage. - let batch_new_seps = Instant::now(); - for (new_sep, index) in &new_seps { - Batch::::batch_insert(storage, &mut batch, new_sep, index) - .map_err(|e| Error::Storage(Box::new(e)))?; + if actual_size > reduced_size { + let index = *tangle.get_pruning_index(); + prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + } else { + break; } - timings.batch_new_seps = batch_new_seps.elapsed(); - - // Replace the old set of SEPs with the new one. - let replace_seps = Instant::now(); - tangle.replace_solid_entry_points(new_seps).await; - timings.replace_seps = replace_seps.elapsed(); - - // Update entry point index - tangle.update_entry_point_index(index); - - let batch_milestones = Instant::now(); - let milestone_data_metrics = - batch::prune_milestone_data(storage, &mut batch, index, config.receipts().enabled())?; - timings.batch_milestone_data = batch_milestones.elapsed(); - - metrics.receipts = milestone_data_metrics.receipts; - - // Add unconfirmed data to the delete batch. - let batch_unconfirmed_data = Instant::now(); - let unconfirmed_data_metrics = batch::batch_prunable_unconfirmed_data(storage, &mut batch, index)?; - timings.batch_unconfirmed_data = batch_unconfirmed_data.elapsed(); - - metrics.messages += unconfirmed_data_metrics.prunable_messages; - metrics.edges += unconfirmed_data_metrics.prunable_edges; - metrics.indexations += unconfirmed_data_metrics.prunable_indexations; - - // Remove old SEPs from the storage. - // - // **WARNING**: This operation must come before the batch is committed! - // - // TODO: consider batching deletes rather than using Truncate. Is one faster than the other? Do we care if its - // atomic or not? - let truncate_old_seps = Instant::now(); - Truncate::::truncate(storage).expect("truncating solid entry points failed"); - timings.truncate_curr_seps = truncate_old_seps.elapsed(); - - // Execute the batch operation. - let batch_commit = Instant::now(); - storage - .batch_commit(batch, true) - .map_err(|e| Error::Storage(Box::new(e)))?; - timings.batch_commit = batch_commit.elapsed(); - - // Update the pruning index. - tangle.update_pruning_index(index); - - // Write the updated snapshot info to the storage. - let timestamp = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("error creating timestamp") - .as_secs(); - let mut snapshot_info = storage::fetch_snapshot_info(storage) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingSnapshotInfo)?; - snapshot_info.update_pruning_index(index); - snapshot_info.update_timestamp(timestamp); - storage::insert_snapshot_info(storage, &snapshot_info).map_err(|e| Error::Storage(Box::new(e)))?; - - timings.full_prune = full_prune.elapsed(); - - debug!("{:?}.", metrics); - debug!("{:?}", confirmed_data_metrics); - debug!("{:?}", unconfirmed_data_metrics); - debug!("{:?}.", timings); - debug!( - "Entry point index now at {} with {} solid entry points..", - index, num_next_seps - ); - debug!("Pruned milestone {}.", index); + } - bus.dispatch(PrunedIndex { index }); + Ok(()) +} + +/// Prunes a single milestone. +async fn prune_milestone( + index: u32, + tangle: &Tangle, + storage: &S, + bus: &Bus<'_>, + timings: &mut Timings, + metrics: &mut PruningMetrics, + config: &PruningConfig, +) -> Result<(), Error> { + let index = MilestoneIndex(index); + + debug!("Pruning milestone {}...", index); + + // Measurement of the full pruning step. + let full_prune = Instant::now(); + + // Get the current set of SEPs. + let get_curr_seps = Instant::now(); + let mut curr_seps = tangle.get_solid_entry_points().await; + timings.get_curr_seps = get_curr_seps.elapsed(); + + metrics.curr_seps = curr_seps.len(); + + // Start a batch to make changes to the storage in a single atomic step. + let mut batch = S::batch_begin(); + + // Add confirmed data to the delete batch. + // NOTE: This is the most costly thing during pruning, because it has to perform a past-cone traversal. + let batch_confirmed_data = Instant::now(); + let (mut new_seps, confirmed_data_metrics) = + batch::batch_prunable_confirmed_data(storage, &mut batch, index, &curr_seps)?; + timings.batch_confirmed_data = batch_confirmed_data.elapsed(); + + metrics.new_seps = new_seps.len(); + metrics.messages = confirmed_data_metrics.prunable_messages; + metrics.edges = confirmed_data_metrics.prunable_edges; + metrics.indexations = confirmed_data_metrics.prunable_indexations; + + // Keep still relevant SEPs. + // + // Note: + // Currently Bee is reliant on the snapshot file generated by Hornet, which stores the confirmation index + // of an SEP along with it. It then keeps it long enough to be (pretty) sure the coordinator would reject a + // message directly referencing it. In Bee, however, we wanted to try a different approach, which doesn't + // trust the Coordinator's tip selection, and stores the highest confirmation index of any of its direct + // approvers instead. + // + // For the first X milestones we keep the initial SEP set (from the snapshot file) around, after that, we keep + // only the necessary SEPs (the ones that will be referenced in future prunings). + let filter_curr_seps = Instant::now(); + if NUM_PRUNINGS.fetch_add(1, Ordering::Relaxed) >= KEEP_INITIAL_SNAPSHOT_SEPS { + curr_seps.retain(|_, v| **v > *index); } + timings.filter_curr_seps = filter_curr_seps.elapsed(); - if start_index == target_index { - info!("Pruned milestone {}.", start_index); - } else { - info!("Pruned from milestone {} to milestone {}.", start_index, target_index); + metrics.kept_seps = curr_seps.len(); + + // Create the union of both sets: + new_seps.extend(curr_seps); + + let num_next_seps = new_seps.len(); + + metrics.next_seps = num_next_seps; + + // Write the new set of SEPs to the storage. + let batch_new_seps = Instant::now(); + for (new_sep, index) in &new_seps { + Batch::::batch_insert(storage, &mut batch, new_sep, index) + .map_err(|e| Error::Storage(Box::new(e)))?; } + timings.batch_new_seps = batch_new_seps.elapsed(); + + // Replace the old set of SEPs with the new one. + let replace_seps = Instant::now(); + tangle.replace_solid_entry_points(new_seps).await; + timings.replace_seps = replace_seps.elapsed(); + + // Update entry point index + tangle.update_entry_point_index(index); + + let batch_milestones = Instant::now(); + let milestone_data_metrics = batch::prune_milestone_data(storage, &mut batch, index, config.receipts().enabled())?; + timings.batch_milestone_data = batch_milestones.elapsed(); + + metrics.receipts = milestone_data_metrics.receipts; + + // Add unconfirmed data to the delete batch. + let batch_unconfirmed_data = Instant::now(); + let unconfirmed_data_metrics = batch::batch_prunable_unconfirmed_data(storage, &mut batch, index)?; + timings.batch_unconfirmed_data = batch_unconfirmed_data.elapsed(); + + metrics.messages += unconfirmed_data_metrics.prunable_messages; + metrics.edges += unconfirmed_data_metrics.prunable_edges; + metrics.indexations += unconfirmed_data_metrics.prunable_indexations; + + // Remove old SEPs from the storage. + // + // **WARNING**: This operation must come before the batch is committed! + // + // TODO: consider batching deletes rather than using Truncate. Is one faster than the other? Do we care if its + // atomic or not? + let truncate_old_seps = Instant::now(); + Truncate::::truncate(storage).expect("truncating solid entry points failed"); + timings.truncate_curr_seps = truncate_old_seps.elapsed(); + + // Execute the batch operation. + let batch_commit = Instant::now(); + storage + .batch_commit(batch, true) + .map_err(|e| Error::Storage(Box::new(e)))?; + timings.batch_commit = batch_commit.elapsed(); + + // Update the pruning index. + tangle.update_pruning_index(index); + + // Write the updated snapshot info to the storage. + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("error creating timestamp") + .as_secs(); + let mut snapshot_info = storage::fetch_snapshot_info(storage) + .map_err(|e| Error::Storage(Box::new(e)))? + .ok_or(Error::MissingSnapshotInfo)?; + snapshot_info.update_pruning_index(index); + snapshot_info.update_timestamp(timestamp); + storage::insert_snapshot_info(storage, &snapshot_info).map_err(|e| Error::Storage(Box::new(e)))?; + + timings.full_prune = full_prune.elapsed(); + + debug!("{:?}.", metrics); + debug!("{:?}", confirmed_data_metrics); + debug!("{:?}", unconfirmed_data_metrics); + debug!("{:?}.", timings); + debug!( + "Entry point index now at {} with {} solid entry points..", + index, num_next_seps + ); + debug!("Pruned milestone {}.", index); + + bus.dispatch(PrunedIndex { index }); Ok(()) } From b498e8e20a2005ea7a1ba7a70a3fcff2ded7def0 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 19 Apr 2022 14:57:24 +0200 Subject: [PATCH 09/39] Enable pruning-by-size; Cleanup --- bee-ledger/src/workers/consensus/worker.rs | 14 +-- bee-ledger/src/workers/error.rs | 27 +----- bee-ledger/src/workers/pruning/batch.rs | 90 ++++++++++++-------- bee-ledger/src/workers/pruning/condition.rs | 43 ++++++---- bee-ledger/src/workers/pruning/error.rs | 2 +- bee-ledger/src/workers/pruning/mod.rs | 2 +- bee-ledger/src/workers/pruning/prune.rs | 20 ++--- bee-ledger/src/workers/snapshot/condition.rs | 8 +- 8 files changed, 109 insertions(+), 97 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index ba63fbc6f8..67d3f54853 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -362,18 +362,20 @@ where ) .await { - error!("Pruning failed: {e:?}."); - // TODO: consider initiating an emergency (but graceful) shutdown instead of just - // panicking. - panic!("Aborting due to an unexpected pruning error."); + error!("Pruning milestone range failed: {}.", e); } } PruningTask::BySize(reduced_size) => { - todo!("pruning by size"); + if let Err(e) = + prune::prune_by_size(&tangle, &storage, &bus, reduced_size, &pruning_config) + .await + { + error!("Pruning by storage size failed: {}.", e); + } } }, Err(reason) => { - debug!("Pruning skipped: {:?}", reason); + debug!("Pruning skipped: {}", reason); } } } diff --git a/bee-ledger/src/workers/error.rs b/bee-ledger/src/workers/error.rs index 3df86cbca4..e28e721083 100644 --- a/bee-ledger/src/workers/error.rs +++ b/bee-ledger/src/workers/error.rs @@ -7,76 +7,57 @@ use bee_message::{address::Address, milestone::MilestoneIndex, Error as MessageE use crate::{ types::{Balance, Error as TypesError, Unspent}, - workers::snapshot::error::Error as SnapshotError, + workers::{pruning::error::PruningError, snapshot::error::Error as SnapshotError}, }; /// Errors occurring during ledger workers operations. #[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] pub enum Error { - /// Snapshot error. #[error("Snapshot error: {0}")] Snapshot(#[from] SnapshotError), - /// Types error. + #[error("Pruning error: {0}")] + Pruning(#[from] PruningError), #[error("Types error: {0}")] Types(#[from] TypesError), - /// Message error. #[error("Message error: {0}")] Message(#[from] MessageError), - /// Missing message in the past cone of the milestone #[error("Message {0} is missing in the past cone of the milestone")] MissingMessage(MessageId), - /// Unsupported input kind. #[error("Unsupported input kind: {0}")] UnsupportedInputKind(u8), - /// Unsupported output kind. #[error("Unsupported output kind: {0}")] UnsupportedOutputKind(u8), - /// Unsupported payload kind. #[error("Unsupported payload kind: {0}")] UnsupportedPayloadKind(u32), - /// Milestone message not found. #[error("Milestone message not found: {0}")] MilestoneMessageNotFound(MessageId), - /// Message payload is not a milestone #[error("Message payload is not a milestone")] NoMilestonePayload, - /// Non contiguous milestones. #[error("Non contiguous milestones: tried to confirm {0} on top of {1}")] NonContiguousMilestones(u32, u32), - /// Merkle proof mismatch. #[error("Merkle proof mismatch on milestone {0}: computed {1} != provided {2}")] MerkleProofMismatch(MilestoneIndex, String, String), - /// Invalid messages count. #[error("Invalid messages count: referenced ({0}) != no transaction ({1}) + conflicting ({2}) + included ({3})")] InvalidMessagesCount(usize, usize, usize, usize), - /// Invalid ledger unspent state. #[error("Invalid ledger unspent state: {0}")] InvalidLedgerUnspentState(u64), - /// Invalid ledger balance state. #[error("Invalid ledger balance state: {0}")] InvalidLedgerBalanceState(u64), - /// Invalid ledger dust state. #[error("Invalid ledger dust state: {0:?} {1:?}")] InvalidLedgerDustState(Address, Balance), - /// Consumed amount overflow. #[error("Consumed amount overflow: {0}.")] ConsumedAmountOverflow(u128), - /// Created amount overflow. #[error("Created amount overflow: {0}.")] CreatedAmountOverflow(u128), - /// Ledger state overflow. #[error("Ledger state overflow: {0}")] LedgerStateOverflow(u128), - /// Non zero balance diff sum. #[error("Non zero balance diff sum: {0}.")] NonZeroBalanceDiffSum(i64), - /// Decreasing receipt migrated at index. #[error("Decreasing receipt migrated at index: {0} < {1}")] DecreasingReceiptMigratedAtIndex(MilestoneIndex, MilestoneIndex), - /// Missing unspent output. #[error("Missing unspent output {0}")] MissingUnspentOutput(Unspent), - /// Storage backend error. #[error("Storage backend error: {0}")] Storage(Box), } diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index b95262713c..998daf276b 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -24,7 +24,7 @@ use crate::{ types::{ConsumedOutput, CreatedOutput, OutputDiff, Receipt}, workers::{ pruning::{ - error::Error, + error::PruningError, metrics::{ConfirmedDataPruningMetrics, MilestoneDataPruningMetrics, UnconfirmedDataPruningMetrics}, }, storage::StorageBackend, @@ -46,7 +46,7 @@ pub fn batch_prunable_confirmed_data( batch: &mut S::Batch, prune_index: MilestoneIndex, current_seps: &Seps, -) -> Result<(Seps, ConfirmedDataPruningMetrics), Error> { +) -> Result<(Seps, ConfirmedDataPruningMetrics), PruningError> { // We keep a list of already visited messages. let mut visited = Messages::with_capacity(512); // We keep a cache of approvers to prevent fetch the same data from the storage more than once. @@ -58,8 +58,8 @@ pub fn batch_prunable_confirmed_data( // Get the `MessageId` of the milestone we are about to prune from the storage. let prune_id = *Fetch::::fetch(storage, &prune_index) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingMilestone(prune_index))? + .map_err(|e| PruningError::Storage(Box::new(e)))? + .ok_or(PruningError::MissingMilestone(prune_index))? .message_id(); // Breadth-first traversal will increase our chances of sorting out redundant messages without querying the storage. @@ -80,8 +80,8 @@ pub fn batch_prunable_confirmed_data( // Get the `Message` for `message_id`. let msg = match Fetch::::fetch(storage, &message_id) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingMessage(message_id)) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .ok_or(PruningError::MissingMessage(message_id)) { Ok(msg) => msg, Err(e) => { @@ -134,8 +134,8 @@ pub fn batch_prunable_confirmed_data( // Fetch its approvers from the storage. let approvers = Fetch::>::fetch(storage, &message_id) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingApprovers(message_id))?; + .map_err(|e| PruningError::Storage(Box::new(e)))? + .ok_or(PruningError::MissingApprovers(message_id))?; // We can safely skip messages whose approvers are all part of the currently pruned cone. If we are lucky // (chances are better with the chosen breadth-first traversal) we've already seen all of its approvers. @@ -163,8 +163,8 @@ pub fn batch_prunable_confirmed_data( metrics.approver_cache_miss += 1; let unvisited_md = Fetch::::fetch(storage, &unvisited_id) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingMetadata(unvisited_id))?; + .map_err(|e| PruningError::Storage(Box::new(e)))? + .ok_or(PruningError::MissingMetadata(unvisited_id))?; // A non-existing milestone index means that a message remained unconfirmed and therefore is neglibable // for its parent in terms of SEP consideration. This can be expressed by assigning the @@ -204,11 +204,11 @@ pub fn batch_prunable_unconfirmed_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, -) -> Result { +) -> Result { let mut metrics = UnconfirmedDataPruningMetrics::default(); let unconf_msgs = match Fetch::>::fetch(storage, &prune_index) - .map_err(|e| Error::Storage(Box::new(e)))? + .map_err(|e| PruningError::Storage(Box::new(e)))? { Some(unconf_msgs) => { if unconf_msgs.is_empty() { @@ -227,7 +227,7 @@ pub fn batch_prunable_unconfirmed_data( // TODO: consider using `MultiFetch` 'next_unconf_msg: for unconf_msg_id in unconf_msgs.iter().map(|unconf_msg| unconf_msg.message_id()) { match Fetch::::fetch(storage, unconf_msg_id) - .map_err(|e| Error::Storage(Box::new(e)))? + .map_err(|e| PruningError::Storage(Box::new(e)))? { Some(msg_meta) => { if msg_meta.flags().is_referenced() { @@ -244,7 +244,9 @@ pub fn batch_prunable_unconfirmed_data( } // Delete those messages that remained unconfirmed. - match Fetch::::fetch(storage, unconf_msg_id).map_err(|e| Error::Storage(Box::new(e)))? { + match Fetch::::fetch(storage, unconf_msg_id) + .map_err(|e| PruningError::Storage(Box::new(e)))? + { Some(msg) => { let payload = msg.payload().as_ref(); let parents = msg.parents(); @@ -282,7 +284,7 @@ pub fn batch_prunable_unconfirmed_data( batch, &(prune_index, (*unconf_msg_id).into()), ) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; metrics.prunable_messages += 1; } @@ -295,7 +297,7 @@ pub fn prune_milestone_data( batch: &mut S::Batch, prune_index: MilestoneIndex, should_prune_receipts: bool, -) -> Result { +) -> Result { let mut metrics = MilestoneDataPruningMetrics::default(); prune_milestone(storage, batch, prune_index)?; @@ -313,10 +315,11 @@ fn prune_message_and_metadata( storage: &S, batch: &mut S::Batch, message_id: &MessageId, -) -> Result<(), Error> { - Batch::::batch_delete(storage, batch, message_id).map_err(|e| Error::Storage(Box::new(e)))?; +) -> Result<(), PruningError> { + Batch::::batch_delete(storage, batch, message_id) + .map_err(|e| PruningError::Storage(Box::new(e)))?; Batch::::batch_delete(storage, batch, message_id) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; Ok(()) } @@ -325,8 +328,9 @@ fn prune_edge( storage: &S, batch: &mut S::Batch, edge: &(MessageId, MessageId), -) -> Result<(), Error> { - Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge).map_err(|e| Error::Storage(Box::new(e)))?; +) -> Result<(), PruningError> { + Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge) + .map_err(|e| PruningError::Storage(Box::new(e)))?; Ok(()) } @@ -335,29 +339,37 @@ fn prune_indexation_data( storage: &S, batch: &mut S::Batch, index_message_id: &(PaddedIndex, MessageId), -) -> Result<(), Error> { +) -> Result<(), PruningError> { Batch::<(PaddedIndex, MessageId), ()>::batch_delete(storage, batch, index_message_id) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; Ok(()) } -fn prune_milestone(storage: &S, batch: &mut S::Batch, index: MilestoneIndex) -> Result<(), Error> { +fn prune_milestone( + storage: &S, + batch: &mut S::Batch, + index: MilestoneIndex, +) -> Result<(), PruningError> { Batch::::batch_delete(storage, batch, &index) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; Ok(()) } -fn prune_output_diff(storage: &S, batch: &mut S::Batch, index: MilestoneIndex) -> Result<(), Error> { +fn prune_output_diff( + storage: &S, + batch: &mut S::Batch, + index: MilestoneIndex, +) -> Result<(), PruningError> { if let Some(output_diff) = - Fetch::::fetch(storage, &index).map_err(|e| Error::Storage(Box::new(e)))? + Fetch::::fetch(storage, &index).map_err(|e| PruningError::Storage(Box::new(e)))? { for consumed_output in output_diff.consumed_outputs() { Batch::::batch_delete(storage, batch, consumed_output) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; Batch::::batch_delete(storage, batch, consumed_output) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; } if let Some(_treasury_diff) = output_diff.treasury_diff() { @@ -366,21 +378,25 @@ fn prune_output_diff(storage: &S, batch: &mut S::Batch, index } Batch::::batch_delete(storage, batch, &index) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; Ok(()) } -fn prune_receipts(storage: &S, batch: &mut S::Batch, index: MilestoneIndex) -> Result { +fn prune_receipts( + storage: &S, + batch: &mut S::Batch, + index: MilestoneIndex, +) -> Result { let receipts = Fetch::>::fetch(storage, &index) - .map_err(|e| Error::Storage(Box::new(e)))? + .map_err(|e| PruningError::Storage(Box::new(e)))? // Fine since Fetch of a Vec<_> always returns Some(Vec<_>). .unwrap(); let mut num = 0; for receipt in receipts.into_iter() { Batch::<(MilestoneIndex, Receipt), ()>::batch_delete(storage, batch, &(index, receipt)) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; num += 1; } @@ -410,11 +426,15 @@ fn unwrap_indexation(payload: Option<&Payload>) -> Option<&IndexationPayload> { // TODO: consider using this instead of 'truncate' #[allow(dead_code)] -fn prune_seps(storage: &S, batch: &mut S::Batch, seps: &[SolidEntryPoint]) -> Result { +fn prune_seps( + storage: &S, + batch: &mut S::Batch, + seps: &[SolidEntryPoint], +) -> Result { let mut num = 0; for sep in seps { Batch::::batch_delete(storage, batch, sep) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; num += 1; } diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 6cb2cffdae..d20c8f4365 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -1,7 +1,7 @@ // Copyright 2020-2021 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::time::Duration; +// use std::time::Duration; use bee_message::milestone::MilestoneIndex; use bee_tangle::{storage::StorageBackend, Tangle}; @@ -11,18 +11,20 @@ use crate::{types::LedgerIndex, workers::pruning::config::PruningConfig}; const PRUNING_BATCH_SIZE_MAX: u32 = 200; /// Reasons for skipping pruning. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub(crate) enum PruningSkipReason { - /// Pruning is disabled in the config. + #[error("Pruning disabled.")] Disabled, - /// The storage in use doesn't support this pruning task. - PruningTaskNotSupported, - /// Not enough history yet to be pruned. - BelowMaxMilestonesToKeepThreshold { reached_in: u32 }, - /// The database has not yet reached its target size. - BelowTargetSize { reached_in: usize }, - /// The cooldown time has not yet been passed. - BelowCooldownTimeThreshold { reached_in: Duration }, + #[error("Pruning by size is not supported by the storage layer.")] + PruningBySizeUnsupported, + #[error("Pruning by size is currently unavailable.")] + PruningBySizeUnavailable, + #[error("Pruning milestones skipped for the next {reached_in} indexes.")] + BelowMilestoneIndexThreshold { reached_in: u32 }, + #[error("Pruning by storage size skipped because current size < {target_size}.")] + BelowStorageSizeThreshold { target_size: usize }, + // #[error("Pruning by storage size is skipped because of cooldown.")] + // BelowCooldownTimeThreshold { reached_in: Duration }, } pub(crate) enum PruningTask { @@ -44,7 +46,7 @@ pub(crate) fn should_prune( let pruning_threshold = pruning_index + milestones_to_keep; if *ledger_index < pruning_threshold { - Err(PruningSkipReason::BelowMaxMilestonesToKeepThreshold { + Err(PruningSkipReason::BelowMilestoneIndexThreshold { reached_in: pruning_threshold - *ledger_index, }) } else { @@ -61,16 +63,23 @@ pub(crate) fn should_prune( } } else if config.size().enabled() { // TODO: return `PruningTAskNotSupported` if we can't get the size of the storage. - let actual_size = storage.size().expect("ok storage size").expect("some storage size"); + let actual_size = { + if let Ok(size) = storage.size() { + if let Some(size) = size { + size + } else { + return Err(PruningSkipReason::PruningBySizeUnavailable); + } + } else { + return Err(PruningSkipReason::PruningBySizeUnsupported); + } + }; let target_size = config.size().target_size(); log::debug!("Storage size: actual {actual_size} target {target_size}"); if actual_size < target_size { - // Panic: cannot underflow. - Err(PruningSkipReason::BelowTargetSize { - reached_in: target_size - actual_size, - }) + Err(PruningSkipReason::BelowStorageSizeThreshold { target_size }) } else { let reduced_size = actual_size - (config.size().threshold_percentage() as f64 / 100.0 * target_size as f64) as usize; diff --git a/bee-ledger/src/workers/pruning/error.rs b/bee-ledger/src/workers/pruning/error.rs index ad6cf01047..f7cd3ba6e3 100644 --- a/bee-ledger/src/workers/pruning/error.rs +++ b/bee-ledger/src/workers/pruning/error.rs @@ -4,7 +4,7 @@ use bee_message::{milestone::MilestoneIndex, MessageId}; #[derive(Debug, thiserror::Error)] -pub enum Error { +pub enum PruningError { #[error("pruning target index {selected} below minimum {minimum}")] InvalidTargetIndex { selected: MilestoneIndex, diff --git a/bee-ledger/src/workers/pruning/mod.rs b/bee-ledger/src/workers/pruning/mod.rs index ce004ff587..cecb48b704 100644 --- a/bee-ledger/src/workers/pruning/mod.rs +++ b/bee-ledger/src/workers/pruning/mod.rs @@ -4,10 +4,10 @@ //! Module that contains the pruning logic. mod batch; -mod error; mod metrics; pub(crate) mod condition; +pub(crate) mod error; pub(crate) mod prune; pub mod config; diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index c290dbeaeb..1ea6a3be61 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -17,7 +17,7 @@ use crate::workers::{ pruning::{ batch, config::PruningConfig, - error::Error, + error::PruningError, metrics::{PruningMetrics, Timings}, }, storage::{self, StorageBackend}, @@ -36,12 +36,12 @@ pub async fn prune_by_range( start_index: MilestoneIndex, target_index: MilestoneIndex, config: &PruningConfig, -) -> Result<(), Error> { +) -> Result<(), PruningError> { let mut timings = Timings::default(); let mut metrics = PruningMetrics::default(); if target_index < start_index { - return Err(Error::InvalidTargetIndex { + return Err(PruningError::InvalidTargetIndex { selected: target_index, minimum: start_index, }); @@ -75,7 +75,7 @@ pub async fn prune_by_size( bus: &Bus<'_>, reduced_size: usize, config: &PruningConfig, -) -> Result<(), Error> { +) -> Result<(), PruningError> { let mut timings = Timings::default(); let mut metrics = PruningMetrics::default(); @@ -102,7 +102,7 @@ async fn prune_milestone( timings: &mut Timings, metrics: &mut PruningMetrics, config: &PruningConfig, -) -> Result<(), Error> { +) -> Result<(), PruningError> { let index = MilestoneIndex(index); debug!("Pruning milestone {}...", index); @@ -162,7 +162,7 @@ async fn prune_milestone( let batch_new_seps = Instant::now(); for (new_sep, index) in &new_seps { Batch::::batch_insert(storage, &mut batch, new_sep, index) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; } timings.batch_new_seps = batch_new_seps.elapsed(); @@ -203,7 +203,7 @@ async fn prune_milestone( let batch_commit = Instant::now(); storage .batch_commit(batch, true) - .map_err(|e| Error::Storage(Box::new(e)))?; + .map_err(|e| PruningError::Storage(Box::new(e)))?; timings.batch_commit = batch_commit.elapsed(); // Update the pruning index. @@ -215,11 +215,11 @@ async fn prune_milestone( .expect("error creating timestamp") .as_secs(); let mut snapshot_info = storage::fetch_snapshot_info(storage) - .map_err(|e| Error::Storage(Box::new(e)))? - .ok_or(Error::MissingSnapshotInfo)?; + .map_err(|e| PruningError::Storage(Box::new(e)))? + .ok_or(PruningError::MissingSnapshotInfo)?; snapshot_info.update_pruning_index(index); snapshot_info.update_timestamp(timestamp); - storage::insert_snapshot_info(storage, &snapshot_info).map_err(|e| Error::Storage(Box::new(e)))?; + storage::insert_snapshot_info(storage, &snapshot_info).map_err(|e| PruningError::Storage(Box::new(e)))?; timings.full_prune = full_prune.elapsed(); diff --git a/bee-ledger/src/workers/snapshot/condition.rs b/bee-ledger/src/workers/snapshot/condition.rs index ac55e643f6..be88063a4b 100644 --- a/bee-ledger/src/workers/snapshot/condition.rs +++ b/bee-ledger/src/workers/snapshot/condition.rs @@ -6,11 +6,11 @@ use bee_tangle::{storage::StorageBackend, Tangle}; use crate::{types::LedgerIndex, workers::snapshot::config::SnapshotConfig}; /// Reasons for skipping snapshotting. -#[derive(Debug)] -pub enum SnapshottingSkipReason { - /// Not enough data yet to create a snapshot. +#[derive(Debug, thiserror::Error)] +pub(crate) enum SnapshottingSkipReason { + #[error("Snapshotting skipped for the next {reached_in} indexes.")] BelowThreshold { reached_in: u32 }, - /// Snapshotting is deferred to a later milestone. + #[error("Snapshotting deferred for the next {next_in} indexes.")] Deferred { next_in: u32 }, } From fde093c6ba8d7a1429e9d665b8a5d815ef56cbf7 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 19 Apr 2022 15:58:43 +0200 Subject: [PATCH 10/39] Increase pruning index --- bee-ledger/src/workers/pruning/condition.rs | 1 - bee-ledger/src/workers/pruning/prune.rs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index d20c8f4365..50e9e3ba53 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -62,7 +62,6 @@ pub(crate) fn should_prune( )) } } else if config.size().enabled() { - // TODO: return `PruningTAskNotSupported` if we can't get the size of the storage. let actual_size = { if let Ok(size) = storage.size() { if let Some(size) = size { diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 1ea6a3be61..e3eb7f6cfc 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -80,10 +80,11 @@ pub async fn prune_by_size( let mut metrics = PruningMetrics::default(); loop { + // Panic: already checked that size() returns a valid result. let actual_size = storage.size().expect("ok storage size").expect("some storage size"); if actual_size > reduced_size { - let index = *tangle.get_pruning_index(); + let index = *tangle.get_pruning_index() + 1; prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; } else { break; From ba3b93eead865fdcc2b646986f35fa1a4acb10e0 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 19 Apr 2022 17:45:26 +0200 Subject: [PATCH 11/39] Log actual size during size pruning --- bee-ledger/src/workers/pruning/prune.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index e3eb7f6cfc..19aa232906 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -10,7 +10,6 @@ use bee_message::milestone::MilestoneIndex; use bee_runtime::event::Bus; use bee_storage::access::{Batch, Truncate}; use bee_tangle::{solid_entry_point::SolidEntryPoint, Tangle}; -use log::{debug, info}; use crate::workers::{ event::PrunedIndex, @@ -48,7 +47,7 @@ pub async fn prune_by_range( } if start_index != target_index { - info!( + log::info!( "Pruning from milestone {} to milestone {}...", start_index, target_index ); @@ -59,9 +58,9 @@ pub async fn prune_by_range( } if start_index == target_index { - info!("Pruned milestone {}.", start_index); + log::info!("Pruned milestone {}.", start_index); } else { - info!("Pruned from milestone {} to milestone {}.", start_index, target_index); + log::info!("Pruned from milestone {} to milestone {}.", start_index, target_index); } Ok(()) @@ -82,6 +81,7 @@ pub async fn prune_by_size( loop { // Panic: already checked that size() returns a valid result. let actual_size = storage.size().expect("ok storage size").expect("some storage size"); + log::debug!("Storage size: actual {actual_size} reduced {reduced_size}"); if actual_size > reduced_size { let index = *tangle.get_pruning_index() + 1; @@ -106,7 +106,7 @@ async fn prune_milestone( ) -> Result<(), PruningError> { let index = MilestoneIndex(index); - debug!("Pruning milestone {}...", index); + log::debug!("Pruning milestone {}...", index); // Measurement of the full pruning step. let full_prune = Instant::now(); @@ -224,15 +224,15 @@ async fn prune_milestone( timings.full_prune = full_prune.elapsed(); - debug!("{:?}.", metrics); - debug!("{:?}", confirmed_data_metrics); - debug!("{:?}", unconfirmed_data_metrics); - debug!("{:?}.", timings); - debug!( + log::debug!("{:?}.", metrics); + log::debug!("{:?}", confirmed_data_metrics); + log::debug!("{:?}", unconfirmed_data_metrics); + log::debug!("{:?}.", timings); + log::debug!( "Entry point index now at {} with {} solid entry points..", index, num_next_seps ); - debug!("Pruned milestone {}.", index); + log::debug!("Pruned milestone {}.", index); bus.dispatch(PrunedIndex { index }); From 356d2a6b4bf6adc9bde41ffb5bd085c60fdc1b93 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 20 Apr 2022 12:09:51 +0200 Subject: [PATCH 12/39] Disable RocksDB WAL for pruning --- bee-ledger/src/workers/pruning/prune.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 19aa232906..dea74cfa5c 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -203,7 +203,7 @@ async fn prune_milestone( // Execute the batch operation. let batch_commit = Instant::now(); storage - .batch_commit(batch, true) + .batch_commit(batch, false) .map_err(|e| PruningError::Storage(Box::new(e)))?; timings.batch_commit = batch_commit.elapsed(); From 0d88d222d7a7b85d793c2cb0e445cad1c2a64bde Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 20 Apr 2022 14:59:21 +0200 Subject: [PATCH 13/39] Calculate batch byte size when pruning by size --- bee-ledger/src/workers/consensus/worker.rs | 13 +- bee-ledger/src/workers/pruning/batch.rs | 212 ++++++++++++++------ bee-ledger/src/workers/pruning/condition.rs | 10 +- bee-ledger/src/workers/pruning/prune.rs | 55 ++--- bee-ledger/src/workers/storage.rs | 2 + 5 files changed, 204 insertions(+), 88 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 67d3f54853..6e47433967 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -365,10 +365,15 @@ where error!("Pruning milestone range failed: {}.", e); } } - PruningTask::BySize(reduced_size) => { - if let Err(e) = - prune::prune_by_size(&tangle, &storage, &bus, reduced_size, &pruning_config) - .await + PruningTask::BySize(num_bytes_to_prune) => { + if let Err(e) = prune::prune_by_size( + &tangle, + &storage, + &bus, + num_bytes_to_prune, + &pruning_config, + ) + .await { error!("Pruning by storage size failed: {}.", e); } diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 998daf276b..cc8e0049c4 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -3,6 +3,7 @@ use std::collections::VecDeque; +use bee_common::packable::Packable; use bee_message::{ milestone::{Milestone, MilestoneIndex}, output::OutputId, @@ -31,22 +32,23 @@ use crate::{ }, }; -pub type Messages = HashSet; -pub type ApproverCache = HashMap; -pub type Seps = HashMap; +pub(crate) type Messages = HashSet; +pub(crate) type ApproverCache = HashMap; +pub(crate) type Seps = HashMap; +pub(crate) type ByteLength = usize; #[derive(Eq, PartialEq, Hash)] -pub struct Edge { - pub from_parent: MessageId, - pub to_child: MessageId, +pub(crate) struct Edge { + pub(crate) from_parent: MessageId, + pub(crate) to_child: MessageId, } -pub fn batch_prunable_confirmed_data( +pub(crate) fn batch_prunable_confirmed_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, current_seps: &Seps, -) -> Result<(Seps, ConfirmedDataPruningMetrics), PruningError> { +) -> Result<(Seps, ConfirmedDataPruningMetrics, ByteLength), PruningError> { // We keep a list of already visited messages. let mut visited = Messages::with_capacity(512); // We keep a cache of approvers to prevent fetch the same data from the storage more than once. @@ -55,12 +57,18 @@ pub fn batch_prunable_confirmed_data( let mut new_seps = Seps::with_capacity(512); // We collect stats during the traversal, and return them as a result of this function. let mut metrics = ConfirmedDataPruningMetrics::default(); + // We count the number of bytes pruned from the storage. + let mut byte_length = 0usize; // Get the `MessageId` of the milestone we are about to prune from the storage. - let prune_id = *Fetch::::fetch(storage, &prune_index) + let milestone_to_prune = Fetch::::fetch(storage, &prune_index) .map_err(|e| PruningError::Storage(Box::new(e)))? - .ok_or(PruningError::MissingMilestone(prune_index))? - .message_id(); + .ok_or(PruningError::MissingMilestone(prune_index))?; + + byte_length += prune_index.packed_len(); + byte_length += milestone_to_prune.packed_len(); + + let prune_id = *milestone_to_prune.message_id(); // Breadth-first traversal will increase our chances of sorting out redundant messages without querying the storage. let mut to_visit: VecDeque<_> = vec![prune_id].into_iter().collect(); @@ -105,14 +113,16 @@ pub fn batch_prunable_confirmed_data( if let Some(indexation) = unwrap_indexation(payload) { let padded_index = indexation.padded_index(); - prune_indexation_data(storage, batch, &(padded_index, message_id))?; + byte_length += prune_indexation_data::<_, BY_SIZE>(storage, batch, &(padded_index, message_id))?; + metrics.prunable_indexations += 1; } // Delete its edges. let parents = msg.parents(); for parent_id in parents.iter() { - prune_edge(storage, batch, &(*parent_id, message_id))?; + byte_length += prune_edge::<_, BY_SIZE>(storage, batch, &(*parent_id, message_id))?; + metrics.prunable_edges += 1; } @@ -123,7 +133,7 @@ pub fn batch_prunable_confirmed_data( visited.insert(message_id); // Delete its associated data. - prune_message_and_metadata(storage, batch, &message_id)?; + byte_length += prune_message_and_metadata::<_, BY_SIZE>(storage, batch, &message_id)?; // --- // Everything that follows is required to decide whether this message's id should be kept as a solid entry @@ -197,14 +207,15 @@ pub fn batch_prunable_confirmed_data( metrics.prunable_messages = visited.len(); metrics.new_seps = new_seps.len(); - Ok((new_seps, metrics)) + Ok((new_seps, metrics, byte_length)) } -pub fn batch_prunable_unconfirmed_data( +pub(crate) fn batch_prunable_unconfirmed_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, -) -> Result { +) -> Result<(ByteLength, UnconfirmedDataPruningMetrics), PruningError> { + let mut byte_length = 0usize; let mut metrics = UnconfirmedDataPruningMetrics::default(); let unconf_msgs = match Fetch::>::fetch(storage, &prune_index) @@ -252,7 +263,7 @@ pub fn batch_prunable_unconfirmed_data( let parents = msg.parents(); // Add message data to the delete batch. - prune_message_and_metadata(storage, batch, unconf_msg_id)?; + byte_length += prune_message_and_metadata::<_, BY_SIZE>(storage, batch, unconf_msg_id)?; log::trace!("Pruned unconfirmed msg {} at {}.", unconf_msg_id, prune_index); @@ -261,14 +272,14 @@ pub fn batch_prunable_unconfirmed_data( let message_id = *unconf_msg_id; // Add prunable indexations to the delete batch. - prune_indexation_data(storage, batch, &(padded_index, message_id))?; + byte_length += prune_indexation_data::<_, BY_SIZE>(storage, batch, &(padded_index, message_id))?; metrics.prunable_indexations += 1; } // Add prunable edges to the delete batch. for parent in parents.iter() { - prune_edge(storage, batch, &(*parent, *unconf_msg_id))?; + byte_length += prune_edge::<_, BY_SIZE>(storage, batch, &(*parent, *unconf_msg_id))?; metrics.prunable_edges += 1; } @@ -279,96 +290,178 @@ pub fn batch_prunable_unconfirmed_data( } } - Batch::<(MilestoneIndex, UnreferencedMessage), ()>::batch_delete( - storage, - batch, - &(prune_index, (*unconf_msg_id).into()), - ) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + byte_length += prune_unreferenced_message::<_, BY_SIZE>(storage, batch, prune_index, (*unconf_msg_id).into())?; metrics.prunable_messages += 1; } - Ok(metrics) + Ok((byte_length, metrics)) } -pub fn prune_milestone_data( +pub(crate) fn prune_milestone_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, should_prune_receipts: bool, -) -> Result { +) -> Result<(ByteLength, MilestoneDataPruningMetrics), PruningError> { + let mut byte_length = 0usize; let mut metrics = MilestoneDataPruningMetrics::default(); - prune_milestone(storage, batch, prune_index)?; - - prune_output_diff(storage, batch, prune_index)?; + byte_length += prune_milestone::<_, BY_SIZE>(storage, batch, prune_index)?; + byte_length += prune_output_diff::<_, BY_SIZE>(storage, batch, prune_index)?; if should_prune_receipts { - metrics.receipts = prune_receipts(storage, batch, prune_index)?; + let (num_receipts, num_bytes) = prune_receipts::<_, BY_SIZE>(storage, batch, prune_index)?; + + metrics.receipts = num_receipts; + byte_length += num_bytes; } - Ok(metrics) + Ok((byte_length, metrics)) } -fn prune_message_and_metadata( +fn prune_message_and_metadata( storage: &S, batch: &mut S::Batch, message_id: &MessageId, -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; + + if BY_SIZE { + let msg = Fetch::::fetch(storage, message_id) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .unwrap(); + byte_length += msg.packed_len(); + + let md = Fetch::::fetch(storage, &message_id) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .unwrap(); + byte_length += md.packed_len(); + } + Batch::::batch_delete(storage, batch, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Ok(()) + Ok(byte_length) } -fn prune_edge( +fn prune_edge( storage: &S, batch: &mut S::Batch, edge: &(MessageId, MessageId), -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; + + if BY_SIZE { + byte_length += edge.0.packed_len() + edge.1.packed_len(); + } + Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Ok(()) + Ok(byte_length) } -fn prune_indexation_data( +fn prune_indexation_data( storage: &S, batch: &mut S::Batch, index_message_id: &(PaddedIndex, MessageId), -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; + + if BY_SIZE { + byte_length += index_message_id.0.packed_len() + index_message_id.1.packed_len(); + } + Batch::<(PaddedIndex, MessageId), ()>::batch_delete(storage, batch, index_message_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Ok(()) + Ok(byte_length) +} + +fn prune_unreferenced_message( + storage: &S, + batch: &mut S::Batch, + prune_index: MilestoneIndex, + unreferenced_message: UnreferencedMessage, +) -> Result { + let mut byte_length = 0usize; + + if BY_SIZE { + byte_length += prune_index.packed_len() + unreferenced_message.packed_len(); + } + + Batch::<(MilestoneIndex, UnreferencedMessage), ()>::batch_delete( + storage, + batch, + &(prune_index, unreferenced_message), + ) + .map_err(|e| PruningError::Storage(Box::new(e)))?; + + Ok(byte_length) } -fn prune_milestone( +fn prune_milestone( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; + + if BY_SIZE { + let ms = Fetch::::fetch(storage, &index) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .unwrap(); + byte_length += ms.packed_len(); + } + Batch::::batch_delete(storage, batch, &index) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Ok(()) + Ok(byte_length) } -fn prune_output_diff( +fn prune_output_diff( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; + if let Some(output_diff) = Fetch::::fetch(storage, &index).map_err(|e| PruningError::Storage(Box::new(e)))? { - for consumed_output in output_diff.consumed_outputs() { - Batch::::batch_delete(storage, batch, consumed_output) + byte_length += index.packed_len(); + byte_length += output_diff.packed_len(); + + for consumed_output_id in output_diff.consumed_outputs() { + if BY_SIZE { + let consumed_output = Fetch::::fetch(storage, consumed_output_id) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .unwrap(); + + byte_length += consumed_output_id.packed_len(); + byte_length += consumed_output.packed_len(); + } + + Batch::::batch_delete(storage, batch, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Batch::::batch_delete(storage, batch, consumed_output) + } + + for created_output_id in output_diff.created_outputs() { + if BY_SIZE { + let created_output = Fetch::::fetch(storage, created_output_id) + .map_err(|e| PruningError::Storage(Box::new(e)))? + .unwrap(); + + byte_length += created_output_id.packed_len(); + byte_length += created_output.packed_len(); + } + + Batch::::batch_delete(storage, batch, created_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; } @@ -380,14 +473,16 @@ fn prune_output_diff( Batch::::batch_delete(storage, batch, &index) .map_err(|e| PruningError::Storage(Box::new(e)))?; - Ok(()) + Ok(byte_length) } -fn prune_receipts( +fn prune_receipts( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, -) -> Result { +) -> Result<(usize, ByteLength), PruningError> { + let mut byte_length = 0usize; + let receipts = Fetch::>::fetch(storage, &index) .map_err(|e| PruningError::Storage(Box::new(e)))? // Fine since Fetch of a Vec<_> always returns Some(Vec<_>). @@ -395,13 +490,18 @@ fn prune_receipts( let mut num = 0; for receipt in receipts.into_iter() { + if BY_SIZE { + byte_length += index.packed_len(); + byte_length += receipt.packed_len(); + } + Batch::<(MilestoneIndex, Receipt), ()>::batch_delete(storage, batch, &(index, receipt)) .map_err(|e| PruningError::Storage(Box::new(e)))?; num += 1; } - Ok(num) + Ok((num, byte_length)) } fn unwrap_indexation(payload: Option<&Payload>) -> Option<&IndexationPayload> { @@ -426,7 +526,7 @@ fn unwrap_indexation(payload: Option<&Payload>) -> Option<&IndexationPayload> { // TODO: consider using this instead of 'truncate' #[allow(dead_code)] -fn prune_seps( +fn prune_seps( storage: &S, batch: &mut S::Batch, seps: &[SolidEntryPoint], diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 50e9e3ba53..c6a0867a5f 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -30,7 +30,7 @@ pub(crate) enum PruningSkipReason { pub(crate) enum PruningTask { // TODO: consider using named structs: start index, target index ByRange(MilestoneIndex, MilestoneIndex), - // TODO: same: reduced size + // TODO: same: num_bytes_to_prune BySize(usize), } @@ -80,12 +80,12 @@ pub(crate) fn should_prune( if actual_size < target_size { Err(PruningSkipReason::BelowStorageSizeThreshold { target_size }) } else { - let reduced_size = - actual_size - (config.size().threshold_percentage() as f64 / 100.0 * target_size as f64) as usize; + let num_bytes_to_prune = + (config.size().threshold_percentage() as f64 / 100.0 * target_size as f64) as usize; - log::debug!("Reduced size: {reduced_size}"); + log::debug!("Num bytes to prune: {num_bytes_to_prune}"); - Ok(PruningTask::BySize(reduced_size)) + Ok(PruningTask::BySize(num_bytes_to_prune)) } } else { Err(PruningSkipReason::Disabled) diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index dea74cfa5c..751d29f024 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -49,12 +49,13 @@ pub async fn prune_by_range( if start_index != target_index { log::info!( "Pruning from milestone {} to milestone {}...", - start_index, target_index + start_index, + target_index ); } for index in *start_index..=*target_index { - prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + prune_milestone::<_, false>(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; } if start_index == target_index { @@ -72,30 +73,26 @@ pub async fn prune_by_size( tangle: &Tangle, storage: &S, bus: &Bus<'_>, - reduced_size: usize, + num_bytes_to_prune: usize, config: &PruningConfig, ) -> Result<(), PruningError> { let mut timings = Timings::default(); let mut metrics = PruningMetrics::default(); + let mut num_pruned_bytes = 0; - loop { - // Panic: already checked that size() returns a valid result. - let actual_size = storage.size().expect("ok storage size").expect("some storage size"); - log::debug!("Storage size: actual {actual_size} reduced {reduced_size}"); - - if actual_size > reduced_size { - let index = *tangle.get_pruning_index() + 1; - prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; - } else { - break; - } + while num_pruned_bytes < num_bytes_to_prune { + log::debug!("Pruned {num_pruned_bytes}/{num_bytes_to_prune} bytes."); + + let index = *tangle.get_pruning_index() + 1; + num_pruned_bytes += + prune_milestone::<_, true>(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; } Ok(()) } /// Prunes a single milestone. -async fn prune_milestone( +async fn prune_milestone( index: u32, tangle: &Tangle, storage: &S, @@ -103,7 +100,8 @@ async fn prune_milestone( timings: &mut Timings, metrics: &mut PruningMetrics, config: &PruningConfig, -) -> Result<(), PruningError> { +) -> Result { + let mut byte_length = 0usize; let index = MilestoneIndex(index); log::debug!("Pruning milestone {}...", index); @@ -124,10 +122,12 @@ async fn prune_milestone( // Add confirmed data to the delete batch. // NOTE: This is the most costly thing during pruning, because it has to perform a past-cone traversal. let batch_confirmed_data = Instant::now(); - let (mut new_seps, confirmed_data_metrics) = - batch::batch_prunable_confirmed_data(storage, &mut batch, index, &curr_seps)?; + let (mut new_seps, confirmed_data_metrics, num_bytes) = + batch::batch_prunable_confirmed_data::<_, BY_SIZE>(storage, &mut batch, index, &curr_seps)?; timings.batch_confirmed_data = batch_confirmed_data.elapsed(); + byte_length += num_bytes; + metrics.new_seps = new_seps.len(); metrics.messages = confirmed_data_metrics.prunable_messages; metrics.edges = confirmed_data_metrics.prunable_edges; @@ -176,16 +176,20 @@ async fn prune_milestone( tangle.update_entry_point_index(index); let batch_milestones = Instant::now(); - let milestone_data_metrics = batch::prune_milestone_data(storage, &mut batch, index, config.receipts().enabled())?; + let (num_bytes, milestone_data_metrics) = + batch::prune_milestone_data::<_, BY_SIZE>(storage, &mut batch, index, config.receipts().enabled())?; timings.batch_milestone_data = batch_milestones.elapsed(); + byte_length += num_bytes; metrics.receipts = milestone_data_metrics.receipts; // Add unconfirmed data to the delete batch. let batch_unconfirmed_data = Instant::now(); - let unconfirmed_data_metrics = batch::batch_prunable_unconfirmed_data(storage, &mut batch, index)?; + let (num_bytes, unconfirmed_data_metrics) = batch::batch_prunable_unconfirmed_data::<_, BY_SIZE>(storage, &mut batch, index)?; timings.batch_unconfirmed_data = batch_unconfirmed_data.elapsed(); + byte_length += num_bytes; + metrics.messages += unconfirmed_data_metrics.prunable_messages; metrics.edges += unconfirmed_data_metrics.prunable_edges; metrics.indexations += unconfirmed_data_metrics.prunable_indexations; @@ -230,11 +234,16 @@ async fn prune_milestone( log::debug!("{:?}.", timings); log::debug!( "Entry point index now at {} with {} solid entry points..", - index, num_next_seps + index, + num_next_seps ); - log::debug!("Pruned milestone {}.", index); + if BY_SIZE { + log::debug!("Pruned milestone {}: {} bytes", index, byte_length); + } else { + log::debug!("Pruned milestone {}", index); + } bus.dispatch(PrunedIndex { index }); - Ok(()) + Ok(byte_length) } diff --git a/bee-ledger/src/workers/storage.rs b/bee-ledger/src/workers/storage.rs index 32987b1905..fc53c94212 100644 --- a/bee-ledger/src/workers/storage.rs +++ b/bee-ledger/src/workers/storage.rs @@ -51,6 +51,7 @@ pub trait StorageBackend: + Exist + Fetch<(), SnapshotInfo> + Fetch + + Fetch + Fetch<(), LedgerIndex> + Fetch + Fetch> @@ -92,6 +93,7 @@ impl StorageBackend for T where + Exist + Fetch<(), SnapshotInfo> + Fetch + + Fetch + Fetch<(), LedgerIndex> + Fetch + Fetch> From 9d4e8fbe39d8f10011e278c787b10de316ae1fe5 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 20 Apr 2022 15:49:51 +0200 Subject: [PATCH 14/39] Impl cooldown --- bee-ledger/src/workers/pruning/batch.rs | 16 ++++++---------- bee-ledger/src/workers/pruning/condition.rs | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index cc8e0049c4..e44928081e 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -445,23 +445,19 @@ fn prune_output_diff( byte_length += consumed_output_id.packed_len(); byte_length += consumed_output.packed_len(); - } - - Batch::::batch_delete(storage, batch, consumed_output_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; - } - for created_output_id in output_diff.created_outputs() { - if BY_SIZE { - let created_output = Fetch::::fetch(storage, created_output_id) + let created_output = Fetch::::fetch(storage, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); - byte_length += created_output_id.packed_len(); + byte_length += consumed_output_id.packed_len(); byte_length += created_output.packed_len(); } - Batch::::batch_delete(storage, batch, created_output_id) + Batch::::batch_delete(storage, batch, consumed_output_id) + .map_err(|e| PruningError::Storage(Box::new(e)))?; + + Batch::::batch_delete(storage, batch, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; } diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index c6a0867a5f..caec58d380 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -3,6 +3,8 @@ // use std::time::Duration; +use std::{time::{Duration, SystemTime, UNIX_EPOCH}, sync::atomic::{AtomicU64, Ordering}}; + use bee_message::milestone::MilestoneIndex; use bee_tangle::{storage::StorageBackend, Tangle}; @@ -10,6 +12,8 @@ use crate::{types::LedgerIndex, workers::pruning::config::PruningConfig}; const PRUNING_BATCH_SIZE_MAX: u32 = 200; +static LAST_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); + /// Reasons for skipping pruning. #[derive(Debug, thiserror::Error)] pub(crate) enum PruningSkipReason { @@ -23,8 +27,8 @@ pub(crate) enum PruningSkipReason { BelowMilestoneIndexThreshold { reached_in: u32 }, #[error("Pruning by storage size skipped because current size < {target_size}.")] BelowStorageSizeThreshold { target_size: usize }, - // #[error("Pruning by storage size is skipped because of cooldown.")] - // BelowCooldownTimeThreshold { reached_in: Duration }, + #[error("Pruning by storage size is skipped because of cooldown.")] + BelowCooldownTimeThreshold, } pub(crate) enum PruningTask { @@ -62,6 +66,14 @@ pub(crate) fn should_prune( )) } } else if config.size().enabled() { + + let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + + if now < last + config.size().cooldown_time() { + return Err(PruningSkipReason::BelowCooldownTimeThreshold); + } + let actual_size = { if let Ok(size) = storage.size() { if let Some(size) = size { @@ -85,6 +97,9 @@ pub(crate) fn should_prune( log::debug!("Num bytes to prune: {num_bytes_to_prune}"); + // Store the time we issued a pruning-by-size. + LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); + Ok(PruningTask::BySize(num_bytes_to_prune)) } } else { From b2f5f7455a2ff6656d99faeb5e962775f0c41a62 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 20 Apr 2022 17:02:14 +0200 Subject: [PATCH 15/39] Format --- bee-ledger/src/workers/pruning/condition.rs | 8 +++++--- bee-ledger/src/workers/pruning/prune.rs | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index caec58d380..1da269822b 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -3,7 +3,10 @@ // use std::time::Duration; -use std::{time::{Duration, SystemTime, UNIX_EPOCH}, sync::atomic::{AtomicU64, Ordering}}; +use std::{ + sync::atomic::{AtomicU64, Ordering}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; use bee_message::milestone::MilestoneIndex; use bee_tangle::{storage::StorageBackend, Tangle}; @@ -66,9 +69,8 @@ pub(crate) fn should_prune( )) } } else if config.size().enabled() { - let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); - let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); if now < last + config.size().cooldown_time() { return Err(PruningSkipReason::BelowCooldownTimeThreshold); diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 751d29f024..05c2a2056d 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -185,7 +185,8 @@ async fn prune_milestone( // Add unconfirmed data to the delete batch. let batch_unconfirmed_data = Instant::now(); - let (num_bytes, unconfirmed_data_metrics) = batch::batch_prunable_unconfirmed_data::<_, BY_SIZE>(storage, &mut batch, index)?; + let (num_bytes, unconfirmed_data_metrics) = + batch::batch_prunable_unconfirmed_data::<_, BY_SIZE>(storage, &mut batch, index)?; timings.batch_unconfirmed_data = batch_unconfirmed_data.elapsed(); byte_length += num_bytes; From f42b599a6c28e69d25e3728e98aa6a01b7d05619 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 20 Apr 2022 17:02:51 +0200 Subject: [PATCH 16/39] Clippy --- bee-ledger/src/workers/pruning/batch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index e44928081e..8e1923211e 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -333,7 +333,7 @@ fn prune_message_and_metadata( .unwrap(); byte_length += msg.packed_len(); - let md = Fetch::::fetch(storage, &message_id) + let md = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); byte_length += md.packed_len(); From dea68214189ee596154b3adb2fc7b7207b490945 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 21 Apr 2022 11:23:09 +0200 Subject: [PATCH 17/39] Prevent pruning-by-size to prune too close to ledger index --- bee-ledger/src/workers/consensus/worker.rs | 1 + bee-ledger/src/workers/pruning/condition.rs | 28 +++++++++---------- bee-ledger/src/workers/pruning/config.rs | 6 ++++- bee-ledger/src/workers/pruning/prune.rs | 30 ++++++++++++++------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 6e47433967..480d828594 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -370,6 +370,7 @@ where &tangle, &storage, &bus, + ledger_index, num_bytes_to_prune, &pruning_config, ) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 1da269822b..cc8589a8a7 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -22,15 +22,15 @@ static LAST_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); pub(crate) enum PruningSkipReason { #[error("Pruning disabled.")] Disabled, - #[error("Pruning by size is not supported by the storage layer.")] + #[error("Pruning by milestone below index threshold.")] + BelowMilestoneIndexThreshold, + #[error("Pruning by size not supported by the storage layer.")] PruningBySizeUnsupported, - #[error("Pruning by size is currently unavailable.")] + #[error("Pruning by size currently unavailable.")] PruningBySizeUnavailable, - #[error("Pruning milestones skipped for the next {reached_in} indexes.")] - BelowMilestoneIndexThreshold { reached_in: u32 }, - #[error("Pruning by storage size skipped because current size < {target_size}.")] - BelowStorageSizeThreshold { target_size: usize }, - #[error("Pruning by storage size is skipped because of cooldown.")] + #[error("Pruning by size below size threshold.")] + BelowStorageSizeThreshold, + #[error("Pruning by size below cooldown threshold.")] BelowCooldownTimeThreshold, } @@ -53,9 +53,7 @@ pub(crate) fn should_prune( let pruning_threshold = pruning_index + milestones_to_keep; if *ledger_index < pruning_threshold { - Err(PruningSkipReason::BelowMilestoneIndexThreshold { - reached_in: pruning_threshold - *ledger_index, - }) + Err(PruningSkipReason::BelowMilestoneIndexThreshold) } else { let target_pruning_index = *ledger_index - milestones_to_keep; @@ -87,15 +85,15 @@ pub(crate) fn should_prune( return Err(PruningSkipReason::PruningBySizeUnsupported); } }; - let target_size = config.size().target_size(); + let threshold_size = config.size().target_size(); - log::debug!("Storage size: actual {actual_size} target {target_size}"); + log::debug!("Storage size: actual {actual_size} limit {threshold_size}"); - if actual_size < target_size { - Err(PruningSkipReason::BelowStorageSizeThreshold { target_size }) + if actual_size < threshold_size { + Err(PruningSkipReason::BelowStorageSizeThreshold) } else { let num_bytes_to_prune = - (config.size().threshold_percentage() as f64 / 100.0 * target_size as f64) as usize; + (config.size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; log::debug!("Num bytes to prune: {num_bytes_to_prune}"); diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 7a8834185f..ff958c4154 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -11,7 +11,8 @@ use serde::Deserialize; const PRUNING_MILESTONES_ENABLED_DEFAULT: bool = true; const PRUNING_SIZE_ENABLED_DEFAULT: bool = true; const PRUNING_RECEIPTS_ENABLED_DEFAULT: bool = false; -const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 99999; //60480; +const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 60480; +pub(crate) const MAX_MILESTONES_TO_KEEP_MINIMUM: u32 = 50; const THRESHOLD_PERCENTAGE_DEFAULT: f32 = 10.0; const COOLDOWN_TIME_DEFAULT: &str = "5m"; const TARGET_SIZE_DEFAULT: &str = "30Gb"; @@ -77,7 +78,10 @@ impl PruningMilestonesConfigBuilder { } /// Sets how many milestones to hold available in the storage. + /// + /// Note: You cannot set a value below [`MAX_MILESTONES_TO_KEEP_MINIMUM`]. pub fn max_milestones_to_keep(mut self, max_milestones_to_keep: u32) -> Self { + let max_milestones_to_keep = max_milestones_to_keep.max(MAX_MILESTONES_TO_KEEP_MINIMUM); self.max_milestones_to_keep.replace(max_milestones_to_keep); self } diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 05c2a2056d..febff901c1 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -11,15 +11,18 @@ use bee_runtime::event::Bus; use bee_storage::access::{Batch, Truncate}; use bee_tangle::{solid_entry_point::SolidEntryPoint, Tangle}; -use crate::workers::{ - event::PrunedIndex, - pruning::{ - batch, - config::PruningConfig, - error::PruningError, - metrics::{PruningMetrics, Timings}, +use crate::{ + types::LedgerIndex, + workers::{ + event::PrunedIndex, + pruning::{ + batch, + config::{PruningConfig, MAX_MILESTONES_TO_KEEP_MINIMUM}, + error::PruningError, + metrics::{PruningMetrics, Timings}, + }, + storage::{self, StorageBackend}, }, - storage::{self, StorageBackend}, }; const KEEP_INITIAL_SNAPSHOT_SEPS: usize = 50; @@ -73,6 +76,7 @@ pub async fn prune_by_size( tangle: &Tangle, storage: &S, bus: &Bus<'_>, + ledger_index: LedgerIndex, num_bytes_to_prune: usize, config: &PruningConfig, ) -> Result<(), PruningError> { @@ -81,11 +85,17 @@ pub async fn prune_by_size( let mut num_pruned_bytes = 0; while num_pruned_bytes < num_bytes_to_prune { - log::debug!("Pruned {num_pruned_bytes}/{num_bytes_to_prune} bytes."); - let index = *tangle.get_pruning_index() + 1; + + if *ledger_index < index + MAX_MILESTONES_TO_KEEP_MINIMUM { + log::debug!("Minimum pruning index reached."); + break; + } + num_pruned_bytes += prune_milestone::<_, true>(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + + log::debug!("Pruned {num_pruned_bytes}/{num_bytes_to_prune} bytes."); } Ok(()) From 56293b16849b0ffaa283492f83889600ff3b7f32 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 21 Apr 2022 18:40:27 +0200 Subject: [PATCH 18/39] Add excess to num_bytes_to_prune --- bee-ledger/src/workers/pruning/condition.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index cc8589a8a7..1ed820219c 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -92,7 +92,9 @@ pub(crate) fn should_prune( if actual_size < threshold_size { Err(PruningSkipReason::BelowStorageSizeThreshold) } else { - let num_bytes_to_prune = + // Panic: cannot underflow due to actual_size >= threshold_size. + let excess = actual_size - threshold_size; + let num_bytes_to_prune = excess + (config.size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; log::debug!("Num bytes to prune: {num_bytes_to_prune}"); From dacd509f9ecb7127ea95de22b07e490f8e1cde56 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 21 Apr 2022 19:13:10 +0200 Subject: [PATCH 19/39] Address some todos --- bee-ledger/src/workers/consensus/worker.rs | 4 ++-- bee-ledger/src/workers/pruning/batch.rs | 6 +---- bee-ledger/src/workers/pruning/condition.rs | 25 ++++++++++++--------- bee-ledger/src/workers/pruning/config.rs | 6 ++--- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 480d828594..b793d625e1 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -351,7 +351,7 @@ where match should_prune(&tangle, &storage, ledger_index, milestones_to_keep, &pruning_config) { Ok(pruning_task) => match pruning_task { - PruningTask::ByRange(start_index, target_index) => { + PruningTask::ByRange { start_index, target_index } => { if let Err(e) = prune::prune_by_range( &tangle, &storage, @@ -365,7 +365,7 @@ where error!("Pruning milestone range failed: {}.", e); } } - PruningTask::BySize(num_bytes_to_prune) => { + PruningTask::BySize { num_bytes_to_prune } => { if let Err(e) = prune::prune_by_size( &tangle, &storage, diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 8e1923211e..af7320b2f0 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -460,10 +460,6 @@ fn prune_output_diff( Batch::::batch_delete(storage, batch, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))?; } - - if let Some(_treasury_diff) = output_diff.treasury_diff() { - // TODO - } } Batch::::batch_delete(storage, batch, &index) @@ -520,7 +516,7 @@ fn unwrap_indexation(payload: Option<&Payload>) -> Option<&IndexationPayload> { } } -// TODO: consider using this instead of 'truncate' +// TODO: consider using this instead of 'truncate'. #[allow(dead_code)] fn prune_seps( storage: &S, diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 1ed820219c..a40117cd4a 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -35,10 +35,13 @@ pub(crate) enum PruningSkipReason { } pub(crate) enum PruningTask { - // TODO: consider using named structs: start index, target index - ByRange(MilestoneIndex, MilestoneIndex), - // TODO: same: num_bytes_to_prune - BySize(usize), + ByRange { + start_index: MilestoneIndex, + target_index: MilestoneIndex, + }, + BySize { + num_bytes_to_prune: usize, + }, } pub(crate) fn should_prune( @@ -57,14 +60,14 @@ pub(crate) fn should_prune( } else { let target_pruning_index = *ledger_index - milestones_to_keep; - Ok(PruningTask::ByRange( - pruning_index.into(), - if target_pruning_index > pruning_index + PRUNING_BATCH_SIZE_MAX { + Ok(PruningTask::ByRange { + start_index: pruning_index.into(), + target_index: if target_pruning_index > pruning_index + PRUNING_BATCH_SIZE_MAX { (pruning_index + PRUNING_BATCH_SIZE_MAX).into() } else { target_pruning_index.into() }, - )) + }) } } else if config.size().enabled() { let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); @@ -94,15 +97,15 @@ pub(crate) fn should_prune( } else { // Panic: cannot underflow due to actual_size >= threshold_size. let excess = actual_size - threshold_size; - let num_bytes_to_prune = excess + - (config.size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; + let num_bytes_to_prune = + excess + (config.size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; log::debug!("Num bytes to prune: {num_bytes_to_prune}"); // Store the time we issued a pruning-by-size. LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); - Ok(PruningTask::BySize(num_bytes_to_prune)) + Ok(PruningTask::BySize { num_bytes_to_prune }) } } else { Err(PruningSkipReason::Disabled) diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index ff958c4154..3ccc73a8b7 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -116,19 +116,19 @@ impl PruningSizeConfigBuilder { self } - /// TODO + /// Sets the target size (i.e. the maximum size) of the database. pub fn target_size(mut self, target_size: String) -> Self { self.target_size.replace(target_size); self } - /// TODO + /// Sets the percentage of the target size that is pruned from the database. pub fn threshold_percentage(mut self, threshold_percentage: f32) -> Self { self.threshold_percentage.replace(threshold_percentage); self } - /// TODO + /// Sets the cooldown time (i.e. the sleep interval) between two subsequent pruning-by-size events. pub fn cooldown_time(mut self, cooldown_time: String) -> Self { self.cooldown_time.replace(cooldown_time); self From abc1ef9100468143000fff707ad4c9f18f7dc5ab Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 22 Apr 2022 12:46:43 +0200 Subject: [PATCH 20/39] Format --- bee-ledger/src/workers/consensus/worker.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index b793d625e1..abbdfa318c 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -351,7 +351,10 @@ where match should_prune(&tangle, &storage, ledger_index, milestones_to_keep, &pruning_config) { Ok(pruning_task) => match pruning_task { - PruningTask::ByRange { start_index, target_index } => { + PruningTask::ByRange { + start_index, + target_index, + } => { if let Err(e) = prune::prune_by_range( &tangle, &storage, From 172493113005c3a6bc67f59c23e029e6b89097c7 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 22 Apr 2022 13:22:01 +0200 Subject: [PATCH 21/39] Cleanup --- bee-ledger/src/workers/pruning/condition.rs | 12 ++--- bee-ledger/src/workers/pruning/config.rs | 51 +++++++++++---------- bee-ledger/src/workers/pruning/prune.rs | 4 +- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index a40117cd4a..b16186a66d 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -69,11 +69,11 @@ pub(crate) fn should_prune( }, }) } - } else if config.size().enabled() { + } else if config.db_size().enabled() { let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - if now < last + config.size().cooldown_time() { + if now < last + config.db_size().cooldown_time() { return Err(PruningSkipReason::BelowCooldownTimeThreshold); } @@ -88,17 +88,17 @@ pub(crate) fn should_prune( return Err(PruningSkipReason::PruningBySizeUnsupported); } }; - let threshold_size = config.size().target_size(); + let threshold_size = config.db_size().target_size(); - log::debug!("Storage size: actual {actual_size} limit {threshold_size}"); + log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); if actual_size < threshold_size { Err(PruningSkipReason::BelowStorageSizeThreshold) } else { // Panic: cannot underflow due to actual_size >= threshold_size. - let excess = actual_size - threshold_size; + let excess_size = actual_size - threshold_size; let num_bytes_to_prune = - excess + (config.size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; + excess_size + (config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; log::debug!("Num bytes to prune: {num_bytes_to_prune}"); diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 3ccc73a8b7..9fcdb8f6b8 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -12,7 +12,7 @@ const PRUNING_MILESTONES_ENABLED_DEFAULT: bool = true; const PRUNING_SIZE_ENABLED_DEFAULT: bool = true; const PRUNING_RECEIPTS_ENABLED_DEFAULT: bool = false; const MAX_MILESTONES_TO_KEEP_DEFAULT: u32 = 60480; -pub(crate) const MAX_MILESTONES_TO_KEEP_MINIMUM: u32 = 50; +pub(crate) const MILESTONES_TO_KEEP_MIN: u32 = 50; const THRESHOLD_PERCENTAGE_DEFAULT: f32 = 10.0; const COOLDOWN_TIME_DEFAULT: &str = "5m"; const TARGET_SIZE_DEFAULT: &str = "30Gb"; @@ -22,7 +22,8 @@ const TARGET_SIZE_DEFAULT: &str = "30Gb"; #[must_use] pub struct PruningConfigBuilder { milestones: Option, - size: Option, + #[serde(rename = "size")] + db_size: Option, receipts: Option, } @@ -38,9 +39,9 @@ impl PruningConfigBuilder { self } - /// Sets the [`PruningSizeConfigBuilder`]. - pub fn size(mut self, builder: PruningSizeConfigBuilder) -> Self { - self.size.replace(builder); + /// Sets the [`PruningDbSizeConfigBuilder`]. + pub fn db_size(mut self, builder: PruningDbSizeConfigBuilder) -> Self { + self.db_size.replace(builder); self } @@ -55,8 +56,8 @@ impl PruningConfigBuilder { pub fn finish(self) -> PruningConfig { PruningConfig { milestones: self.milestones.unwrap_or_default().finish(), + db_size: self.db_size.unwrap_or_default().finish(), receipts: self.receipts.unwrap_or_default().finish(), - size: self.size.unwrap_or_default().finish(), } } } @@ -79,9 +80,9 @@ impl PruningMilestonesConfigBuilder { /// Sets how many milestones to hold available in the storage. /// - /// Note: You cannot set a value below [`MAX_MILESTONES_TO_KEEP_MINIMUM`]. + /// Note: values below [`MILESTONES_TO_KEEP_MIN`] are not accepted. pub fn max_milestones_to_keep(mut self, max_milestones_to_keep: u32) -> Self { - let max_milestones_to_keep = max_milestones_to_keep.max(MAX_MILESTONES_TO_KEEP_MINIMUM); + let max_milestones_to_keep = max_milestones_to_keep.max(MILESTONES_TO_KEEP_MIN); self.max_milestones_to_keep.replace(max_milestones_to_keep); self } @@ -99,7 +100,7 @@ impl PruningMilestonesConfigBuilder { /// Builder for a [`PruningSizeConfig`]. #[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] -pub struct PruningSizeConfigBuilder { +pub struct PruningDbSizeConfigBuilder { enabled: Option, #[serde(alias = "targetSize")] target_size: Option, @@ -109,7 +110,7 @@ pub struct PruningSizeConfigBuilder { cooldown_time: Option, } -impl PruningSizeConfigBuilder { +impl PruningDbSizeConfigBuilder { /// Sets whether pruning based on storage size is enabled. pub fn enabled(mut self, enabled: bool) -> Self { self.enabled.replace(enabled); @@ -136,7 +137,7 @@ impl PruningSizeConfigBuilder { /// Finishes this builder into a [`PruningSizeConfig`]. #[must_use] - pub fn finish(self) -> PruningSizeConfig { + pub fn finish(self) -> PruningDbSizeConfig { let target_size = self.target_size.unwrap_or_else(|| TARGET_SIZE_DEFAULT.to_string()); let target_size = target_size .parse::() @@ -147,7 +148,7 @@ impl PruningSizeConfigBuilder { let cooldown_time = duration::parse(cooldown_time.as_ref()).expect("parse human-readable pruning cooldown time"); - PruningSizeConfig { + PruningDbSizeConfig { enabled: self.enabled.unwrap_or(PRUNING_SIZE_ENABLED_DEFAULT), target_size, threshold_percentage: self.threshold_percentage.unwrap_or(THRESHOLD_PERCENTAGE_DEFAULT), @@ -183,7 +184,7 @@ impl PruningReceiptsConfigBuilder { #[derive(Clone, Debug)] pub struct PruningConfig { milestones: PruningMilestonesConfig, - size: PruningSizeConfig, + db_size: PruningDbSizeConfig, receipts: PruningReceiptsConfig, } @@ -201,8 +202,8 @@ impl PruningConfig { /// Returns the `[PruningSizeConfig`]. #[inline(always)] - pub fn size(&self) -> &PruningSizeConfig { - &self.size + pub fn db_size(&self) -> &PruningDbSizeConfig { + &self.db_size } /// Returns the `[PruningReceiptsConfig`]. @@ -233,14 +234,14 @@ impl PruningMilestonesConfig { /// The config associated with storage size based pruning. #[derive(Clone, Debug)] -pub struct PruningSizeConfig { +pub struct PruningDbSizeConfig { enabled: bool, target_size: usize, threshold_percentage: f32, cooldown_time: Duration, } -impl PruningSizeConfig { +impl PruningDbSizeConfig { /// Returns whether pruning based on a target storage size is enabled. pub fn enabled(&self) -> bool { self.enabled @@ -357,18 +358,18 @@ mod tests { assert!(!json_config.milestones().enabled()); assert_eq!(json_config.milestones().max_milestones_to_keep(), 200); - assert_eq!(json_config.size().target_size(), 500000000); - assert_eq!(json_config.size().threshold_percentage(), 20.0); - assert_eq!(json_config.size().cooldown_time(), Duration::from_secs(60)); - assert!(!json_config.size().enabled()); + assert_eq!(json_config.db_size().target_size(), 500000000); + assert_eq!(json_config.db_size().threshold_percentage(), 20.0); + assert_eq!(json_config.db_size().cooldown_time(), Duration::from_secs(60)); + assert!(!json_config.db_size().enabled()); assert!(json_config.receipts().enabled()); assert!(!toml_config.milestones().enabled()); assert_eq!(toml_config.milestones().max_milestones_to_keep(), 200); - assert_eq!(toml_config.size().target_size(), 500000000); - assert_eq!(toml_config.size().threshold_percentage(), 20.0); - assert_eq!(toml_config.size().cooldown_time(), Duration::from_secs(60)); - assert!(!toml_config.size().enabled()); + assert_eq!(toml_config.db_size().target_size(), 500000000); + assert_eq!(toml_config.db_size().threshold_percentage(), 20.0); + assert_eq!(toml_config.db_size().cooldown_time(), Duration::from_secs(60)); + assert!(!toml_config.db_size().enabled()); assert!(toml_config.receipts().enabled()); } } diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index febff901c1..f40c62aef9 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -17,7 +17,7 @@ use crate::{ event::PrunedIndex, pruning::{ batch, - config::{PruningConfig, MAX_MILESTONES_TO_KEEP_MINIMUM}, + config::{PruningConfig, MILESTONES_TO_KEEP_MIN}, error::PruningError, metrics::{PruningMetrics, Timings}, }, @@ -87,7 +87,7 @@ pub async fn prune_by_size( while num_pruned_bytes < num_bytes_to_prune { let index = *tangle.get_pruning_index() + 1; - if *ledger_index < index + MAX_MILESTONES_TO_KEEP_MINIMUM { + if *ledger_index < index + MILESTONES_TO_KEEP_MIN { log::debug!("Minimum pruning index reached."); break; } From 1ade79a433f4928dab6989fda7d9d4c13b0267ad Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 25 Apr 2022 15:53:15 +0200 Subject: [PATCH 22/39] Const gen param to fn param --- bee-ledger/src/workers/pruning/batch.rs | 66 ++++++++++++++----------- bee-ledger/src/workers/pruning/prune.rs | 16 +++--- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index af7320b2f0..885588736f 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -43,11 +43,12 @@ pub(crate) struct Edge { pub(crate) to_child: MessageId, } -pub(crate) fn batch_prunable_confirmed_data( +pub(crate) fn batch_prunable_confirmed_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, current_seps: &Seps, + by_size: bool, ) -> Result<(Seps, ConfirmedDataPruningMetrics, ByteLength), PruningError> { // We keep a list of already visited messages. let mut visited = Messages::with_capacity(512); @@ -113,7 +114,7 @@ pub(crate) fn batch_prunable_confirmed_data(storage, batch, &(padded_index, message_id))?; + byte_length += prune_indexation_data(storage, batch, &(padded_index, message_id), by_size)?; metrics.prunable_indexations += 1; } @@ -121,7 +122,7 @@ pub(crate) fn batch_prunable_confirmed_data(storage, batch, &(*parent_id, message_id))?; + byte_length += prune_edge(storage, batch, &(*parent_id, message_id), by_size)?; metrics.prunable_edges += 1; } @@ -133,7 +134,7 @@ pub(crate) fn batch_prunable_confirmed_data(storage, batch, &message_id)?; + byte_length += prune_message_and_metadata(storage, batch, &message_id, by_size)?; // --- // Everything that follows is required to decide whether this message's id should be kept as a solid entry @@ -210,10 +211,11 @@ pub(crate) fn batch_prunable_confirmed_data( +pub(crate) fn batch_prunable_unconfirmed_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, + by_size: bool, ) -> Result<(ByteLength, UnconfirmedDataPruningMetrics), PruningError> { let mut byte_length = 0usize; let mut metrics = UnconfirmedDataPruningMetrics::default(); @@ -263,7 +265,7 @@ pub(crate) fn batch_prunable_unconfirmed_data(storage, batch, unconf_msg_id)?; + byte_length += prune_message_and_metadata(storage, batch, unconf_msg_id, by_size)?; log::trace!("Pruned unconfirmed msg {} at {}.", unconf_msg_id, prune_index); @@ -272,14 +274,14 @@ pub(crate) fn batch_prunable_unconfirmed_data(storage, batch, &(padded_index, message_id))?; + byte_length += prune_indexation_data(storage, batch, &(padded_index, message_id), by_size)?; metrics.prunable_indexations += 1; } // Add prunable edges to the delete batch. for parent in parents.iter() { - byte_length += prune_edge::<_, BY_SIZE>(storage, batch, &(*parent, *unconf_msg_id))?; + byte_length += prune_edge(storage, batch, &(*parent, *unconf_msg_id), by_size)?; metrics.prunable_edges += 1; } @@ -290,7 +292,7 @@ pub(crate) fn batch_prunable_unconfirmed_data(storage, batch, prune_index, (*unconf_msg_id).into())?; + byte_length += prune_unreferenced_message(storage, batch, prune_index, (*unconf_msg_id).into(), by_size)?; metrics.prunable_messages += 1; } @@ -298,20 +300,21 @@ pub(crate) fn batch_prunable_unconfirmed_data( +pub(crate) fn prune_milestone_data( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, should_prune_receipts: bool, + by_size: bool, ) -> Result<(ByteLength, MilestoneDataPruningMetrics), PruningError> { let mut byte_length = 0usize; let mut metrics = MilestoneDataPruningMetrics::default(); - byte_length += prune_milestone::<_, BY_SIZE>(storage, batch, prune_index)?; - byte_length += prune_output_diff::<_, BY_SIZE>(storage, batch, prune_index)?; + byte_length += prune_milestone(storage, batch, prune_index, by_size)?; + byte_length += prune_output_diff(storage, batch, prune_index, by_size)?; if should_prune_receipts { - let (num_receipts, num_bytes) = prune_receipts::<_, BY_SIZE>(storage, batch, prune_index)?; + let (num_receipts, num_bytes) = prune_receipts(storage, batch, prune_index, by_size)?; metrics.receipts = num_receipts; byte_length += num_bytes; @@ -320,14 +323,15 @@ pub(crate) fn prune_milestone_data( Ok((byte_length, metrics)) } -fn prune_message_and_metadata( +fn prune_message_and_metadata( storage: &S, batch: &mut S::Batch, message_id: &MessageId, + by_size: bool, ) -> Result { let mut byte_length = 0usize; - if BY_SIZE { + if by_size { let msg = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -348,14 +352,15 @@ fn prune_message_and_metadata( Ok(byte_length) } -fn prune_edge( +fn prune_edge( storage: &S, batch: &mut S::Batch, edge: &(MessageId, MessageId), + by_size: bool, ) -> Result { let mut byte_length = 0usize; - if BY_SIZE { + if by_size { byte_length += edge.0.packed_len() + edge.1.packed_len(); } @@ -365,14 +370,15 @@ fn prune_edge( Ok(byte_length) } -fn prune_indexation_data( +fn prune_indexation_data( storage: &S, batch: &mut S::Batch, index_message_id: &(PaddedIndex, MessageId), + by_size: bool, ) -> Result { let mut byte_length = 0usize; - if BY_SIZE { + if by_size { byte_length += index_message_id.0.packed_len() + index_message_id.1.packed_len(); } @@ -382,15 +388,16 @@ fn prune_indexation_data( Ok(byte_length) } -fn prune_unreferenced_message( +fn prune_unreferenced_message( storage: &S, batch: &mut S::Batch, prune_index: MilestoneIndex, unreferenced_message: UnreferencedMessage, + by_size: bool, ) -> Result { let mut byte_length = 0usize; - if BY_SIZE { + if by_size { byte_length += prune_index.packed_len() + unreferenced_message.packed_len(); } @@ -404,14 +411,15 @@ fn prune_unreferenced_message( Ok(byte_length) } -fn prune_milestone( +fn prune_milestone( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, + by_size: bool, ) -> Result { let mut byte_length = 0usize; - if BY_SIZE { + if by_size { let ms = Fetch::::fetch(storage, &index) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -424,10 +432,11 @@ fn prune_milestone( Ok(byte_length) } -fn prune_output_diff( +fn prune_output_diff( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, + by_size: bool, ) -> Result { let mut byte_length = 0usize; @@ -438,7 +447,7 @@ fn prune_output_diff( byte_length += output_diff.packed_len(); for consumed_output_id in output_diff.consumed_outputs() { - if BY_SIZE { + if by_size { let consumed_output = Fetch::::fetch(storage, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -468,10 +477,11 @@ fn prune_output_diff( Ok(byte_length) } -fn prune_receipts( +fn prune_receipts( storage: &S, batch: &mut S::Batch, index: MilestoneIndex, + by_size: bool, ) -> Result<(usize, ByteLength), PruningError> { let mut byte_length = 0usize; @@ -482,7 +492,7 @@ fn prune_receipts( let mut num = 0; for receipt in receipts.into_iter() { - if BY_SIZE { + if by_size { byte_length += index.packed_len(); byte_length += receipt.packed_len(); } @@ -518,7 +528,7 @@ fn unwrap_indexation(payload: Option<&Payload>) -> Option<&IndexationPayload> { // TODO: consider using this instead of 'truncate'. #[allow(dead_code)] -fn prune_seps( +fn prune_seps( storage: &S, batch: &mut S::Batch, seps: &[SolidEntryPoint], diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index f40c62aef9..1f48a3590d 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -58,7 +58,7 @@ pub async fn prune_by_range( } for index in *start_index..=*target_index { - prune_milestone::<_, false>(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config, false).await?; } if start_index == target_index { @@ -93,7 +93,7 @@ pub async fn prune_by_size( } num_pruned_bytes += - prune_milestone::<_, true>(index, tangle, storage, bus, &mut timings, &mut metrics, config).await?; + prune_milestone(index, tangle, storage, bus, &mut timings, &mut metrics, config, true).await?; log::debug!("Pruned {num_pruned_bytes}/{num_bytes_to_prune} bytes."); } @@ -102,7 +102,8 @@ pub async fn prune_by_size( } /// Prunes a single milestone. -async fn prune_milestone( +#[allow(clippy::too_many_arguments)] +async fn prune_milestone( index: u32, tangle: &Tangle, storage: &S, @@ -110,6 +111,7 @@ async fn prune_milestone( timings: &mut Timings, metrics: &mut PruningMetrics, config: &PruningConfig, + by_size: bool, ) -> Result { let mut byte_length = 0usize; let index = MilestoneIndex(index); @@ -133,7 +135,7 @@ async fn prune_milestone( // NOTE: This is the most costly thing during pruning, because it has to perform a past-cone traversal. let batch_confirmed_data = Instant::now(); let (mut new_seps, confirmed_data_metrics, num_bytes) = - batch::batch_prunable_confirmed_data::<_, BY_SIZE>(storage, &mut batch, index, &curr_seps)?; + batch::batch_prunable_confirmed_data(storage, &mut batch, index, &curr_seps, by_size)?; timings.batch_confirmed_data = batch_confirmed_data.elapsed(); byte_length += num_bytes; @@ -187,7 +189,7 @@ async fn prune_milestone( let batch_milestones = Instant::now(); let (num_bytes, milestone_data_metrics) = - batch::prune_milestone_data::<_, BY_SIZE>(storage, &mut batch, index, config.receipts().enabled())?; + batch::prune_milestone_data(storage, &mut batch, index, config.receipts().enabled(), by_size)?; timings.batch_milestone_data = batch_milestones.elapsed(); byte_length += num_bytes; @@ -196,7 +198,7 @@ async fn prune_milestone( // Add unconfirmed data to the delete batch. let batch_unconfirmed_data = Instant::now(); let (num_bytes, unconfirmed_data_metrics) = - batch::batch_prunable_unconfirmed_data::<_, BY_SIZE>(storage, &mut batch, index)?; + batch::batch_prunable_unconfirmed_data(storage, &mut batch, index, by_size)?; timings.batch_unconfirmed_data = batch_unconfirmed_data.elapsed(); byte_length += num_bytes; @@ -248,7 +250,7 @@ async fn prune_milestone( index, num_next_seps ); - if BY_SIZE { + if by_size { log::debug!("Pruned milestone {}: {} bytes", index, byte_length); } else { log::debug!("Pruned milestone {}", index); From 4cf4f7313b598ac7c4e24eb3d4b6ecbbc9bd3b07 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 25 Apr 2022 18:20:16 +0200 Subject: [PATCH 23/39] Fix intra-doc links --- bee-ledger/src/workers/pruning/config.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bee-ledger/src/workers/pruning/config.rs b/bee-ledger/src/workers/pruning/config.rs index 9fcdb8f6b8..1e184ad2cc 100644 --- a/bee-ledger/src/workers/pruning/config.rs +++ b/bee-ledger/src/workers/pruning/config.rs @@ -79,8 +79,6 @@ impl PruningMilestonesConfigBuilder { } /// Sets how many milestones to hold available in the storage. - /// - /// Note: values below [`MILESTONES_TO_KEEP_MIN`] are not accepted. pub fn max_milestones_to_keep(mut self, max_milestones_to_keep: u32) -> Self { let max_milestones_to_keep = max_milestones_to_keep.max(MILESTONES_TO_KEEP_MIN); self.max_milestones_to_keep.replace(max_milestones_to_keep); @@ -97,7 +95,7 @@ impl PruningMilestonesConfigBuilder { } } -/// Builder for a [`PruningSizeConfig`]. +/// Builder for a [`PruningDbSizeConfig`]. #[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] pub struct PruningDbSizeConfigBuilder { @@ -135,7 +133,7 @@ impl PruningDbSizeConfigBuilder { self } - /// Finishes this builder into a [`PruningSizeConfig`]. + /// Finishes this builder into a [`PruningDbSizeConfig`]. #[must_use] pub fn finish(self) -> PruningDbSizeConfig { let target_size = self.target_size.unwrap_or_else(|| TARGET_SIZE_DEFAULT.to_string()); From 87b6d554f25a2b221418a301d195765cae325243 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 25 Apr 2022 18:31:05 +0200 Subject: [PATCH 24/39] Document unwraps --- bee-ledger/src/workers/pruning/batch.rs | 7 ++++++- bee-ledger/src/workers/pruning/condition.rs | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 885588736f..5e4f6eeb3d 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -332,11 +332,13 @@ fn prune_message_and_metadata( let mut byte_length = 0usize; if by_size { + // Panic: we know the message is in the database. let msg = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); byte_length += msg.packed_len(); + // Panic: we know the message metadata is in the database. let md = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -420,6 +422,7 @@ fn prune_milestone( let mut byte_length = 0usize; if by_size { + // Panic: we know the milestone is in the database. let ms = Fetch::::fetch(storage, &index) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -448,6 +451,7 @@ fn prune_output_diff( for consumed_output_id in output_diff.consumed_outputs() { if by_size { + // Panic: we know the output is in this database table. let consumed_output = Fetch::::fetch(storage, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -455,6 +459,7 @@ fn prune_output_diff( byte_length += consumed_output_id.packed_len(); byte_length += consumed_output.packed_len(); + // Panic: we know the output is in this database table. let created_output = Fetch::::fetch(storage, consumed_output_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); @@ -485,9 +490,9 @@ fn prune_receipts( ) -> Result<(usize, ByteLength), PruningError> { let mut byte_length = 0usize; + // Panic: Fine since Fetch of a Vec<_> always returns Some(Vec<_>). let receipts = Fetch::>::fetch(storage, &index) .map_err(|e| PruningError::Storage(Box::new(e)))? - // Fine since Fetch of a Vec<_> always returns Some(Vec<_>). .unwrap(); let mut num = 0; diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index b16186a66d..dc25ba9628 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -71,6 +71,7 @@ pub(crate) fn should_prune( } } else if config.db_size().enabled() { let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); + // Panic: should not cause problems on properly set up hosts. let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); if now < last + config.db_size().cooldown_time() { From 74aac6ba8cc316902a1ad504c7709e6dfdefe6cd Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 25 Apr 2022 18:35:55 +0200 Subject: [PATCH 25/39] Make humanize-rs optional with no default features --- bee-ledger/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bee-ledger/Cargo.toml b/bee-ledger/Cargo.toml index 6388c03ae1..cba80ce85b 100644 --- a/bee-ledger/Cargo.toml +++ b/bee-ledger/Cargo.toml @@ -23,7 +23,7 @@ digest = { version = "0.9.0", default-features = false, optional = true } futures = { version = "0.3.17", default-features = false, optional = true } hashbrown = { version = "0.11.2", default-features = false, optional = true } hex = { version = "0.4.3", default-features = false, optional = true } -humanize-rs = "0.1.5" +humanize-rs = { version = "0.1.5", default-features = false, optional = true } iota-crypto = { version = "0.10.0", default-features = false, features = [ "blake2b" ], optional = true } log = { version = "0.4.14", default-features = false, optional = true } ref-cast = { version = "1.0.6", default-features = false, optional = true } @@ -52,6 +52,7 @@ workers = [ "futures", "hashbrown", "hex", + "humanize-rs", "iota-crypto", "log", "ref-cast", From 30655246ec28c0fae19edce8ddbb22011a06b01e Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 25 Apr 2022 20:30:23 +0200 Subject: [PATCH 26/39] Small renaming of enum variants --- bee-ledger/src/workers/consensus/worker.rs | 4 ++-- bee-ledger/src/workers/pruning/condition.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index abbdfa318c..c2ee370399 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -351,7 +351,7 @@ where match should_prune(&tangle, &storage, ledger_index, milestones_to_keep, &pruning_config) { Ok(pruning_task) => match pruning_task { - PruningTask::ByRange { + PruningTask::ByIndexRange { start_index, target_index, } => { @@ -368,7 +368,7 @@ where error!("Pruning milestone range failed: {}.", e); } } - PruningTask::BySize { num_bytes_to_prune } => { + PruningTask::ByDbSize { num_bytes_to_prune } => { if let Err(e) = prune::prune_by_size( &tangle, &storage, diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index dc25ba9628..628c914462 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -22,7 +22,7 @@ static LAST_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); pub(crate) enum PruningSkipReason { #[error("Pruning disabled.")] Disabled, - #[error("Pruning by milestone below index threshold.")] + #[error("Pruning by index range below index threshold.")] BelowMilestoneIndexThreshold, #[error("Pruning by size not supported by the storage layer.")] PruningBySizeUnsupported, @@ -35,11 +35,11 @@ pub(crate) enum PruningSkipReason { } pub(crate) enum PruningTask { - ByRange { + ByIndexRange { start_index: MilestoneIndex, target_index: MilestoneIndex, }, - BySize { + ByDbSize { num_bytes_to_prune: usize, }, } @@ -60,7 +60,7 @@ pub(crate) fn should_prune( } else { let target_pruning_index = *ledger_index - milestones_to_keep; - Ok(PruningTask::ByRange { + Ok(PruningTask::ByIndexRange { start_index: pruning_index.into(), target_index: if target_pruning_index > pruning_index + PRUNING_BATCH_SIZE_MAX { (pruning_index + PRUNING_BATCH_SIZE_MAX).into() @@ -106,7 +106,7 @@ pub(crate) fn should_prune( // Store the time we issued a pruning-by-size. LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); - Ok(PruningTask::BySize { num_bytes_to_prune }) + Ok(PruningTask::ByDbSize { num_bytes_to_prune }) } } else { Err(PruningSkipReason::Disabled) From 6d68c57998003df02c1622474e5942350e647686 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 26 Apr 2022 10:29:21 +0200 Subject: [PATCH 27/39] Make pruning options not mutually exclusive --- bee-ledger/src/workers/pruning/condition.rs | 96 ++++++++++++--------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 628c914462..378df13246 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -51,7 +51,60 @@ pub(crate) fn should_prune( milestones_to_keep: u32, config: &PruningConfig, ) -> Result { - if config.milestones().enabled() { + if !config.milestones().enabled() && !config.db_size().enabled() { + return Err(PruningSkipReason::Disabled); + } + + let pruning_by_size = if config.db_size().enabled() { + let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); + // Panic: should not cause problems on properly set up hosts. + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + + if now < last + config.db_size().cooldown_time() { + Err(PruningSkipReason::BelowCooldownTimeThreshold) + } else { + let actual_size = { + if let Ok(size) = storage.size() { + if let Some(size) = size { + Ok(size) + } else { + Err(PruningSkipReason::PruningBySizeUnavailable) + } + } else { + Err(PruningSkipReason::PruningBySizeUnsupported) + } + }; + + match actual_size { + Ok(actual_size) => { + let threshold_size = config.db_size().target_size(); + + log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); + + if actual_size < threshold_size { + Err(PruningSkipReason::BelowStorageSizeThreshold) + } else { + // Panic: cannot underflow due to actual_size >= threshold_size. + let excess_size = actual_size - threshold_size; + let num_bytes_to_prune = excess_size + + (config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; + + log::debug!("Num bytes to prune: {num_bytes_to_prune}"); + + // Store the time we issued a pruning-by-size. + LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); + + Ok(PruningTask::ByDbSize { num_bytes_to_prune }) + } + } + Err(reason) => Err(reason), + } + } + } else { + Err(PruningSkipReason::Disabled) + }; + + if pruning_by_size.is_err() && config.milestones().enabled() { let pruning_index = *tangle.get_pruning_index() + 1; let pruning_threshold = pruning_index + milestones_to_keep; @@ -69,46 +122,7 @@ pub(crate) fn should_prune( }, }) } - } else if config.db_size().enabled() { - let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); - // Panic: should not cause problems on properly set up hosts. - let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - - if now < last + config.db_size().cooldown_time() { - return Err(PruningSkipReason::BelowCooldownTimeThreshold); - } - - let actual_size = { - if let Ok(size) = storage.size() { - if let Some(size) = size { - size - } else { - return Err(PruningSkipReason::PruningBySizeUnavailable); - } - } else { - return Err(PruningSkipReason::PruningBySizeUnsupported); - } - }; - let threshold_size = config.db_size().target_size(); - - log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); - - if actual_size < threshold_size { - Err(PruningSkipReason::BelowStorageSizeThreshold) - } else { - // Panic: cannot underflow due to actual_size >= threshold_size. - let excess_size = actual_size - threshold_size; - let num_bytes_to_prune = - excess_size + (config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; - - log::debug!("Num bytes to prune: {num_bytes_to_prune}"); - - // Store the time we issued a pruning-by-size. - LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); - - Ok(PruningTask::ByDbSize { num_bytes_to_prune }) - } } else { - Err(PruningSkipReason::Disabled) + pruning_by_size } } From c4cc8cf102cf0e2dd5555f36c4a4cdc96cdcda87 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 26 Apr 2022 12:08:49 +0200 Subject: [PATCH 28/39] Refactor snapshotting config and skip reasons --- bee-ledger/src/workers/consensus/worker.rs | 6 +- bee-ledger/src/workers/pruning/condition.rs | 15 +- bee-ledger/src/workers/snapshot/condition.rs | 29 ++-- bee-ledger/src/workers/snapshot/config.rs | 138 ++++++++++++------ .../bee-node/config.chrysalis-devnet.json | 11 +- .../bee-node/config.chrysalis-devnet.toml | 8 +- .../bee-node/config.chrysalis-mainnet.json | 11 +- .../bee-node/config.chrysalis-mainnet.toml | 18 ++- 8 files changed, 152 insertions(+), 84 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index c2ee370399..1d0b87903b 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -283,15 +283,15 @@ where let bmd = tangle.config().below_max_depth(); let snapshot_depth_min = bmd + EXTRA_SNAPSHOT_DEPTH; - let snapshot_depth = if snapshot_config.depth() < snapshot_depth_min { + let snapshot_depth = if snapshot_config.snapshotting().depth() < snapshot_depth_min { warn!( "Configuration value for \"snapshot.depth\" is too low ({}), value changed to {}.", - snapshot_config.depth(), + snapshot_config.snapshotting().depth(), snapshot_depth_min ); snapshot_depth_min } else { - snapshot_config.depth() + snapshot_config.snapshotting().depth() }; let snapshot_pruning_delta = bmd + EXTRA_PRUNING_DEPTH; diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 378df13246..aab3646bc3 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -49,18 +49,18 @@ pub(crate) fn should_prune( storage: &S, ledger_index: LedgerIndex, milestones_to_keep: u32, - config: &PruningConfig, + pruning_config: &PruningConfig, ) -> Result { - if !config.milestones().enabled() && !config.db_size().enabled() { + if !pruning_config.milestones().enabled() && !pruning_config.db_size().enabled() { return Err(PruningSkipReason::Disabled); } - let pruning_by_size = if config.db_size().enabled() { + let pruning_by_size = if pruning_config.db_size().enabled() { let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); // Panic: should not cause problems on properly set up hosts. let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - if now < last + config.db_size().cooldown_time() { + if now < last + pruning_config.db_size().cooldown_time() { Err(PruningSkipReason::BelowCooldownTimeThreshold) } else { let actual_size = { @@ -77,7 +77,7 @@ pub(crate) fn should_prune( match actual_size { Ok(actual_size) => { - let threshold_size = config.db_size().target_size(); + let threshold_size = pruning_config.db_size().target_size(); log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); @@ -87,7 +87,8 @@ pub(crate) fn should_prune( // Panic: cannot underflow due to actual_size >= threshold_size. let excess_size = actual_size - threshold_size; let num_bytes_to_prune = excess_size - + (config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) as usize; + + (pruning_config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) + as usize; log::debug!("Num bytes to prune: {num_bytes_to_prune}"); @@ -104,7 +105,7 @@ pub(crate) fn should_prune( Err(PruningSkipReason::Disabled) }; - if pruning_by_size.is_err() && config.milestones().enabled() { + if pruning_by_size.is_err() && pruning_config.milestones().enabled() { let pruning_index = *tangle.get_pruning_index() + 1; let pruning_threshold = pruning_index + milestones_to_keep; diff --git a/bee-ledger/src/workers/snapshot/condition.rs b/bee-ledger/src/workers/snapshot/condition.rs index be88063a4b..15cf30b14d 100644 --- a/bee-ledger/src/workers/snapshot/condition.rs +++ b/bee-ledger/src/workers/snapshot/condition.rs @@ -8,10 +8,12 @@ use crate::{types::LedgerIndex, workers::snapshot::config::SnapshotConfig}; /// Reasons for skipping snapshotting. #[derive(Debug, thiserror::Error)] pub(crate) enum SnapshottingSkipReason { - #[error("Snapshotting skipped for the next {reached_in} indexes.")] - BelowThreshold { reached_in: u32 }, - #[error("Snapshotting deferred for the next {next_in} indexes.")] - Deferred { next_in: u32 }, + #[error("Snapshotting disabled.")] + Disabled, + #[error("Ledger index below snapshotting depth.")] + BelowDepth, + #[error("Ledger index below next snapshot index {next_snapshot_index}.")] + BelowNextSnapshotIndex { next_snapshot_index: u32 }, } pub(crate) fn should_snapshot( @@ -20,21 +22,24 @@ pub(crate) fn should_snapshot( snapshot_depth: u32, snapshot_config: &SnapshotConfig, ) -> Result<(), SnapshottingSkipReason> { + if !snapshot_config.snapshotting().enabled() { + return Err(SnapshottingSkipReason::Disabled); + } + + // Get the index of the last snapshot. let snapshot_index = *tangle.get_snapshot_index(); let snapshot_interval = if tangle.is_synced() { - snapshot_config.interval_synced() + snapshot_config.snapshotting().interval_synced() } else { - snapshot_config.interval_unsynced() + snapshot_config.snapshotting().interval_unsynced() }; - if *ledger_index < snapshot_depth { - Err(SnapshottingSkipReason::BelowThreshold { - reached_in: snapshot_depth - *ledger_index, - }) + if *ledger_index < snapshot_index + snapshot_depth { + Err(SnapshottingSkipReason::BelowDepth) } else if *ledger_index < snapshot_index + snapshot_interval { - Err(SnapshottingSkipReason::Deferred { - next_in: (snapshot_index + snapshot_interval) - *ledger_index, + Err(SnapshottingSkipReason::BelowNextSnapshotIndex { + next_snapshot_index: snapshot_index + snapshot_interval, }) } else { Ok(()) diff --git a/bee-ledger/src/workers/snapshot/config.rs b/bee-ledger/src/workers/snapshot/config.rs index 0927ed4051..1f3bc19444 100644 --- a/bee-ledger/src/workers/snapshot/config.rs +++ b/bee-ledger/src/workers/snapshot/config.rs @@ -8,14 +8,15 @@ use std::path::{Path, PathBuf}; use serde::Deserialize; use url::Url; -const DEFAULT_FULL_PATH: &str = "./snapshots/mainnet/latest-full_snapshot.bin"; -const DEFAULT_DOWNLOAD_URLS: Vec = Vec::new(); +const DEFAULT_ENABLED: bool = false; const DEFAULT_DEPTH: u32 = 50; const DEFAULT_INTERVAL_SYNCED: u32 = 50; const DEFAULT_INTERVAL_UNSYNCED: u32 = 1000; +const DEFAULT_DOWNLOAD_URLS: Vec = Vec::new(); +const DEFAULT_FULL_PATH: &str = "./snapshots/mainnet/latest-full_snapshot.bin"; /// Contains URLs to download the full and delta snapshot files. -#[derive(Clone, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq)] pub struct DownloadUrls { full: Url, delta: Url, @@ -33,16 +34,11 @@ impl DownloadUrls { } } -/// Builder for a `SnapshotConfig`. -#[derive(Default, Deserialize, PartialEq)] +/// Builder for a [`SnapshottingConfig`] which is part of the [`SnapshotConfig`]. +#[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] -pub struct SnapshotConfigBuilder { - #[serde(alias = "fullPath")] - full_path: Option, - #[serde(alias = "deltaPath")] - delta_path: Option, - #[serde(alias = "downloadUrls")] - download_urls: Option>, +pub struct SnapshottingConfigBuilder { + enabled: Option, depth: Option, #[serde(alias = "intervalSynced")] interval_synced: Option, @@ -50,49 +46,88 @@ pub struct SnapshotConfigBuilder { interval_unsynced: Option, } +impl SnapshottingConfigBuilder { + /// Sets whether snapshotting is enabled. + pub fn enabled(mut self, enabled: bool) -> Self { + self.enabled.replace(enabled); + self + } + + /// Sets the snapshotting depth. + pub fn depth(mut self, depth: u32) -> Self { + self.depth.replace(depth); + self + } + + /// Sets the snapshotting interval for a synced node. + pub fn interval_synced(mut self, interval_synced: u32) -> Self { + self.interval_synced.replace(interval_synced); + self + } + + /// Sets the snapshotting interval for an unsynced node. + pub fn interval_unsynced(mut self, interval_unsynced: u32) -> Self { + self.interval_unsynced.replace(interval_unsynced); + self + } + + /// Produces a [`SnapshottingConfig`] from this builder. + #[must_use] + pub fn finish(self) -> SnapshottingConfig { + SnapshottingConfig { + enabled: self.enabled.unwrap_or(DEFAULT_ENABLED), + depth: self.depth.unwrap_or(DEFAULT_DEPTH), + interval_synced: self.interval_synced.unwrap_or(DEFAULT_INTERVAL_SYNCED), + interval_unsynced: self.interval_unsynced.unwrap_or(DEFAULT_INTERVAL_UNSYNCED), + } + } +} + +/// Builder for a [`SnapshotConfig`] that can also be deserialized from some source. +#[derive(Default, Debug, Deserialize, PartialEq)] +#[must_use] +pub struct SnapshotConfigBuilder { + #[serde(alias = "downloadUrls")] + download_urls: Option>, + #[serde(alias = "fullPath")] + full_path: Option, + #[serde(alias = "deltaPath")] + delta_path: Option, + #[serde(alias = "create")] + snapshotting: Option, +} + impl SnapshotConfigBuilder { - /// Creates a new `SnapshotConfigBuilder`. + /// Creates a new builder. pub fn new() -> Self { Self::default() } - /// Sets the full path of the `SnapshotConfigBuilder`. + /// Sets the path of full snapshots. pub fn full_path(mut self, full_path: PathBuf) -> Self { self.full_path.replace(full_path); self } - /// Sets the delta path of the `SnapshotConfigBuilder`. + /// Sets the path of delta snapshots. pub fn delta_path(mut self, delta_path: PathBuf) -> Self { self.delta_path.replace(delta_path); self } - /// Sets the download URLs of the `SnapshotConfigBuilder`. + /// Sets the URLs for downloading remotely produced snapshots. pub fn download_urls(mut self, download_urls: Vec) -> Self { self.download_urls.replace(download_urls); self } - /// Sets the depth of the `SnapshotConfigBuilder`. - pub fn depth(mut self, depth: u32) -> Self { - self.depth.replace(depth); - self - } - - /// Sets the synced interval of the `SnapshotConfigBuilder`. - pub fn interval_synced(mut self, interval_synced: u32) -> Self { - self.interval_synced.replace(interval_synced); - self - } - - /// Sets the unsynced interval of the `SnapshotConfigBuilder`. - pub fn interval_unsynced(mut self, interval_unsynced: u32) -> Self { - self.interval_unsynced.replace(interval_unsynced); + /// Sets the `[SnapshottingConfigBuilder`]. + pub fn snapshotting(mut self, builder: SnapshottingConfigBuilder) -> Self { + self.snapshotting.replace(builder); self } - /// Finishes the `SnapshotConfigBuilder` into a `SnapshotConfig`. + /// Produces a [`SnapshotConfig`] from this builder. #[must_use] pub fn finish(self) -> SnapshotConfig { SnapshotConfig { @@ -101,22 +136,18 @@ impl SnapshotConfigBuilder { .unwrap_or_else(|| PathBuf::from(DEFAULT_FULL_PATH.to_string())), delta_path: self.delta_path, download_urls: self.download_urls.unwrap_or(DEFAULT_DOWNLOAD_URLS), - depth: self.depth.unwrap_or(DEFAULT_DEPTH), - interval_synced: self.interval_synced.unwrap_or(DEFAULT_INTERVAL_SYNCED), - interval_unsynced: self.interval_unsynced.unwrap_or(DEFAULT_INTERVAL_UNSYNCED), + snapshotting: self.snapshotting.unwrap_or_default().finish(), } } } -/// A snapshot configuration. -#[derive(Clone)] +/// The configuration of downloading and creating snapshots. +#[derive(Clone, Debug)] pub struct SnapshotConfig { full_path: PathBuf, delta_path: Option, download_urls: Vec, - depth: u32, - interval_synced: u32, - interval_unsynced: u32, + snapshotting: SnapshottingConfig, } impl SnapshotConfig { @@ -140,17 +171,38 @@ impl SnapshotConfig { &self.download_urls } - /// Returns the depth of the `SnapshotConfig`. + /// Returns the [`SnapshottingConfig`]. + pub fn snapshotting(&self) -> &SnapshottingConfig { + &self.snapshotting + } +} + +/// The configuration for creating snapshots. +#[derive(Clone, Debug)] +pub struct SnapshottingConfig { + enabled: bool, + depth: u32, + interval_synced: u32, + interval_unsynced: u32, +} + +impl SnapshottingConfig { + /// Returns whether pruning based on milestone indexes is enabled. + pub fn enabled(&self) -> bool { + self.enabled + } + + /// Returns the snapshot depth. pub fn depth(&self) -> u32 { self.depth } - /// Returns the synced interval of the `SnapshotConfig`. + /// Returns the snapshot interval for a synced node. pub fn interval_synced(&self) -> u32 { self.interval_synced } - /// Returns the unsynced interval of the `SnapshotConfig`. + /// Returns the snapshot interval for an unsynced node. pub fn interval_unsynced(&self) -> u32 { self.interval_unsynced } diff --git a/bee-node/bee-node/config.chrysalis-devnet.json b/bee-node/bee-node/config.chrysalis-devnet.json index 14b3f74218..62708824f4 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.json +++ b/bee-node/bee-node/config.chrysalis-devnet.json @@ -80,9 +80,6 @@ "whiteFlagSolidificationTimeout": 2 }, "snapshot": { - "depth": 50, - "intervalSynced": 50, - "intervalUnsynced": 1000, "fullPath": "./snapshots/devnet/full_snapshot.bin", "deltaPath": "./snapshots/devnet/delta_snapshot.bin", "downloadUrls": [ @@ -90,7 +87,13 @@ "full": "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest-full_snapshot.bin", "delta": "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest-delta_snapshot.bin" } - ] + ], + "create": { + "enabled": false, + "depth": 50, + "intervalSynced": 50, + "intervalUnsynced": 1000 + } }, "pruning": { "milestones": { diff --git a/bee-node/bee-node/config.chrysalis-devnet.toml b/bee-node/bee-node/config.chrysalis-devnet.toml index 9b3fec5440..61db8d582c 100644 --- a/bee-node/bee-node/config.chrysalis-devnet.toml +++ b/bee-node/bee-node/config.chrysalis-devnet.toml @@ -76,14 +76,16 @@ feature_proof_of_work = true white_flag_solidification_timeout = 2 [snapshot] -depth = 50 -interval_synced = 50 -interval_unsynced = 1000 full_path = "./snapshots/devnet/full_snapshot.bin" delta_path = "./snapshots/devnet/delta_snapshot.bin" [[snapshot.download_urls]] full = "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest-full_snapshot.bin" delta = "http://dbfiles.chrysalis-devnet.iota.cafe/snapshots/hornet/latest-delta_snapshot.bin" +[snapshot.create] +enabled = false +depth = 50 +interval_synced = 50 +interval_unsynced = 1000 [pruning] [pruning.milestones] diff --git a/bee-node/bee-node/config.chrysalis-mainnet.json b/bee-node/bee-node/config.chrysalis-mainnet.json index b9e8ca8df8..5d2993cb56 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.json +++ b/bee-node/bee-node/config.chrysalis-mainnet.json @@ -115,9 +115,6 @@ "whiteFlagSolidificationTimeout": 2 }, "snapshot": { - "depth": 50, - "intervalSynced": 50, - "intervalUnsynced": 1000, "fullPath": "./snapshots/mainnet/full_snapshot.bin", "deltaPath": "./snapshots/mainnet/delta_snapshot.bin", "downloadUrls": [ @@ -129,7 +126,13 @@ "full": "https://cdn.tanglebay.com/snapshots/mainnet/full_snapshot.bin", "delta": "https://cdn.tanglebay.com/snapshots/mainnet/delta_snapshot.bin" } - ] + ], + "create": { + "enabled": false, + "depth": 50, + "intervalSynced": 50, + "intervalUnsynced": 1000 + } }, "pruning": { "milestones": { diff --git a/bee-node/bee-node/config.chrysalis-mainnet.toml b/bee-node/bee-node/config.chrysalis-mainnet.toml index 56fcee02cb..69213631cd 100644 --- a/bee-node/bee-node/config.chrysalis-mainnet.toml +++ b/bee-node/bee-node/config.chrysalis-mainnet.toml @@ -106,17 +106,19 @@ feature_proof_of_work = true white_flag_solidification_timeout = 2 [snapshot] +full_path = "./snapshots/mainnet/full_snapshot.bin" +delta_path = "./snapshots/mainnet/delta_snapshot.bin" +[[snapshot.download_urls]] +full = "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-full_snapshot.bin" +delta = "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-delta_snapshot.bin" +[[snapshot.download_urls]] +full = "https://cdn.tanglebay.com/snapshots/mainnet/full_snapshot.bin" +delta = "https://cdn.tanglebay.com/snapshots/mainnet/delta_snapshot.bin" +[snapshot.create] +enabled = false depth = 50 interval_synced = 50 interval_unsynced = 1000 -full_path = "./snapshots/mainnet/full_snapshot.bin" -delta_path = "./snapshots/mainnet/delta_snapshot.bin" -[[snapshot.download_urls]] -full = "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-full_snapshot.bin" -delta = "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-delta_snapshot.bin" -[[snapshot.download_urls]] -full = "https://cdn.tanglebay.com/snapshots/mainnet/full_snapshot.bin" -delta = "https://cdn.tanglebay.com/snapshots/mainnet/delta_snapshot.bin" [pruning] [pruning.milestones] From 9fe501669688fe5ede813d223846a5cb38d10f87 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 26 Apr 2022 13:51:31 +0200 Subject: [PATCH 29/39] Change log --- bee-ledger/src/workers/pruning/prune.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 1f48a3590d..75f521c790 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -88,7 +88,7 @@ pub async fn prune_by_size( let index = *tangle.get_pruning_index() + 1; if *ledger_index < index + MILESTONES_TO_KEEP_MIN { - log::debug!("Minimum pruning index reached."); + log::debug!("Minimum number of milestones to keep reached."); break; } From b26b28a811beb6c113036df49dcbd2b9c26a7462 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 26 Apr 2022 14:06:42 +0200 Subject: [PATCH 30/39] Better skip reason naming and consistent messages --- bee-ledger/src/workers/consensus/worker.rs | 8 ++--- bee-ledger/src/workers/pruning/condition.rs | 36 ++++++++++---------- bee-ledger/src/workers/snapshot/condition.rs | 6 ++-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/bee-ledger/src/workers/consensus/worker.rs b/bee-ledger/src/workers/consensus/worker.rs index 1d0b87903b..a337c9f0a7 100644 --- a/bee-ledger/src/workers/consensus/worker.rs +++ b/bee-ledger/src/workers/consensus/worker.rs @@ -345,7 +345,7 @@ where // } } Err(reason) => { - debug!("Snapshotting skipped: {:?}", reason); + debug!("Snapshotting skipped: {reason}"); } } @@ -365,7 +365,7 @@ where ) .await { - error!("Pruning milestone range failed: {}.", e); + error!("Pruning milestone range failed: {e}."); } } PruningTask::ByDbSize { num_bytes_to_prune } => { @@ -379,12 +379,12 @@ where ) .await { - error!("Pruning by storage size failed: {}.", e); + error!("Pruning by storage size failed: {e}."); } } }, Err(reason) => { - debug!("Pruning skipped: {}", reason); + debug!("Pruning skipped: {reason}"); } } } diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index aab3646bc3..0dfc093f2d 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -20,18 +20,18 @@ static LAST_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); /// Reasons for skipping pruning. #[derive(Debug, thiserror::Error)] pub(crate) enum PruningSkipReason { - #[error("Pruning disabled.")] + #[error("disabled")] Disabled, - #[error("Pruning by index range below index threshold.")] - BelowMilestoneIndexThreshold, - #[error("Pruning by size not supported by the storage layer.")] - PruningBySizeUnsupported, - #[error("Pruning by size currently unavailable.")] - PruningBySizeUnavailable, - #[error("Pruning by size below size threshold.")] - BelowStorageSizeThreshold, - #[error("Pruning by size below cooldown threshold.")] - BelowCooldownTimeThreshold, + #[error("ledger index < target index threshold")] + BelowTargetIndexThreshold, + #[error("size metric not supported by the storage layer")] + SizeMetricUnsupported, + #[error("size metric currently unavailable")] + SizeMetricUnavailable, + #[error("current size < target size threshold")] + BelowTargetSizeThreshold, + #[error("cooldown")] + Cooldown, } pub(crate) enum PruningTask { @@ -61,17 +61,17 @@ pub(crate) fn should_prune( let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); if now < last + pruning_config.db_size().cooldown_time() { - Err(PruningSkipReason::BelowCooldownTimeThreshold) + Err(PruningSkipReason::Cooldown) } else { let actual_size = { if let Ok(size) = storage.size() { if let Some(size) = size { Ok(size) } else { - Err(PruningSkipReason::PruningBySizeUnavailable) + Err(PruningSkipReason::SizeMetricUnavailable) } } else { - Err(PruningSkipReason::PruningBySizeUnsupported) + Err(PruningSkipReason::SizeMetricUnsupported) } }; @@ -82,7 +82,7 @@ pub(crate) fn should_prune( log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); if actual_size < threshold_size { - Err(PruningSkipReason::BelowStorageSizeThreshold) + Err(PruningSkipReason::BelowTargetSizeThreshold) } else { // Panic: cannot underflow due to actual_size >= threshold_size. let excess_size = actual_size - threshold_size; @@ -107,10 +107,10 @@ pub(crate) fn should_prune( if pruning_by_size.is_err() && pruning_config.milestones().enabled() { let pruning_index = *tangle.get_pruning_index() + 1; - let pruning_threshold = pruning_index + milestones_to_keep; + let target_index_threshold = pruning_index + milestones_to_keep; - if *ledger_index < pruning_threshold { - Err(PruningSkipReason::BelowMilestoneIndexThreshold) + if *ledger_index < target_index_threshold { + Err(PruningSkipReason::BelowTargetIndexThreshold) } else { let target_pruning_index = *ledger_index - milestones_to_keep; diff --git a/bee-ledger/src/workers/snapshot/condition.rs b/bee-ledger/src/workers/snapshot/condition.rs index 15cf30b14d..1cc2186fb9 100644 --- a/bee-ledger/src/workers/snapshot/condition.rs +++ b/bee-ledger/src/workers/snapshot/condition.rs @@ -8,11 +8,11 @@ use crate::{types::LedgerIndex, workers::snapshot::config::SnapshotConfig}; /// Reasons for skipping snapshotting. #[derive(Debug, thiserror::Error)] pub(crate) enum SnapshottingSkipReason { - #[error("Snapshotting disabled.")] + #[error("disabled")] Disabled, - #[error("Ledger index below snapshotting depth.")] + #[error("ledger index < snapshotting depth")] BelowDepth, - #[error("Ledger index below next snapshot index {next_snapshot_index}.")] + #[error("ledger index < next snapshot index {next_snapshot_index}")] BelowNextSnapshotIndex { next_snapshot_index: u32 }, } From 3a124f4285ad71330af6726746a07a00af183a0d Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 26 Apr 2022 21:16:02 +0200 Subject: [PATCH 31/39] Fix doc --- bee-ledger/src/workers/snapshot/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bee-ledger/src/workers/snapshot/config.rs b/bee-ledger/src/workers/snapshot/config.rs index 1f3bc19444..c7846dc001 100644 --- a/bee-ledger/src/workers/snapshot/config.rs +++ b/bee-ledger/src/workers/snapshot/config.rs @@ -187,7 +187,7 @@ pub struct SnapshottingConfig { } impl SnapshottingConfig { - /// Returns whether pruning based on milestone indexes is enabled. + /// Returns whether snapshotting is enabled. pub fn enabled(&self) -> bool { self.enabled } From 1117a996abbf51231891c7b460d447c3532662ac Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 29 Apr 2022 13:27:33 +0200 Subject: [PATCH 32/39] Better panic explanations --- bee-ledger/src/workers/pruning/batch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 5e4f6eeb3d..146bbe117a 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -332,13 +332,13 @@ fn prune_message_and_metadata( let mut byte_length = 0usize; if by_size { - // Panic: we know the message is in the database. + // Panic: we know the message is in the database, because the caller already made sure of that. let msg = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); byte_length += msg.packed_len(); - // Panic: we know the message metadata is in the database. + // Panic: we know the message metadata is in the database for the same reason as above. let md = Fetch::::fetch(storage, message_id) .map_err(|e| PruningError::Storage(Box::new(e)))? .unwrap(); From 54687d02654929fd373dbee9a5c3b4d792e74eef Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 29 Apr 2022 13:34:20 +0200 Subject: [PATCH 33/39] previous instead of last --- bee-ledger/src/workers/pruning/condition.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 0dfc093f2d..a92ce03cf8 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -15,7 +15,7 @@ use crate::{types::LedgerIndex, workers::pruning::config::PruningConfig}; const PRUNING_BATCH_SIZE_MAX: u32 = 200; -static LAST_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); +static PREVIOUS_PRUNING_BY_SIZE: AtomicU64 = AtomicU64::new(0); /// Reasons for skipping pruning. #[derive(Debug, thiserror::Error)] @@ -56,11 +56,11 @@ pub(crate) fn should_prune( } let pruning_by_size = if pruning_config.db_size().enabled() { - let last = Duration::from_secs(LAST_PRUNING_BY_SIZE.load(Ordering::Relaxed)); + let prev = Duration::from_secs(PREVIOUS_PRUNING_BY_SIZE.load(Ordering::Relaxed)); // Panic: should not cause problems on properly set up hosts. let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - if now < last + pruning_config.db_size().cooldown_time() { + if now < prev + pruning_config.db_size().cooldown_time() { Err(PruningSkipReason::Cooldown) } else { let actual_size = { @@ -93,7 +93,7 @@ pub(crate) fn should_prune( log::debug!("Num bytes to prune: {num_bytes_to_prune}"); // Store the time we issued a pruning-by-size. - LAST_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); + PREVIOUS_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); Ok(PruningTask::ByDbSize { num_bytes_to_prune }) } From 74b99951aa09a38f9d61a67b4a75cdad0189ed74 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 29 Apr 2022 13:39:18 +0200 Subject: [PATCH 34/39] Underflow panic doc --- bee-ledger/src/workers/pruning/condition.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index a92ce03cf8..91d30a30cd 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -112,6 +112,8 @@ pub(crate) fn should_prune( if *ledger_index < target_index_threshold { Err(PruningSkipReason::BelowTargetIndexThreshold) } else { + // Panic: cannot underflow due to ledger_size >= target_index_threshold = pruning_index + + // milestones_to_keep. let target_pruning_index = *ledger_index - milestones_to_keep; Ok(PruningTask::ByIndexRange { From 37ca08a77bfc92f0468f665300da89e12ac2d472 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 29 Apr 2022 13:45:26 +0200 Subject: [PATCH 35/39] Reorder struct fields --- bee-ledger/src/workers/snapshot/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bee-ledger/src/workers/snapshot/config.rs b/bee-ledger/src/workers/snapshot/config.rs index c7846dc001..815416a755 100644 --- a/bee-ledger/src/workers/snapshot/config.rs +++ b/bee-ledger/src/workers/snapshot/config.rs @@ -87,12 +87,12 @@ impl SnapshottingConfigBuilder { #[derive(Default, Debug, Deserialize, PartialEq)] #[must_use] pub struct SnapshotConfigBuilder { - #[serde(alias = "downloadUrls")] - download_urls: Option>, #[serde(alias = "fullPath")] full_path: Option, #[serde(alias = "deltaPath")] delta_path: Option, + #[serde(alias = "downloadUrls")] + download_urls: Option>, #[serde(alias = "create")] snapshotting: Option, } From 54b34aaccc77f928c26a7f09e23630bc31640394 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 5 May 2022 11:42:53 +0200 Subject: [PATCH 36/39] Address some comments --- bee-ledger/src/workers/pruning/batch.rs | 2 +- bee-ledger/src/workers/pruning/condition.rs | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 146bbe117a..0413f5b733 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -363,7 +363,7 @@ fn prune_edge( let mut byte_length = 0usize; if by_size { - byte_length += edge.0.packed_len() + edge.1.packed_len(); + byte_length += edge.packed_len(); } Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 91d30a30cd..a04843ecae 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -65,11 +65,7 @@ pub(crate) fn should_prune( } else { let actual_size = { if let Ok(size) = storage.size() { - if let Some(size) = size { - Ok(size) - } else { - Err(PruningSkipReason::SizeMetricUnavailable) - } + size.ok_or(PruningSkipReason::SizeMetricUnavailable) } else { Err(PruningSkipReason::SizeMetricUnsupported) } From e019d882842743ff5968ce820c4d142f2889ab11 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 6 May 2022 13:14:04 +0200 Subject: [PATCH 37/39] Add boxing helper to PruningError enum; remove redundant closures --- bee-ledger/src/workers/pruning/batch.rs | 60 +++++++--------- bee-ledger/src/workers/pruning/error.rs | 6 ++ bee-ledger/src/workers/pruning/metrics.rs | 85 ++++++++++++----------- bee-ledger/src/workers/pruning/prune.rs | 10 ++- 4 files changed, 78 insertions(+), 83 deletions(-) diff --git a/bee-ledger/src/workers/pruning/batch.rs b/bee-ledger/src/workers/pruning/batch.rs index 0413f5b733..ab09d95d1b 100644 --- a/bee-ledger/src/workers/pruning/batch.rs +++ b/bee-ledger/src/workers/pruning/batch.rs @@ -63,7 +63,7 @@ pub(crate) fn batch_prunable_confirmed_data( // Get the `MessageId` of the milestone we are about to prune from the storage. let milestone_to_prune = Fetch::::fetch(storage, &prune_index) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .ok_or(PruningError::MissingMilestone(prune_index))?; byte_length += prune_index.packed_len(); @@ -89,7 +89,7 @@ pub(crate) fn batch_prunable_confirmed_data( // Get the `Message` for `message_id`. let msg = match Fetch::::fetch(storage, &message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .ok_or(PruningError::MissingMessage(message_id)) { Ok(msg) => msg, @@ -145,7 +145,7 @@ pub(crate) fn batch_prunable_confirmed_data( // Fetch its approvers from the storage. let approvers = Fetch::>::fetch(storage, &message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .ok_or(PruningError::MissingApprovers(message_id))?; // We can safely skip messages whose approvers are all part of the currently pruned cone. If we are lucky @@ -174,7 +174,7 @@ pub(crate) fn batch_prunable_confirmed_data( metrics.approver_cache_miss += 1; let unvisited_md = Fetch::::fetch(storage, &unvisited_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .ok_or(PruningError::MissingMetadata(unvisited_id))?; // A non-existing milestone index means that a message remained unconfirmed and therefore is neglibable @@ -221,7 +221,7 @@ pub(crate) fn batch_prunable_unconfirmed_data( let mut metrics = UnconfirmedDataPruningMetrics::default(); let unconf_msgs = match Fetch::>::fetch(storage, &prune_index) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? { Some(unconf_msgs) => { if unconf_msgs.is_empty() { @@ -239,9 +239,7 @@ pub(crate) fn batch_prunable_unconfirmed_data( // TODO: consider using `MultiFetch` 'next_unconf_msg: for unconf_msg_id in unconf_msgs.iter().map(|unconf_msg| unconf_msg.message_id()) { - match Fetch::::fetch(storage, unconf_msg_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? - { + match Fetch::::fetch(storage, unconf_msg_id).map_err(PruningError::storage)? { Some(msg_meta) => { if msg_meta.flags().is_referenced() { metrics.were_confirmed += 1; @@ -257,9 +255,7 @@ pub(crate) fn batch_prunable_unconfirmed_data( } // Delete those messages that remained unconfirmed. - match Fetch::::fetch(storage, unconf_msg_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? - { + match Fetch::::fetch(storage, unconf_msg_id).map_err(PruningError::storage)? { Some(msg) => { let payload = msg.payload().as_ref(); let parents = msg.parents(); @@ -334,22 +330,20 @@ fn prune_message_and_metadata( if by_size { // Panic: we know the message is in the database, because the caller already made sure of that. let msg = Fetch::::fetch(storage, message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); byte_length += msg.packed_len(); // Panic: we know the message metadata is in the database for the same reason as above. let md = Fetch::::fetch(storage, message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); byte_length += md.packed_len(); } - Batch::::batch_delete(storage, batch, message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, message_id).map_err(PruningError::storage)?; - Batch::::batch_delete(storage, batch, message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, message_id).map_err(PruningError::storage)?; Ok(byte_length) } @@ -366,8 +360,7 @@ fn prune_edge( byte_length += edge.packed_len(); } - Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::<(MessageId, MessageId), ()>::batch_delete(storage, batch, edge).map_err(PruningError::storage)?; Ok(byte_length) } @@ -385,7 +378,7 @@ fn prune_indexation_data( } Batch::<(PaddedIndex, MessageId), ()>::batch_delete(storage, batch, index_message_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; Ok(byte_length) } @@ -408,7 +401,7 @@ fn prune_unreferenced_message( batch, &(prune_index, unreferenced_message), ) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; Ok(byte_length) } @@ -424,13 +417,12 @@ fn prune_milestone( if by_size { // Panic: we know the milestone is in the database. let ms = Fetch::::fetch(storage, &index) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); byte_length += ms.packed_len(); } - Batch::::batch_delete(storage, batch, &index) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, &index).map_err(PruningError::storage)?; Ok(byte_length) } @@ -444,7 +436,7 @@ fn prune_output_diff( let mut byte_length = 0usize; if let Some(output_diff) = - Fetch::::fetch(storage, &index).map_err(|e| PruningError::Storage(Box::new(e)))? + Fetch::::fetch(storage, &index).map_err(PruningError::storage)? { byte_length += index.packed_len(); byte_length += output_diff.packed_len(); @@ -453,7 +445,7 @@ fn prune_output_diff( if by_size { // Panic: we know the output is in this database table. let consumed_output = Fetch::::fetch(storage, consumed_output_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); byte_length += consumed_output_id.packed_len(); @@ -461,7 +453,7 @@ fn prune_output_diff( // Panic: we know the output is in this database table. let created_output = Fetch::::fetch(storage, consumed_output_id) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); byte_length += consumed_output_id.packed_len(); @@ -469,15 +461,14 @@ fn prune_output_diff( } Batch::::batch_delete(storage, batch, consumed_output_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; Batch::::batch_delete(storage, batch, consumed_output_id) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; } } - Batch::::batch_delete(storage, batch, &index) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, &index).map_err(PruningError::storage)?; Ok(byte_length) } @@ -492,7 +483,7 @@ fn prune_receipts( // Panic: Fine since Fetch of a Vec<_> always returns Some(Vec<_>). let receipts = Fetch::>::fetch(storage, &index) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .unwrap(); let mut num = 0; @@ -503,7 +494,7 @@ fn prune_receipts( } Batch::<(MilestoneIndex, Receipt), ()>::batch_delete(storage, batch, &(index, receipt)) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; num += 1; } @@ -540,8 +531,7 @@ fn prune_seps( ) -> Result { let mut num = 0; for sep in seps { - Batch::::batch_delete(storage, batch, sep) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + Batch::::batch_delete(storage, batch, sep).map_err(PruningError::storage)?; num += 1; } diff --git a/bee-ledger/src/workers/pruning/error.rs b/bee-ledger/src/workers/pruning/error.rs index f7cd3ba6e3..f422669366 100644 --- a/bee-ledger/src/workers/pruning/error.rs +++ b/bee-ledger/src/workers/pruning/error.rs @@ -23,3 +23,9 @@ pub enum PruningError { #[error("storage operation failed due to: {0:?}")] Storage(Box), } + +impl PruningError { + pub(crate) fn storage(e: impl std::error::Error + Send + 'static) -> Self { + Self::Storage(Box::new(e)) + } +} diff --git a/bee-ledger/src/workers/pruning/metrics.rs b/bee-ledger/src/workers/pruning/metrics.rs index c74b3f7e9d..8c3ea9e957 100644 --- a/bee-ledger/src/workers/pruning/metrics.rs +++ b/bee-ledger/src/workers/pruning/metrics.rs @@ -4,58 +4,59 @@ use std::time::Duration; #[derive(Debug, Default)] -pub struct PruningMetrics { - pub curr_seps: usize, - pub new_seps: usize, - pub kept_seps: usize, - pub next_seps: usize, - pub messages: usize, - pub edges: usize, - pub indexations: usize, - pub output_diffs: bool, - pub receipts: usize, +pub(crate) struct PruningMetrics { + pub(crate) curr_seps: usize, + pub(crate) new_seps: usize, + pub(crate) kept_seps: usize, + pub(crate) next_seps: usize, + pub(crate) messages: usize, + pub(crate) edges: usize, + pub(crate) indexations: usize, + // TODO + // pub(crate) output_diffs: bool, + pub(crate) receipts: usize, } #[derive(Debug, Default)] -pub struct ConfirmedDataPruningMetrics { - pub msg_already_visited: usize, - pub references_sep: usize, - pub approver_cache_miss: usize, - pub approver_cache_hit: usize, - pub all_approvers_visited: usize, - pub not_all_approvers_visited: usize, - pub found_seps: usize, - pub prunable_messages: usize, - pub prunable_edges: usize, - pub prunable_indexations: usize, - pub new_seps: usize, +pub(crate) struct ConfirmedDataPruningMetrics { + pub(crate) msg_already_visited: usize, + pub(crate) references_sep: usize, + pub(crate) approver_cache_miss: usize, + pub(crate) approver_cache_hit: usize, + pub(crate) all_approvers_visited: usize, + pub(crate) not_all_approvers_visited: usize, + pub(crate) found_seps: usize, + pub(crate) prunable_messages: usize, + pub(crate) prunable_edges: usize, + pub(crate) prunable_indexations: usize, + pub(crate) new_seps: usize, } #[derive(Debug, Default)] -pub struct UnconfirmedDataPruningMetrics { - pub none_received: bool, - pub prunable_messages: usize, - pub prunable_edges: usize, - pub prunable_indexations: usize, - pub already_pruned: usize, - pub were_confirmed: usize, +pub(crate) struct UnconfirmedDataPruningMetrics { + pub(crate) none_received: bool, + pub(crate) prunable_messages: usize, + pub(crate) prunable_edges: usize, + pub(crate) prunable_indexations: usize, + pub(crate) already_pruned: usize, + pub(crate) were_confirmed: usize, } #[derive(Debug, Default)] -pub struct MilestoneDataPruningMetrics { - pub receipts: usize, +pub(crate) struct MilestoneDataPruningMetrics { + pub(crate) receipts: usize, } #[derive(Debug, Default)] -pub struct Timings { - pub full_prune: Duration, - pub get_curr_seps: Duration, - pub filter_curr_seps: Duration, - pub replace_seps: Duration, - pub batch_confirmed_data: Duration, - pub batch_unconfirmed_data: Duration, - pub batch_milestone_data: Duration, - pub batch_new_seps: Duration, - pub truncate_curr_seps: Duration, - pub batch_commit: Duration, +pub(crate) struct Timings { + pub(crate) full_prune: Duration, + pub(crate) get_curr_seps: Duration, + pub(crate) filter_curr_seps: Duration, + pub(crate) replace_seps: Duration, + pub(crate) batch_confirmed_data: Duration, + pub(crate) batch_unconfirmed_data: Duration, + pub(crate) batch_milestone_data: Duration, + pub(crate) batch_new_seps: Duration, + pub(crate) truncate_curr_seps: Duration, + pub(crate) batch_commit: Duration, } diff --git a/bee-ledger/src/workers/pruning/prune.rs b/bee-ledger/src/workers/pruning/prune.rs index 75f521c790..4b33314f72 100644 --- a/bee-ledger/src/workers/pruning/prune.rs +++ b/bee-ledger/src/workers/pruning/prune.rs @@ -175,7 +175,7 @@ async fn prune_milestone( let batch_new_seps = Instant::now(); for (new_sep, index) in &new_seps { Batch::::batch_insert(storage, &mut batch, new_sep, index) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + .map_err(PruningError::storage)?; } timings.batch_new_seps = batch_new_seps.elapsed(); @@ -219,9 +219,7 @@ async fn prune_milestone( // Execute the batch operation. let batch_commit = Instant::now(); - storage - .batch_commit(batch, false) - .map_err(|e| PruningError::Storage(Box::new(e)))?; + storage.batch_commit(batch, false).map_err(PruningError::storage)?; timings.batch_commit = batch_commit.elapsed(); // Update the pruning index. @@ -233,11 +231,11 @@ async fn prune_milestone( .expect("error creating timestamp") .as_secs(); let mut snapshot_info = storage::fetch_snapshot_info(storage) - .map_err(|e| PruningError::Storage(Box::new(e)))? + .map_err(PruningError::storage)? .ok_or(PruningError::MissingSnapshotInfo)?; snapshot_info.update_pruning_index(index); snapshot_info.update_timestamp(timestamp); - storage::insert_snapshot_info(storage, &snapshot_info).map_err(|e| PruningError::Storage(Box::new(e)))?; + storage::insert_snapshot_info(storage, &snapshot_info).map_err(PruningError::storage)?; timings.full_prune = full_prune.elapsed(); From 9f6590e50fc479a74656dca79f12814896e09d58 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 6 May 2022 13:33:30 +0200 Subject: [PATCH 38/39] Applied Christian awesomeness --- bee-ledger/src/workers/pruning/condition.rs | 49 +++++++++------------ 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index a04843ecae..8c54862066 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -63,39 +63,30 @@ pub(crate) fn should_prune( if now < prev + pruning_config.db_size().cooldown_time() { Err(PruningSkipReason::Cooldown) } else { - let actual_size = { - if let Ok(size) = storage.size() { - size.ok_or(PruningSkipReason::SizeMetricUnavailable) - } else { - Err(PruningSkipReason::SizeMetricUnsupported) - } - }; - - match actual_size { - Ok(actual_size) => { + storage + .size() + .map_err(|_| PruningSkipReason::SizeMetricUnsupported) + .and_then(|a| { + let actual_size = a.ok_or(PruningSkipReason::SizeMetricUnavailable)?; let threshold_size = pruning_config.db_size().target_size(); log::debug!("Storage size: actual {actual_size} threshold {threshold_size}"); - if actual_size < threshold_size { - Err(PruningSkipReason::BelowTargetSizeThreshold) - } else { - // Panic: cannot underflow due to actual_size >= threshold_size. - let excess_size = actual_size - threshold_size; - let num_bytes_to_prune = excess_size - + (pruning_config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) - as usize; - - log::debug!("Num bytes to prune: {num_bytes_to_prune}"); - - // Store the time we issued a pruning-by-size. - PREVIOUS_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); - - Ok(PruningTask::ByDbSize { num_bytes_to_prune }) - } - } - Err(reason) => Err(reason), - } + let excess_size = actual_size + .checked_sub(threshold_size) + .ok_or(PruningSkipReason::BelowTargetSizeThreshold)?; + + let num_bytes_to_prune = excess_size + + (pruning_config.db_size().threshold_percentage() as f64 / 100.0 * threshold_size as f64) + as usize; + + log::debug!("Num bytes to prune: {num_bytes_to_prune}"); + + // Store the time we issued a pruning-by-size. + PREVIOUS_PRUNING_BY_SIZE.store(now.as_secs(), Ordering::Relaxed); + + Ok(PruningTask::ByDbSize { num_bytes_to_prune }) + }) } } else { Err(PruningSkipReason::Disabled) From 680540c684e3e3e3bbf7941a075f600168a51d3b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 6 May 2022 21:00:55 +0200 Subject: [PATCH 39/39] Rename param --- bee-ledger/src/workers/pruning/condition.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bee-ledger/src/workers/pruning/condition.rs b/bee-ledger/src/workers/pruning/condition.rs index 8c54862066..9ede5f9ba3 100644 --- a/bee-ledger/src/workers/pruning/condition.rs +++ b/bee-ledger/src/workers/pruning/condition.rs @@ -66,8 +66,8 @@ pub(crate) fn should_prune( storage .size() .map_err(|_| PruningSkipReason::SizeMetricUnsupported) - .and_then(|a| { - let actual_size = a.ok_or(PruningSkipReason::SizeMetricUnavailable)?; + .and_then(|actual_size| { + let actual_size = actual_size.ok_or(PruningSkipReason::SizeMetricUnavailable)?; let threshold_size = pruning_config.db_size().target_size(); log::debug!("Storage size: actual {actual_size} threshold {threshold_size}");