Merge branch 'development' into research/transaction-fee

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace.package]
 authors = [ "Analog Devs <https://github.com/Analog-Labs>" ]
 edition = "2021"
-version = "0.6.0"
+version = "0.7.0"
 homepage = "https://analog.one/"
 license = "GPL-3.0-only"
 readme = "README.md"
@@ -57,10 +57,10 @@ scale-decode = { version = "0.13.1", default-features = false, features = [ "der
 scale-info = { version = "2.11.3", default-features = false, features = [ "derive" ] }
 
 # main substrate sdk
-polkadot-sdk = { git = "https://github.com/Analog-Labs/polkadot-sdk", tag = "v1.14.0-anlog0", default-features = false }
+polkadot-sdk = { git = "https://github.com/Analog-Labs/polkadot-sdk", tag = "v1.15.1-anlog0", default-features = false }
 
 # specialized wasm builder
-substrate-wasm-builder = { git = "https://github.com/analog-labs/polkadot-sdk", tag = "v1.14.0-anlog0", features = [ "metadata-hash" ] }
+substrate-wasm-builder = { git = "https://github.com/analog-labs/polkadot-sdk", tag = "v1.15.1-anlog0", features = [ "metadata-hash" ] }
 
 # chain connectors
 rosetta-client = { git = "https://github.com/analog-labs/chain-connectors" }

flake.nix

@@ -59,7 +59,7 @@
         );
 
       # Create developer shell for combination of build and target package set
-      mkDevShell = pkgs: tpkgs: pkgs.mkShell {
+      mkDevShell = pkgs: tpkgs: tpkgs.mkShell {
         # Provide target platform to cargo via env var
         CARGO_BUILD_TARGET = tpkgs.rust.toRustTarget tpkgs.stdenv.targetPlatform;
 
@@ -68,8 +68,8 @@
           # - Customized rust toolchain (via path)
           (mkRustToolchain tpkgs)
           # - Some helpers to improve compatibility
-          tpkgs.pkg-config
-          tpkgs.rustPlatform.bindgenHook
+          pkgs.gcc
+          pkgs.pkg-config
         ];
 
         # - Protobuf compiler (via env var)

node/src/chains/internal.keys.json

@@ -121,7 +121,8 @@
   ],
   "controller": "anAFMd7NCCekftTGrkhv9oymmZ4biPrmbMGj5mtvfXqeqkSST",
   "councils": [
-    "an9CbHiXeALyA5pDGQNfJ8YKF5k8eW5YTHWRsn8ZBEH8tano9"
+    "an9CbHiXeALyA5pDGQNfJ8YKF5k8eW5YTHWRsn8ZBEH8tano9",
+    "an5jsJBqi2whFgKyZqBcgS7iY7qjvG1DGVuQsaVcGZaVu7uio"
   ],
   "endowments": [
     ["an5xJhYKVepTVKH3JqeRc126R4iAzV8As9DuXLW9EQkyzhgFf", 100000],

node/src/cli.rs

@@ -11,6 +11,14 @@ pub struct Cli {
 	#[clap(flatten)]
 	pub run: sc_cli::RunCmd,
 
+	/// Specify the staging chain.
+	///
+	/// This flag sets `--chain=sta`, `--force-authoring`, `--rpc-cors=all`,
+	/// `--alice`, and `--tmp` flags, unless explicitly overridden.
+	/// It also disables local peer discovery (see --no-mdns and --discover-local)
+	#[arg(long, conflicts_with_all = &["dev", "chain"])]
+	pub sta: bool,
+
 	/// Disable automatic hardware benchmarks.
 	///
 	/// By default these benchmarks are automatically ran at startup and measure

node/src/command.rs

@@ -86,7 +86,7 @@ impl SubstrateCli for Cli {
 
 #[allow(clippy::extra_unused_type_parameters)]
 /// Parse command line arguments into service configuration.
-pub fn run_with<Runtime, RuntimeApi>(cli: Cli) -> sc_cli::Result<()>
+pub fn run_with<Runtime, RuntimeApi>(mut cli: Cli) -> sc_cli::Result<()>
 where
 	Runtime: frame_system::Config + pallet_transaction_payment::Config + Send + Sync,
 	RuntimeApi: sp_api::ConstructRuntimeApi<Block, FullClient<RuntimeApi>> + Send + Sync + 'static,
@@ -102,6 +102,13 @@ where
 		+ frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Nonce>
 		+ pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance>,
 {
+	// Support for custom "--sta" flag
+	if cli.sta {
+		cli.run.shared_params.dev = true;
+		cli.run.shared_params.chain = Some("sta".to_string());
+	}
+
+	// Parse subcommand to determine what to run
 	match &cli.subcommand {
 		None => {
 			let runner = cli.create_runner(&cli.run)?;

pallets/elections/src/lib.rs

@@ -153,6 +153,20 @@ pub mod pallet {
 		ThresholdLargerThanSize,
 	}
 
+	#[pallet::hooks]
+	impl<T: Config> Hooks<BlockNumberFor<T>> for Pallet<T> {
+		fn on_initialize(_: BlockNumberFor<T>) -> Weight {
+			let mut weight = Weight::default();
+			for (network, _, _) in Unassigned::<T>::iter() {
+				weight = weight
+					// 1 Read of Unassigned per Loop
+					.saturating_add(T::DbWeight::get().reads(1))
+					.saturating_add(Self::try_elect_shard(network));
+			}
+			weight
+		}
+	}
+
 	/// The pallet provides mechanisms to configure and manage shards, elect members to shards,
 	/// and handle member events. It includes storage items to keep track of shard sizes,
 	/// thresholds, and electable members. It also defines events and errors related to shard
@@ -208,15 +222,13 @@ pub mod pallet {
 		///    1. Checks if the member is not already a shard member.
 		///    2. Checks if the member is electable or if there are no electable members defined.
 		///    3. Inserts the member into the [`Unassigned`] storage for the given network.
-		///    4. Attempts to elect a new shard for the network by calling `try_elect_shard`.
-		///    5. Notifies the `Shards` interface about the member coming online.
+		///    4. Notifies the `Shards` interface about the member coming online.
 		fn member_online(member: &AccountId, network: NetworkId) {
-			if !T::Shards::is_shard_member(member) {
-				if Electable::<T>::iter().next().is_none() || Electable::<T>::get(member).is_some()
-				{
-					Unassigned::<T>::insert(network, member, ());
-				}
-				Self::try_elect_shard(network);
+			if !T::Shards::is_shard_member(member)
+				&& (Electable::<T>::iter().next().is_none()
+					|| Electable::<T>::get(member).is_some())
+			{
+				Unassigned::<T>::insert(network, member, ());
 			}
 			T::Shards::member_online(member, network);
 		}
@@ -237,10 +249,8 @@ pub mod pallet {
 		///  Handles the event when a shard goes offline.
 		/// # Flow
 		///    1. Inserts each member of the offline shard into the [`Unassigned`] storage for the given network.
-		///    2. Attempts to elect a new shard for the network by calling `try_elect_shard`.
 		fn shard_offline(network: NetworkId, members: Vec<AccountId>) {
 			members.into_iter().for_each(|m| Unassigned::<T>::insert(network, m, ()));
-			Self::try_elect_shard(network);
 		}
 
 		///  Retrieves the default shard size.
@@ -257,10 +267,17 @@ pub mod pallet {
 		///    1. Calls `new_shard_members` to get a list of new shard members.
 		///    2. If a new shard can be formed, removes the selected members from [`Unassigned`] storage.
 		///    3. Creates a new shard using the `Shards` interface with the selected members and current shard threshold.
-		fn try_elect_shard(network: NetworkId) {
-			if let Some(members) = Self::new_shard_members(network) {
-				members.iter().for_each(|m| Unassigned::<T>::remove(network, m));
-				T::Shards::create_shard(network, members, ShardThreshold::<T>::get());
+		fn try_elect_shard(network: NetworkId) -> Weight {
+			match Self::new_shard_members(network) {
+				(Some(members), r) => {
+					members.iter().for_each(|m| Unassigned::<T>::remove(network, m));
+					let weight = T::DbWeight::get()
+						.reads_writes(r, members.len().try_into().unwrap_or_default());
+					T::Shards::create_shard(network, members, ShardThreshold::<T>::get())
+						.1
+						.saturating_add(weight)
+				},
+				(None, r) => T::DbWeight::get().reads(r),
 			}
 		}
 
@@ -271,17 +288,21 @@ pub mod pallet {
 		///    3. Returns `None` if there are not enough members to form a shard.
 		///    4. If there are just enough members, returns the list of members.
 		///    5. If there are more members than needed, sorts them by their stake and selects the top members to form the shard.
-		fn new_shard_members(network: NetworkId) -> Option<Vec<AccountId>> {
+		fn new_shard_members(network: NetworkId) -> (Option<Vec<AccountId>>, u64) {
+			let mut reads: u64 = 0;
 			let shard_members_len = ShardSize::<T>::get() as usize;
-			let mut members = Unassigned::<T>::iter_prefix(network)
-				.map(|(m, _)| m)
-				.filter(T::Members::is_member_online)
-				.collect::<Vec<_>>();
+			let mut members = Vec::new();
+			for (m, _) in Unassigned::<T>::iter_prefix(network) {
+				if T::Members::is_member_online(&m) {
+					members.push(m);
+				}
+				reads = reads.saturating_add(2);
+			}
 			if members.len() < shard_members_len {
-				return None;
+				return (None, reads);
 			}
 			if members.len() == shard_members_len {
-				return Some(members);
+				return (Some(members), reads);
 			}
 			// else members.len() > shard_members_len:
 			members.sort_unstable_by(|a, b| {
@@ -291,7 +312,7 @@ pub mod pallet {
 					.then_with(|| a.cmp(b))
 					.reverse()
 			});
-			Some(members.into_iter().take(shard_members_len).collect())
+			(Some(members.into_iter().take(shard_members_len).collect()), reads)
 		}
 	}
 }

pallets/elections/src/mock.rs

@@ -3,6 +3,7 @@ use crate::{self as pallet_elections};
 use polkadot_sdk::*;
 
 use frame_support::derive_impl;
+use frame_support::traits::OnInitialize;
 use sp_core::{ConstU128, ConstU64};
 use sp_runtime::{
 	traits::{IdentifyAccount, IdentityLookup, Verify},
@@ -142,3 +143,16 @@ pub fn new_test_ext() -> sp_io::TestExternalities {
 fn acc_pub(acc_num: u8) -> sp_core::sr25519::Public {
 	sp_core::sr25519::Public::from_raw([acc_num; 32])
 }
+
+pub fn roll(n: u64) {
+	for _ in 0..n {
+		next_block();
+	}
+}
+
+fn next_block() {
+	let mut now = System::block_number();
+	now += 1;
+	System::set_block_number(now);
+	Elections::on_initialize(now);
+}

pallets/elections/src/tests.rs

@@ -25,10 +25,13 @@ fn shard_size_new_members_online_creates_shard() {
 	let c: AccountId = [3u8; 32].into();
 	new_test_ext().execute_with(|| {
 		Elections::member_online(&a, ETHEREUM);
+		roll(1);
 		assert!(Unassigned::<Test>::get(ETHEREUM, &a).is_some());
 		Elections::member_online(&b, ETHEREUM);
+		roll(1);
 		assert!(Unassigned::<Test>::get(ETHEREUM, &b).is_some());
 		Elections::member_online(&c, ETHEREUM);
+		roll(1);
 		System::assert_last_event(pallet_shards::Event::<Test>::ShardCreated(0, ETHEREUM).into());
 		for member in [a, b, c] {
 			assert!(Unassigned::<Test>::get(ETHEREUM, member).is_none());
@@ -57,8 +60,10 @@ fn shard_offline_automatically_creates_new_shard() {
 		Elections::member_online(&a, ETHEREUM);
 		Elections::member_online(&b, ETHEREUM);
 		Elections::member_online(&c, ETHEREUM);
+		roll(1);
 		System::assert_last_event(pallet_shards::Event::<Test>::ShardCreated(0, ETHEREUM).into());
 		Elections::shard_offline(ETHEREUM, [a, b, c].to_vec());
+		roll(1);
 		System::assert_last_event(pallet_shards::Event::<Test>::ShardCreated(1, ETHEREUM).into());
 	});
 }

pallets/governance/src/lib.rs

@@ -6,7 +6,7 @@
 //! Currently only wraps a few important root call to lower the required privilege level
 //! to a custom origin.
 //!
-//! See [`Calls`] for a list of wrapped extrinsics.
+//! See [`Call`] for a list of wrapped extrinsics.
 
 pub use pallet::*;
 

pallets/shards/src/lib.rs

@@ -363,9 +363,7 @@ pub mod pallet {
 					reads += 1;
 				}
 			});
-			T::DbWeight::get()
-				.writes(writes)
-				.saturating_add(T::DbWeight::get().reads(reads))
+			T::DbWeight::get().reads_writes(reads, writes)
 		}
 	}
 
@@ -563,18 +561,33 @@ pub mod pallet {
 		///   6. Inserts each member into ShardMembers and associates them with [`MemberStatus::Added`].
 		///   7. Registers each member in `MemberShard` with the `shard_id`.
 		///   8. Emits a [`Event::ShardCreated`] event with the `shard_id` and network.
-		fn create_shard(network: NetworkId, members: Vec<AccountId>, threshold: u16) {
+		fn create_shard(
+			network: NetworkId,
+			members: Vec<AccountId>,
+			threshold: u16,
+		) -> (ShardId, Weight) {
+			let (mut reads, mut writes) = (0, 0);
 			let shard_id = <ShardIdCounter<T>>::get();
 			<ShardIdCounter<T>>::put(shard_id + 1);
 			<ShardNetwork<T>>::insert(shard_id, network);
 			<ShardState<T>>::insert(shard_id, ShardStatus::Created);
 			<DkgTimeout<T>>::insert(shard_id, frame_system::Pallet::<T>::block_number());
 			<ShardThreshold<T>>::insert(shard_id, threshold);
+			// ShardIdCounter, frame_system::Pallet::<T>::block_number()
+			reads = reads.saturating_add(2);
+			// ShardIdCounter, ShardNetwork, ShardState, DkgTimeout, ShardThreshold
+			writes = writes.saturating_add(5);
 			for member in &members {
 				ShardMembers::<T>::insert(shard_id, member, MemberStatus::Added);
 				MemberShard::<T>::insert(member, shard_id);
+				// ShardMembers, MemberShard
+				writes = writes.saturating_add(2);
 			}
 			Self::deposit_event(Event::ShardCreated(shard_id, network));
+			// Event Emission
+			writes = writes.saturating_plus_one();
+			let weight = T::DbWeight::get().reads_writes(reads, writes);
+			(shard_id, weight)
 		}
 		/// Retrieves the public key of the next signer for the specified shard, updating the signer index.
 		///

pallets/tasks/src/benchmarking.rs

@@ -1,4 +1,4 @@
-use crate::{Call, Config, Pallet, TaskShard, TaskSigner};
+use crate::{Call, Config, Pallet, ShardRegistered, TaskShard, TaskSigner};
 
 use polkadot_sdk::{
 	frame_benchmarking, frame_support, frame_system, pallet_balances, pallet_treasury, sp_core,
@@ -85,6 +85,7 @@ fn create_simple_task<T: Config + pallet_shards::Config>() -> Result<T::AccountI
 	);
 	ShardState::<T>::insert(0, ShardStatus::Online);
 	Pallet::<T>::shard_online(0, ETHEREUM);
+	ShardRegistered::<T>::insert(0, ());
 	let caller = whitelisted_caller();
 	pallet_balances::Pallet::<T>::resolve_creating(
 		&caller,

pallets/tasks/src/lib.rs

@@ -1301,6 +1301,10 @@ pub mod pallet {
 			let mut reads = 0;
 			let mut shards = Vec::new();
 			for (network, shard, _) in NetworkShards::<T>::iter() {
+				let is_registered = ShardRegistered::<T>::get(shard).is_some();
+				if !is_registered {
+					continue;
+				}
 				let task_limit =
 					ShardTaskLimit::<T>::get(network).unwrap_or(DEFAULT_SHARD_TASK_LIMIT) as usize;
 				let tasks = ShardTasks::<T>::iter_prefix(shard).count();
@@ -1334,20 +1338,11 @@ pub mod pallet {
 		///   8. Assign each task to the shard using `Self::assign_task(network, shard_id, index, task)`.
 		fn schedule_tasks_shard(network: NetworkId, shard_id: ShardId, capacity: usize) -> Weight {
 			let mut reads = 0;
-			let shard_size = T::Shards::shard_members(shard_id).len() as u16;
-			let is_registered = ShardRegistered::<T>::get(shard_id).is_some();
-			let system_tasks = Self::prioritized_unassigned_tasks(network).get_n(
-				capacity,
-				shard_size,
-				is_registered,
-			);
+			let system_tasks = Self::prioritized_unassigned_tasks(network).get_n(capacity);
 			let tasks = if let Some(non_system_capacity) = capacity.checked_sub(system_tasks.len())
 			{
-				let non_system_tasks = Self::remaining_unassigned_tasks(network).get_n(
-					non_system_capacity,
-					shard_size,
-					is_registered,
-				);
+				let non_system_tasks =
+					Self::remaining_unassigned_tasks(network).get_n(non_system_capacity);
 				// reads: remaining_unassigned_tasks
 				reads = reads.saturating_plus_one();
 				system_tasks.into_iter().chain(non_system_tasks).collect::<Vec<_>>()

pallets/tasks/src/mock.rs

@@ -110,6 +110,7 @@ thread_local! {
 }
 
 /// set status for a given payment id
+#[allow(dead_code)]
 fn set_status(id: u64, s: PaymentStatus) {
 	STATUS.with(|m| m.borrow_mut().insert(id, s));
 }