From 872e59d5f0b61603b4f5fa075e44dc62e8e0f2f3 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Fri, 10 Jan 2025 23:16:47 +0300 Subject: [PATCH 1/6] Add documentation for the Indexed Merkle Map API --- docs/zkapps/o1js/indexed-merkle-map.mdx | 263 ++++++++++++++++++++++++ sidebars.js | 5 +- 2 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 docs/zkapps/o1js/indexed-merkle-map.mdx diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx new file mode 100644 index 000000000..3a6fd5e5f --- /dev/null +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -0,0 +1,263 @@ +--- +title: Indexed Merkle Map +hide_title: true +description: + A comprehensive guide on how to use Indexed Merkle Map to reference off-chain data in zkApps on Mina. + - zkapp + - o1js + - merkle map + - merkle tree + - zkapps + - mina blockchain + - storage + - off-chain data + - blockchain technology + - data structures +--- + +:::experimental + +The Indexed Merkle Map API is currently an experimental feature. + +::: + +# Indexed Merkle Map + +Similar to a Merkle Tree, a Merkle Map allows referencing off-chain data by storing a single hash, also known as the root. + +A Merkle Map is a wrapper around a [Merkle Tree](/zkapps/o1js/merkle-tree). Both data structures are analogous, but instead of using an index to set a leaf in a tree, a Merkle Map uses a key in a map. + +## Design + +The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutorials/common-types-and-functions#merkle-map), offering enhanced efficiency and usability: + +- **Reduced Constraints:** Uses 4-8x fewer constraints than `MerkleMap`. +- **Support for Dummy Updates:** Handles dummy updates for keys like `0` and `-1`. +- **Provable Code Integration:** Unlike `MerkleTree` and `MerkleMap`, the high-level API of `IndexedMerkleMap` is usable within provable code. +- **Comprehensive Methods:** Comes with a rich set of methods that enable a wide range of operations. + +## Utilizing Indexed Merkle Map + +### Prerequisites + +The `IndexedMerkleMap` API is accessible within the `Experimental` namespace. To use the API, import `Experimental` from o1js version 1.5.0 or higher. + +```ts +import { Experimental } from 'o1js'; + +const { IndexedMerkleMap } = Experimental; +``` + +### Instantiating an Indexed Merkle Map + +Given a height, you can instantiate an Indexed Merkle Map by extending the base class. +The height determines the capacity of the map; the maximum number of leaf nodes it can contain. + +```ts +const height = 31; +class IndexedMerkleMap31 extends IndexedMerkleMap(height) {} +``` + +In this example, `IndexedMerkleMap31` is a Merkle map capable of holding up to 2(31−1) leaves; approximately 1 billion entries. + +### Utilizing IndexedMerkleMap in a smart contract + +Developers can integrate the `IndexedMerkleMap` into their zkApps by passing a Merkle map as a method parameter and invoking its methods as needed. + +This is possible because the `IndexedMerkleMap` can be used within provable code. + +```ts +class MyContract extends SmartContract { + @state(Field) mapRoot = State(); + + init() { + super.init(); + this.mapRoot.set(new IndexedMerkleMap31().root); + } + + @method async insertNewLeaf( + newKey: Field, + newValue: Field, + indexedMerkleMap: IndexedMerkleMap31 + ) { + // Validate the integrity of the Merkle Map input + const currentRoot = this.mapRoot.getAndRequireEquals(); + currentRoot.assertEquals( + indexedMerkleMap.root, + 'Off-chain Indexed Merkle Map is out of sync!' + ); + + // Insert a new key-value pair + indexedMerkleMap = indexedMerkleMap.clone(); + indexedMerkleMap.insert(newKey, newValue); + + // Update the on-chain map root + const newMapRoot = indexedMerkleMap.root; + this.mapRoot.set(newMapRoot); + } + + @method async updateExistingLeaf( + key: Field, + newValue: Field, + indexedMerkleMap: IndexedMerkleMap31 + ) { + // Validate integrity of the Merkle Map input + const currentRoot = this.mapRoot.getAndRequireEquals(); + currentRoot.assertEquals( + indexedMerkleMap.root, + 'Off-chain Indexed Merkle Map is out of sync!' + ); + + indexedMerkleMap = indexedMerkleMap.clone(); + + /** + * Proves that the key exists + * Updates an existing leaf + * Returns the previous value + */ + indexedMerkleMap.update(key, newValue); + + // Update the on-chain map root + const newMapRoot = indexedMerkleMap.root; + this.mapRoot.set(newMapRoot); + } +} +``` + +:warning: Direct modification of a method input can lead to errors. To avoid this, perform changes on a cloned Merkle Map rather than altering the map passed as a method input. + +### Interacting with a smart contract utilizing an indexed Merkle Map + +To interact with a zkapp that utilizes an `IndexedMerkleMap`, instantiate a map instance and pass it as an argument to the contract’s method within a Mina transaction. + +In addition, ensure you synchronize your off-chain map with on-chain updates to maintain data integrity and prepare for subsequent interactions. + +```ts +// Instantiate a new Indexed Merkle Map +indexedMerkleMap = new IndexedMerkleMap31(); + +const insertTx = await Mina.transaction(userPubKey, async () => { + await contract.insertNewLeaf(Field(1), Field(1234), indexedMerkleMap); +}); + +await insertTx.prove(); +await insertTx.sign([userKey]).send(); + +// Synchornize the off-chain Indexed Merkle Map to match the on-chain state root +indexedMerkleMap.insert(Field(1), Field(1234)); + +console.log( + indexedMerkleMap.root.toBigInt() === contract.mapRoot.get().toBigInt() +); +console.log(indexedMerkleMap.data.get().sortedLeaves); + +const updateTx = await Mina.transaction(userPubKey, async () => { + await contract.updateExistingLeaf(Field(1), Field(5678), indexedMerkleMap); +}); + +await updateTx.prove(); +await updateTx.sign([userKey]).send(); + +// Synchronize the off-chain Indexed Merkle Map to match the on-chain state root +indexedMerkleMap.update(Field(1), Field(5678)); + +console.log( + indexedMerkleMap.root.toBigInt() === contract.mapRoot.get().toBigInt() +); +console.log(indexedMerkleMap.data.get().sortedLeaves); +``` + +## Indexed Merkle Map - API reference + +The Indexed Merkle Map API provides a comprehensive set of methods that enable developers to perform a wide range of operations within provable code. + +```ts +/** + * Clone the entire Merkle map. + * + * This method is provable. + */ +clone(): IndexedMerkleMapBase; +/** + * Overwrite the entire Merkle map with another one. + * + * This method is provable. + */ +overwrite(other: IndexedMerkleMapBase): void; +/** + * Overwrite the entire Merkle map with another one, if the condition is true. + * + * This method is provable. + */ +overwriteIf(condition: Bool | boolean, other: IndexedMerkleMapBase): void; +/** + * Insert a new leaf `(key, value)`. + * + * Proves that `key` doesn't exist yet. + */ +insert(key: Field | bigint, value: Field | bigint): void; +/** + * Update an existing leaf `(key, value)`. + * + * Proves that the `key` exists. + * + * Returns the previous value. + */ +update(key: Field | bigint, value: Field | bigint): Field; +/** + * Perform _either_ an insertion or update, depending on whether the key exists. + * + * Note: This method is handling both the `insert()` and `update()` case at the same time, so you + * can use it if you don't know whether the key exists or not. + * + * However, this comes at an efficiency cost, so prefer to use `insert()` or `update()` if you know whether the key exists. + * + * Returns the previous value, as an option (which is `None` if the key didn't exist before). + */ +set(key: Field | bigint, value: Field | bigint): Option; +/** + * Perform an insertion or update, if the enabling condition is true. + * + * If the condition is false, we instead set the 0 key to the value 0. + * This is the initial value and for typical uses of `IndexedMerkleMap`, it is guaranteed to be a no-op because the 0 key is never used. + * + * **Warning**: Only use this method if you are sure that the 0 key is not used in your application. + * Otherwise, you might accidentally overwrite a valid key-value pair. + */ +setIf(condition: Bool | boolean, key: Field | bigint, value: Field | bigint): Option; +/** + * Get a value from a key. + * + * Proves that the key already exists in the map yet and fails otherwise. + */ +get(key: Field | bigint): Field; +/** + * Get a value from a key. + * + * Returns an option which is `None` if the key doesn't exist. (In that case, the option's value is unconstrained.) + * + * Note that this is more flexible than `get()` and allows you to handle the case where the key doesn't exist. + * However, it uses about twice as many constraints for that reason. + */ +getOption(key: Field | bigint): Option; +/** + * Prove that the given key exists in the map. + */ +assertIncluded(key: Field | bigint, message?: string): void; +/** + * Prove that the given key does not exist in the map. + */ +assertNotIncluded(key: Field | bigint, message?: string): void; +/** + * Check whether the given key exists in the map. + */ +isIncluded(key: Field | bigint): Bool; +``` + +## Additional Resources + +For more details and examples, please refer to the following GitHub resources: + +- [Indexed Merkle Tree: o1js PR#1666](https://github.com/o1-labs/o1js/pull/1666) +- [IndexedMerkleMap: Support 0 and -1 Keys: o1js PR#1671](https://github.com/o1-labs/o1js/pull/1671) +- [Mastermind zkApp Example Using Indexed Merkle Map](https://github.com/o1-labs-XT/mastermind-zkApp/tree/level3) diff --git a/sidebars.js b/sidebars.js index 72c2aa286..f56d09a22 100644 --- a/sidebars.js +++ b/sidebars.js @@ -75,6 +75,7 @@ module.exports = { 'zkapps/o1js/bitwise-operations', 'zkapps/o1js/foreign-fields', 'zkapps/o1js/merkle-tree', + 'zkapps/o1js/indexed-merkle-map', 'zkapps/o1js/keccak', 'zkapps/o1js/ecdsa', 'zkapps/o1js/sha256', @@ -150,9 +151,7 @@ module.exports = { type: 'doc', id: 'zkapps/front-end-integration-guides/angular', }, - items: [ - 'zkapps/front-end-integration-guides/angular', - ], + items: ['zkapps/front-end-integration-guides/angular'], }, { type: 'category', From aa74239bc34a1cd9d1343d393d9f02144372a269 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Tue, 14 Jan 2025 21:58:18 +0300 Subject: [PATCH 2/6] Add note about height --- docs/zkapps/o1js/indexed-merkle-map.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx index 3a6fd5e5f..de027c044 100644 --- a/docs/zkapps/o1js/indexed-merkle-map.mdx +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -36,6 +36,11 @@ The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutoria - **Provable Code Integration:** Unlike `MerkleTree` and `MerkleMap`, the high-level API of `IndexedMerkleMap` is usable within provable code. - **Comprehensive Methods:** Comes with a rich set of methods that enable a wide range of operations. +:::note + +The `Indexed Merkle Map` can have a height of at most `52`, whereas the `Merkle Map` has a larger height fixed at `256`. +::: + ## Utilizing Indexed Merkle Map ### Prerequisites From 19d30d15608b41e747b471e0b347e7a4d2c3a861 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Mon, 20 Jan 2025 19:08:56 +0300 Subject: [PATCH 3/6] Remove smart contract example --- docs/zkapps/o1js/indexed-merkle-map.mdx | 195 +----------------------- 1 file changed, 3 insertions(+), 192 deletions(-) diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx index de027c044..1ad9191ee 100644 --- a/docs/zkapps/o1js/indexed-merkle-map.mdx +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -39,6 +39,7 @@ The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutoria :::note The `Indexed Merkle Map` can have a height of at most `52`, whereas the `Merkle Map` has a larger height fixed at `256`. + ::: ## Utilizing Indexed Merkle Map @@ -65,199 +66,9 @@ class IndexedMerkleMap31 extends IndexedMerkleMap(height) {} In this example, `IndexedMerkleMap31` is a Merkle map capable of holding up to 2(31−1) leaves; approximately 1 billion entries. -### Utilizing IndexedMerkleMap in a smart contract - -Developers can integrate the `IndexedMerkleMap` into their zkApps by passing a Merkle map as a method parameter and invoking its methods as needed. - -This is possible because the `IndexedMerkleMap` can be used within provable code. - -```ts -class MyContract extends SmartContract { - @state(Field) mapRoot = State(); - - init() { - super.init(); - this.mapRoot.set(new IndexedMerkleMap31().root); - } - - @method async insertNewLeaf( - newKey: Field, - newValue: Field, - indexedMerkleMap: IndexedMerkleMap31 - ) { - // Validate the integrity of the Merkle Map input - const currentRoot = this.mapRoot.getAndRequireEquals(); - currentRoot.assertEquals( - indexedMerkleMap.root, - 'Off-chain Indexed Merkle Map is out of sync!' - ); - - // Insert a new key-value pair - indexedMerkleMap = indexedMerkleMap.clone(); - indexedMerkleMap.insert(newKey, newValue); - - // Update the on-chain map root - const newMapRoot = indexedMerkleMap.root; - this.mapRoot.set(newMapRoot); - } - - @method async updateExistingLeaf( - key: Field, - newValue: Field, - indexedMerkleMap: IndexedMerkleMap31 - ) { - // Validate integrity of the Merkle Map input - const currentRoot = this.mapRoot.getAndRequireEquals(); - currentRoot.assertEquals( - indexedMerkleMap.root, - 'Off-chain Indexed Merkle Map is out of sync!' - ); - - indexedMerkleMap = indexedMerkleMap.clone(); - - /** - * Proves that the key exists - * Updates an existing leaf - * Returns the previous value - */ - indexedMerkleMap.update(key, newValue); - - // Update the on-chain map root - const newMapRoot = indexedMerkleMap.root; - this.mapRoot.set(newMapRoot); - } -} -``` - -:warning: Direct modification of a method input can lead to errors. To avoid this, perform changes on a cloned Merkle Map rather than altering the map passed as a method input. - -### Interacting with a smart contract utilizing an indexed Merkle Map - -To interact with a zkapp that utilizes an `IndexedMerkleMap`, instantiate a map instance and pass it as an argument to the contract’s method within a Mina transaction. +### Indexed Merkle Map - API reference -In addition, ensure you synchronize your off-chain map with on-chain updates to maintain data integrity and prepare for subsequent interactions. - -```ts -// Instantiate a new Indexed Merkle Map -indexedMerkleMap = new IndexedMerkleMap31(); - -const insertTx = await Mina.transaction(userPubKey, async () => { - await contract.insertNewLeaf(Field(1), Field(1234), indexedMerkleMap); -}); - -await insertTx.prove(); -await insertTx.sign([userKey]).send(); - -// Synchornize the off-chain Indexed Merkle Map to match the on-chain state root -indexedMerkleMap.insert(Field(1), Field(1234)); - -console.log( - indexedMerkleMap.root.toBigInt() === contract.mapRoot.get().toBigInt() -); -console.log(indexedMerkleMap.data.get().sortedLeaves); - -const updateTx = await Mina.transaction(userPubKey, async () => { - await contract.updateExistingLeaf(Field(1), Field(5678), indexedMerkleMap); -}); - -await updateTx.prove(); -await updateTx.sign([userKey]).send(); - -// Synchronize the off-chain Indexed Merkle Map to match the on-chain state root -indexedMerkleMap.update(Field(1), Field(5678)); - -console.log( - indexedMerkleMap.root.toBigInt() === contract.mapRoot.get().toBigInt() -); -console.log(indexedMerkleMap.data.get().sortedLeaves); -``` - -## Indexed Merkle Map - API reference - -The Indexed Merkle Map API provides a comprehensive set of methods that enable developers to perform a wide range of operations within provable code. - -```ts -/** - * Clone the entire Merkle map. - * - * This method is provable. - */ -clone(): IndexedMerkleMapBase; -/** - * Overwrite the entire Merkle map with another one. - * - * This method is provable. - */ -overwrite(other: IndexedMerkleMapBase): void; -/** - * Overwrite the entire Merkle map with another one, if the condition is true. - * - * This method is provable. - */ -overwriteIf(condition: Bool | boolean, other: IndexedMerkleMapBase): void; -/** - * Insert a new leaf `(key, value)`. - * - * Proves that `key` doesn't exist yet. - */ -insert(key: Field | bigint, value: Field | bigint): void; -/** - * Update an existing leaf `(key, value)`. - * - * Proves that the `key` exists. - * - * Returns the previous value. - */ -update(key: Field | bigint, value: Field | bigint): Field; -/** - * Perform _either_ an insertion or update, depending on whether the key exists. - * - * Note: This method is handling both the `insert()` and `update()` case at the same time, so you - * can use it if you don't know whether the key exists or not. - * - * However, this comes at an efficiency cost, so prefer to use `insert()` or `update()` if you know whether the key exists. - * - * Returns the previous value, as an option (which is `None` if the key didn't exist before). - */ -set(key: Field | bigint, value: Field | bigint): Option; -/** - * Perform an insertion or update, if the enabling condition is true. - * - * If the condition is false, we instead set the 0 key to the value 0. - * This is the initial value and for typical uses of `IndexedMerkleMap`, it is guaranteed to be a no-op because the 0 key is never used. - * - * **Warning**: Only use this method if you are sure that the 0 key is not used in your application. - * Otherwise, you might accidentally overwrite a valid key-value pair. - */ -setIf(condition: Bool | boolean, key: Field | bigint, value: Field | bigint): Option; -/** - * Get a value from a key. - * - * Proves that the key already exists in the map yet and fails otherwise. - */ -get(key: Field | bigint): Field; -/** - * Get a value from a key. - * - * Returns an option which is `None` if the key doesn't exist. (In that case, the option's value is unconstrained.) - * - * Note that this is more flexible than `get()` and allows you to handle the case where the key doesn't exist. - * However, it uses about twice as many constraints for that reason. - */ -getOption(key: Field | bigint): Option; -/** - * Prove that the given key exists in the map. - */ -assertIncluded(key: Field | bigint, message?: string): void; -/** - * Prove that the given key does not exist in the map. - */ -assertNotIncluded(key: Field | bigint, message?: string): void; -/** - * Check whether the given key exists in the map. - */ -isIncluded(key: Field | bigint): Bool; -``` +For an example, see the `IndexedMerkleMap` [API reference](/zkapps/o1js-reference/namespaces/Experimental/functions/IndexedMerkleMap) in o1js. ## Additional Resources From a2a84c096244c0f1098a3135e0e293b2f4b84777 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Tue, 21 Jan 2025 21:05:43 +0300 Subject: [PATCH 4/6] Remove methods line from the Design section --- docs/zkapps/o1js/indexed-merkle-map.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx index 1ad9191ee..3ebee109f 100644 --- a/docs/zkapps/o1js/indexed-merkle-map.mdx +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -34,7 +34,6 @@ The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutoria - **Reduced Constraints:** Uses 4-8x fewer constraints than `MerkleMap`. - **Support for Dummy Updates:** Handles dummy updates for keys like `0` and `-1`. - **Provable Code Integration:** Unlike `MerkleTree` and `MerkleMap`, the high-level API of `IndexedMerkleMap` is usable within provable code. -- **Comprehensive Methods:** Comes with a rich set of methods that enable a wide range of operations. :::note From 7a6b989d39a1ec8530c413a1630fccd07a6eb643 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Tue, 21 Jan 2025 21:31:16 +0300 Subject: [PATCH 5/6] Elaborate on dummy updates in the Design section --- docs/zkapps/o1js/indexed-merkle-map.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx index 3ebee109f..193b0e35b 100644 --- a/docs/zkapps/o1js/indexed-merkle-map.mdx +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -32,7 +32,7 @@ A Merkle Map is a wrapper around a [Merkle Tree](/zkapps/o1js/merkle-tree). Both The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutorials/common-types-and-functions#merkle-map), offering enhanced efficiency and usability: - **Reduced Constraints:** Uses 4-8x fewer constraints than `MerkleMap`. -- **Support for Dummy Updates:** Handles dummy updates for keys like `0` and `-1`. +- **Support for Dummy Updates:** Handles updates that don't affect the root for keys like `0` and `-1`. - **Provable Code Integration:** Unlike `MerkleTree` and `MerkleMap`, the high-level API of `IndexedMerkleMap` is usable within provable code. :::note From ffef25e6ea9bc18272eae92538bd4097a00f8514 Mon Sep 17 00:00:00 2001 From: Shigoto-dev19 Date: Wed, 22 Jan 2025 23:25:45 +0300 Subject: [PATCH 6/6] Remove dummy updates line from the Design section --- docs/zkapps/o1js/indexed-merkle-map.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zkapps/o1js/indexed-merkle-map.mdx b/docs/zkapps/o1js/indexed-merkle-map.mdx index 193b0e35b..1e27fc587 100644 --- a/docs/zkapps/o1js/indexed-merkle-map.mdx +++ b/docs/zkapps/o1js/indexed-merkle-map.mdx @@ -32,7 +32,6 @@ A Merkle Map is a wrapper around a [Merkle Tree](/zkapps/o1js/merkle-tree). Both The Indexed Merkle Map is an improved version of the [MerkleMap](/zkapps/tutorials/common-types-and-functions#merkle-map), offering enhanced efficiency and usability: - **Reduced Constraints:** Uses 4-8x fewer constraints than `MerkleMap`. -- **Support for Dummy Updates:** Handles updates that don't affect the root for keys like `0` and `-1`. - **Provable Code Integration:** Unlike `MerkleTree` and `MerkleMap`, the high-level API of `IndexedMerkleMap` is usable within provable code. :::note