From 1fbebe9397a039ca7a29460ced064660b1d53bb3 Mon Sep 17 00:00:00 2001 From: nuts-rice Date: Tue, 7 Jan 2025 09:00:10 -0500 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=94=A7=20feat:=20binary=20db=20config?= =?UTF-8?q?=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/src/commands/new.rs | 1 + core/src/database/postgres/setup.rs | 3 +++ core/src/manifest/storage.rs | 13 +++++++++++++ 3 files changed, 17 insertions(+) diff --git a/cli/src/commands/new.rs b/cli/src/commands/new.rs index 5b32f8dd..8245a1cc 100644 --- a/cli/src/commands/new.rs +++ b/cli/src/commands/new.rs @@ -170,6 +170,7 @@ pub fn handle_new_command( postgres: if postgres_enabled { Some(PostgresDetails { enabled: true, + binary_storage: None, drop_each_run: None, relationships: None, indexes: None, diff --git a/core/src/database/postgres/setup.rs b/core/src/database/postgres/setup.rs index 5226d3a4..6e2fe338 100644 --- a/core/src/database/postgres/setup.rs +++ b/core/src/database/postgres/setup.rs @@ -41,6 +41,9 @@ pub async fn setup_postgres( client.batch_execute(sql.as_str()).await?; info!("Dropped all data for {}", manifest.name); } + if manifest.storage.binary_storage() { + info!("storing data in bytes") + } if !disable_event_tables { info!("Creating tables for {}", manifest.name); diff --git a/core/src/manifest/storage.rs b/core/src/manifest/storage.rs index 7138188e..f7f12d92 100644 --- a/core/src/manifest/storage.rs +++ b/core/src/manifest/storage.rs @@ -79,6 +79,9 @@ pub struct PostgresDetails { #[serde(default, skip_serializing_if = "Option::is_none")] pub drop_each_run: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub binary_storage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub relationships: Option>, @@ -147,6 +150,16 @@ impl Storage { .map_or(false, |details| details.disable_create_tables.unwrap_or_default()) } + pub fn binary_storage(&self) -> bool { + let enabled = self.postgres_enabled(); + if !enabled { + return false; + } + self.postgres + .as_ref() + .map_or(false, |details| details.binary_storage.unwrap_or_default()) + } + pub fn postgres_drop_each_run(&self) -> bool { let enabled = self.postgres_enabled(); if !enabled { From 32f7731990d1e28bfdc80afd1386edf529b59b56 Mon Sep 17 00:00:00 2001 From: nuts-rice Date: Tue, 7 Jan 2025 11:09:44 -0500 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20binary=5Fmode=20check?= =?UTF-8?q?=20+=20wrapping,=20fmt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/database/postgres/sql_type_wrapper.rs | 44 +++++++++++++++++++ core/src/manifest/storage.rs | 4 +- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/core/src/database/postgres/sql_type_wrapper.rs b/core/src/database/postgres/sql_type_wrapper.rs index 0711b851..12004a5d 100644 --- a/core/src/database/postgres/sql_type_wrapper.rs +++ b/core/src/database/postgres/sql_type_wrapper.rs @@ -944,6 +944,7 @@ pub fn solidity_type_to_ethereum_sql_type_wrapper( pub fn map_log_params_to_ethereum_wrapper( abi_inputs: &[ABIInput], params: &[LogParam], + binary_mode: bool, ) -> Vec { let mut wrappers = vec![]; @@ -951,6 +952,24 @@ pub fn map_log_params_to_ethereum_wrapper( if let Some(abi_input) = abi_inputs.get(index) { match ¶m.value { Token::Tuple(tuple) => { + if binary_mode { + wrappers.extend(process_tuple( + abi_input + .components + .as_ref() + .expect("tuple should have a component ABI on"), + tuple, + )); + map_ethereum_wrapper_to_byte( + abi_input + .components + .as_ref() + .expect("tuple should have component ABI on"), + &wrappers, + true, + ); + todo!() + } wrappers.extend(process_tuple( abi_input .components @@ -960,6 +979,20 @@ pub fn map_log_params_to_ethereum_wrapper( )); } _ => { + if binary_mode { + wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, ¶m.value)); + map_ethereum_wrapper_to_byte( + abi_input + .components + .as_ref() + .expect("value should have component ABI on"), + &wrappers, + false, + ); + + todo!() + } + wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, ¶m.value)); } } @@ -971,6 +1004,17 @@ pub fn map_log_params_to_ethereum_wrapper( wrappers } +pub fn map_ethereum_wrapper_to_byte( + abi_inputs: &[ABIInput], + wrapper: &[EthereumSqlTypeWrapper], + // transaction_information: &TxInformation, + is_within_tuple: bool, +) -> Bytes { + let mut buf = bytes::BytesMut::new(); + for abi_input in abi_inputs.iter() {} + unimplemented!() +} + fn process_tuple(abi_inputs: &[ABIInput], tokens: &[Token]) -> Vec { let mut wrappers = vec![]; diff --git a/core/src/manifest/storage.rs b/core/src/manifest/storage.rs index f7f12d92..379a559c 100644 --- a/core/src/manifest/storage.rs +++ b/core/src/manifest/storage.rs @@ -155,9 +155,7 @@ impl Storage { if !enabled { return false; } - self.postgres - .as_ref() - .map_or(false, |details| details.binary_storage.unwrap_or_default()) + self.postgres.as_ref().map_or(false, |details| details.binary_storage.unwrap_or_default()) } pub fn postgres_drop_each_run(&self) -> bool { From 8dceec61bed6252d0d8dbbfac7cb4cf7bf8d4213 Mon Sep 17 00:00:00 2001 From: nuts-rice Date: Tue, 7 Jan 2025 16:00:42 -0500 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20map=20types=20to=20by?= =?UTF-8?q?tes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/database/postgres/sql_type_wrapper.rs | 160 +++++++++++++++++- 1 file changed, 155 insertions(+), 5 deletions(-) diff --git a/core/src/database/postgres/sql_type_wrapper.rs b/core/src/database/postgres/sql_type_wrapper.rs index 12004a5d..ac147a0f 100644 --- a/core/src/database/postgres/sql_type_wrapper.rs +++ b/core/src/database/postgres/sql_type_wrapper.rs @@ -1,5 +1,6 @@ use std::str::FromStr; +use async_std::io::ReadExt; use bytes::{BufMut, BytesMut}; use chrono::{DateTime, Utc}; use ethers::{ @@ -1006,13 +1007,162 @@ pub fn map_log_params_to_ethereum_wrapper( pub fn map_ethereum_wrapper_to_byte( abi_inputs: &[ABIInput], - wrapper: &[EthereumSqlTypeWrapper], + wrappers: &[EthereumSqlTypeWrapper], // transaction_information: &TxInformation, is_within_tuple: bool, -) -> Bytes { - let mut buf = bytes::BytesMut::new(); - for abi_input in abi_inputs.iter() {} - unimplemented!() +) -> bytes::Bytes { + let mut buf = vec![]; + let mut current_wrapper_index = 0; + let mut wrappers_index_processed = Vec::new(); + + for abi_input in abi_inputs.iter() { + // tuples will take in multiple wrapper indexes, so we need to skip them if processed + if wrappers_index_processed.contains(¤t_wrapper_index) { + continue; + } + if let Some(wrapper) = wrappers.get(current_wrapper_index) { + if abi_input.type_ == "tuple" { + let components = + abi_input.components.as_ref().expect("Tuple should have components defined"); + let total_properties = count_components(components); + let tuple_value = map_ethereum_wrapper_to_byte( + components, + &wrappers[current_wrapper_index..total_properties], + // transaction_information, + true, + ); + buf.extend_from_slice(&abi_input.name.clone().into_bytes()); + buf.extend_from_slice(&tuple_value); + for i in current_wrapper_index..total_properties { + wrappers_index_processed.push(i); + } + current_wrapper_index = total_properties; + } else { + let byte = match wrapper { + EthereumSqlTypeWrapper::U64(u) => { + let mut bytes = vec![]; + u.to_big_endian(&mut bytes); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecU64(u64s) => { + let mut bytes = vec![]; + for u in u64s { + u.to_big_endian(&mut bytes); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I64(i) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&i.to_be_bytes()); + let bytes = bytes::Bytes::from(bytes); + bytes + } + _ => todo!(), + // EthereumSqlTypeWrapper::VecI64(i64s) => (i64s.iter().map(|i| + // i.to_be_bytes()).collect::>()), + // EthereumSqlTypeWrapper::U128(u) => (u.to_be_bytes()), + // EthereumSqlTypeWrapper::VecU128(u128s) => { + // json!(u128s.iter().map(|u| u.to_string()).collect::>()) + // } + // EthereumSqlTypeWrapper::I128(i) => (i.to_be_bytes()), + // EthereumSqlTypeWrapper::VecI128(i128s) => { + // json!(i128s.iter().map(|i| i.to_string()).collect::>()) + // } + // EthereumSqlTypeWrapper::U256(u) | + // EthereumSqlTypeWrapper::U256Bytes(u) | + // EthereumSqlTypeWrapper::U256Nullable(u) | + // EthereumSqlTypeWrapper::U256BytesNullable(u) => { + // (u.low_u64().to_be_bytes()) + // } + // EthereumSqlTypeWrapper::VecU256(u256s) | + // EthereumSqlTypeWrapper::VecU256Bytes(u256s) => { + // json!(u256s.iter().map(|u| u).collect::>()) + // } + // EthereumSqlTypeWrapper::I256(i) | + // EthereumSqlTypeWrapper::I256Bytes(i) | + // EthereumSqlTypeWrapper::I256Nullable(i) | + // EthereumSqlTypeWrapper::I256BytesNullable(i) => { + // json!(i.to_string()) + // } + // EthereumSqlTypeWrapper::VecI256(i256s) | + // EthereumSqlTypeWrapper::VecI256Bytes(i256s) => { + // json!(i256s.iter().map(|i| i.to_string()).collect::>()) + // } + // EthereumSqlTypeWrapper::U512(u) => json!(u.to_string()), + // EthereumSqlTypeWrapper::VecU512(u512s) => { + // json!(u512s.iter().map(|u| u.to_string()).collect::>()) + // } + // EthereumSqlTypeWrapper::H128(h) => json!(h), + // EthereumSqlTypeWrapper::VecH128(h128s) => json!(h128s), + // EthereumSqlTypeWrapper::H160(h) => json!(h), + // EthereumSqlTypeWrapper::VecH160(h160s) => json!(h160s), + // EthereumSqlTypeWrapper::H256(h) | EthereumSqlTypeWrapper::H256Bytes(h) => { + // json!(h) + // } + // EthereumSqlTypeWrapper::VecH256(h256s) | + // EthereumSqlTypeWrapper::VecH256Bytes(h256s) => json!(h256s), + // EthereumSqlTypeWrapper::H512(h) => json!(h), + // EthereumSqlTypeWrapper::VecH512(h512s) => json!(h512s), + // EthereumSqlTypeWrapper::Address(address) | + // EthereumSqlTypeWrapper::AddressBytes(address) | + // EthereumSqlTypeWrapper::AddressBytesNullable(address) | + // EthereumSqlTypeWrapper::AddressNullable(address) => json!(address), + // EthereumSqlTypeWrapper::VecAddress(addresses) | + // EthereumSqlTypeWrapper::VecAddressBytes(addresses) => json!(addresses), + // EthereumSqlTypeWrapper::Bool(b) => json!(b), + // EthereumSqlTypeWrapper::VecBool(bools) => json!(bools), + // EthereumSqlTypeWrapper::U32(u) => json!(u), + // EthereumSqlTypeWrapper::VecU32(u32s) => json!(u32s), + // EthereumSqlTypeWrapper::I32(i) => json!(i), + // EthereumSqlTypeWrapper::VecI32(i32s) => json!(i32s), + // EthereumSqlTypeWrapper::U16(u) => json!(u), + // EthereumSqlTypeWrapper::VecU16(u16s) => json!(u16s), + // EthereumSqlTypeWrapper::I16(i) => json!(i), + // EthereumSqlTypeWrapper::VecI16(i16s) => json!(i16s), + // EthereumSqlTypeWrapper::U8(u) => json!(u), + // EthereumSqlTypeWrapper::VecU8(u8s) => json!(u8s), + // EthereumSqlTypeWrapper::I8(i) => json!(i), + // EthereumSqlTypeWrapper::VecI8(i8s) => json!(i8s), + // EthereumSqlTypeWrapper::String(s) | + // EthereumSqlTypeWrapper::StringNullable(s) | + // EthereumSqlTypeWrapper::StringVarchar(s) | + // EthereumSqlTypeWrapper::StringVarcharNullable(s) | + // EthereumSqlTypeWrapper::StringChar(s) | + // EthereumSqlTypeWrapper::StringCharNullable(s) => json!(s), + // EthereumSqlTypeWrapper::VecString(strings) | + // EthereumSqlTypeWrapper::VecStringVarchar(strings) | + // EthereumSqlTypeWrapper::VecStringChar(strings) => json!(strings), + // EthereumSqlTypeWrapper::Bytes(bytes) | + // EthereumSqlTypeWrapper::BytesNullable(bytes) => json!(hex::encode(bytes)), + // EthereumSqlTypeWrapper::VecBytes(bytes) => { + // json!(bytes.iter().map(hex::encode).collect::>()) + // } + // EthereumSqlTypeWrapper::DateTime(date_time) => { + // json!(date_time.to_rfc3339()) + // } + // EthereumSqlTypeWrapper::JSONB(json) => json.clone(), + }; + buf.extend_from_slice(&abi_input.name.clone().into_bytes()); + buf.extend_from_slice(&byte); + + wrappers_index_processed.push(current_wrapper_index); + current_wrapper_index += 1; + } + } else { + panic!("No wrapper found for ABI input at index: {}", current_wrapper_index) + } + } + // if !is_within_tuple { + // let mut bytes: Vec = vec![]; + // buf.extend_from_slice("transaction_information".to_bytes()); + // buf.extend_from_slice(transaction_information.transaction_hash.as_bytes()); + // let bytes = bytes::Bytes::from(buf); + // return bytes + + // } + bytes::Bytes::from(buf) } fn process_tuple(abi_inputs: &[ABIInput], tokens: &[Token]) -> Vec { From 6d9fe736a94c98e886cbf502077e54e7c4949a00 Mon Sep 17 00:00:00 2001 From: nuts-rice Date: Tue, 7 Jan 2025 17:10:53 -0500 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20feat:=20map=20ether?= =?UTF-8?q?eum=20wrapper=20type=20to=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/database/postgres/sql_type_wrapper.rs | 375 ++++++++++++++---- 1 file changed, 291 insertions(+), 84 deletions(-) diff --git a/core/src/database/postgres/sql_type_wrapper.rs b/core/src/database/postgres/sql_type_wrapper.rs index ac147a0f..55c86b3e 100644 --- a/core/src/database/postgres/sql_type_wrapper.rs +++ b/core/src/database/postgres/sql_type_wrapper.rs @@ -1059,90 +1059,297 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(bytes); bytes } - _ => todo!(), - // EthereumSqlTypeWrapper::VecI64(i64s) => (i64s.iter().map(|i| - // i.to_be_bytes()).collect::>()), - // EthereumSqlTypeWrapper::U128(u) => (u.to_be_bytes()), - // EthereumSqlTypeWrapper::VecU128(u128s) => { - // json!(u128s.iter().map(|u| u.to_string()).collect::>()) - // } - // EthereumSqlTypeWrapper::I128(i) => (i.to_be_bytes()), - // EthereumSqlTypeWrapper::VecI128(i128s) => { - // json!(i128s.iter().map(|i| i.to_string()).collect::>()) - // } - // EthereumSqlTypeWrapper::U256(u) | - // EthereumSqlTypeWrapper::U256Bytes(u) | - // EthereumSqlTypeWrapper::U256Nullable(u) | - // EthereumSqlTypeWrapper::U256BytesNullable(u) => { - // (u.low_u64().to_be_bytes()) - // } - // EthereumSqlTypeWrapper::VecU256(u256s) | - // EthereumSqlTypeWrapper::VecU256Bytes(u256s) => { - // json!(u256s.iter().map(|u| u).collect::>()) - // } - // EthereumSqlTypeWrapper::I256(i) | - // EthereumSqlTypeWrapper::I256Bytes(i) | - // EthereumSqlTypeWrapper::I256Nullable(i) | - // EthereumSqlTypeWrapper::I256BytesNullable(i) => { - // json!(i.to_string()) - // } - // EthereumSqlTypeWrapper::VecI256(i256s) | - // EthereumSqlTypeWrapper::VecI256Bytes(i256s) => { - // json!(i256s.iter().map(|i| i.to_string()).collect::>()) - // } - // EthereumSqlTypeWrapper::U512(u) => json!(u.to_string()), - // EthereumSqlTypeWrapper::VecU512(u512s) => { - // json!(u512s.iter().map(|u| u.to_string()).collect::>()) - // } - // EthereumSqlTypeWrapper::H128(h) => json!(h), - // EthereumSqlTypeWrapper::VecH128(h128s) => json!(h128s), - // EthereumSqlTypeWrapper::H160(h) => json!(h), - // EthereumSqlTypeWrapper::VecH160(h160s) => json!(h160s), - // EthereumSqlTypeWrapper::H256(h) | EthereumSqlTypeWrapper::H256Bytes(h) => { - // json!(h) - // } - // EthereumSqlTypeWrapper::VecH256(h256s) | - // EthereumSqlTypeWrapper::VecH256Bytes(h256s) => json!(h256s), - // EthereumSqlTypeWrapper::H512(h) => json!(h), - // EthereumSqlTypeWrapper::VecH512(h512s) => json!(h512s), - // EthereumSqlTypeWrapper::Address(address) | - // EthereumSqlTypeWrapper::AddressBytes(address) | - // EthereumSqlTypeWrapper::AddressBytesNullable(address) | - // EthereumSqlTypeWrapper::AddressNullable(address) => json!(address), - // EthereumSqlTypeWrapper::VecAddress(addresses) | - // EthereumSqlTypeWrapper::VecAddressBytes(addresses) => json!(addresses), - // EthereumSqlTypeWrapper::Bool(b) => json!(b), - // EthereumSqlTypeWrapper::VecBool(bools) => json!(bools), - // EthereumSqlTypeWrapper::U32(u) => json!(u), - // EthereumSqlTypeWrapper::VecU32(u32s) => json!(u32s), - // EthereumSqlTypeWrapper::I32(i) => json!(i), - // EthereumSqlTypeWrapper::VecI32(i32s) => json!(i32s), - // EthereumSqlTypeWrapper::U16(u) => json!(u), - // EthereumSqlTypeWrapper::VecU16(u16s) => json!(u16s), - // EthereumSqlTypeWrapper::I16(i) => json!(i), - // EthereumSqlTypeWrapper::VecI16(i16s) => json!(i16s), - // EthereumSqlTypeWrapper::U8(u) => json!(u), - // EthereumSqlTypeWrapper::VecU8(u8s) => json!(u8s), - // EthereumSqlTypeWrapper::I8(i) => json!(i), - // EthereumSqlTypeWrapper::VecI8(i8s) => json!(i8s), - // EthereumSqlTypeWrapper::String(s) | - // EthereumSqlTypeWrapper::StringNullable(s) | - // EthereumSqlTypeWrapper::StringVarchar(s) | - // EthereumSqlTypeWrapper::StringVarcharNullable(s) | - // EthereumSqlTypeWrapper::StringChar(s) | - // EthereumSqlTypeWrapper::StringCharNullable(s) => json!(s), - // EthereumSqlTypeWrapper::VecString(strings) | - // EthereumSqlTypeWrapper::VecStringVarchar(strings) | - // EthereumSqlTypeWrapper::VecStringChar(strings) => json!(strings), - // EthereumSqlTypeWrapper::Bytes(bytes) | - // EthereumSqlTypeWrapper::BytesNullable(bytes) => json!(hex::encode(bytes)), - // EthereumSqlTypeWrapper::VecBytes(bytes) => { - // json!(bytes.iter().map(hex::encode).collect::>()) - // } - // EthereumSqlTypeWrapper::DateTime(date_time) => { - // json!(date_time.to_rfc3339()) - // } - // EthereumSqlTypeWrapper::JSONB(json) => json.clone(), + EthereumSqlTypeWrapper::VecI64(i64s) => { + let mut bytes = vec![]; + for i in i64s { + bytes.extend_from_slice(&i.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U128(u) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&u.to_be_bytes()); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecU128(u128s) => { + let mut bytes = vec![]; + for u in u128s { + bytes.extend_from_slice(&u.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I128(i) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&i.to_be_bytes()); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecI128(i128s) => { + let mut bytes = vec![]; + for i in i128s { + bytes.extend_from_slice(&i.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U256(u) | + EthereumSqlTypeWrapper::U256Bytes(u) | + EthereumSqlTypeWrapper::U256Nullable(u) | + EthereumSqlTypeWrapper::U256BytesNullable(u) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&u.low_u64().to_be_bytes()); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecU256(u256s) | + EthereumSqlTypeWrapper::VecU256Bytes(u256s) => { + let mut bytes = vec![]; + for u in u256s { + bytes.extend_from_slice(&u.low_u64().to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I256(i) | + EthereumSqlTypeWrapper::I256Bytes(i) | + EthereumSqlTypeWrapper::I256Nullable(i) | + EthereumSqlTypeWrapper::I256BytesNullable(i) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&i.low_u64().to_be_bytes()); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecI256(i256s) | + EthereumSqlTypeWrapper::VecI256Bytes(i256s) => { + let mut bytes = vec![]; + for i in i256s { + bytes.extend_from_slice(&i.low_u64().to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U512(u) => { + let mut bytes = vec![]; + u.to_big_endian(&mut bytes); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecU512(u512s) => { + let mut bytes = vec![]; + for u in u512s { + u.to_big_endian(&mut bytes); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::H128(h) => { + let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecH128(h128s) => { + let mut bytes = vec![]; + for h in h128s { + bytes.extend_from_slice(h.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::H160(h) => { + let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecH160(h160s) => { + let mut bytes = vec![]; + for h in h160s { + bytes.extend_from_slice(h.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::H256(h) => { + let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecH256(h256s) => { + let mut bytes = vec![]; + for h in h256s { + bytes.extend_from_slice(h.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::H256Bytes(h) => { + let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecH256Bytes(h256s) => { + let mut bytes = vec![]; + for h in h256s { + bytes.extend_from_slice(h.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::H512(h) => { + let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecH512(h512s) => { + let mut bytes = vec![]; + for h in h512s { + bytes.extend_from_slice(h.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::Address(address) | + EthereumSqlTypeWrapper::AddressBytes(address) | + EthereumSqlTypeWrapper::AddressBytesNullable(address) | + EthereumSqlTypeWrapper::AddressNullable(address) => { + let bytes = bytes::Bytes::from(address.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecAddress(addresses) | + EthereumSqlTypeWrapper::VecAddressBytes(addresses) => { + let mut bytes = vec![]; + for address in addresses { + bytes.extend_from_slice(address.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::Bool(b) => { + let mut bytes = vec![]; + bytes.extend_from_slice(&[*b as u8]); + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::VecBool(bools) => { + let mut bytes = vec![]; + for b in bools { + bytes.push(*b as u8); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U32(u) => { + let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecU32(u32s) => { + let mut bytes = vec![]; + for u in u32s { + bytes.extend_from_slice(&u.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I32(i) => { + let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecI32(i32s) => { + let mut bytes = vec![]; + for i in i32s { + bytes.extend_from_slice(&i.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U16(u) => { + let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecU16(u16s) => { + let mut bytes = vec![]; + for u in u16s { + bytes.extend_from_slice(&u.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I16(i) => { + let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecI16(i16s) => { + let mut bytes = vec![]; + for i in i16s { + bytes.extend_from_slice(&i.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::U8(u) => { + let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecU8(u8s) => { + let mut bytes = vec![]; + for u in u8s { + bytes.extend_from_slice(&u.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::I8(i) => { + let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecI8(i8s) => { + let mut bytes = vec![]; + for i in i8s { + bytes.extend_from_slice(&i.to_be_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::String(s) | + EthereumSqlTypeWrapper::StringNullable(s) | + EthereumSqlTypeWrapper::StringVarchar(s) | + EthereumSqlTypeWrapper::StringVarcharNullable(s) | + EthereumSqlTypeWrapper::StringChar(s) | + EthereumSqlTypeWrapper::StringCharNullable(s) => { + let bytes = bytes::Bytes::from(s.as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::VecString(strings) | + EthereumSqlTypeWrapper::VecStringVarchar(strings) | + EthereumSqlTypeWrapper::VecStringChar(strings) => { + let mut bytes = vec![]; + for s in strings { + bytes.extend_from_slice(s.as_bytes()); + } + let bytes = bytes::Bytes::from(bytes); + bytes + } + EthereumSqlTypeWrapper::Bytes(bytes) | + EthereumSqlTypeWrapper::BytesNullable(bytes) => { + for byte in bytes { + buf.push(*byte); + } + let bytes = bytes::Bytes::from(buf.clone()); + bytes + } + EthereumSqlTypeWrapper::VecBytes(bytes) => { + let mut buf = vec![]; + for byte in bytes { + buf.extend_from_slice(byte); + } + let bytes = bytes::Bytes::from(buf); + bytes + } + EthereumSqlTypeWrapper::DateTime(date_time) => { + let bytes = bytes::Bytes::from(date_time.to_rfc3339().as_bytes().to_vec()); + bytes + } + EthereumSqlTypeWrapper::JSONB(json) => { + let bytes = bytes::Bytes::from(json.to_string().as_bytes().to_vec()); + bytes + } }; buf.extend_from_slice(&abi_input.name.clone().into_bytes()); buf.extend_from_slice(&byte); From e6dfe4a17e6e70bf8cfed622714219de5f3ebdee Mon Sep 17 00:00:00 2001 From: nuts-rice Date: Wed, 8 Jan 2025 19:11:32 -0500 Subject: [PATCH 5/8] =?UTF-8?q?=E2=9C=A8=20feat:=20binary=5Fmode=20param?= =?UTF-8?q?=20for=20no-code=20db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/database/postgres/generate.rs | 13 ++++++++++++- core/src/database/postgres/setup.rs | 3 ++- core/src/indexer/no_code.rs | 10 ++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/core/src/database/postgres/generate.rs b/core/src/database/postgres/generate.rs index b8929fc5..172add68 100644 --- a/core/src/database/postgres/generate.rs +++ b/core/src/database/postgres/generate.rs @@ -43,6 +43,7 @@ fn generate_event_table_sql_with_comments( abi_inputs: &[EventInfo], contract_name: &str, schema_name: &str, + binary_mode: bool, apply_full_name_comment_for_events: Vec, ) -> String { abi_inputs @@ -53,7 +54,15 @@ fn generate_event_table_sql_with_comments( let event_columns = if event_info.inputs.is_empty() { "".to_string() } else { - generate_columns_with_data_types(&event_info.inputs).join(", ") + "," + if binary_mode { + generate_columns_with_data_types(&event_info.inputs) + .iter() + .map(|column| format!("{} BYTEA", column)) + .collect::>() + .join(", ") + } else { + generate_columns_with_data_types(&event_info.inputs).join(", ") + } }; let create_table_sql = format!( @@ -160,6 +169,7 @@ pub fn generate_tables_for_indexer_sql( project_path: &Path, indexer: &Indexer, disable_event_tables: bool, + binary_mode: bool, ) -> Result { let mut sql = "CREATE SCHEMA IF NOT EXISTS rindexer_internal;".to_string(); @@ -185,6 +195,7 @@ pub fn generate_tables_for_indexer_sql( &event_names, &contract.name, &schema_name, + binary_mode, event_matching_name_on_other, )); } diff --git a/core/src/database/postgres/setup.rs b/core/src/database/postgres/setup.rs index 6e2fe338..87949bb3 100644 --- a/core/src/database/postgres/setup.rs +++ b/core/src/database/postgres/setup.rs @@ -31,7 +31,7 @@ pub async fn setup_postgres( let client = PostgresClient::new().await?; let disable_event_tables = manifest.storage.postgres_disable_create_tables(); - + let binary_mode = manifest.storage.binary_storage(); if manifest.storage.postgres_drop_each_run() { info!( "`drop_each_run` enabled so dropping all data for {} before starting", @@ -54,6 +54,7 @@ pub async fn setup_postgres( project_path, &manifest.to_indexer(), disable_event_tables, + binary_mode, )?; debug!("{}", sql); client.batch_execute(sql.as_str()).await?; diff --git a/core/src/indexer/no_code.rs b/core/src/indexer/no_code.rs index 02dfef20..22e35960 100644 --- a/core/src/indexer/no_code.rs +++ b/core/src/indexer/no_code.rs @@ -134,6 +134,7 @@ struct NoCodeCallbackParams { contract_name: String, event: Event, index_event_in_order: bool, + binary_mode: bool, csv: Option>, postgres: Option>, postgres_event_table_name: String, @@ -193,7 +194,11 @@ fn no_code_callback(params: Arc) -> EventCallbackType { let log_index = result.tx_information.log_index; let event_parameters: Vec = - map_log_params_to_ethereum_wrapper(¶ms.event_info.inputs, &log.params); + map_log_params_to_ethereum_wrapper( + ¶ms.event_info.inputs, + &log.params, + params.binary_mode, + ); let contract_address = EthereumSqlTypeWrapper::Address(address); let end_global_parameters = vec![ @@ -542,7 +547,7 @@ pub async fn process_events( .index_event_in_order .as_ref() .map_or(false, |vec| vec.contains(&event_info.name)); - + let binary_mode = manifest.storage.binary_storage(); let event = EventCallbackRegistryInformation { id: generate_random_id(10), indexer_name: manifest.name.clone(), @@ -556,6 +561,7 @@ pub async fn process_events( contract_name: contract.name.clone(), event: event.clone(), index_event_in_order, + binary_mode, csv, postgres: postgres.clone(), postgres_event_table_name, From 651aafcd671cc5e13100944b8f0a727b1eecbea2 Mon Sep 17 00:00:00 2001 From: squid_kid Date: Sat, 11 Jan 2025 16:48:47 -0500 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=A9=B9=20feat(max=5Foptimisation):=20?= =?UTF-8?q?types=20and=20naming?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/src/commands/new.rs | 2 +- core/src/database/postgres/generate.rs | 7 +- core/src/database/postgres/setup.rs | 4 +- .../src/database/postgres/sql_type_wrapper.rs | 274 ++---------------- core/src/indexer/no_code.rs | 8 +- core/src/manifest/storage.rs | 6 +- 6 files changed, 32 insertions(+), 269 deletions(-) diff --git a/cli/src/commands/new.rs b/cli/src/commands/new.rs index 8245a1cc..c3d59329 100644 --- a/cli/src/commands/new.rs +++ b/cli/src/commands/new.rs @@ -170,7 +170,7 @@ pub fn handle_new_command( postgres: if postgres_enabled { Some(PostgresDetails { enabled: true, - binary_storage: None, + max_optimisation: None, drop_each_run: None, relationships: None, indexes: None, diff --git a/core/src/database/postgres/generate.rs b/core/src/database/postgres/generate.rs index 172add68..82fe302b 100644 --- a/core/src/database/postgres/generate.rs +++ b/core/src/database/postgres/generate.rs @@ -43,7 +43,7 @@ fn generate_event_table_sql_with_comments( abi_inputs: &[EventInfo], contract_name: &str, schema_name: &str, - binary_mode: bool, + max_optimisation: bool, apply_full_name_comment_for_events: Vec, ) -> String { abi_inputs @@ -54,10 +54,11 @@ fn generate_event_table_sql_with_comments( let event_columns = if event_info.inputs.is_empty() { "".to_string() } else { - if binary_mode { + if max_optimisation { generate_columns_with_data_types(&event_info.inputs) .iter() - .map(|column| format!("{} BYTEA", column)) + //TODO: filter types to those convertable to bytes + .map(|column| format!("{} BYTEA[]", column)) .collect::>() .join(", ") } else { diff --git a/core/src/database/postgres/setup.rs b/core/src/database/postgres/setup.rs index 87949bb3..ae00307f 100644 --- a/core/src/database/postgres/setup.rs +++ b/core/src/database/postgres/setup.rs @@ -31,7 +31,7 @@ pub async fn setup_postgres( let client = PostgresClient::new().await?; let disable_event_tables = manifest.storage.postgres_disable_create_tables(); - let binary_mode = manifest.storage.binary_storage(); + let binary_mode = manifest.storage.max_optimisation(); if manifest.storage.postgres_drop_each_run() { info!( "`drop_each_run` enabled so dropping all data for {} before starting", @@ -41,7 +41,7 @@ pub async fn setup_postgres( client.batch_execute(sql.as_str()).await?; info!("Dropped all data for {}", manifest.name); } - if manifest.storage.binary_storage() { + if manifest.storage.max_optimisation() { info!("storing data in bytes") } diff --git a/core/src/database/postgres/sql_type_wrapper.rs b/core/src/database/postgres/sql_type_wrapper.rs index 55c86b3e..4b16b159 100644 --- a/core/src/database/postgres/sql_type_wrapper.rs +++ b/core/src/database/postgres/sql_type_wrapper.rs @@ -945,7 +945,7 @@ pub fn solidity_type_to_ethereum_sql_type_wrapper( pub fn map_log_params_to_ethereum_wrapper( abi_inputs: &[ABIInput], params: &[LogParam], - binary_mode: bool, + max_optimisation: bool, ) -> Vec { let mut wrappers = vec![]; @@ -953,14 +953,14 @@ pub fn map_log_params_to_ethereum_wrapper( if let Some(abi_input) = abi_inputs.get(index) { match ¶m.value { Token::Tuple(tuple) => { - if binary_mode { - wrappers.extend(process_tuple( - abi_input - .components - .as_ref() - .expect("tuple should have a component ABI on"), - tuple, - )); + wrappers.extend(process_tuple( + abi_input + .components + .as_ref() + .expect("tuple should have a component ABI on"), + tuple, + )); + if max_optimisation { map_ethereum_wrapper_to_byte( abi_input .components @@ -969,7 +969,6 @@ pub fn map_log_params_to_ethereum_wrapper( &wrappers, true, ); - todo!() } wrappers.extend(process_tuple( abi_input @@ -980,8 +979,8 @@ pub fn map_log_params_to_ethereum_wrapper( )); } _ => { - if binary_mode { - wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, ¶m.value)); + wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, ¶m.value)); + if max_optimisation { map_ethereum_wrapper_to_byte( abi_input .components @@ -990,8 +989,6 @@ pub fn map_log_params_to_ethereum_wrapper( &wrappers, false, ); - - todo!() } wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, ¶m.value)); @@ -1039,72 +1036,13 @@ pub fn map_ethereum_wrapper_to_byte( current_wrapper_index = total_properties; } else { let byte = match wrapper { - EthereumSqlTypeWrapper::U64(u) => { - let mut bytes = vec![]; - u.to_big_endian(&mut bytes); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecU64(u64s) => { - let mut bytes = vec![]; - for u in u64s { - u.to_big_endian(&mut bytes); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::I64(i) => { - let mut bytes = vec![]; - bytes.extend_from_slice(&i.to_be_bytes()); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecI64(i64s) => { - let mut bytes = vec![]; - for i in i64s { - bytes.extend_from_slice(&i.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::U128(u) => { - let mut bytes = vec![]; - bytes.extend_from_slice(&u.to_be_bytes()); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecU128(u128s) => { - let mut bytes = vec![]; - for u in u128s { - bytes.extend_from_slice(&u.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::I128(i) => { - let mut bytes = vec![]; - bytes.extend_from_slice(&i.to_be_bytes()); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecI128(i128s) => { - let mut bytes = vec![]; - for i in i128s { - bytes.extend_from_slice(&i.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::U256(u) | EthereumSqlTypeWrapper::U256Bytes(u) | - EthereumSqlTypeWrapper::U256Nullable(u) | EthereumSqlTypeWrapper::U256BytesNullable(u) => { let mut bytes = vec![]; bytes.extend_from_slice(&u.low_u64().to_be_bytes()); let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::VecU256(u256s) | EthereumSqlTypeWrapper::VecU256Bytes(u256s) => { let mut bytes = vec![]; for u in u256s { @@ -1113,16 +1051,14 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::I256(i) | + EthereumSqlTypeWrapper::I256Bytes(i) | - EthereumSqlTypeWrapper::I256Nullable(i) | EthereumSqlTypeWrapper::I256BytesNullable(i) => { let mut bytes = vec![]; bytes.extend_from_slice(&i.low_u64().to_be_bytes()); let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::VecI256(i256s) | EthereumSqlTypeWrapper::VecI256Bytes(i256s) => { let mut bytes = vec![]; for i in i256s { @@ -1131,56 +1067,6 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::U512(u) => { - let mut bytes = vec![]; - u.to_big_endian(&mut bytes); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecU512(u512s) => { - let mut bytes = vec![]; - for u in u512s { - u.to_big_endian(&mut bytes); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::H128(h) => { - let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecH128(h128s) => { - let mut bytes = vec![]; - for h in h128s { - bytes.extend_from_slice(h.as_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::H160(h) => { - let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecH160(h160s) => { - let mut bytes = vec![]; - for h in h160s { - bytes.extend_from_slice(h.as_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::H256(h) => { - let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecH256(h256s) => { - let mut bytes = vec![]; - for h in h256s { - bytes.extend_from_slice(h.as_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } EthereumSqlTypeWrapper::H256Bytes(h) => { let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); bytes @@ -1193,26 +1079,13 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::H512(h) => { - let bytes = bytes::Bytes::from(h.as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecH512(h512s) => { - let mut bytes = vec![]; - for h in h512s { - bytes.extend_from_slice(h.as_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::Address(address) | + EthereumSqlTypeWrapper::AddressBytes(address) | - EthereumSqlTypeWrapper::AddressBytesNullable(address) | - EthereumSqlTypeWrapper::AddressNullable(address) => { + EthereumSqlTypeWrapper::AddressBytesNullable(address) => { let bytes = bytes::Bytes::from(address.as_bytes().to_vec()); bytes } - EthereumSqlTypeWrapper::VecAddress(addresses) | + EthereumSqlTypeWrapper::VecAddressBytes(addresses) => { let mut bytes = vec![]; for address in addresses { @@ -1221,111 +1094,7 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(bytes); bytes } - EthereumSqlTypeWrapper::Bool(b) => { - let mut bytes = vec![]; - bytes.extend_from_slice(&[*b as u8]); - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::VecBool(bools) => { - let mut bytes = vec![]; - for b in bools { - bytes.push(*b as u8); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::U32(u) => { - let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecU32(u32s) => { - let mut bytes = vec![]; - for u in u32s { - bytes.extend_from_slice(&u.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::I32(i) => { - let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecI32(i32s) => { - let mut bytes = vec![]; - for i in i32s { - bytes.extend_from_slice(&i.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::U16(u) => { - let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecU16(u16s) => { - let mut bytes = vec![]; - for u in u16s { - bytes.extend_from_slice(&u.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::I16(i) => { - let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecI16(i16s) => { - let mut bytes = vec![]; - for i in i16s { - bytes.extend_from_slice(&i.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::U8(u) => { - let bytes = bytes::Bytes::from(u.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecU8(u8s) => { - let mut bytes = vec![]; - for u in u8s { - bytes.extend_from_slice(&u.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::I8(i) => { - let bytes = bytes::Bytes::from(i.to_be_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecI8(i8s) => { - let mut bytes = vec![]; - for i in i8s { - bytes.extend_from_slice(&i.to_be_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } - EthereumSqlTypeWrapper::String(s) | - EthereumSqlTypeWrapper::StringNullable(s) | - EthereumSqlTypeWrapper::StringVarchar(s) | - EthereumSqlTypeWrapper::StringVarcharNullable(s) | - EthereumSqlTypeWrapper::StringChar(s) | - EthereumSqlTypeWrapper::StringCharNullable(s) => { - let bytes = bytes::Bytes::from(s.as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::VecString(strings) | - EthereumSqlTypeWrapper::VecStringVarchar(strings) | - EthereumSqlTypeWrapper::VecStringChar(strings) => { - let mut bytes = vec![]; - for s in strings { - bytes.extend_from_slice(s.as_bytes()); - } - let bytes = bytes::Bytes::from(bytes); - bytes - } + EthereumSqlTypeWrapper::Bytes(bytes) | EthereumSqlTypeWrapper::BytesNullable(bytes) => { for byte in bytes { @@ -1342,16 +1111,9 @@ pub fn map_ethereum_wrapper_to_byte( let bytes = bytes::Bytes::from(buf); bytes } - EthereumSqlTypeWrapper::DateTime(date_time) => { - let bytes = bytes::Bytes::from(date_time.to_rfc3339().as_bytes().to_vec()); - bytes - } - EthereumSqlTypeWrapper::JSONB(json) => { - let bytes = bytes::Bytes::from(json.to_string().as_bytes().to_vec()); - bytes - } + _ => panic!("Wrapper types can't be converted to bytes"), }; - buf.extend_from_slice(&abi_input.name.clone().into_bytes()); + buf.extend_from_slice(&byte); wrappers_index_processed.push(current_wrapper_index); diff --git a/core/src/indexer/no_code.rs b/core/src/indexer/no_code.rs index 22e35960..8504418e 100644 --- a/core/src/indexer/no_code.rs +++ b/core/src/indexer/no_code.rs @@ -134,7 +134,7 @@ struct NoCodeCallbackParams { contract_name: String, event: Event, index_event_in_order: bool, - binary_mode: bool, + max_optimisation: bool, csv: Option>, postgres: Option>, postgres_event_table_name: String, @@ -197,7 +197,7 @@ fn no_code_callback(params: Arc) -> EventCallbackType { map_log_params_to_ethereum_wrapper( ¶ms.event_info.inputs, &log.params, - params.binary_mode, + params.max_optimisation, ); let contract_address = EthereumSqlTypeWrapper::Address(address); @@ -547,7 +547,7 @@ pub async fn process_events( .index_event_in_order .as_ref() .map_or(false, |vec| vec.contains(&event_info.name)); - let binary_mode = manifest.storage.binary_storage(); + let max_optimisation = manifest.storage.max_optimisation(); let event = EventCallbackRegistryInformation { id: generate_random_id(10), indexer_name: manifest.name.clone(), @@ -561,7 +561,7 @@ pub async fn process_events( contract_name: contract.name.clone(), event: event.clone(), index_event_in_order, - binary_mode, + max_optimisation, csv, postgres: postgres.clone(), postgres_event_table_name, diff --git a/core/src/manifest/storage.rs b/core/src/manifest/storage.rs index 379a559c..f8385920 100644 --- a/core/src/manifest/storage.rs +++ b/core/src/manifest/storage.rs @@ -80,7 +80,7 @@ pub struct PostgresDetails { pub drop_each_run: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub binary_storage: Option, + pub max_optimisation: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub relationships: Option>, @@ -150,12 +150,12 @@ impl Storage { .map_or(false, |details| details.disable_create_tables.unwrap_or_default()) } - pub fn binary_storage(&self) -> bool { + pub fn max_optimisation(&self) -> bool { let enabled = self.postgres_enabled(); if !enabled { return false; } - self.postgres.as_ref().map_or(false, |details| details.binary_storage.unwrap_or_default()) + self.postgres.as_ref().map_or(false, |details| details.max_optimisation.unwrap_or_default()) } pub fn postgres_drop_each_run(&self) -> bool { From 2f59b2545120b76b1b1a1573e49fb4e8ac21ef41 Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Wed, 15 Jan 2025 10:41:26 -0500 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20filtering=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/database/postgres/generate.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/database/postgres/generate.rs b/core/src/database/postgres/generate.rs index 82fe302b..3a9b89dd 100644 --- a/core/src/database/postgres/generate.rs +++ b/core/src/database/postgres/generate.rs @@ -39,6 +39,7 @@ pub fn generate_column_names_only_with_base_properties(inputs: &[ABIInput]) -> V column_names } + fn generate_event_table_sql_with_comments( abi_inputs: &[EventInfo], contract_name: &str, @@ -57,8 +58,13 @@ fn generate_event_table_sql_with_comments( if max_optimisation { generate_columns_with_data_types(&event_info.inputs) .iter() - //TODO: filter types to those convertable to bytes - .map(|column| format!("{} BYTEA[]", column)) + .filter_map(|( column)|{ + if column.contains("CHAR") { + Some(format!("{} BYTEA[]", column)) + } else { + None + } + }) .collect::>() .join(", ") } else { @@ -97,6 +103,7 @@ fn generate_event_table_sql_with_comments( .join("\n") } + fn generate_internal_event_table_sql( abi_inputs: &[EventInfo], schema_name: &str, From ad375adc39d98a70112069fda4682bc76ac9a286 Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Wed, 15 Jan 2025 17:19:49 -0500 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20feat(binary=20stora?= =?UTF-8?q?ge=20db):=20sql=20table=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/abi.rs | 18 +++++++++++++++- core/src/database/postgres/generate.rs | 30 ++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/core/src/abi.rs b/core/src/abi.rs index 363020e8..cff4db81 100644 --- a/core/src/abi.rs +++ b/core/src/abi.rs @@ -39,6 +39,7 @@ pub enum ParamTypeError { #[derive(PartialEq)] pub enum GenerateAbiPropertiesType { PostgresWithDataTypes, + PostgresWithOptimizedDataTypes, PostgresColumnsNamesOnly, CsvHeaderNames, Object, @@ -101,12 +102,26 @@ impl ABIInput { ) } else { match properties_type { + GenerateAbiPropertiesType::PostgresWithOptimizedDataTypes => { + let value = format!( + "\"{}{}\" {}", + prefix.map_or_else(|| "".to_string(), |p| format!("{}_", p)), + camel_to_snake(&input.name), + solidity_type_to_db_type(&input.type_, true) + ); + + vec![GenerateAbiNamePropertiesResult::new( + value, + &input.name, + &input.type_, + )] + } GenerateAbiPropertiesType::PostgresWithDataTypes => { let value = format!( "\"{}{}\" {}", prefix.map_or_else(|| "".to_string(), |p| format!("{}_", p)), camel_to_snake(&input.name), - solidity_type_to_db_type(&input.type_) + solidity_type_to_db_type(&input.type_, false) ); vec![GenerateAbiNamePropertiesResult::new( @@ -115,6 +130,7 @@ impl ABIInput { &input.type_, )] } + GenerateAbiPropertiesType::PostgresColumnsNamesOnly | GenerateAbiPropertiesType::CsvHeaderNames => { let value = format!( diff --git a/core/src/database/postgres/generate.rs b/core/src/database/postgres/generate.rs index 3a9b89dd..521e66a7 100644 --- a/core/src/database/postgres/generate.rs +++ b/core/src/database/postgres/generate.rs @@ -39,7 +39,6 @@ pub fn generate_column_names_only_with_base_properties(inputs: &[ABIInput]) -> V column_names } - fn generate_event_table_sql_with_comments( abi_inputs: &[EventInfo], contract_name: &str, @@ -58,7 +57,7 @@ fn generate_event_table_sql_with_comments( if max_optimisation { generate_columns_with_data_types(&event_info.inputs) .iter() - .filter_map(|( column)|{ + .filter_map(|(column)| { if column.contains("CHAR") { Some(format!("{} BYTEA[]", column)) } else { @@ -103,7 +102,6 @@ fn generate_event_table_sql_with_comments( .join("\n") } - fn generate_internal_event_table_sql( abi_inputs: &[EventInfo], schema_name: &str, @@ -285,9 +283,33 @@ pub fn drop_tables_for_indexer_sql(project_path: &Path, indexer: &Indexer) -> Co } #[allow(clippy::manual_strip)] -pub fn solidity_type_to_db_type(abi_type: &str) -> String { +pub fn solidity_type_to_db_type(abi_type: &str, max_optimisation: bool) -> String { let is_array = abi_type.ends_with("[]"); let base_type = abi_type.trim_end_matches("[]"); + if max_optimisation { + let sql_type: &str = match base_type { + "address" => "BYTEA[]", + "bool" => "BOOLEAN", + "string" => "TEXT", + t if t.starts_with("bytes") => "BYTEA", + t if t.starts_with("int") || t.starts_with("uint") => { + let (prefix, size): (&str, usize) = if t.starts_with("int") { + ("int", t[3..].parse().expect("Invalid intN type")) + } else { + ("uint", t[4..].parse().expect("Invalid uintN type")) + }; + match size { + 8 | 16 => "SMALLINT", + 24 | 32 => "INTEGER", + 40 | 48 | 56 | 64 | 72 | 80 | 88 | 96 | 104 | 112 | 120 | 128 => "NUMERIC", + 136 | 144 | 152 | 160 | 168 | 176 | 184 | 192 | 200 | 208 | 216 | 224 | + 232 | 240 | 248 | 256 => "NUMERIC(80)", + _ => panic!("Unsupported {}N size: {}", prefix, size), + } + } + _ => panic!("Unsupported type: {}", base_type), + }; + } let sql_type = match base_type { "address" => "CHAR(42)",