Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: binary storage db #153

Draft
wants to merge 8 commits into
base: master
Choose a base branch
from
Draft

feat: binary storage db #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1fbebe9
🔧 feat: binary db config check
nuts-rice Jan 7, 2025
32f7731
🚧 wip: binary_mode check + wrapping, fmt
nuts-rice Jan 7, 2025
8dceec6
🚧 wip: map types to bytes
nuts-rice Jan 7, 2025
6d9fe73
🏷️ feat: map ethereum wrapper type to bytes
nuts-rice Jan 7, 2025
e6dfe4a
✨ feat: binary_mode param for no-code db
nuts-rice Jan 9, 2025
651aafc
🩹 feat(max_optimisation): types and naming
Jan 11, 2025
2f59b25
🚧 wip: filtering types
nuts-rice Jan 15, 2025
ad375ad
🏷️ feat(binary storage db): sql table types
nuts-rice Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cli/src/commands/new.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ pub fn handle_new_command(
postgres: if postgres_enabled {
Some(PostgresDetails {
enabled: true,
max_optimisation: None,
drop_each_run: None,
relationships: None,
indexes: None,
Expand Down
18 changes: 17 additions & 1 deletion core/src/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ pub enum ParamTypeError {
#[derive(PartialEq)]
pub enum GenerateAbiPropertiesType {
PostgresWithDataTypes,
PostgresWithOptimizedDataTypes,
PostgresColumnsNamesOnly,
CsvHeaderNames,
Object,
Expand Down Expand Up @@ -101,12 +102,26 @@ impl ABIInput {
)
} else {
match properties_type {
GenerateAbiPropertiesType::PostgresWithOptimizedDataTypes => {
let value = format!(
"\"{}{}\" {}",
prefix.map_or_else(|| "".to_string(), |p| format!("{}_", p)),
camel_to_snake(&input.name),
solidity_type_to_db_type(&input.type_, true)
);

vec![GenerateAbiNamePropertiesResult::new(
value,
&input.name,
&input.type_,
)]
}
GenerateAbiPropertiesType::PostgresWithDataTypes => {
let value = format!(
"\"{}{}\" {}",
prefix.map_or_else(|| "".to_string(), |p| format!("{}_", p)),
camel_to_snake(&input.name),
solidity_type_to_db_type(&input.type_)
solidity_type_to_db_type(&input.type_, false)
);

vec![GenerateAbiNamePropertiesResult::new(
Expand All @@ -115,6 +130,7 @@ impl ABIInput {
&input.type_,
)]
}

GenerateAbiPropertiesType::PostgresColumnsNamesOnly |
GenerateAbiPropertiesType::CsvHeaderNames => {
let value = format!(
Expand Down
45 changes: 43 additions & 2 deletions core/src/database/postgres/generate.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ fn generate_event_table_sql_with_comments(
abi_inputs: &[EventInfo],
contract_name: &str,
schema_name: &str,
max_optimisation: bool,
apply_full_name_comment_for_events: Vec<String>,
) -> String {
abi_inputs
Expand All @@ -53,7 +54,21 @@ fn generate_event_table_sql_with_comments(
let event_columns = if event_info.inputs.is_empty() {
"".to_string()
} else {
generate_columns_with_data_types(&event_info.inputs).join(", ") + ","
if max_optimisation {
generate_columns_with_data_types(&event_info.inputs)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not everything can be stored in bytes like booleans etc we only want to store hex stuff as bytes so i think you need to filter out based on the solidity type which ones to do?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we use a .filter() or .filter_map() for this? I'm not sure exactly

.iter()
.filter_map(|(column)| {
if column.contains("CHAR") {
Some(format!("{} BYTEA[]", column))
} else {
None
}
})
.collect::<Vec<_>>()
.join(", ")
} else {
generate_columns_with_data_types(&event_info.inputs).join(", ")
}
};

let create_table_sql = format!(
Expand Down Expand Up @@ -160,6 +175,7 @@ pub fn generate_tables_for_indexer_sql(
project_path: &Path,
indexer: &Indexer,
disable_event_tables: bool,
binary_mode: bool,
) -> Result<Code, GenerateTablesForIndexerSqlError> {
let mut sql = "CREATE SCHEMA IF NOT EXISTS rindexer_internal;".to_string();

Expand All @@ -185,6 +201,7 @@ pub fn generate_tables_for_indexer_sql(
&event_names,
&contract.name,
&schema_name,
binary_mode,
event_matching_name_on_other,
));
}
Expand Down Expand Up @@ -266,9 +283,33 @@ pub fn drop_tables_for_indexer_sql(project_path: &Path, indexer: &Indexer) -> Co
}

#[allow(clippy::manual_strip)]
pub fn solidity_type_to_db_type(abi_type: &str) -> String {
pub fn solidity_type_to_db_type(abi_type: &str, max_optimisation: bool) -> String {
let is_array = abi_type.ends_with("[]");
let base_type = abi_type.trim_end_matches("[]");
if max_optimisation {
let sql_type: &str = match base_type {
"address" => "BYTEA[]",
"bool" => "BOOLEAN",
"string" => "TEXT",
t if t.starts_with("bytes") => "BYTEA",
t if t.starts_with("int") || t.starts_with("uint") => {
let (prefix, size): (&str, usize) = if t.starts_with("int") {
("int", t[3..].parse().expect("Invalid intN type"))
} else {
("uint", t[4..].parse().expect("Invalid uintN type"))
};
match size {
8 | 16 => "SMALLINT",
24 | 32 => "INTEGER",
40 | 48 | 56 | 64 | 72 | 80 | 88 | 96 | 104 | 112 | 120 | 128 => "NUMERIC",
136 | 144 | 152 | 160 | 168 | 176 | 184 | 192 | 200 | 208 | 216 | 224 |
232 | 240 | 248 | 256 => "NUMERIC(80)",
_ => panic!("Unsupported {}N size: {}", prefix, size),
}
}
_ => panic!("Unsupported type: {}", base_type),
};
}

let sql_type = match base_type {
"address" => "CHAR(42)",
Expand Down
6 changes: 5 additions & 1 deletion core/src/database/postgres/setup.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub async fn setup_postgres(
let client = PostgresClient::new().await?;

let disable_event_tables = manifest.storage.postgres_disable_create_tables();

let binary_mode = manifest.storage.max_optimisation();
if manifest.storage.postgres_drop_each_run() {
info!(
"`drop_each_run` enabled so dropping all data for {} before starting",
Expand All @@ -41,6 +41,9 @@ pub async fn setup_postgres(
client.batch_execute(sql.as_str()).await?;
info!("Dropped all data for {}", manifest.name);
}
if manifest.storage.max_optimisation() {
info!("storing data in bytes")
}

if !disable_event_tables {
info!("Creating tables for {}", manifest.name);
Expand All @@ -51,6 +54,7 @@ pub async fn setup_postgres(
project_path,
&manifest.to_indexer(),
disable_event_tables,
binary_mode,
)?;
debug!("{}", sql);
client.batch_execute(sql.as_str()).await?;
Expand Down
163 changes: 163 additions & 0 deletions core/src/database/postgres/sql_type_wrapper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::str::FromStr;

use async_std::io::ReadExt;
use bytes::{BufMut, BytesMut};
use chrono::{DateTime, Utc};
use ethers::{
Expand Down Expand Up @@ -944,6 +945,7 @@ pub fn solidity_type_to_ethereum_sql_type_wrapper(
pub fn map_log_params_to_ethereum_wrapper(
abi_inputs: &[ABIInput],
params: &[LogParam],
max_optimisation: bool,
) -> Vec<EthereumSqlTypeWrapper> {
let mut wrappers = vec![];

Expand All @@ -958,9 +960,38 @@ pub fn map_log_params_to_ethereum_wrapper(
.expect("tuple should have a component ABI on"),
tuple,
));
if max_optimisation {
map_ethereum_wrapper_to_byte(
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be in wrappers.extend

abi_input
.components
.as_ref()
.expect("tuple should have component ABI on"),
&wrappers,
true,
);
}
wrappers.extend(process_tuple(
abi_input
.components
.as_ref()
.expect("tuple should have a component ABI on"),
tuple,
));
}
_ => {
wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, &param.value));
if max_optimisation {
map_ethereum_wrapper_to_byte(
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be in wrappers.extend

abi_input
.components
.as_ref()
.expect("value should have component ABI on"),
&wrappers,
false,
);
}

wrappers.push(map_log_token_to_ethereum_wrapper(abi_input, &param.value));
}
}
} else {
Expand All @@ -971,6 +1002,138 @@ pub fn map_log_params_to_ethereum_wrapper(
wrappers
}

pub fn map_ethereum_wrapper_to_byte(
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the only thing which can be bytes are address (handle array), bytes, + main tables tx hash, block hash, contract address so this is why i suggest above we turn this to max_optimisation we can then make:

uint256 NUMBER(80)

and then pass in optimisations on the solidity_type_to_db_type

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so should solidity_type_to_db_type have the max_optimisation field too?

abi_inputs: &[ABIInput],
wrappers: &[EthereumSqlTypeWrapper],
// transaction_information: &TxInformation,
is_within_tuple: bool,
) -> bytes::Bytes {
let mut buf = vec![];
let mut current_wrapper_index = 0;
let mut wrappers_index_processed = Vec::new();

for abi_input in abi_inputs.iter() {
// tuples will take in multiple wrapper indexes, so we need to skip them if processed
if wrappers_index_processed.contains(&current_wrapper_index) {
continue;
}
if let Some(wrapper) = wrappers.get(current_wrapper_index) {
if abi_input.type_ == "tuple" {
let components =
abi_input.components.as_ref().expect("Tuple should have components defined");
let total_properties = count_components(components);
let tuple_value = map_ethereum_wrapper_to_byte(
components,
&wrappers[current_wrapper_index..total_properties],
// transaction_information,
true,
);
buf.extend_from_slice(&abi_input.name.clone().into_bytes());
buf.extend_from_slice(&tuple_value);
for i in current_wrapper_index..total_properties {
wrappers_index_processed.push(i);
}
current_wrapper_index = total_properties;
} else {
let byte = match wrapper {
EthereumSqlTypeWrapper::U256Bytes(u) |
EthereumSqlTypeWrapper::U256BytesNullable(u) => {
let mut bytes = vec![];
bytes.extend_from_slice(&u.low_u64().to_be_bytes());
let bytes = bytes::Bytes::from(bytes);
bytes
}
EthereumSqlTypeWrapper::VecU256Bytes(u256s) => {
let mut bytes = vec![];
for u in u256s {
bytes.extend_from_slice(&u.low_u64().to_be_bytes());
}
let bytes = bytes::Bytes::from(bytes);
bytes
}

EthereumSqlTypeWrapper::I256Bytes(i) |
EthereumSqlTypeWrapper::I256BytesNullable(i) => {
let mut bytes = vec![];
bytes.extend_from_slice(&i.low_u64().to_be_bytes());
let bytes = bytes::Bytes::from(bytes);
bytes
}
EthereumSqlTypeWrapper::VecI256Bytes(i256s) => {
let mut bytes = vec![];
for i in i256s {
bytes.extend_from_slice(&i.low_u64().to_be_bytes());
}
let bytes = bytes::Bytes::from(bytes);
bytes
}
EthereumSqlTypeWrapper::H256Bytes(h) => {
let bytes = bytes::Bytes::from(h.as_bytes().to_vec());
bytes
}
EthereumSqlTypeWrapper::VecH256Bytes(h256s) => {
let mut bytes = vec![];
for h in h256s {
bytes.extend_from_slice(h.as_bytes());
}
let bytes = bytes::Bytes::from(bytes);
bytes
}

EthereumSqlTypeWrapper::AddressBytes(address) |
EthereumSqlTypeWrapper::AddressBytesNullable(address) => {
let bytes = bytes::Bytes::from(address.as_bytes().to_vec());
bytes
}

EthereumSqlTypeWrapper::VecAddressBytes(addresses) => {
let mut bytes = vec![];
for address in addresses {
bytes.extend_from_slice(address.as_bytes());
}
let bytes = bytes::Bytes::from(bytes);
bytes
}

EthereumSqlTypeWrapper::Bytes(bytes) |
EthereumSqlTypeWrapper::BytesNullable(bytes) => {
for byte in bytes {
buf.push(*byte);
}
let bytes = bytes::Bytes::from(buf.clone());
bytes
}
EthereumSqlTypeWrapper::VecBytes(bytes) => {
let mut buf = vec![];
for byte in bytes {
buf.extend_from_slice(byte);
}
let bytes = bytes::Bytes::from(buf);
bytes
}
_ => panic!("Wrapper types can't be converted to bytes"),
};

buf.extend_from_slice(&byte);

wrappers_index_processed.push(current_wrapper_index);
current_wrapper_index += 1;
}
} else {
panic!("No wrapper found for ABI input at index: {}", current_wrapper_index)
}
}
// if !is_within_tuple {
// let mut bytes: Vec<bytes::Bytes> = vec![];
// buf.extend_from_slice("transaction_information".to_bytes());
// buf.extend_from_slice(transaction_information.transaction_hash.as_bytes());
// let bytes = bytes::Bytes::from(buf);
// return bytes

// }
bytes::Bytes::from(buf)
}

fn process_tuple(abi_inputs: &[ABIInput], tokens: &[Token]) -> Vec<EthereumSqlTypeWrapper> {
let mut wrappers = vec![];

Expand Down
10 changes: 8 additions & 2 deletions core/src/indexer/no_code.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ struct NoCodeCallbackParams {
contract_name: String,
event: Event,
index_event_in_order: bool,
max_optimisation: bool,
csv: Option<Arc<AsyncCsvAppender>>,
postgres: Option<Arc<PostgresClient>>,
postgres_event_table_name: String,
Expand Down Expand Up @@ -193,7 +194,11 @@ fn no_code_callback(params: Arc<NoCodeCallbackParams>) -> EventCallbackType {
let log_index = result.tx_information.log_index;

let event_parameters: Vec<EthereumSqlTypeWrapper> =
map_log_params_to_ethereum_wrapper(&params.event_info.inputs, &log.params);
map_log_params_to_ethereum_wrapper(
&params.event_info.inputs,
&log.params,
params.max_optimisation,
);

let contract_address = EthereumSqlTypeWrapper::Address(address);
let end_global_parameters = vec![
Expand Down Expand Up @@ -542,7 +547,7 @@ pub async fn process_events(
.index_event_in_order
.as_ref()
.map_or(false, |vec| vec.contains(&event_info.name));

let max_optimisation = manifest.storage.max_optimisation();
let event = EventCallbackRegistryInformation {
id: generate_random_id(10),
indexer_name: manifest.name.clone(),
Expand All @@ -556,6 +561,7 @@ pub async fn process_events(
contract_name: contract.name.clone(),
event: event.clone(),
index_event_in_order,
max_optimisation,
csv,
postgres: postgres.clone(),
postgres_event_table_name,
Expand Down
Loading