Support efficient query for related account with method filter

.changelog/809.feature.md

@@ -0,0 +1,6 @@
+Support efficient query for related account with method filter
+
+- Promote 'likely_native_transfer' to a first-class database field.
+
+- Denormalize 'method' and 'likely_native_transfer' to create
+efficient indexes for account filtering

analyzer/consensus/consensus.go

@@ -588,6 +588,16 @@ func (m *processor) queueTransactionInserts(batch *storage.QueryBatch, data *con
 // Enqueue DB statements to store events that were generated as the result of a TX execution.
 func (m *processor) queueTxEventInserts(batch *storage.QueryBatch, data *consensusBlockData) error {
 	for i, txr := range data.TransactionsWithResults {
+		tx, err := OpenSignedTxNoVerify(&txr.Transaction)
+		if err != nil {
+			m.logger.Info("couldn't parse transaction",
+				"err", err,
+				"height", data.Height,
+				"tx_index", i,
+			)
+			continue
+		}
+
 		txAccounts := []staking.Address{
 			// Always insert sender as a related address, some transactions (e.g. failed ones) might not have
 			// any events associated.
@@ -620,6 +630,7 @@ func (m *processor) queueTxEventInserts(batch *storage.QueryBatch, data *consens
 		for _, addr := range uniqueTxAccounts {
 			batch.Queue(queries.ConsensusAccountRelatedTransactionInsert,
 				addr,
+				tx.Method,
 				data.Height,
 				i,
 			)

analyzer/queries/queries.go

@@ -299,8 +299,8 @@ var (
       result = excluded.result`
 
 	ConsensusAccountRelatedTransactionInsert = `
-    INSERT INTO chain.accounts_related_transactions (account_address, tx_block, tx_index)
-      VALUES ($1, $2, $3)`
+    INSERT INTO chain.accounts_related_transactions (account_address, method, tx_block, tx_index)
+      VALUES ($1, $2, $3, $4)`
 
 	ConsensusAccountRelatedEventInsert = `
     INSERT INTO chain.accounts_related_events (account_address, event_block, tx_index, tx_hash, type, body)
@@ -558,8 +558,8 @@ var (
       VALUES ($1, $2, $3, $4, $5, $6)`
 
 	RuntimeRelatedTransactionInsert = `
-    INSERT INTO chain.runtime_related_transactions (runtime, account_address, tx_round, tx_index)
-      VALUES ($1, $2, $3, $4)`
+    INSERT INTO chain.runtime_related_transactions (runtime, account_address, tx_round, tx_index, method, likely_native_transfer)
+      VALUES ($1, $2, $3, $4, $5, $6)`
 
 	RuntimeAccountNumTxsUpsert = `
     INSERT INTO chain.runtime_accounts as accounts (runtime, address, num_txs)
@@ -619,8 +619,8 @@ var (
       SET total_received = EXCLUDED.total_received`
 
 	RuntimeTransactionInsert = `
-    INSERT INTO chain.runtime_transactions (runtime, round, tx_index, tx_hash, tx_eth_hash, fee, fee_symbol, fee_proxy_module, fee_proxy_id, gas_limit, gas_used, size, timestamp, oasis_encrypted_format, oasis_encrypted_public_key, oasis_encrypted_data_nonce, oasis_encrypted_data_data, oasis_encrypted_result_nonce, oasis_encrypted_result_data, method, body, "to", amount, amount_symbol, evm_encrypted_format, evm_encrypted_public_key, evm_encrypted_data_nonce, evm_encrypted_data_data, evm_encrypted_result_nonce, evm_encrypted_result_data, success, error_module, error_code, error_message_raw, error_message)
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35)`
+    INSERT INTO chain.runtime_transactions (runtime, round, tx_index, tx_hash, tx_eth_hash, fee, fee_symbol, fee_proxy_module, fee_proxy_id, gas_limit, gas_used, size, timestamp, oasis_encrypted_format, oasis_encrypted_public_key, oasis_encrypted_data_nonce, oasis_encrypted_data_data, oasis_encrypted_result_nonce, oasis_encrypted_result_data, method, body, "to", amount, amount_symbol, evm_encrypted_format, evm_encrypted_public_key, evm_encrypted_data_nonce, evm_encrypted_data_data, evm_encrypted_result_nonce, evm_encrypted_result_data, success, error_module, error_code, error_message_raw, error_message, likely_native_transfer)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`
 
 	// We use COALESCE here to avoid overwriting existing data with null values.
 	RuntimeTransactionEvmParsedFieldsUpdate = `

analyzer/runtime/extract.go

@@ -77,6 +77,7 @@ type BlockTransactionData struct {
 	EVMContract             *evm.EVMContractData
 	Success                 *bool
 	Error                   *TxError
+	IsLikelyTokenTransfer   bool
 }
 
 type TxError struct {
@@ -320,6 +321,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []nodeapi.Runtime
 			var amount quantity.Quantity
 			if err = VisitCall(&tx.Call, &txr.Result, &CallHandler{
 				AccountsTransfer: func(body *accounts.Transfer) error {
+					blockTransactionData.IsLikelyTokenTransfer = true
 					blockTransactionData.Body = body
 					amount = body.Amount.Amount
 					blockTransactionData.AmountSymbol = common.Ptr(stringifyDenomination(sdkPT, body.Amount.Denomination))
@@ -469,6 +471,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []nodeapi.Runtime
 					return nil
 				},
 				EVMCall: func(body *sdkEVM.Call, ok *[]byte) error {
+					blockTransactionData.IsLikelyTokenTransfer = len(body.Data) == 0 // Calls with empty body are likely native token transfers.
 					blockTransactionData.Body = body
 					amount = uncategorized.QuantityFromBytes(body.Value)
 					if to, err = addresses.RegisterRelatedEthAddress(blockData.AddressPreimages, blockTransactionData.RelatedAccountAddresses, body.Address); err != nil {

analyzer/runtime/runtime.go

@@ -325,6 +325,7 @@ func (m *processor) queueTransactionInsert(batch *storage.QueryBatch, round uint
 		errorCode,
 		errorMessageRaw,
 		errorMessage,
+		transactionData.IsLikelyTokenTransfer,
 	)
 }
 
@@ -362,7 +363,7 @@ func (m *processor) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) {
 			)
 		}
 		for addr := range transactionData.RelatedAccountAddresses {
-			batch.Queue(queries.RuntimeRelatedTransactionInsert, m.runtime, addr, data.Header.Round, transactionData.Index)
+			batch.Queue(queries.RuntimeRelatedTransactionInsert, m.runtime, addr, data.Header.Round, transactionData.Index, transactionData.Method, transactionData.IsLikelyTokenTransfer)
 			if m.mode != analyzer.FastSyncMode {
 				// We do not dead-reckon the number of transactions for accounts in fast sync mode because there are some
 				// "heavy hitter" accounts (system, etc) that are involved in a large fraction of transactions, resulting in

api/spec/v1.yaml

@@ -2873,7 +2873,7 @@ components:
     RuntimeTransaction:
       type: object
       # NOTE: Not guaranteed to be present: eth_hash, to, amount.
-      required: [round, index, timestamp, hash, signers, sender_0, nonce_0, fee, fee_symbol, charged_fee, gas_limit, gas_used, size]
+      required: [round, index, timestamp, hash, signers, sender_0, nonce_0, fee, fee_symbol, charged_fee, gas_limit, gas_used, size, is_likely_native_token_transfer]
       properties:
         round:
           type: integer

storage/client/client.go

@@ -1516,6 +1516,7 @@ func (c *StorageClient) RuntimeTransactions(ctx context.Context, p apiTypes.GetR
 			&oasisEncryptionEnvelope.ResultNonce,
 			&oasisEncryptionEnvelope.Result,
 			&t.Method,
+			&t.IsLikelyNativeTokenTransfer,
 			&t.Body,
 			&t.To,
 			&toPreimageContextIdentifier,
@@ -1580,19 +1581,6 @@ func (c *StorageClient) RuntimeTransactions(ctx context.Context, p apiTypes.GetR
 			t.ToEth = EthChecksumAddrFromPreimage(*toPreimageContextIdentifier, *toPreimageContextVersion, toPreimageData)
 		}
 
-		// Heuristically decide if this is a native runtime token transfer.
-		// TODO: Similarly to above, this application logic doesn't belong here (= the DB layer);
-		// move it out if we establish a separate app/logic layer.
-		if t.Method != nil {
-			if *t.Method == "accounts.Transfer" && t.AmountSymbol != nil && *t.AmountSymbol == c.nativeTokenSymbol(runtimeFromCtx(ctx)) {
-				t.IsLikelyNativeTokenTransfer = common.Ptr(true)
-			} else if *t.Method == "evm.Call" && t.Body != nil && (*t.Body)["data"] == "" {
-				// Note: This demands that the body.data key does exist (as we expect from evm.Call tx bodies),
-				// but has an empty value.
-				t.IsLikelyNativeTokenTransfer = common.Ptr(true)
-			}
-		}
-
 		// Try extracting parsed PCS quote from rofl.Register transaction body.
 		if t.Method != nil && *t.Method == "rofl.Register" {
 			nb, err := extractPCSQuote(t.Body)

storage/client/queries/queries.go

@@ -69,34 +69,38 @@ const (
 
 	Transactions = `
 		SELECT
-				chain.transactions.block as block,
-				chain.transactions.tx_index as tx_index,
-				chain.transactions.tx_hash as tx_hash,
-				chain.transactions.sender as sender,
-				chain.transactions.nonce as nonce,
-				chain.transactions.fee_amount as fee_amount,
-				chain.transactions.max_gas as gas_limit,
-				chain.transactions.method as method,
-				chain.transactions.body as body,
-				chain.transactions.code as code,
-				chain.transactions.module as module,
-				chain.transactions.message as message,
-				chain.blocks.time as time
-			FROM chain.transactions
-			JOIN chain.blocks ON chain.transactions.block = chain.blocks.height
-			LEFT JOIN chain.accounts_related_transactions ON chain.transactions.block = chain.accounts_related_transactions.tx_block
-				AND chain.transactions.tx_index = chain.accounts_related_transactions.tx_index
+				t.block as block,
+				t.tx_index as tx_index,
+				t.tx_hash as tx_hash,
+				t.sender as sender,
+				t.nonce as nonce,
+				t.fee_amount as fee_amount,
+				t.max_gas as gas_limit,
+				t.method as method,
+				t.body as body,
+				t.code as code,
+				t.module as module,
+				t.message as message,
+				b.time as time
+			FROM chain.transactions AS t
+			JOIN chain.blocks AS b ON
+				t.block = b.height
+			LEFT JOIN chain.accounts_related_transactions AS art ON
+				(t.block = art.tx_block) AND
+				(t.tx_index = art.tx_index) AND
+				($3::text IS NULL OR t.method = art.method) AND
 				-- When related_address ($5) is NULL and hence we do no filtering on it, avoid the join altogether.
 				-- Otherwise, every tx will be returned as many times as there are related addresses for it.
-				AND $5::text IS NOT NULL
-			WHERE ($1::text IS NULL OR chain.transactions.tx_hash = $1::text) AND
-					($2::bigint IS NULL OR chain.transactions.block = $2::bigint) AND
-					($3::text IS NULL OR chain.transactions.method = $3::text) AND
-					($4::text IS NULL OR chain.transactions.sender = $4::text) AND
-					($5::text IS NULL OR chain.accounts_related_transactions.account_address = $5::text) AND
-					($6::timestamptz IS NULL OR chain.blocks.time >= $6::timestamptz) AND
-					($7::timestamptz IS NULL OR chain.blocks.time < $7::timestamptz)
-			ORDER BY chain.transactions.block DESC, chain.transactions.tx_index
+				($5::text IS NOT NULL)
+			WHERE
+				($1::text IS NULL OR t.tx_hash = $1::text) AND
+				($2::bigint IS NULL OR t.block = $2::bigint) AND
+				($3::text IS NULL OR t.method = $3::text) AND
+				($4::text IS NULL OR t.sender = $4::text) AND
+				($5::text IS NULL OR art.account_address = $5::text) AND
+				($6::timestamptz IS NULL OR b.time >= $6::timestamptz) AND
+				($7::timestamptz IS NULL OR b.time < $7::timestamptz)
+			ORDER BY t.block DESC, t.tx_index
 			LIMIT $8::bigint
 			OFFSET $9::bigint`
 
@@ -486,6 +490,7 @@ const (
 			txs.oasis_encrypted_result_nonce,
 			txs.oasis_encrypted_result_data,
 			txs.method,
+			txs.likely_native_transfer,
 			txs.body,
 			txs.to,
 			to_preimage.context_identifier AS to_preimage_context_identifier,
@@ -507,7 +512,7 @@ const (
 			txs.error_message,
 			txs.error_params
 		FROM chain.runtime_transactions AS txs
-		LEFT JOIN chain.runtime_transaction_signers AS signers ON
+		JOIN chain.runtime_transaction_signers AS signers ON
 			(signers.runtime = txs.runtime) AND
 			(signers.round = txs.round) AND
 			(signers.tx_index = txs.tx_index)
@@ -526,9 +531,10 @@ const (
 			(to_preimage.context_identifier = 'oasis-runtime-sdk/address: secp256k1eth') AND
 			(to_preimage.context_version = 0)
 		LEFT JOIN chain.runtime_related_transactions AS rel ON
+			(txs.runtime = rel.runtime) AND
 			(txs.round = rel.tx_round) AND
 			(txs.tx_index = rel.tx_index) AND
-			(txs.runtime = rel.runtime) AND
+			($5::text IS NULL OR txs.method = rel.method AND txs.likely_native_transfer = rel.likely_native_transfer) AND
 			-- When related_address ($4) is NULL and hence we do no filtering on it, avoid the join altogether.
 			-- Otherwise, every tx will be returned as many times as there are related addresses for it.
 			($4::text IS NOT NULL)
@@ -544,10 +550,10 @@ const (
 						TRUE
 					-- Special case to return are 'likely to be native transfers'.
 					WHEN $5::text = 'native_transfers' THEN
-						(txs.method = 'accounts.Transfer' OR (txs.method = 'evm.Call' AND (txs.body ->> 'data') = ''))
+						(txs.likely_native_transfer = TRUE)
 					-- Special case to return all evm.Calls that are likely not native transfers.
 					WHEN $5::text = 'evm.Call_no_native' THEN
-						(txs.method = 'evm.Call' AND (txs.body ->> 'data') != '')
+						(txs.method = 'evm.Call' AND txs.likely_native_transfer = FALSE)
 					-- Regular case.
 					ELSE
 						(txs.method = $5::text)
@@ -575,6 +581,7 @@ const (
 			txs.oasis_encrypted_result_nonce,
 			txs.oasis_encrypted_result_data,
 			txs.method,
+			txs.likely_native_transfer,
 			txs.body,
 			txs.to,
 			to_preimage.context_identifier,

storage/migrations/13_related_transactions_method_denorm.up.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+-- Update consensus account related transactions to include the method field.
+ALTER TABLE chain.accounts_related_transactions
+	ADD COLUMN method TEXT;
+
+UPDATE chain.accounts_related_transactions AS art
+	SET method = t.method
+	FROM chain.transactions AS t
+	WHERE art.tx_block = t.block AND art.tx_index = t.tx_index;
+
+CREATE INDEX ix_accounts_related_transactions_address_method_block_tx_index ON chain.accounts_related_transactions (account_address, method, tx_block DESC, tx_index);
+
+ALTER TABLE chain.accounts_related_transactions ALTER COLUMN method SET NOT NULL;
+
+COMMIT;

storage/migrations/14_related_runtime_transactions_method_denom.up.sql

@@ -0,0 +1,36 @@
+BEGIN;
+
+-- Add 'likely native transfer' field to runtime transactions.
+ALTER TABLE chain.runtime_transactions
+	ADD COLUMN likely_native_transfer BOOLEAN NOT NULL DEFAULT FALSE;
+UPDATE chain.runtime_transactions AS rt
+	SET likely_native_transfer = (rt.method = 'accounts.transfer' OR (rt.method = 'evm.Call' AND (rt.body ->> 'data') = ''));
+
+-- Drop unneeded indexes which were previously used to filter for 'likely native transfer'.
+DROP INDEX chain.ix_runtime_transactions_evm_call_empty_data;
+DROP INDEX chain.ix_runtime_transactions_evm_call_non_empty_data;
+
+-- Create indexes for native transfer and method fields.
+CREATE INDEX ix_runtime_transactions_native_transfer_round ON chain.runtime_transactions (runtime, likely_native_transfer, round, tx_index);
+-- This combination is needed, since explorer needs to obtain evm.Call's which are not native transfers.
+CREATE INDEX ix_runtime_transactions_method_native_transfer_round ON chain.runtime_transactions (runtime, method, likely_native_transfer, round, tx_index);
+
+-- Update runtime related transactions table to include the method and likely_native_transfer fields.
+-- We have this denormalized data since efficient filtering by method for a specific account is important.
+ALTER TABLE chain.runtime_related_transactions
+	ADD COLUMN method TEXT,
+	ADD COLUMN likely_native_transfer BOOLEAN NOT NULL DEFAULT FALSE;
+
+UPDATE chain.runtime_related_transactions AS rrt
+	SET
+		method = rt.method,
+		likely_native_transfer = rt.likely_native_transfer
+	FROM chain.runtime_transactions AS rt
+	WHERE rrt.runtime = rt.runtime AND rrt.tx_round = rt.round AND rrt.tx_index = rt.tx_index;
+
+CREATE INDEX ix_runtime_related_transactions_address_method_round ON chain.runtime_related_transactions (runtime, account_address, method, tx_round, tx_index);
+CREATE INDEX ix_runtime_related_transactions_address_native_transfer_round ON chain.runtime_related_transactions (runtime, account_address, likely_native_transfer, tx_round, tx_index);
+-- This combination is needed, since explorer needs to obtain evm.Call's which are not native transfers.
+CREATE INDEX ix_runtime_related_transactions_address_method_native_transfer_round_evm_call ON chain.runtime_related_transactions (runtime, account_address, method, likely_native_transfer, tx_round, tx_index);
+
+COMMIT;

tests/e2e_regression/damask/expected/db__account_related_txs.csv

@@ -1 +1 @@
-runtime,account_address,tx_round,tx_index
+runtime,account_address,tx_round,tx_index,method,likely_native_transfer