diff --git a/astar/src/mappings/mappingHandlers.ts b/astar/src/mappings/mappingHandlers.ts index c54c390..63ba435 100644 --- a/astar/src/mappings/mappingHandlers.ts +++ b/astar/src/mappings/mappingHandlers.ts @@ -12,9 +12,9 @@ import { EvmLog as EvmLogModel } from "../types"; import FrontierEvmDatasourcePlugin, { FrontierEvmCall } from "@subql/frontier-evm-processor/"; -import {inputToFunctionSighash, isZero, getSelector, wrapExtrinsics, wrapEvents} from "../utils"; +import {inputToFunctionSighash, isZero, wrapExtrinsics, wrapEvents} from "../utils"; import {ApiPromise} from "@polkadot/api"; - +import {u8, Vec} from "@polkadot/types"; let specVersion: SpecVersion; @@ -67,23 +67,17 @@ export async function handleBlock(block: SubstrateBlock): Promise { evmTransactions.push(handleEvmTransaction(call.idx.toString(),frontierEvmCall)) } }) - // seems there is a concurrent limitation for promise.all and bulkCreate work together, - // the last entity upsertion are missed - // We will put them into two promise for now. - await Promise.all([ - store.bulkCreate('Event', events), - store.bulkCreate('ContractEmitted', contractEmittedEvents), - store.bulkCreate('EvmLog', evmLogs), - ]); - await Promise.all([ - store.bulkCreate('Extrinsic', calls), - store.bulkCreate('ContractsCall', contractCalls), - store.bulkCreate('EvmTransaction', evmTransactions) - ]); + // seems there is a concurrent limitation for promise.all and bulkCreate work together + await store.bulkCreate('Event', events) + await store.bulkCreate('ContractEmitted', contractEmittedEvents) + await store.bulkCreate('EvmLog', evmLogs) + await store.bulkCreate('Extrinsic', calls) + await store.bulkCreate('ContractsCall', contractCalls) + await store.bulkCreate('EvmTransaction', evmTransactions) } export function handleEvent(event: SubstrateEvent): Event { - const newEvent = new Event(`${event.block.block.header.number.toString()}-${event.idx}`); + const newEvent = new Event(`${event.block.block.header.number.toString()}-${event.idx.toString()}`); newEvent.blockHeight = event.block.block.header.number.toBigInt(); newEvent.module = event.event.section; newEvent.event = event.event.method; @@ -102,8 +96,7 @@ export function handleCall(extrinsic: SubstrateExtrinsic): Extrinsic { function handleEvmEvent(event: SubstrateEvent): EvmLogModel { const [{address, data, topics}] = event.event.data as unknown as [EvmLog]; - - const evmLog = new EvmLogModel(`${event.block.block.header.number.toString()}-${event.idx}`) + const evmLog = new EvmLogModel(`${event.block.block.header.number.toString()}-${event.idx.toString()}`) evmLog.address = address.toString() evmLog.blockHeight= event.block.block.header.number.toBigInt(); evmLog.topics0= topics[0].toHex(); @@ -118,7 +111,7 @@ export function handleEvmTransaction(idx: string, tx: FrontierEvmCall): EvmTrans return; } const func = isZero(tx.data) ? undefined : inputToFunctionSighash(tx.data); - const evmTransaction = new EvmTransaction(`${tx.blockNumber.toString()}-${idx}`) + const evmTransaction = new EvmTransaction(`${tx.blockNumber.toString()}-${idx.toString()}`) evmTransaction.txHash = tx.hash; evmTransaction.from = tx.from; evmTransaction.to= tx.to; @@ -130,22 +123,23 @@ export function handleEvmTransaction(idx: string, tx: FrontierEvmCall): EvmTrans export function handleContractCalls(call: SubstrateExtrinsic): ContractsCall { - const [dest,,,, data] = call.extrinsic.method.args; - const contractCall = new ContractsCall(`${call.block.block.header.number.toString()}-${call.idx}`) + const dest = call.extrinsic.args[0]; + const data = call.extrinsic.args[4]; + const contractCall = new ContractsCall(`${call.block.block.header.number.toString()}-${call.idx.toString()}`) contractCall.from = call.extrinsic.isSigned? call.extrinsic.signer.toString(): undefined; contractCall.success = !call.events.find( (evt) => evt.event.section === 'system' && evt.event.method === 'ExtrinsicFailed' ); contractCall.dest = (dest as Address).toString(); contractCall.blockHeight = call.block.block.header.number.toBigInt(); - contractCall.selector = getSelector(data.toU8a()) + contractCall.selector =(data as Vec).toHex().slice(0, 10) return contractCall; } export function handleContractsEmitted(event: SubstrateEvent):ContractEmitted{ const [contract, data] = event.event.data as unknown as ContractEmittedResult; - const contractEmitted = new ContractEmitted(`${event.block.block.header.number.toString()}-${event.event.index.toString()}`); + const contractEmitted = new ContractEmitted(`${event.block.block.header.number.toString()}-${event.idx.toString()}`); contractEmitted.blockHeight = event.block.block.header.number.toBigInt(); contractEmitted.contract= contract.toString(); contractEmitted.from= event.extrinsic.extrinsic.isSigned? event.extrinsic.extrinsic.signer.toString(): EMPTY_ADDRESS; diff --git a/astar/src/utils.ts b/astar/src/utils.ts index dbaa4f5..c9a2cfd 100644 --- a/astar/src/utils.ts +++ b/astar/src/utils.ts @@ -43,14 +43,6 @@ function getExtrinsicSuccess(events: EventRecord[]): boolean { ); } -export function getSelector(data: Uint8Array): string { - //This should align with https://github.com/polkadot-js/api/blob/0b6f7861080c920407a346e2a3dbe64adcb07a1e/packages/api-contract/src/Abi/index.ts#L249 - const [, trimmed] = compactStripLength(data); - return u8aToHex(trimmed.subarray(0, 4)); -} - - - export function wrapEvents( extrinsics: SubstrateExtrinsic[], events: EventRecord[], diff --git a/edgeware/.gitignore b/edgeware/.gitignore new file mode 100644 index 0000000..5140c32 --- /dev/null +++ b/edgeware/.gitignore @@ -0,0 +1,53 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ +yarn.lock + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ +src/types + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +.data \ No newline at end of file diff --git a/edgeware/README.md b/edgeware/README.md new file mode 100644 index 0000000..1237e25 --- /dev/null +++ b/edgeware/README.md @@ -0,0 +1,28 @@ +# SubQuery - Dictionary + +This special SubQuery Project provides a dictionary of data that pre-indexes events on chain to dramatically improve indexing the performance of your own SubQuery Project, sometimes up to 10x faster. + +It scans over the network, and simply records the module and method for every event/extrinsic on each block - please see the standard entities in `schema.graphql`. + +**If you want to create your SubQuery Dictionary to speed up indexing of your own Substrate chain, fork this project and let us know** + +# Geting Started +### 1. Install dependencies +```shell +yarn +``` + +### 2. Generate types +```shell +yarn codegen +``` + +### 3. Build +```shell +yarn build +``` + +### 4. Run locally +```shell +yarn start:docker +``` diff --git a/edgeware/docker-compose.yml b/edgeware/docker-compose.yml new file mode 100644 index 0000000..9d56aeb --- /dev/null +++ b/edgeware/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3' + +services: + postgres: + image: postgres:12-alpine + ports: + - 5432:5432 + volumes: + - .data/postgres:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: postgres + + subquery-node: + image: onfinality/subql-node:latest + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + volumes: + - ./:/app + command: + - -f=/app + + graphql-engine: + image: onfinality/subql-query:latest + ports: + - 3000:3000 + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + command: + - --name=app + - --playground diff --git a/edgeware/package.json b/edgeware/package.json new file mode 100644 index 0000000..f58120e --- /dev/null +++ b/edgeware/package.json @@ -0,0 +1,34 @@ +{ + "name": "subql-dictionary", + "version": "1.0.0", + "description": "A SubQuery Dictionary Project that provides increased indexing speed to all projects", + "main": "dist/index.js", + "scripts": { + "build": "subql build", + "prepack": "rm -rf dist && npm build", + "test": "jest", + "codegen": "./node_modules/.bin/subql codegen", + "start:docker": "docker-compose pull && docker-compose up --remove-orphans" + }, + "homepage": "https://github.com/subquery/subql-dictionary", + "repository": "github:subquery/subql-dictionary", + "files": [ + "dist", + "schema.graphql", + "project.yaml" + ], + "author": "SubQuery Network", + "license": "Apache-2.0", + "resolutions": { + "@polkadot/api": "9.2.4", + "ipfs-unixfs": "6.0.6" + }, + "devDependencies": { + "@polkadot/api": "9.2.4", + "@subql/cli": "latest", + "@subql/frontier-evm-processor": "0.1.2", + "@subql/types": "latest", + "typescript": "^4.4.4", + "abab": "latest" + } +} diff --git a/edgeware/project.yaml b/edgeware/project.yaml new file mode 100644 index 0000000..ae63fae --- /dev/null +++ b/edgeware/project.yaml @@ -0,0 +1,27 @@ +specVersion: 1.0.0 +name: SubQuery Dictionary - Edgeware +version: 1.0.0 +runner: + node: + name: '@subql/node' + version: '>=0.32.0' + query: + name: '@subql/query' + version: '*' +description: >- + A SubQuery Dictionary Project that provides increased indexing speed to all + projects +repository: 'https://github.com/subquery/subql-dictionary' +schema: + file: ./schema.graphql +network: + chainId: '0x742a2ca70c2fda6cee4f8df98d64c4c670a052d9568058982dad9d5a7a135c5b' + endpoint: 'wss://edgeware.api.onfinality.io/public-ws' +dataSources: + - kind: substrate/Runtime + startBlock: 1 + mapping: + file: ./dist/index.js + handlers: + - handler: handleBlock + kind: substrate/BlockHandler diff --git a/edgeware/schema.graphql b/edgeware/schema.graphql new file mode 100644 index 0000000..cda0043 --- /dev/null +++ b/edgeware/schema.graphql @@ -0,0 +1,100 @@ +type SpecVersion @entity { + + id: ID! #specVersion + + blockHeight: BigInt! + +} + +type Event @entity { + + id: ID! + + module: String! @index + + event: String! @index + + blockHeight: BigInt! @index +} + +type Extrinsic @entity { + + id: ID! + + module: String! @index + + call: String! @index + + blockHeight: BigInt! @index + + success: Boolean! + + isSigned: Boolean! + +} + +type EvmTransaction @entity { + + id: ID! + + txHash: String! @index + + from: String! @index + + to: String! @index + + func: String @index + + blockHeight: BigInt! @index + + success: Boolean! + +} + +type EvmLog @entity { + + id: ID! + + address: String! @index + + blockHeight: BigInt! @index + + topics0: String! + + topics1: String + + topics2: String + + topics3: String + +} + +type ContractsCall @entity { + + id: ID! + + from: String! @index + + dest: String! @index #contract address + + selector: String! @index + + blockHeight: BigInt! @index + + success: Boolean! + +} + +type ContractEmitted @entity { + + id: ID! + + contract: String! @index + + from: String !@index + + eventIndex: Int! @index + + blockHeight: BigInt! @index + +} diff --git a/edgeware/src/index.ts b/edgeware/src/index.ts new file mode 100644 index 0000000..c38faa4 --- /dev/null +++ b/edgeware/src/index.ts @@ -0,0 +1,5 @@ +//Exports all handler functions +import {atob} from 'abab'; +global.atob = atob; +export * from "./mappings/mappingHandlers"; +import "@polkadot/api-augment"; diff --git a/edgeware/src/mappings/mappingHandlers.ts b/edgeware/src/mappings/mappingHandlers.ts new file mode 100644 index 0000000..c54c390 --- /dev/null +++ b/edgeware/src/mappings/mappingHandlers.ts @@ -0,0 +1,154 @@ +import {Bytes} from '@polkadot/types'; +const EMPTY_ADDRESS = '0x0000000000000000000000000000000000000000'; +import {TransactionV2, EthTransaction ,AccountId, Address, EvmLog} from "@polkadot/types/interfaces" +import {SubstrateExtrinsic,SubstrateBlock,SubstrateEvent} from "@subql/types"; +import { + SpecVersion, + Event, + Extrinsic, + EvmTransaction, + ContractEmitted, + ContractsCall, + EvmLog as EvmLogModel +} from "../types"; +import FrontierEvmDatasourcePlugin, { FrontierEvmCall } from "@subql/frontier-evm-processor/"; +import {inputToFunctionSighash, isZero, getSelector, wrapExtrinsics, wrapEvents} from "../utils"; +import {ApiPromise} from "@polkadot/api"; + + +let specVersion: SpecVersion; + +export type ContractEmittedResult = [AccountId, Bytes] + +export async function handleBlock(block: SubstrateBlock): Promise { + if (!specVersion) { + specVersion = await SpecVersion.get(block.specVersion.toString()); + } + if(!specVersion || specVersion.id !== block.specVersion.toString()){ + specVersion = new SpecVersion(block.specVersion.toString()); + specVersion.blockHeight = block.block.header.number.toBigInt(); + await specVersion.save(); + } + const wrappedCalls = wrapExtrinsics(block); + const wrappedEvents = wrapEvents(wrappedCalls,block.events.filter( + (evt) => + !(evt.event.section === "system" && + evt.event.method === "ExtrinsicSuccess") + ),block) + let events=[] + let contractEmittedEvents=[]; + let evmLogs=[]; + wrappedEvents.filter(evt => evt.event.section!=='system' && evt.event.method!=='ExtrinsicSuccess').map(event=>{ + events.push(handleEvent(event)) + if (event.event.section === 'contracts' && (event.event.method === 'ContractEmitted' || event.event.method === 'ContractExecution')) { + contractEmittedEvents.push(handleContractsEmitted(event)); + } + if(event.event.section === 'evm' && event.event.method === 'Log'){ + evmLogs.push(handleEvmEvent(event)); + } + }) + + let calls=[] + let contractCalls=[]; + let evmTransactions=[]; + + wrappedCalls.map(async call => { + calls.push(handleCall(call)) + if (call.extrinsic.method.section === 'contracts' && call.extrinsic.method.method === 'call') { + contractCalls.push(handleContractCalls(call)); + } + if (call.extrinsic.method.section === 'ethereum' && call.extrinsic.method.method === 'transact') { + const [frontierEvmCall] = await FrontierEvmDatasourcePlugin.handlerProcessors['substrate/FrontierEvmCall'].transformer({ + input: call as SubstrateExtrinsic<[TransactionV2 | EthTransaction]>, + ds: {} as any, + filter: undefined, + api: api as ApiPromise + }) + evmTransactions.push(handleEvmTransaction(call.idx.toString(),frontierEvmCall)) + } + }) + // seems there is a concurrent limitation for promise.all and bulkCreate work together, + // the last entity upsertion are missed + // We will put them into two promise for now. + await Promise.all([ + store.bulkCreate('Event', events), + store.bulkCreate('ContractEmitted', contractEmittedEvents), + store.bulkCreate('EvmLog', evmLogs), + ]); + await Promise.all([ + store.bulkCreate('Extrinsic', calls), + store.bulkCreate('ContractsCall', contractCalls), + store.bulkCreate('EvmTransaction', evmTransactions) + ]); +} + +export function handleEvent(event: SubstrateEvent): Event { + const newEvent = new Event(`${event.block.block.header.number.toString()}-${event.idx}`); + newEvent.blockHeight = event.block.block.header.number.toBigInt(); + newEvent.module = event.event.section; + newEvent.event = event.event.method; + return newEvent; +} + +export function handleCall(extrinsic: SubstrateExtrinsic): Extrinsic { + const newExtrinsic = new Extrinsic(`${extrinsic.block.block.header.number.toString()}-${extrinsic.idx.toString()}`); + newExtrinsic.module = extrinsic.extrinsic.method.section; + newExtrinsic.call = extrinsic.extrinsic.method.method; + newExtrinsic.blockHeight = extrinsic.block.block.header.number.toBigInt(); + newExtrinsic.success = extrinsic.success; + newExtrinsic.isSigned = extrinsic.extrinsic.isSigned; + return newExtrinsic; +} + +function handleEvmEvent(event: SubstrateEvent): EvmLogModel { + const [{address, data, topics}] = event.event.data as unknown as [EvmLog]; + + const evmLog = new EvmLogModel(`${event.block.block.header.number.toString()}-${event.idx}`) + evmLog.address = address.toString() + evmLog.blockHeight= event.block.block.header.number.toBigInt(); + evmLog.topics0= topics[0].toHex(); + evmLog.topics1= topics[1]?.toHex(); + evmLog.topics2= topics[2]?.toHex(); + evmLog.topics3= topics[3]?.toHex(); + return evmLog +} + +export function handleEvmTransaction(idx: string, tx: FrontierEvmCall): EvmTransaction { + if (!tx.hash) { + return; + } + const func = isZero(tx.data) ? undefined : inputToFunctionSighash(tx.data); + const evmTransaction = new EvmTransaction(`${tx.blockNumber.toString()}-${idx}`) + evmTransaction.txHash = tx.hash; + evmTransaction.from = tx.from; + evmTransaction.to= tx.to; + evmTransaction.func = func; + evmTransaction.blockHeight = BigInt(tx.blockNumber.toString()); + evmTransaction.success = tx.success; + return evmTransaction +} + + +export function handleContractCalls(call: SubstrateExtrinsic): ContractsCall { + const [dest,,,, data] = call.extrinsic.method.args; + const contractCall = new ContractsCall(`${call.block.block.header.number.toString()}-${call.idx}`) + contractCall.from = call.extrinsic.isSigned? call.extrinsic.signer.toString(): undefined; + contractCall.success = !call.events.find( + (evt) => evt.event.section === 'system' && evt.event.method === 'ExtrinsicFailed' + ); + contractCall.dest = (dest as Address).toString(); + contractCall.blockHeight = call.block.block.header.number.toBigInt(); + contractCall.selector = getSelector(data.toU8a()) + return contractCall; + +} + +export function handleContractsEmitted(event: SubstrateEvent):ContractEmitted{ + const [contract, data] = event.event.data as unknown as ContractEmittedResult; + const contractEmitted = new ContractEmitted(`${event.block.block.header.number.toString()}-${event.event.index.toString()}`); + contractEmitted.blockHeight = event.block.block.header.number.toBigInt(); + contractEmitted.contract= contract.toString(); + contractEmitted.from= event.extrinsic.extrinsic.isSigned? event.extrinsic.extrinsic.signer.toString(): EMPTY_ADDRESS; + contractEmitted.eventIndex= data[0] + return contractEmitted; +} diff --git a/edgeware/src/utils.ts b/edgeware/src/utils.ts new file mode 100644 index 0000000..dbaa4f5 --- /dev/null +++ b/edgeware/src/utils.ts @@ -0,0 +1,68 @@ +import {hexDataSlice, stripZeros} from '@ethersproject/bytes'; +import {EventRecord} from "@polkadot/types/interfaces" +import {SubstrateBlock, SubstrateEvent, SubstrateExtrinsic} from "@subql/types"; +import {compactStripLength, u8aToHex} from '@polkadot/util'; +import { merge } from 'lodash'; + +export function inputToFunctionSighash(input: string): string { + return hexDataSlice(input, 0, 4); +} + +export function isZero(input: string): boolean { + return stripZeros(input).length === 0; +} + +function filterExtrinsicEvents( + extrinsicIdx: number, + events: EventRecord[], +): EventRecord[] { + return events.filter( + ({ phase }) => + phase.isApplyExtrinsic && phase.asApplyExtrinsic.eqn(extrinsicIdx), + ); +} + +export function wrapExtrinsics( + wrappedBlock: SubstrateBlock, +): SubstrateExtrinsic[] { + return wrappedBlock.block.extrinsics.map((extrinsic, idx) => { + const events = filterExtrinsicEvents(idx, wrappedBlock.events); + return { + idx, + extrinsic, + block: wrappedBlock, + events, + success: getExtrinsicSuccess(events), + }; + }); +} + +function getExtrinsicSuccess(events: EventRecord[]): boolean { + return ( + events.findIndex((evt) => evt.event.method === 'ExtrinsicSuccess') > -1 + ); +} + +export function getSelector(data: Uint8Array): string { + //This should align with https://github.com/polkadot-js/api/blob/0b6f7861080c920407a346e2a3dbe64adcb07a1e/packages/api-contract/src/Abi/index.ts#L249 + const [, trimmed] = compactStripLength(data); + return u8aToHex(trimmed.subarray(0, 4)); +} + + + +export function wrapEvents( + extrinsics: SubstrateExtrinsic[], + events: EventRecord[], + block: SubstrateBlock, +): SubstrateEvent[] { + return events.reduce((acc, event, idx) => { + const { phase } = event; + const wrappedEvent: SubstrateEvent = merge(event, { idx, block }); + if (phase.isApplyExtrinsic) { + wrappedEvent.extrinsic = extrinsics[phase.asApplyExtrinsic.toNumber()]; + } + acc.push(wrappedEvent); + return acc; + }, [] as SubstrateEvent[]); +} diff --git a/edgeware/tsconfig.json b/edgeware/tsconfig.json new file mode 100644 index 0000000..9b95212 --- /dev/null +++ b/edgeware/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "esModuleInterop": true, + "declaration": true, + "importHelpers": true, + "resolveJsonModule": true, + "module": "commonjs", + "outDir": "dist", + "rootDir": "src", + "target": "es2017" + }, + "include": [ + "src/**/*", + "node_modules/@subql/types/dist/global.d.ts" + ] +}