HydraDX dict

.github/workflows/cli-deploy.yml

@@ -28,7 +28,7 @@ jobs:
     - name: Setup Node.js environment
       uses: actions/setup-node@v2
       with:
-        node-version: 16
+        node-version: 18
     - run: yarn
       working-directory: ./${{github.event.inputs.directory}}
     - name: Codegen

.github/workflows/scripts/publish-deploy.sh

@@ -14,13 +14,13 @@ done
 
 cd "$FILE" || exit
 
-if [ "$YAML" != undefined ]
-  then
-    IPFSCID=$(npx subql publish -o -f"$ROOTPATH/$FILE/$YAML")
-  else
+# Check if YAML is defined and set IPFSCID and ENDPOINT accordingly
+if [ -n "$YAML" ]; then
+    IPFSCID=$(npx subql publish -o -f "$ROOTPATH/$FILE/$YAML")
+    ENDPOINT=$(yq e '.network.endpoint' "$ROOTPATH/$FILE/$YAML")
+else
     IPFSCID=$(npx subql publish -o -f "$ROOTPATH/$FILE")
+    ENDPOINT=$(yq e '.network.endpoint' "$ROOTPATH/$FILE/project.yaml")
 fi
 
-IPFSCID=$(npx subql publish -o -f "$ROOTPATH/$FILE")
-
-npx subql deployment:deploy -d --ipfsCID="$IPFSCID" --projectName="${PROJECTNAME}" --org="${ORG%/*}"
+npx subql deployment:deploy -d --ipfsCID="$IPFSCID" --projectName="${PROJECTNAME}" --org="${ORG%/*}" --endpoint="${ENDPOINT}"

hydradx/.gitignore

@@ -0,0 +1,53 @@
+# These are some examples of commonly ignored file patterns.
+# You should customize this list as applicable to your project.
+# Learn more about .gitignore:
+#     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
+
+# Node artifact files
+node_modules/
+dist/
+yarn.lock
+
+# Compiled Java class files
+*.class
+
+# Compiled Python bytecode
+*.py[cod]
+
+# Log files
+*.log
+
+# Package files
+*.jar
+
+# Maven
+target/
+dist/
+src/types
+
+# JetBrains IDE
+.idea/
+
+# Unit test reports
+TEST*.xml
+
+# Generated by MacOS
+.DS_Store
+
+# Generated by Windows
+Thumbs.db
+
+# Applications
+*.app
+*.exe
+*.war
+
+# Large media files
+*.mp4
+*.tiff
+*.avi
+*.flv
+*.mov
+*.wmv
+
+.data

hydradx/README.md

@@ -0,0 +1,28 @@
+# SubQuery - Dictionary 
+
+This special SubQuery Project provides a dictionary of data that pre-indexes events on chain to dramatically improve indexing the performance of your own SubQuery Project, sometimes up to 10x faster.
+
+It scans over the network, and simply records the module and method for every event/extrinsic on each block - please see the standard entities in `schema.graphql`.
+
+**If you want to create your SubQuery Dictionary to speed up indexing of your own Substrate chain, fork this project and let us know**
+
+# Geting Started
+### 1. Install dependencies
+```shell
+yarn
+```
+
+### 2. Generate types
+```shell
+yarn codegen
+```
+
+### 3. Build
+```shell
+yarn build
+```
+
+### 4. Run locally
+```shell
+yarn start:docker
+```

hydradx/docker-compose.yml

@@ -0,0 +1,45 @@
+version: '3'
+
+services:
+  postgres:
+    image: postgres:12-alpine
+    ports:
+      - 5432:5432
+    volumes:
+      - .data/postgres:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  subquery-node:
+    image: onfinality/subql-node:latest
+    depends_on:
+      - "postgres"
+    restart: always
+    environment:
+      DB_USER: postgres
+      DB_PASS: postgres
+      DB_DATABASE: postgres
+      DB_HOST: postgres
+      DB_PORT: 5432
+    volumes:
+      - ./:/app
+    command:
+      - -f=/app
+      - --disable-historical=true
+
+  graphql-engine:
+    image: onfinality/subql-query:latest
+    ports:
+      - 3000:3000
+    depends_on:
+      - "postgres"
+    restart: always
+    environment:
+      DB_USER: postgres
+      DB_PASS: postgres
+      DB_DATABASE: postgres
+      DB_HOST: postgres
+      DB_PORT: 5432
+    command:
+      - --name=app
+      - --playground

hydradx/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "hydradx-dictionary",
+  "version": "1.0.0",
+  "description": "A SubQuery Dictionary Project that provides increased indexing speed to all projects",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "subql build",
+    "prepack": "rm -rf dist && npm build",
+    "test": "jest",
+    "codegen": "./node_modules/.bin/subql codegen",
+    "publish:all": "bash scripts/publish-manifest.sh",
+    "start:docker": "docker-compose pull && docker-compose up --remove-orphans"
+  },
+  "homepage": "https://github.com/subquery/subql-dictionary",
+  "repository": "github:subquery/subql-dictionary",
+  "files": [
+    "dist",
+    "schema.graphql",
+    "project.yaml"
+  ],
+  "author": "SubQuery Network",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@polkadot/api": "^9",
+    "@subql/types": "latest",
+    "typescript": "^4.1.3",
+    "@subql/cli": "latest",
+    "@parallel-finance/type-definitions": "latest"
+  },
+  "resolutions": {
+    "ipfs-unixfs": "6.0.6"
+  }
+}

hydradx/project.yaml

@@ -0,0 +1,30 @@
+specVersion: 1.0.0
+name: SubQuery Dictionary - HydraDX
+version: 1.0.0
+runner:
+  node:
+    name: '@subql/node'
+    version: '>=3.0.0'
+    options:
+      unfinalizedBlocks: false
+      historical: false
+  query:
+    name: '@subql/query'
+    version: '*'
+description: >-
+  A SubQuery Dictionary Project that provides increased indexing speed to all
+  projects
+repository: 'https://github.com/subquery/subql-dictionary'
+schema:
+  file: ./schema.graphql
+network:
+  chainId: '0xafdc188f45c71dacbaa0b62e16a91f726c7b8699a9748cdf715459de6b7f366d'
+  endpoint: 'wss://rpc.helikon.io/hydradx'
+dataSources:
+  - kind: substrate/Runtime
+    startBlock: 1
+    mapping:
+      file: ./dist/index.js
+      handlers:
+        - handler: handleBlock
+          kind: substrate/BlockHandler

hydradx/schema.graphql

@@ -0,0 +1,21 @@
+type SpecVersion @entity {
+  id: ID! #specVersion
+  blockHeight: BigInt!
+}
+
+type Event @entity {
+  id: ID!
+  module: String! @index
+  event: String! @index
+  blockHeight: BigInt! @index
+}
+
+type Extrinsic @entity {
+  id: ID!
+  txHash: String!
+  module: String! @index
+  call: String! @index
+  blockHeight: BigInt! @index
+  success: Boolean!
+  isSigned: Boolean!
+}

hydradx/scripts/publish-manifest.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for file in ./project*.yaml; do
+    echo "Publishing $file..."
+    npx subql publish -f "$file"
+done

hydradx/src/index.ts

@@ -0,0 +1,3 @@
+//Exports all handler functions
+export * from "./mappings/mappingHandlers";
+import "@polkadot/api-augment";

hydradx/src/mappings/mappingHandlers.ts

@@ -0,0 +1,86 @@
+import { EventRecord } from "@polkadot/types/interfaces";
+import { SubstrateExtrinsic, SubstrateBlock } from "@subql/types";
+import { SpecVersion, Event, Extrinsic } from "../types";
+
+let specVersion: SpecVersion;
+export async function handleBlock(block: SubstrateBlock): Promise<void> {
+  // Initialise Spec Version
+  if (!specVersion) {
+    specVersion = await SpecVersion.get(block.specVersion.toString());
+  }
+
+  // Check for updates to Spec Version
+  if (!specVersion || specVersion.id !== block.specVersion.toString()) {
+    specVersion = SpecVersion.create({
+      id: block.specVersion.toString(),
+      blockHeight: block.block.header.number.toBigInt()
+    });
+    await specVersion.save();
+  }
+
+  // Process all events in block
+  const events = block.events
+    .filter(
+      (evt) =>
+        !(evt.event.section === "system" &&
+        evt.event.method === "ExtrinsicSuccess")
+    )
+    .map((evt, idx) =>
+      handleEvent(block.block.header.number.toString(), idx, evt)
+    );
+
+  // Process all calls in block
+  const calls = wrapExtrinsics(block).map((ext, idx) =>
+    handleCall(`${block.block.header.number.toString()}-${idx}`, ext)
+  );
+
+  // Save all data
+  // All save order should always follow this structure
+  for (const event of events) {
+    await event.save()
+  }
+  for (const call of calls) {
+    await call.save()
+  }
+}
+
+function handleEvent(
+  blockNumber: string,
+  eventIdx: number,
+  event: EventRecord
+): Event {
+  return Event.create({
+    id: `${blockNumber}-${eventIdx}`,
+    blockHeight: BigInt(blockNumber),
+    module : event.event.section,
+    event : event.event.method
+  });
+}
+
+function handleCall(idx: string, extrinsic: SubstrateExtrinsic): Extrinsic {
+  return Extrinsic.create({
+    id: idx,
+    txHash : extrinsic.extrinsic.hash.toString(),
+    module : extrinsic.extrinsic.method.section,
+    call : extrinsic.extrinsic.method.method,
+    blockHeight : extrinsic.block.block.header.number.toBigInt(),
+    success : extrinsic.success,
+    isSigned : extrinsic.extrinsic.isSigned,
+  });
+}
+
+function wrapExtrinsics(wrappedBlock: SubstrateBlock): SubstrateExtrinsic[] {
+  return wrappedBlock.block.extrinsics.map((extrinsic, idx) => {
+    const events = wrappedBlock.events.filter(
+      ({ phase }) => phase.isApplyExtrinsic && phase.asApplyExtrinsic.eqn(idx)
+    );
+    return {
+      idx,
+      extrinsic,
+      block: wrappedBlock,
+      events,
+      success:
+        events.findIndex((evt) => evt.event.method === "ExtrinsicSuccess") > -1,
+    };
+  });
+}

hydradx/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "esModuleInterop": true,
+    "declaration": true,
+    "importHelpers": true,
+    "resolveJsonModule": true,
+    "module": "commonjs",
+    "outDir": "dist",
+    "rootDir": "src",
+    "target": "es2017"
+  },
+  "include": [
+    "src/**/*",
+    "node_modules/@subql/types/dist/global.d.ts"
+  ]
+}