From 66c023f01f11516131147fb8b6ff0e2cc87a7f5b Mon Sep 17 00:00:00 2001 From: pdp2121 <71317875+pdp2121@users.noreply.github.com> Date: Mon, 23 Dec 2024 12:10:37 -0500 Subject: [PATCH] feat: Use feature for amendment name (#291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## High Level Overview of Change Resolve: https://github.com/ripple/validator-history-service/issues/278 ### Context of Change Missing new amendments due to the change in rippled code structure to parse name: ![Screenshot 2024-12-19 at 11 32 43 AM](https://github.com/user-attachments/assets/0da97dff-9d6d-4840-b8cf-6b44ba628053) ### Type of Change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Refactor (non-breaking change that only restructures code) - [ ] Tests (You added tests for code that already exists, or your new feature included in this PR) - [ ] Documentation Updates - [ ] Release ## Before / After Data should show up: ![Screenshot 2024-12-19 at 11 58 52 AM](https://github.com/user-attachments/assets/5198638f-4c69-4158-9a31-fa004aab78c4) VHS is lacking tests at the moment. There will be a separate ticket to add comprehensive tests throughout the code --- package-lock.json | 20 ---- package.json | 2 - src/connection-manager/wsHandling.ts | 10 +- src/shared/database/amendments.ts | 166 ++++++++++++++++++--------- 4 files changed, 118 insertions(+), 80 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1d965a8..556af2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "@types/bunyan": "^1.8.7", "axios": "^0.21.1", "bunyan": "^1.8.15", - "create-hash": "^1.2.0", "dotenv": "^16.3.1", "express": "4.21.2", "knex": "2.5.1", @@ -30,7 +29,6 @@ }, "devDependencies": { "@types/axios": "^0.14.0", - "@types/create-hash": "^1.2.2", "@types/express": "4.17.21", "@types/jest": "^26.0.19", "@types/nconf": "^0.10.0", @@ -1476,15 +1474,6 @@ "@types/node": "*" } }, - "node_modules/@types/create-hash": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@types/create-hash/-/create-hash-1.2.6.tgz", - "integrity": "sha512-/VIViZZAK3rAFvfGbWmcLaxwKfmU213W/XL2cr5VE0ac44jE5ky7+sHl54OJhd+bTz7sqi+Ev//8RU1F/S/ZJQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/express": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", @@ -12273,15 +12262,6 @@ "@types/node": "*" } }, - "@types/create-hash": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@types/create-hash/-/create-hash-1.2.6.tgz", - "integrity": "sha512-/VIViZZAK3rAFvfGbWmcLaxwKfmU213W/XL2cr5VE0ac44jE5ky7+sHl54OJhd+bTz7sqi+Ev//8RU1F/S/ZJQ==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/express": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", diff --git a/package.json b/package.json index ca80fff..bd84b6d 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,6 @@ "homepage": "https://github.com/ripple/validator-history-service#readme", "devDependencies": { "@types/axios": "^0.14.0", - "@types/create-hash": "^1.2.2", "@types/express": "4.17.21", "@types/jest": "^26.0.19", "@types/nconf": "^0.10.0", @@ -62,7 +61,6 @@ "@types/bunyan": "^1.8.7", "axios": "^0.21.1", "bunyan": "^1.8.15", - "create-hash": "^1.2.0", "dotenv": "^16.3.1", "express": "4.21.2", "knex": "2.5.1", diff --git a/src/connection-manager/wsHandling.ts b/src/connection-manager/wsHandling.ts index 637d60d..86fb450 100644 --- a/src/connection-manager/wsHandling.ts +++ b/src/connection-manager/wsHandling.ts @@ -14,7 +14,10 @@ import { saveAmendmentStatus, saveAmendmentsStatus, } from '../shared/database' -import { deleteAmendmentStatus } from '../shared/database/amendments' +import { + NETWORKS_HOSTS, + deleteAmendmentStatus, +} from '../shared/database/amendments' import { AmendmentStatus, DatabaseValidator, @@ -32,11 +35,6 @@ const LEDGER_HASHES_SIZE = 10 const GOT_MAJORITY_FLAG = 65536 const LOST_MAJORITY_FLAG = 131072 const FOURTEEN_DAYS_IN_MILLISECONDS = 14 * 24 * 60 * 60 * 1000 -const NETWORKS_HOSTS = new Map([ - ['main', 'ws://s2.ripple.com:51233'], - ['test', 'wss://s.altnet.rippletest.net:51233'], - ['dev', 'wss://s.devnet.rippletest.net:51233'], -]) const log = logger({ name: 'connections' }) diff --git a/src/shared/database/amendments.ts b/src/shared/database/amendments.ts index a9ff837..a637ecf 100644 --- a/src/shared/database/amendments.ts +++ b/src/shared/database/amendments.ts @@ -1,5 +1,9 @@ import axios from 'axios' -import createHash from 'create-hash' +import { Client, ErrorResponse } from 'xrpl' +import { + FeatureAllResponse, + FeatureOneResponse, +} from 'xrpl/dist/npm/models/methods/feature' import { AmendmentInfo } from '../types' import logger from '../utils/logger' @@ -9,78 +13,135 @@ import { query } from './utils' const log = logger({ name: 'amendments' }) const amendmentIDs = new Map() +const votingAmendmentsToTrack = new Set() const rippledVersions = new Map() - -const ACTIVE_AMENDMENT_REGEX = - /^\s*REGISTER_F[A-Z]+\s*\((?\S+),\s*.*$/u -const RETIRED_AMENDMENT_REGEX = - /^ .*retireFeature\("(?\S+)"\)[,;].*$/u +// TODO: Use feature RPC instead when this issue is fixed and released: +// https://github.com/XRPLF/rippled/issues/4730 +const RETIRED_AMENDMENTS = [ + 'MultiSign', + 'TrustSetAuth', + 'FeeEscalation', + 'PayChan', + 'CryptoConditions', + 'TickSize', + 'fix1368', + 'Escrow', + 'fix1373', + 'EnforceInvariants', + 'SortedDirectories', + 'fix1201', + 'fix1512', + 'fix1523', + 'fix1528', +] const AMENDMENT_VERSION_REGEX = /\| \[(?[a-zA-Z0-9_]+)\][^\n]+\| (?v[0-9]*\.[0-9]*\.[0-9]*|TBD) *\|/u -// TODO: Clean this up when this PR is merged: -// https://github.com/XRPLF/rippled/pull/4781 +export const NETWORKS_HOSTS = new Map([ + ['main', 'ws://s2.ripple.com:51233'], + ['test', 'wss://s.altnet.rippletest.net:51233'], + ['dev', 'wss://s.devnet.rippletest.net:51233'], +]) + /** - * Fetch a list of amendments names from rippled file. + * Fetch amendments information including id, name, and deprecated status. * - * @returns The list of amendment names. + * @returns Void. */ -async function fetchAmendmentNames(): Promise | null> { +async function fetchAmendmentsList(): Promise { + for (const [network, url] of NETWORKS_HOSTS) { + await fetchNetworkAmendments(network, url) + } +} + +/** + * Fetch amendments information including id, name, and deprecated status of a network. + * + * @param network - The network being retrieved. + * @param url - The Faucet URL of the network. + * + * @returns Void. + */ +async function fetchNetworkAmendments( + network: string, + url: string, +): Promise { try { - const response = await axios.get( - 'https://raw.githubusercontent.com/XRPLF/rippled/develop/src/libxrpl/protocol/Feature.cpp', - ) - const text = response.data - const amendmentNames: Map = new Map() - text.split('\n').forEach((line: string) => { - const name = ACTIVE_AMENDMENT_REGEX.exec(line) - if (name) { - amendmentNames.set(name[1], name[0].includes('VoteBehavior::Obsolete')) - } else { - const name2 = RETIRED_AMENDMENT_REGEX.exec(line) - if (name2) { - amendmentNames.set(name2[1], true) - } - } + log.info(`Updating amendment info for ${network}...`) + const client = new Client(url) + await client.connect() + const featureAllResponse: FeatureAllResponse = await client.request({ + command: 'feature', }) - return amendmentNames - } catch (err) { - log.error('Error getting amendment names', err) - return null + + const featuresAll = featureAllResponse.result.features + + for (const id of Object.keys(featuresAll)) { + addAmendmentToCache(id, featuresAll[id].name) + } + + // Some amendments in voting are not available in feature all request. + // This loop tries to fetch them in feature one. + for (const amendment_id of votingAmendmentsToTrack) { + const featureOneResponse: FeatureOneResponse | ErrorResponse = + await client.request({ + command: 'feature', + feature: amendment_id, + }) + + // eslint-disable-next-line max-depth -- The depth is only 2, try catch should not count. + if ('result' in featureOneResponse) { + const feature = featureOneResponse.result[amendment_id] + addAmendmentToCache(amendment_id, feature.name) + } + } + + await client.disconnect() + + log.info(`Finished updating amendment info for ${network}...`) + } catch (error) { + log.error( + `Failed to update amendment info for ${network} due to error: ${String( + error, + )}`, + ) } } /** - * Extracts Amendment ID from Amendment name inside a buffer. + * Add an amendment to amendmentIds cache and remove it from the votingAmendmentToTrack cache. * - * @param buffer -- The buffer containing the amendment name. - * - * @returns The amendment ID string. + * @param id - The id of the amendment to add. + * @param name - The name of the amendment to add. */ -function sha512Half(buffer: Buffer): string { - return createHash('sha512') - .update(buffer) - .digest('hex') - .toUpperCase() - .slice(0, 64) +function addAmendmentToCache(id: string, name: string): void { + amendmentIDs.set(id, { + name, + deprecated: RETIRED_AMENDMENTS.includes(name), + }) + votingAmendmentsToTrack.delete(id) } /** - * Maps the id of Amendments to its corresponding names. + * Fetch amendments in voting. * * @returns Void. */ -async function nameOfAmendmentID(): Promise { - // The Amendment ID is the hash of the Amendment name - const amendmentNames = await fetchAmendmentNames() - if (amendmentNames !== null) { - amendmentNames.forEach((deprecated, name) => { - amendmentIDs.set(sha512Half(Buffer.from(name, 'ascii')), { - name, - deprecated, - }) - }) +async function fetchVotingAmendments(): Promise { + const votingDb = await query('ballot') + .select('amendments') + .then(async (res) => + res.map((vote: { amendments: string | null }) => vote.amendments), + ) + for (const amendmentsDb of votingDb) { + if (!amendmentsDb) { + continue + } + const amendments = amendmentsDb.split(',') + for (const amendment of amendments) { + votingAmendmentsToTrack.add(amendment) + } } } @@ -145,7 +206,8 @@ export async function deleteAmendmentStatus( export async function fetchAmendmentInfo(): Promise { log.info('Fetch amendments info from data sources...') - await nameOfAmendmentID() + await fetchVotingAmendments() + await fetchAmendmentsList() await fetchMinRippledVersions() amendmentIDs.forEach(async (value, id) => { const amendment: AmendmentInfo = {