From 0bc73cba5dcba6b47a2b43bb0af6dcadf3888742 Mon Sep 17 00:00:00 2001 From: classicalliu Date: Thu, 19 Aug 2021 13:06:32 +0800 Subject: [PATCH] feat: Migrate hashmap db to sql --- README.md | 9 +- package.json | 1 + packages/api-server/package.json | 5 + packages/api-server/src/migrate-accounts.ts | 50 ++++++++ yarn.lock | 123 +++++++++++++++++++- 5 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 packages/api-server/src/migrate-accounts.ts diff --git a/README.md b/README.md index e1b652df..0d55ddeb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,14 @@ $ yarn $ yarn run migrate:latest -// Only for test purpose +# Migrate accounts data from hashmap db to sql if need +# relative hashmap db path is relative to packages/api-server +# and will use packages/api-server/lib/hashmap-db as default. +$ yarn run migrate-accounts +For example: +$ yarn run migrate-accounts ./hashmap-db + +# Only for test purpose $ yarn workspace @godwoken-web3/api-server reset_database ``` diff --git a/package.json b/package.json index e64e5154..bc870aae 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "knex": "yarn workspace @godwoken-web3/api-server run knex", "migrate:make": "yarn workspace @godwoken-web3/api-server run migrate:make", "migrate:latest": "yarn workspace @godwoken-web3/api-server run migrate:latest", + "migrate-accounts": "yarn workspace @godwoken-web3/api-server run migrate-accounts", "start": "yarn workspace @godwoken-web3/api-server run start" }, "devDependencies": { diff --git a/packages/api-server/package.json b/packages/api-server/package.json index 384b422a..e41d17c4 100644 --- a/packages/api-server/package.json +++ b/packages/api-server/package.json @@ -11,6 +11,7 @@ "knex": "knex", "migrate:make": "knex migrate:make", "migrate:latest": "knex migrate:latest", + "migrate-accounts": "ts-node ./src/migrate-accounts.ts", "build": "tsc" }, "dependencies": { @@ -30,12 +31,16 @@ "jayson": "^3.6.4", "keccak256": "^1.0.2", "knex": "^0.95.7", + "leveldown": "^6.0.1", + "levelup": "^5.0.1", "morgan": "~1.9.1", "pg": "^8.5.1", "rlp": "^2.2.6", "secp256k1": "^4.0.2" }, "devDependencies": { + "@types/leveldown": "^4.0.3", + "@types/levelup": "^4.3.3", "@types/secp256k1": "^4.0.2", "ava": "^3.15.0", "concurrently": "^6.0.0", diff --git a/packages/api-server/src/migrate-accounts.ts b/packages/api-server/src/migrate-accounts.ts new file mode 100644 index 00000000..823d490d --- /dev/null +++ b/packages/api-server/src/migrate-accounts.ts @@ -0,0 +1,50 @@ +import levelup from "levelup"; +import leveldown from "leveldown"; +import path from "path"; +import { Query } from "./db"; +import { envConfig } from "./base/env-config"; + +async function main() { + const argvArgs = process.argv.slice(2); + const defaultStorePath = path.resolve(__dirname, "..", "lib", "./hashmap-db"); + let storePath = argvArgs[0]; + if (storePath == null) { + console.log(`store path not provided, using ${defaultStorePath}`); + storePath = defaultStorePath; + } else { + storePath = path.resolve(__dirname, "..", storePath); + console.log(`Using store path: ${storePath}`); + } + + const db = levelup(leveldown(storePath)); + + const query = new Query(envConfig.databaseUrl); + + const arr: { shortAddress: string; ethAddress: string }[] = []; + db.createReadStream() + .on("data", function (data) { + const shortAddress = data.key.toString(); + const ethAddress = data.value.toString(); + arr.push({ + shortAddress, + ethAddress, + }); + }) + .on("error", function (err) { + console.error("Migrate error:", err); + }) + .on("close", async function () { + for (let a of arr) { + const { shortAddress, ethAddress } = a; + await query.accounts.save(ethAddress, shortAddress); + console.log( + `insert one record, short_address: ${shortAddress}, eth_address: ${ethAddress}` + ); + } + console.log("Stream closed"); + process.exit(0); + }) + .on("end", function () {}); +} + +main(); diff --git a/yarn.lock b/yarn.lock index 3069348e..36be6486 100644 --- a/yarn.lock +++ b/yarn.lock @@ -326,6 +326,11 @@ dependencies: defer-to-connect "^1.0.1" +"@types/abstract-leveldown@*": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@types/abstract-leveldown/-/abstract-leveldown-5.0.2.tgz#ee81917fe38f770e29eec8139b6f16ee4a8b0a5f" + integrity sha512-+jA1XXF3jsz+Z7FcuiNqgK53hTa/luglT2TyTpKPqoYbxVY+mCPF22Rm+q3KPBrMHJwNXFrTViHszBOfU4vftQ== + "@types/bn.js@^4.11.3", "@types/bn.js@^4.11.5": version "4.11.6" resolved "https://registry.yarnpkg.com/@types/bn.js/-/bn.js-4.11.6.tgz#c306c70d9358aaea33cd4eda092a742b9505967c" @@ -361,6 +366,28 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.8.tgz#edf1bf1dbf4e04413ca8e5b17b3b7d7d54b59818" integrity sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg== +"@types/level-errors@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/level-errors/-/level-errors-3.0.0.tgz#15c1f4915a5ef763b51651b15e90f6dc081b96a8" + integrity sha512-/lMtoq/Cf/2DVOm6zE6ORyOM+3ZVm/BvzEZVxUhf6bgh8ZHglXlBqxbxSlJeVp8FCbD3IVvk/VbsaNmDjrQvqQ== + +"@types/leveldown@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@types/leveldown/-/leveldown-4.0.3.tgz#4b868fd747808d378df6ffb27de7f889cae46aad" + integrity sha512-fzIXOuUCSZQKkRadWhURwu6mMYbfqi/nRDA+yBwz8kj7AK/L7L//u6A0MqSv0gsilzo7N/5+FlZOx8G6m03EVQ== + dependencies: + "@types/abstract-leveldown" "*" + "@types/node" "*" + +"@types/levelup@^4.3.3": + version "4.3.3" + resolved "https://registry.yarnpkg.com/@types/levelup/-/levelup-4.3.3.tgz#4dc2b77db079b1cf855562ad52321aa4241b8ef4" + integrity sha512-K+OTIjJcZHVlZQN1HmU64VtrC0jC3dXWQozuEIR9zVvltIk90zaGPM2AgT+fIkChpzHhFE3YnvFLCbLtzAmexA== + dependencies: + "@types/abstract-leveldown" "*" + "@types/level-errors" "*" + "@types/node" "*" + "@types/lodash@^4.14.159": version "4.14.171" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.171.tgz#f01b3a5fe3499e34b622c362a46a609fdb23573b" @@ -499,6 +526,17 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abstract-leveldown@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz#1a8bc3b07f502793804d456a881dc15cedb9bc5d" + integrity sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ== + dependencies: + buffer "^6.0.3" + is-buffer "^2.0.5" + level-concat-iterator "^3.0.0" + level-supports "^2.0.0" + queue-microtask "^1.2.3" + accepts@~1.3.7: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" @@ -1078,6 +1116,11 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +catering@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/catering/-/catering-2.0.0.tgz#15ce31bcbffafbf62855ea7677b0e5d23581233d" + integrity sha512-aD/WmxhGwUGsVPrj8C80vH7C7GphJilYVSdudoV4u16XdrLF7CVyfBmENsc4tLTVsJJzCRid8GbwJ7mcPLee6Q== + chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1570,6 +1613,14 @@ defer-to-connect@^1.0.1: resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== +deferred-leveldown@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-6.0.0.tgz#364dc436682eccbd4a25a5fd5585614770d0d3d6" + integrity sha512-F6CLAZzNeURojlH4MCigZr54tNz+xDSi06YXsDr5uLSKeF3JKnvnQWTqd+RETh2hbWTJR3qDzGicQOWS5ZQ1BQ== + dependencies: + abstract-leveldown "^7.0.0" + inherits "^2.0.3" + define-properties@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" @@ -1741,6 +1792,13 @@ equal-length@^1.0.0: resolved "https://registry.yarnpkg.com/equal-length/-/equal-length-1.0.1.tgz#21ca112d48ab24b4e1e7ffc0e5339d31fdfc274c" integrity sha1-IcoRLUirJLTh5//A5TOdMf38J0w= +errno@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/errno/-/errno-1.0.0.tgz#0ea47d701864accf996412f09e29b4dc2cf3856d" + integrity sha512-3zV5mFS1E8/1bPxt/B0xxzI1snsg3uSCIh6Zo1qKg6iMw93hzPANk9oBFzSFBFrwuVoQuE3rLoouAUfwOAj1wQ== + dependencies: + prr "~1.0.1" + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -2831,6 +2889,11 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" + integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== + is-callable@^1.1.4, is-callable@^1.2.3: version "1.2.4" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.4.tgz#47301d58dd0259407865547853df6d61fe471945" @@ -3240,6 +3303,52 @@ latest-version@^5.0.0, latest-version@^5.1.0: dependencies: package-json "^6.3.0" +level-concat-iterator@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz#416ddaf0c2ed834f006aa3124ee68906eb4769d4" + integrity sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg== + +level-errors@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/level-errors/-/level-errors-3.0.0.tgz#5719f2ad9061d9f2bd7cf22e4a7def893e6d2e60" + integrity sha512-MZXOQT061uEjxxxq4C/Jf+M3RdEKK9e3NbxlN7yOp1LDYoLVAhE2i1j0b7XqXfl8FjFtUL7phwr3Sn0wXXoMqA== + dependencies: + errno "^1.0.0" + +level-iterator-stream@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz#85b3438e1b4c54ce5aa8c0eb973cfb628117df9e" + integrity sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA== + dependencies: + inherits "^2.0.4" + readable-stream "^3.4.0" + +level-supports@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-2.0.0.tgz#b0b9f63f30c4175fb2612217144f03f3b77580d9" + integrity sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw== + +leveldown@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-6.0.1.tgz#d4758e4e91c1c5ff90d3e1cd29fb1bf5ce19ae22" + integrity sha512-D1fz5G58UrLBg5pXcN1pNxWtNLXwBiZsYt06XqjhD5zBHj/ws7xvZe3K0ZAhCbYuL9b4nbRCim8Z15HYWL2exQ== + dependencies: + abstract-leveldown "^7.0.0" + napi-macros "~2.0.0" + node-gyp-build "~4.2.1" + +levelup@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/levelup/-/levelup-5.0.1.tgz#a871c3cb0943f4fc0280bc4d71c5d60950b0d570" + integrity sha512-MJvQgBRQmB+E5+d6Qbxqm05N4U9NzOxGNhXx0rR8maRBwmVuVV+m4IV3N4HzZJW8JwiJ0jj92RZaytcD+Hr1CA== + dependencies: + catering "^2.0.0" + deferred-leveldown "^6.0.0" + level-errors "^3.0.0" + level-iterator-stream "^5.0.0" + level-supports "^2.0.0" + queue-microtask "^1.2.3" + levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" @@ -3603,6 +3712,11 @@ nanoassert@^1.0.0: resolved "https://registry.yarnpkg.com/nanoassert/-/nanoassert-1.1.0.tgz#4f3152e09540fde28c76f44b19bbcd1d5a42478d" integrity sha1-TzFS4JVA/eKMdvRLGbvNHVpCR40= +napi-macros@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" + integrity sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg== + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -3628,7 +3742,7 @@ node-fetch@2.6.1: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== -node-gyp-build@^4.2.0: +node-gyp-build@^4.2.0, node-gyp-build@~4.2.1: version "4.2.3" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" integrity sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg== @@ -4120,6 +4234,11 @@ proxy-addr@~2.0.5: forwarded "0.2.0" ipaddr.js "1.9.1" +prr@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" + integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= + psl@^1.1.28: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -4186,7 +4305,7 @@ query-string@^5.0.1: object-assign "^4.1.0" strict-uri-encode "^1.0.0" -queue-microtask@^1.2.2: +queue-microtask@^1.2.2, queue-microtask@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==