From 2ba434d7406b29af11f0626247a08712ad573d6e Mon Sep 17 00:00:00 2001 From: boyang Date: Fri, 28 May 2021 20:27:54 +0800 Subject: [PATCH] feat: oss sync and secrets --- .eslintrc.js | 1 + .gitignore | 3 ++ package.json | 4 ++- scripts/compile-archives.ts | 57 +++++++++++++++++++++++++++++++++++-- scripts/migrate-backend2.ts | 1 + scripts/package.json | 6 +++- tsconfig.json | 8 +++--- 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 6fed7be..f814534 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -9,5 +9,6 @@ module.exports = { 'no-param-reassign': 0, 'max-classes-per-file': 0, 'no-plusplus': 0, + 'no-unused-vars': ['warn', { 'varsIgnorePattern': '^_' }] }, }; diff --git a/.gitignore b/.gitignore index fbe5313..0699e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# secrets +/scripts/secrets.json + # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. ossutil_output/ # dependencies diff --git a/package.json b/package.json index e0097f9..e5ecf49 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,14 @@ "start:no-ui": "cross-env UMI_UI=none umi dev", "start:pre": "cross-env REACT_APP_ENV=pre umi dev", "start:test": "cross-env REACT_APP_ENV=test MOCK=none umi dev", - "archives:sync": "git pull" + + "archives:sync": "echo TODO", "archives": "yarn archives:compile && yarn archives:add && yarn archives:upload", "archives:compile": "node --experimental-specifier-resolution=node --loader ts-node/esm ./scripts/compile-archives.ts", "archives:add": "git stash; git pull --rebase; git stash pop; git add ./archives; git commit --no-verify -m \"DATA: archives `date \"+%Y-%m-%d-%H:%M:%S\"`\" && git push", "archives:upload": "ossutilmac64 cp -r -u ./archives oss://lanting-public/archives", "archives:migrate-backend2": "node --experimental-specifier-resolution=node --loader ts-node/esm ./scripts/migrate-backend2.ts", + "deploy": "yarn build && yarn deploy:ec2 && yarn deploy:oss", "build": "UMI_ENV=prod umi build", "analyze": "cross-env ANALYZE=1 umi build", diff --git a/scripts/compile-archives.ts b/scripts/compile-archives.ts index 7e8e799..45f703a 100644 --- a/scripts/compile-archives.ts +++ b/scripts/compile-archives.ts @@ -2,12 +2,23 @@ import * as fs from 'fs'; import { dirname } from 'path'; import { fileURLToPath } from 'url'; +import OSS from 'ali-oss'; import { Archive, Archives } from './data'; +import secrets from './secrets.json'; + +const client = new OSS({ + region: 'oss-cn-beijing', + accessKeyId: secrets.oss.accessKeyId, + accessKeySecret: secrets.oss.accessKeySecret, + bucket: 'lanting-public' +}); // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention const __dirname = dirname(fileURLToPath(import.meta.url)); -const ARCHIVE_DIR = `${__dirname}/../../archives`; -const currentOrigs = fs.readdirSync(`${ARCHIVE_DIR}/origs`); +const ARCHIVE_DIR = `${__dirname}/../archives`; +// XXX boyang: old way +// const currentOrigs = fs.readdirSync(`${ARCHIVE_DIR}/origs`); +let currentOrigs: string[] = []; function setField(archive: Archive, field: string, fileContent: string, archives: Archives) { const regexArr = toFieldRegex(field).exec(fileContent); @@ -61,7 +72,40 @@ function getIdFromCommentFilename(f: string) { return f.substring(0, f.indexOf('-')); } -function main() { +async function init() { + currentOrigs = (await client.list({ prefix: 'archives/origs/', 'max-keys': 1000 }, {})) + .objects + .map(o => o.name.replace(/^archives\/origs\//, '')); +} + +async function origsMap() { + const commentsFiles = fs.readdirSync(`${ARCHIVE_DIR}/comments`); + const archives = commentsFiles.map((f) => { + console.log('Processing: ', f); + + const archive = new Archive(); + archive.id = getIdFromCommentFilename(f); + + const foundOrigs = currentOrigs.filter((orig) => { + const parts = orig.split('.'); + let id; + if (parts[0].includes('-')) { + id = +parts[0].split('-')[0]; + } else { + id = +parts[0]; + } + return id === +archive.id; + }); + archive.origs = foundOrigs; + return archive; + }); + const noOrig = archives.filter(a => (a.origs || []).length === 0).map(a => a.id); + const noComment = currentOrigs.filter(o => !archives.find(a => `${ a.id}` === o.split('.')[0])); + + console.log('XXXTEMP noOrig noComment', noOrig, noComment); +} + +async function compileArchives() { const compiledArchives = new Archives(); const commentsFiles = fs.readdirSync(`${ARCHIVE_DIR}/comments`); @@ -102,4 +146,11 @@ function main() { fs.writeFileSync(`${ARCHIVE_DIR}/archives.json`, JSON.stringify(compiledArchives)); } + +async function main() { + await init(); + await origsMap(); + // await compileArchives(); +} + main(); diff --git a/scripts/migrate-backend2.ts b/scripts/migrate-backend2.ts index e185839..a6a26a5 100644 --- a/scripts/migrate-backend2.ts +++ b/scripts/migrate-backend2.ts @@ -3,6 +3,7 @@ import { dirname } from 'path'; import { fileURLToPath } from 'url'; // eslint-disable-next-line import/no-extraneous-dependencies import mysql from 'mysql'; +import secrets from './secrets.json'; // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention const __dirname = dirname(fileURLToPath(import.meta.url)); diff --git a/scripts/package.json b/scripts/package.json index b66a343..7704135 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,5 +3,9 @@ "version": "1.0.0", "description": "", "author": "boyang", - "type": "module" + "type": "module", + "dependencies": { + "@types/ali-oss": "^6.0.8", + "ali-oss": "^6.15.2" + } } diff --git a/tsconfig.json b/tsconfig.json index 38d1042..888e75c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noImplicitReturns": true, "suppressImplicitAnyIndexErrors": true, - "noUnusedLocals": true, + "noUnusedLocals": false, "allowJs": true, "skipLibCheck": true, "experimentalDecorators": true, @@ -21,11 +21,11 @@ "paths": { "@/*": ["./src/*"], "@@/*": ["./src/.umi/*"] - } + }, }, "include": [ "src/**/*", - "scripts/**/*", + "scripts/compile-archives.ts", "tests/**/*", "test/**/*", "__test__/**/*", @@ -36,5 +36,5 @@ ".prettierrc.js", "jest.config.js" ], - "exclude": ["node_modules", "build", "dist", "scripts", "src/.umi/*", "webpack", "jest"] + "exclude": ["node_modules", "build", "dist", "src/.umi/*", "webpack", "jest"] }