feat: oss sync and secrets

.eslintrc.js

@@ -9,5 +9,6 @@ module.exports = {
     'no-param-reassign': 0,
     'max-classes-per-file': 0,
     'no-plusplus': 0,
+    'no-unused-vars': ['warn', { 'varsIgnorePattern': '^_' }]
   },
 };

.gitignore

@@ -1,3 +1,6 @@
+# secrets
+/scripts/secrets.json
+
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 ossutil_output/
 # dependencies

package.json

@@ -16,12 +16,14 @@
     "start:no-ui": "cross-env UMI_UI=none umi dev",
     "start:pre": "cross-env REACT_APP_ENV=pre umi dev",
     "start:test": "cross-env REACT_APP_ENV=test MOCK=none umi dev",
-    "archives:sync": "git pull"
+
+    "archives:sync": "echo TODO",
     "archives": "yarn archives:compile && yarn archives:add && yarn archives:upload",
     "archives:compile": "node --experimental-specifier-resolution=node --loader ts-node/esm ./scripts/compile-archives.ts",
     "archives:add": "git stash; git pull --rebase; git stash pop; git add ./archives; git commit --no-verify -m \"DATA: archives `date \"+%Y-%m-%d-%H:%M:%S\"`\" && git push",
     "archives:upload": "ossutilmac64 cp -r -u ./archives oss://lanting-public/archives",
     "archives:migrate-backend2": "node --experimental-specifier-resolution=node --loader ts-node/esm ./scripts/migrate-backend2.ts",
+
     "deploy": "yarn build && yarn deploy:ec2 && yarn deploy:oss",
     "build": "UMI_ENV=prod umi build",
     "analyze": "cross-env ANALYZE=1 umi build",

scripts/compile-archives.ts

@@ -2,12 +2,23 @@
 import * as fs from 'fs';
 import { dirname } from 'path';
 import { fileURLToPath } from 'url';
+import OSS from 'ali-oss';
 import { Archive, Archives } from './data';
+import secrets from './secrets.json';
+
+const client = new OSS({
+  region: 'oss-cn-beijing',
+  accessKeyId: secrets.oss.accessKeyId,
+  accessKeySecret: secrets.oss.accessKeySecret,
+  bucket: 'lanting-public'
+});
 
 // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const ARCHIVE_DIR = `${__dirname}/../../archives`;
-const currentOrigs = fs.readdirSync(`${ARCHIVE_DIR}/origs`);
+const ARCHIVE_DIR = `${__dirname}/../archives`;
+// XXX boyang: old way
+// const currentOrigs = fs.readdirSync(`${ARCHIVE_DIR}/origs`);
+let currentOrigs: string[] = [];
 
 function setField(archive: Archive, field: string, fileContent: string, archives: Archives) {
   const regexArr = toFieldRegex(field).exec(fileContent);
@@ -61,7 +72,40 @@ function getIdFromCommentFilename(f: string) {
   return f.substring(0, f.indexOf('-'));
 }
 
-function main() {
+async function init() {
+  currentOrigs = (await client.list({ prefix: 'archives/origs/', 'max-keys': 1000 }, {}))
+    .objects
+    .map(o => o.name.replace(/^archives\/origs\//, ''));
+}
+
+async function origsMap() {
+  const commentsFiles = fs.readdirSync(`${ARCHIVE_DIR}/comments`);
+  const archives = commentsFiles.map((f) => {
+    console.log('Processing: ', f);
+
+    const archive = new Archive();
+    archive.id = getIdFromCommentFilename(f);
+
+    const foundOrigs = currentOrigs.filter((orig) => {
+      const parts = orig.split('.');
+      let id;
+      if (parts[0].includes('-')) {
+        id = +parts[0].split('-')[0];
+      } else {
+        id = +parts[0];
+      }
+      return id === +archive.id;
+    });
+    archive.origs = foundOrigs;
+    return archive;
+  });
+  const noOrig = archives.filter(a => (a.origs || []).length === 0).map(a => a.id);
+  const noComment = currentOrigs.filter(o => !archives.find(a => `${  a.id}` === o.split('.')[0]));
+
+  console.log('XXXTEMP noOrig noComment', noOrig, noComment);
+}
+
+async function compileArchives() {
   const compiledArchives = new Archives();
 
   const commentsFiles = fs.readdirSync(`${ARCHIVE_DIR}/comments`);
@@ -102,4 +146,11 @@ function main() {
 
   fs.writeFileSync(`${ARCHIVE_DIR}/archives.json`, JSON.stringify(compiledArchives));
 }
+
+async function main() {
+  await init();
+  await origsMap();
+  // await compileArchives();
+}
+
 main();

scripts/migrate-backend2.ts

@@ -3,6 +3,7 @@ import { dirname } from 'path';
 import { fileURLToPath } from 'url';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import mysql from 'mysql';
+import secrets from './secrets.json';
 
 // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention
 const __dirname = dirname(fileURLToPath(import.meta.url));

scripts/package.json

@@ -3,5 +3,9 @@
   "version": "1.0.0",
   "description": "",
   "author": "boyang",
-  "type": "module"
+  "type": "module",
+  "dependencies": {
+    "@types/ali-oss": "^6.0.8",
+    "ali-oss": "^6.15.2"
+  }
 }

tsconfig.json

@@ -13,19 +13,19 @@
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "suppressImplicitAnyIndexErrors": true,
-    "noUnusedLocals": true,
+    "noUnusedLocals": false,
     "allowJs": true,
     "skipLibCheck": true,
     "experimentalDecorators": true,
     "strict": true,
     "paths": {
       "@/*": ["./src/*"],
       "@@/*": ["./src/.umi/*"]
-    }
+    },
   },
   "include": [
     "src/**/*",
-    "scripts/**/*",
+    "scripts/compile-archives.ts",
     "tests/**/*",
     "test/**/*",
     "__test__/**/*",
@@ -36,5 +36,5 @@
     ".prettierrc.js",
     "jest.config.js"
   ],
-  "exclude": ["node_modules", "build", "dist", "scripts", "src/.umi/*", "webpack", "jest"]
+  "exclude": ["node_modules", "build", "dist", "src/.umi/*", "webpack", "jest"]
 }