From 02ff3785284b71209018bad0126dfc92754c90f3 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 5 Feb 2025 17:35:41 +0200 Subject: [PATCH 1/9] =?UTF-8?q?move=20js=20transpilation=20to=C2=A0worker?= =?UTF-8?q?=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/compiler/CubeSymbols.js | 4 +- .../src/compiler/DataSchemaCompiler.js | 68 ++++++++++++------- .../src/compiler/ErrorReporter.ts | 16 +++++ .../src/compiler/PrepareCompiler.ts | 2 + .../transpilers/CubePropContextTranspiler.ts | 13 ++-- .../LightweightNodeCubeDictionary.ts | 10 +++ .../transpilers/LightweightSymbolResolver.ts | 24 +++++++ .../transpilers/transpiler.interface.ts | 9 +++ .../compiler/transpilers/transpiler_worker.ts | 62 +++++++++++++++++ 9 files changed, 179 insertions(+), 29 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts create mode 100644 packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts create mode 100644 packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js index e8c59a8fb2972..6602713f70ae9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js @@ -8,7 +8,7 @@ import { camelizeCube } from './utils'; import { BaseQuery } from '../adapter'; const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/; -const CONTEXT_SYMBOLS = { +export const CONTEXT_SYMBOLS = { SECURITY_CONTEXT: 'securityContext', // SECURITY_CONTEXT has been deprecated, however security_context (lowecase) // is allowed in RBAC policies for query-time attribute matching @@ -19,7 +19,7 @@ const CONTEXT_SYMBOLS = { SQL_UTILS: 'sqlUtils' }; -const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; +export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; export class CubeSymbols { constructor(evaluateViews) { diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 8ad847bd7f351..562edd760c04e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -1,10 +1,8 @@ import vm from 'vm'; import fs from 'fs'; import path from 'path'; +import { Worker } from 'worker_threads'; import syntaxCheck from 'syntax-error'; -import { parse } from '@babel/parser'; -import babelGenerator from '@babel/generator'; -import babelTraverse from '@babel/traverse'; import R from 'ramda'; import { isNativeSupported } from '@cubejs-backend/shared'; @@ -28,6 +26,8 @@ export class DataSchemaCompiler { this.viewCompilationGate = options.viewCompilationGate; this.cubeNameCompilers = options.cubeNameCompilers || []; this.extensions = options.extensions || {}; + this.cubeDictionary = options.cubeDictionary; + this.cubeSymbols = options.cubeSymbols; this.cubeFactory = options.cubeFactory; this.filesToCompile = options.filesToCompile; this.omitErrors = options.omitErrors; @@ -90,9 +90,22 @@ export class DataSchemaCompiler { this.errorsReport = errorsReport; // TODO: required in order to get pre transpile compilation work - const transpile = () => toCompile.map(f => this.transpileFile(f, errorsReport)).filter(f => !!f); + // const transpile = () => toCompile.map(f => this.transpileFile(f, errorsReport)).filter(f => !!f); + const transpile = async () => { + const cubeNames = Object.keys(this.cubeDictionary.byId); + const cubeSymbolsNames = Object.fromEntries( + Object.entries(this.cubeSymbols.symbols) + .map( + ([key, value]) => [key, Object.fromEntries( + Object.keys(value).map((k) => [k, true]) + )] + ) + ); + const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }))); + return results.filter(f => !!f); + }; - const compilePhase = (compilers) => this.compileCubeFiles(compilers, transpile(), errorsReport); + const compilePhase = async (compilers) => this.compileCubeFiles(compilers, await transpile(), errorsReport); return compilePhase({ cubeCompilers: this.cubeNameCompilers }) .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) })) @@ -118,7 +131,7 @@ export class DataSchemaCompiler { return this.compilePromise; } - transpileFile(file, errorsReport) { + async transpileFile(file, errorsReport, options) { if (R.endsWith('.jinja', file.fileName) || (R.endsWith('.yml', file.fileName) || R.endsWith('.yaml', file.fileName)) // TODO do Jinja syntax check with jinja compiler @@ -137,31 +150,40 @@ export class DataSchemaCompiler { } else if (R.endsWith('.yml', file.fileName) || R.endsWith('.yaml', file.fileName)) { return file; } else if (R.endsWith('.js', file.fileName)) { - return this.transpileJsFile(file, errorsReport); + return this.transpileJsFile(file, errorsReport, options); } else { return file; } } - transpileJsFile(file, errorsReport) { + async transpileJsFile(file, errorsReport, options) { try { - const ast = parse( - file.content, - { - sourceFilename: file.fileName, - sourceType: 'module', - plugins: ['objectRestSpread'] - }, - ); - - this.transpilers.forEach((t) => { - errorsReport.inFile(file); - babelTraverse(ast, t.traverseObject(errorsReport)); - errorsReport.exitFile(); + const res = await new Promise((resolve, reject) => { + const { cubeNames, cubeSymbolsNames } = options; + const worker = new Worker( + path.join(__dirname, 'transpilers/transpiler_worker'), + { workerData: { cubeNames, cubeSymbolsNames } }, + ); + const data = { + fileName: file.fileName, + content: file.content, + transpilers: this.transpilers.map(t => t.constructor.name) + }; + worker.postMessage(data); + + worker.on('message', resolve); + worker.on('error', reject); + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker thread exited with code ${code}`)); + } + }); }); - const content = babelGenerator(ast, {}, file.content).code; - return Object.assign({}, file, { content }); + errorsReport.addErrors(res.errors); + errorsReport.addWarnings(res.warnings); + + return Object.assign({}, file, { content: res.content }); } catch (e) { if (e.toString().indexOf('SyntaxError') !== -1) { const line = file.content.split('\n')[e.loc.line - 1]; diff --git a/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts b/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts index 8407dd8605453..bf48792f4a3ab 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts @@ -137,6 +137,22 @@ export class ErrorReporter { } } + public getErrors() { + return this.rootReporter().errors; + } + + public addErrors(errors: CompilerErrorInterface[]) { + this.rootReporter().errors.push(...errors); + } + + public getWarnings() { + return this.rootReporter().warnings; + } + + public addWarnings(warnings: SyntaxErrorInterface[]) { + this.rootReporter().warnings.push(...warnings); + } + protected rootReporter(): ErrorReporter { return this.parent ? this.parent.rootReporter() : this; } diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 9605b1dac3d1e..70f9f3c454704 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -69,6 +69,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp contextCompilers: [contextEvaluator], cubeFactory: cubeSymbols.createCube.bind(cubeSymbols), compilerCache, + cubeDictionary, + cubeSymbols, extensions: { Funnels, RefreshKeys, diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 2f2574f2d8ed9..2ffc96058b3ef 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -2,7 +2,12 @@ import * as t from '@babel/types'; import R from 'ramda'; import type { NodePath } from '@babel/traverse'; -import type { TranspilerInterface, TraverseObject } from './transpiler.interface'; +import { + TranspilerCubeResolver, + TranspilerInterface, + TranspilerSymbolResolver, + TraverseObject +} from './transpiler.interface'; import type { CubeSymbols } from '../CubeSymbols'; import type { CubeDictionary } from '../CubeDictionary'; @@ -39,9 +44,9 @@ transpiledFieldsPatterns?.forEach((r) => { export class CubePropContextTranspiler implements TranspilerInterface { public constructor( - protected readonly cubeSymbols: CubeSymbols, - protected readonly cubeDictionary: CubeDictionary, - protected readonly viewCompiler: CubeSymbols, + protected readonly cubeSymbols: TranspilerSymbolResolver, + protected readonly cubeDictionary: TranspilerCubeResolver, + protected readonly viewCompiler: TranspilerSymbolResolver, ) { } diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts new file mode 100644 index 0000000000000..c65eb0ffd7c3e --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts @@ -0,0 +1,10 @@ +import { TranspilerCubeResolver } from './transpiler.interface'; + +export class LightweightNodeCubeDictionary implements TranspilerCubeResolver { + public constructor(private readonly cubeNames: string[] = []) { + } + + public resolveCube(name: string): boolean { + return this.cubeNames.includes(name); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts new file mode 100644 index 0000000000000..8b0619ee99abc --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts @@ -0,0 +1,24 @@ +import { TranspilerSymbolResolver } from './transpiler.interface'; +import { CONTEXT_SYMBOLS, CURRENT_CUBE_CONSTANTS } from '../CubeSymbols'; + +export class LightweightSymbolResolver implements TranspilerSymbolResolver { + public constructor(private readonly symbols: any) { + } + + public isCurrentCube(name): boolean { + return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0; + } + + public resolveSymbol(cubeName, name): any { + if (name === 'USER_CONTEXT') { + throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); + } + + if (CONTEXT_SYMBOLS[name]) { + return true; + } + + const cube = this.symbols[this.isCurrentCube(name) ? cubeName : name]; + return cube || (this.symbols[cubeName] && this.symbols[cubeName][name]); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts index 0f15c6f8401a8..803a1c0fa4082 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts @@ -6,3 +6,12 @@ export type TraverseObject = TraverseOptions; export interface TranspilerInterface { traverseObject(reporter: ErrorReporter): TraverseObject; } + +export interface TranspilerSymbolResolver { + resolveSymbol(cubeName, name): any; + isCurrentCube(name): boolean; +} + +export interface TranspilerCubeResolver { + resolveCube(name): boolean; +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts new file mode 100644 index 0000000000000..31fea4ac65b25 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -0,0 +1,62 @@ +import { parentPort, workerData } from 'worker_threads'; + +import { parse } from '@babel/parser'; +import babelGenerator from '@babel/generator'; +import babelTraverse from '@babel/traverse'; + +import { ValidationTranspiler } from './ValidationTranspiler'; +import { ImportExportTranspiler } from './ImportExportTranspiler'; +import { CubeCheckDuplicatePropTranspiler } from './CubeCheckDuplicatePropTranspiler'; +import { CubePropContextTranspiler } from './CubePropContextTranspiler'; +import { ErrorReporter } from '../ErrorReporter'; +import { LightweightSymbolResolver } from './LightweightSymbolResolver'; +import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; + +type FileContent = { + fileName: string; + content: string; + transpilers: string[]; +}; + +const cubeDictionary = new LightweightNodeCubeDictionary(workerData.cubeNames); +const cubeSymbols = new LightweightSymbolResolver(workerData.cubeSymbolsNames); +const errorsReport = new ErrorReporter(null, []); + +const transpilers = { + ValidationTranspiler: new ValidationTranspiler(), + ImportExportTranspiler: new ImportExportTranspiler(), + CubeCheckDuplicatePropTranspiler: new CubeCheckDuplicatePropTranspiler(), + CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols), +}; + +if (parentPort) { + parentPort.on('message', (file: FileContent) => { + const ast = parse( + file.content, + { + sourceFilename: file.fileName, + sourceType: 'module', + plugins: ['objectRestSpread'] + }, + ); + + file.transpilers.forEach(transpilerName => { + if (transpilers[transpilerName]) { + errorsReport.inFile(file); + babelTraverse(ast, transpilers[transpilerName].traverseObject(errorsReport)); + errorsReport.exitFile(); + } else { + throw new Error(`Transpiler ${transpilerName} not supported`); + } + }); + + const content = babelGenerator(ast, {}, file.content).code; + + // @ts-ignore + parentPort.postMessage({ + content, + errors: errorsReport.getErrors(), + warnings: errorsReport.getWarnings() + }); + }); +} From df854cb40e09baa397ae97d7e3b2ed445659b349 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 5 Feb 2025 21:10:15 +0200 Subject: [PATCH 2/9] =?UTF-8?q?hide=20worker=20threads=20transpile=20under?= =?UTF-8?q?=20the=C2=A0flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cubejs-backend-shared/src/env.ts | 3 + .../src/compiler/DataSchemaCompiler.js | 111 ++++++++++++------ 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 33d1854ff7c73..a6d1541234dc2 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -196,6 +196,9 @@ const variables: Record any> = { nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR') .default('false') .asBoolStrict(), + workerThreadsTranspilation: () => get('CUBEJS_WORKER_THREADS_TRANSPILATION') + .default('false') + .asBoolStrict(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 562edd760c04e..2cd33ae043bf5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -3,9 +3,12 @@ import fs from 'fs'; import path from 'path'; import { Worker } from 'worker_threads'; import syntaxCheck from 'syntax-error'; +import { parse } from '@babel/parser'; +import babelGenerator from '@babel/generator'; +import babelTraverse from '@babel/traverse'; import R from 'ramda'; -import { isNativeSupported } from '@cubejs-backend/shared'; +import { getEnv, isNativeSupported } from '@cubejs-backend/shared'; import { UserError } from './UserError'; import { ErrorReporter } from './ErrorReporter'; @@ -89,20 +92,31 @@ export class DataSchemaCompiler { const errorsReport = new ErrorReporter(null, [], this.errorReport); this.errorsReport = errorsReport; - // TODO: required in order to get pre transpile compilation work - // const transpile = () => toCompile.map(f => this.transpileFile(f, errorsReport)).filter(f => !!f); const transpile = async () => { - const cubeNames = Object.keys(this.cubeDictionary.byId); - const cubeSymbolsNames = Object.fromEntries( - Object.entries(this.cubeSymbols.symbols) - .map( - ([key, value]) => [key, Object.fromEntries( - Object.keys(value).map((k) => [k, true]) - )] - ) - ); - const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }))); - return results.filter(f => !!f); + let cubeNames; + let cubeSymbolsNames; + + if (getEnv('workerThreadsTranspilation')) { + cubeNames = Object.keys(this.cubeDictionary.byId); + cubeSymbolsNames = Object.fromEntries( + Object.entries(this.cubeSymbols.symbols) + .map( + ([key, value]) => [key, Object.fromEntries( + Object.keys(value).map((k) => [k, true]), + )], + ), + ); + } + // const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }))); + const results = []; + for (const f of toCompile) { + const result = await this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }); + if (result) { + results.push(result); + } + } + // return results.filter(f => !!f); + return results; }; const compilePhase = async (compilers) => this.compileCubeFiles(compilers, await transpile(), errorsReport); @@ -158,32 +172,55 @@ export class DataSchemaCompiler { async transpileJsFile(file, errorsReport, options) { try { - const res = await new Promise((resolve, reject) => { - const { cubeNames, cubeSymbolsNames } = options; - const worker = new Worker( - path.join(__dirname, 'transpilers/transpiler_worker'), - { workerData: { cubeNames, cubeSymbolsNames } }, - ); - const data = { - fileName: file.fileName, - content: file.content, - transpilers: this.transpilers.map(t => t.constructor.name) - }; - worker.postMessage(data); - - worker.on('message', resolve); - worker.on('error', reject); - worker.on('exit', (code) => { - if (code !== 0) { - reject(new Error(`Worker thread exited with code ${code}`)); - } + if (getEnv('workerThreadsTranspilation')) { + const res = await new Promise((resolve, reject) => { + const { cubeNames, cubeSymbolsNames } = options; + + const worker = new Worker( + path.join(__dirname, 'transpilers/transpiler_worker'), + { workerData: { cubeNames, cubeSymbolsNames } }, + ); + + const data = { + fileName: file.fileName, + content: file.content, + transpilers: this.transpilers.map(t => t.constructor.name), + }; + + worker.postMessage(data); + + worker.on('message', resolve); + worker.on('error', reject); + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker thread exited with code ${code}`)); + } + }); }); - }); - errorsReport.addErrors(res.errors); - errorsReport.addWarnings(res.warnings); + errorsReport.addErrors(res.errors); + errorsReport.addWarnings(res.warnings); + + return Object.assign({}, file, { content: res.content }); + } else { + const ast = parse( + file.content, + { + sourceFilename: file.fileName, + sourceType: 'module', + plugins: ['objectRestSpread'], + }, + ); - return Object.assign({}, file, { content: res.content }); + this.transpilers.forEach((t) => { + errorsReport.inFile(file); + babelTraverse(ast, t.traverseObject(errorsReport)); + errorsReport.exitFile(); + }); + + const content = babelGenerator(ast, {}, file.content).code; + return Object.assign({}, file, { content }); + } } catch (e) { if (e.toString().indexOf('SyntaxError') !== -1) { const line = file.content.split('\n')[e.loc.line - 1]; From 4656b7a387a74eeabf962a58ab329ebd45d3810c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 5 Feb 2025 21:10:31 +0200 Subject: [PATCH 3/9] add useful comment --- .../cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 2cd33ae043bf5..a0dc2ba1e0e10 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -98,6 +98,10 @@ export class DataSchemaCompiler { if (getEnv('workerThreadsTranspilation')) { cubeNames = Object.keys(this.cubeDictionary.byId); + // We need only cubes and all its member names for transpiling. + // Communication between main and worker threads uses + // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm) + // which doesn't allow passing any function objects, so we need to sanitize the symbols. cubeSymbolsNames = Object.fromEntries( Object.entries(this.cubeSymbols.symbols) .map( From ebe5dc61313bfb05bc2628159dff785e481816db Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 5 Feb 2025 22:18:13 +0200 Subject: [PATCH 4/9] =?UTF-8?q?tune=20Push=20CI=20to=C2=A0test=20with=20CU?= =?UTF-8?q?BEJS=5FWORKER=5FTHREADS=5FTRANSPILATION=3Dtrue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ecb81eecc03c7..5eb44dd36e41b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -54,8 +54,11 @@ jobs: # Current docker version + next LTS node-version: [20.x, 22.x] python-version: [3.11] + transpile-worker-threads: [false, true] fail-fast: false + env: + CUBEJS_WORKER_THREADS_TRANSPILATION: ${{ matrix.transpile-worker-threads }} steps: - id: get-tag-out run: echo "$OUT" From e2c617749a0fd039ad54b1a6201fab490c8946d3 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 5 Feb 2025 23:41:37 +0200 Subject: [PATCH 5/9] add workerpool pkg --- packages/cubejs-schema-compiler/package.json | 3 +- yarn.lock | 352 ++----------------- 2 files changed, 28 insertions(+), 327 deletions(-) diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index 22e04f72e3300..c73e7d90da286 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -54,7 +54,8 @@ "node-dijkstra": "^2.5.0", "ramda": "^0.27.2", "syntax-error": "^1.3.0", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "workerpool": "^9.2.0" }, "devDependencies": { "@clickhouse/client": "^1.7.0", diff --git a/yarn.lock b/yarn.lock index 959fdedf9d13e..678c3e55ea242 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8699,11 +8699,6 @@ resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== -"@sindresorhus/is@^0.14.0": - version "0.14.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" - integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== - "@sinonjs/commons@^1.7.0": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -9280,13 +9275,6 @@ dependencies: tslib "^2.4.0" -"@szmarczak/http-timer@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421" - integrity sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA== - dependencies: - defer-to-connect "^1.0.1" - "@techteamer/ocsp@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@techteamer/ocsp/-/ocsp-1.0.1.tgz#420f80c64ff0f74a70b65c88e4031c03a9da6ded" @@ -11555,13 +11543,6 @@ anser@^2.1.1: resolved "https://registry.yarnpkg.com/anser/-/anser-2.1.1.tgz#8afae28d345424c82de89cc0e4d1348eb0c5af7c" integrity sha512-nqLm4HxOTpeLOxcmB3QWmV5TcDFhW9y/fyQ+hivtDFcK4OQ+pQ5fzPnXHM1Mfcm0VkLtvVi1TCPr++Qy0Q/3EQ== -ansi-align@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59" - integrity sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w== - dependencies: - string-width "^4.1.0" - ansi-colors@4.1.1, ansi-colors@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" @@ -12619,20 +12600,6 @@ bowser@^2.11.0: resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f" integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA== -boxen@^5.0.0: - version "5.1.2" - resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" - integrity sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ== - dependencies: - ansi-align "^3.0.0" - camelcase "^6.2.0" - chalk "^4.1.0" - cli-boxes "^2.2.1" - string-width "^4.2.2" - type-fest "^0.20.2" - widest-line "^3.1.0" - wrap-ansi "^7.0.0" - brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -12929,19 +12896,6 @@ cacache@^18.0.3: tar "^6.1.11" unique-filename "^3.0.0" -cacheable-request@^6.0.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" - integrity sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg== - dependencies: - clone-response "^1.0.2" - get-stream "^5.1.0" - http-cache-semantics "^4.0.0" - keyv "^3.0.0" - lowercase-keys "^2.0.0" - normalize-url "^4.1.0" - responselike "^1.0.2" - cachedir@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" @@ -13256,11 +13210,6 @@ clean-stack@^3.0.0: dependencies: escape-string-regexp "4.0.0" -cli-boxes@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" - integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== - cli-cursor@3.1.0, cli-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" @@ -13402,13 +13351,6 @@ clone-deep@4.0.1, clone-deep@^4.0.1: kind-of "^6.0.2" shallow-clone "^3.0.0" -clone-response@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b" - integrity sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws= - dependencies: - mimic-response "^1.0.0" - clone@^1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" @@ -13722,18 +13664,6 @@ config-chain@^1.1.12, config-chain@^1.1.13: ini "^1.3.4" proto-list "~1.2.1" -configstore@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" - integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA== - dependencies: - dot-prop "^5.2.0" - graceful-fs "^4.1.2" - make-dir "^3.0.0" - unique-string "^2.0.0" - write-file-atomic "^3.0.0" - xdg-basedir "^4.0.0" - confusing-browser-globals@^1.0.10, confusing-browser-globals@^1.0.5: version "1.0.10" resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.10.tgz#30d1e7f3d1b882b25ec4933d1d1adac353d20a59" @@ -14837,13 +14767,6 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= -decompress-response@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" - integrity sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M= - dependencies: - mimic-response "^1.0.0" - decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -14967,11 +14890,6 @@ defaults@^1.0.3: dependencies: clone "^1.0.2" -defer-to-connect@^1.0.1: - version "1.1.3" - resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" - integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== - define-data-property@^1.0.1, define-data-property@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.2.tgz#f3c33b4f0102360cd7c0f5f28700f5678510b63a" @@ -15292,7 +15210,7 @@ dot-case@^3.0.4: no-case "^3.0.4" tslib "^2.0.3" -dot-prop@^5.1.0, dot-prop@^5.2.0: +dot-prop@^5.1.0: version "5.3.0" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88" integrity sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q== @@ -15344,11 +15262,6 @@ duckdb@^1.1.3: node-addon-api "^7.0.0" node-gyp "^9.3.0" -duplexer3@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" - integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI= - duplexer@^0.1.1, duplexer@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" @@ -16176,11 +16089,6 @@ escalade@^3.2.0: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== -escape-goat@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" - integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q== - escape-html@^1.0.3, escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" @@ -17151,7 +17059,6 @@ firebolt-sdk@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/firebolt-sdk/-/firebolt-sdk-1.10.0.tgz#915a35155a58ecb2bc431db73ac3d3d18639f0da" integrity sha512-MShjlTGpzoW69A/4eymWmUGTWZxME9g00W5p/hm17s5ewTLCx2hq0XcnTt2nzL58ZgVA4qnmLmwWF20D95//uQ== - dependencies: "@types/json-bigint" "^1.0.1" abort-controller "^3.0.0" @@ -17581,7 +17488,7 @@ get-stream@^3.0.0: resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" integrity sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ= -get-stream@^4.0.0, get-stream@^4.1.0: +get-stream@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w== @@ -17878,23 +17785,6 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -got@^9.6.0: - version "9.6.0" - resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" - integrity sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q== - dependencies: - "@sindresorhus/is" "^0.14.0" - "@szmarczak/http-timer" "^1.1.2" - cacheable-request "^6.0.0" - decompress-response "^3.3.0" - duplexer3 "^0.1.4" - get-stream "^4.1.0" - lowercase-keys "^1.0.1" - mimic-response "^1.0.1" - p-cancelable "^1.0.0" - to-readable-stream "^1.0.0" - url-parse-lax "^3.0.0" - graceful-fs@4.2.11, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -18094,11 +17984,6 @@ has-unicode@2.0.1, has-unicode@^2.0.1: resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk= -has-yarn@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77" - integrity sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw== - has@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" @@ -18329,7 +18214,7 @@ htmlparser2@^6.1.0: domutils "^2.5.2" entities "^2.0.0" -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -18581,11 +18466,6 @@ ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore-by-default@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" - integrity sha1-SMptcvbGo68Aqa1K5odr44ieKwk= - ignore-walk@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-4.0.1.tgz#fc840e8346cf88a3a9380c5b17933cd8f4d39fa3" @@ -18663,11 +18543,6 @@ import-fresh@^3.0.0, import-fresh@^3.2.1, import-fresh@^3.3.0: parent-module "^1.0.0" resolve-from "^4.0.0" -import-lazy@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" - integrity sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM= - import-local@3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.1.0.tgz#b4479df8a5fd44f6cdce24070675676063c95cb4" @@ -19058,7 +18933,7 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-installed-globally@^0.4.0, is-installed-globally@~0.4.0: +is-installed-globally@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== @@ -19096,11 +18971,6 @@ is-negative-zero@^2.0.3: resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.3.tgz#ced903a027aca6381b777a5743069d7376a49747" integrity sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw== -is-npm@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-5.0.0.tgz#43e8d65cc56e1b67f8d47262cf667099193f45a8" - integrity sha512-WW/rQLOazUq+ST/bCAVBp/2oMERWLsR7OrKyt052dNDk4DHcDE0/7QSXITlmi+VBcV13DfIbysG3tZJm5RfdBA== - is-number-object@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.6.tgz#6a7aaf838c7f0686a50b4553f7e54a96494e89f0" @@ -19291,11 +19161,6 @@ is-wsl@^2.1.1, is-wsl@^2.2.0: dependencies: is-docker "^2.0.0" -is-yarn-global@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232" - integrity sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw== - is@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" @@ -20219,11 +20084,6 @@ json-bigint@^1.0.0: dependencies: bignumber.js "^9.0.0" -json-buffer@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" - integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg= - json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -20450,13 +20310,6 @@ karma-source-map-support@1.4.0: dependencies: source-map-support "^0.5.5" -keyv@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9" - integrity sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA== - dependencies: - json-buffer "3.0.0" - kind-of@^3.0.2: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -20501,13 +20354,6 @@ language-tags@^1.0.5: dependencies: language-subtag-registry "~0.3.2" -latest-version@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" - integrity sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA== - dependencies: - package-json "^6.3.0" - launch-editor-middleware@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/launch-editor-middleware/-/launch-editor-middleware-2.2.1.tgz#e14b07e6c7154b0a4b86a0fd345784e45804c157" @@ -21069,16 +20915,6 @@ lower-case@^2.0.2: dependencies: tslib "^2.0.3" -lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" - integrity sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA== - -lowercase-keys@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" - integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== - lru-cache@^10.0.1: version "10.1.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.1.0.tgz#2098d41c2dc56500e6c88584aa656c84de7d0484" @@ -21477,11 +21313,6 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -mimic-response@^1.0.0, mimic-response@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" - integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== - mimic-response@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" @@ -22227,23 +22058,7 @@ node-releases@^2.0.19: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.19.tgz#9e445a52950951ec4d177d843af370b411caf314" integrity sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw== -nodemon@^2.0.3: - version "2.0.15" - resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.15.tgz#504516ce3b43d9dc9a955ccd9ec57550a31a8d4e" - integrity sha512-gdHMNx47Gw7b3kWxJV64NI+Q5nfl0y5DgDbiVtShiwa7Z0IZ07Ll4RLFo6AjrhzMtoEZn5PDE3/c2AbVsiCkpA== - dependencies: - chokidar "^3.5.2" - debug "^3.2.7" - ignore-by-default "^1.0.1" - minimatch "^3.0.4" - pstree.remy "^1.1.8" - semver "^5.7.1" - supports-color "^5.5.0" - touch "^3.1.0" - undefsafe "^2.0.5" - update-notifier "^5.1.0" - -nopt@1.0.10, nopt@~1.0.10: +nopt@1.0.10: version "1.0.10" resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" integrity sha1-bd0hvSoxQXuScn3Vhfim83YI6+4= @@ -22322,11 +22137,6 @@ normalize-range@^0.1.2: resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942" integrity sha1-LRDAa9/TEuqXd2laTShDlFa3WUI= -normalize-url@^4.1.0: - version "4.5.1" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" - integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== - normalize-url@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" @@ -22497,14 +22307,6 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" -npm-watch@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/npm-watch/-/npm-watch-0.7.0.tgz#fe8f87f18dc6767051a83d005370be70a5cbd7a0" - integrity sha512-AN2scNyMljMGkn0mIkaRRk19I7Vx0qTK6GmsIcDblX5YRbSsoJORTAtrceICSx7Om9q48NWcwm/R0t6E7F4Ocg== - dependencies: - nodemon "^2.0.3" - through2 "^2.0.0" - npmlog@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0" @@ -22881,11 +22683,6 @@ ospath@^1.2.2: resolved "https://registry.yarnpkg.com/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" integrity sha1-EnZjl3Sj+O8lcvf+QoDg6kVQwHs= -p-cancelable@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" - integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== - p-finally@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" @@ -23014,16 +22811,6 @@ package-json-from-dist@^1.0.0: resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00" integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw== -package-json@^6.3.0: - version "6.5.0" - resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" - integrity sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ== - dependencies: - got "^9.6.0" - registry-auth-token "^4.0.0" - registry-url "^5.0.0" - semver "^6.2.0" - packet-reader@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" @@ -24534,11 +24321,6 @@ prelude-ls@~1.1.2: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= -prepend-http@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" - integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= - presto-client@^0.12.2: version "0.12.2" resolved "https://registry.yarnpkg.com/presto-client/-/presto-client-0.12.2.tgz#9719064bed0bd98166ae4bf2ac2c08c0f6996544" @@ -24780,11 +24562,6 @@ psl@^1.1.28, psl@^1.1.33: resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== -pstree.remy@^1.1.8: - version "1.1.8" - resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" - integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w== - pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -24808,13 +24585,6 @@ punycode@^2.3.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== -pupa@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" - integrity sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A== - dependencies: - escape-goat "^2.0.0" - python-struct@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/python-struct/-/python-struct-1.1.3.tgz#f0ff1845ec520408e94dd8492bfd770aad26cae3" @@ -25396,7 +25166,7 @@ rc-virtual-list@^3.0.1, rc-virtual-list@^3.2.0: rc-resize-observer "^1.0.0" rc-util "^5.0.7" -rc@^1.2.7, rc@^1.2.8: +rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -25955,20 +25725,6 @@ regexpu-core@^6.2.0: unicode-match-property-ecmascript "^2.0.0" unicode-match-property-value-ecmascript "^2.1.0" -registry-auth-token@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.1.tgz#6d7b4006441918972ccd5fedcd41dc322c79b250" - integrity sha512-6gkSb4U6aWJB4SF2ZvLb76yCBjcvufXBqvvEx1HbmKPkutswjW1xNVRY0+daljIYRbogN7O0etYSlbiaEQyMyw== - dependencies: - rc "^1.2.8" - -registry-url@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/registry-url/-/registry-url-5.1.0.tgz#e98334b50d5434b81136b44ec638d9c2009c5009" - integrity sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw== - dependencies: - rc "^1.2.8" - regjsgen@^0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.5.2.tgz#92ff295fb1deecbf6ecdab2543d207e91aa33733" @@ -26148,13 +25904,6 @@ response-iterator@^0.2.6: resolved "https://registry.yarnpkg.com/response-iterator/-/response-iterator-0.2.6.tgz#249005fb14d2e4eeb478a3f735a28fd8b4c9f3da" integrity sha512-pVzEEzrsg23Sh053rmDUvLSkGXluZio0qu8VT6ukrYuvtjVfCbDZH9d6PGXb8HZfzdNZt8feXv/jvUzlhRgLnw== -responselike@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7" - integrity sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec= - dependencies: - lowercase-keys "^1.0.0" - restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf" @@ -26311,6 +26060,13 @@ rxjs@^7.2.0: dependencies: tslib "~2.1.0" +rxjs@~7.5.0: + version "7.5.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.5.7.tgz#2ec0d57fdc89ece220d2e702730ae8f1e49def39" + integrity sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA== + dependencies: + tslib "^2.1.0" + safe-array-concat@^1.1.0, safe-array-concat@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.2.tgz#81d77ee0c4e8b863635227c721278dd524c20edb" @@ -26503,19 +26259,12 @@ semver-compare@^1.0.0: resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" integrity sha1-De4hahyUGrN+nvsXiPavxf9VN/w= -semver-diff@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-3.1.1.tgz#05f77ce59f325e00e2706afd67bb506ddb1ca32b" - integrity sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg== - dependencies: - semver "^6.3.0" - semver-regex@^3.1.2: version "3.1.3" resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-3.1.3.tgz#b2bcc6f97f63269f286994e297e229b6245d0dc3" integrity sha512-Aqi54Mk9uYTjVexLnR67rTyBusmwd04cLkHy9hNvk3+G3nT2Oyg7E0l4XVbOaNwIvQ3hHeYxGcyEy+mKreyBFQ== -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -26537,7 +26286,7 @@ semver@7.3.5, semver@^7.0.0, semver@^7.1.1, semver@^7.2.1, semver@^7.3.2, semver dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: +semver@^6.0.0, semver@^6.1.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== @@ -27488,7 +27237,7 @@ string-length@^5.0.1: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -27794,7 +27543,7 @@ supports-color@^2.0.0: resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" integrity sha1-U10EXOa2Nj+kARcIRimZXp3zJMc= -supports-color@^5.3.0, supports-color@^5.4.0, supports-color@^5.5.0: +supports-color@^5.3.0, supports-color@^5.4.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== @@ -28345,11 +28094,6 @@ to-fast-properties@^2.0.0: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= -to-readable-stream@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" - integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q== - to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -28382,13 +28126,6 @@ totalist@^3.0.0: resolved "https://registry.yarnpkg.com/totalist/-/totalist-3.0.1.tgz#ba3a3d600c915b1a97872348f79c127475f6acf8" integrity sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ== -touch@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" - integrity sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA== - dependencies: - nopt "~1.0.10" - tough-cookie@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" @@ -28515,11 +28252,6 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tsickle@^0.39.1: - version "0.39.1" - resolved "https://registry.yarnpkg.com/tsickle/-/tsickle-0.39.1.tgz#7ccf672cde5b430f5dd0b281ee49e170ef390ff9" - integrity sha512-CCc9cZhZbKoNizVM+K3Uqgit/go8GacjpqTv1cpwG/n2P0gB9GMoWZbxrUULDE9Wz26Lh86CGf6QyIPUVV1lnQ== - tslib@2.3.1, tslib@^2, tslib@^2.0.0, tslib@^2.0.3, tslib@^2.2.0, tslib@^2.3.0, tslib@^2.3.1, tslib@~2.3.0: version "2.3.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" @@ -28721,10 +28453,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.7.3.tgz#919b44a7dbb8583a9b856d162be24a54bf80073e" integrity sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw== -typescript@~4.5.5: - version "4.5.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" - integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== +typescript@~4.6.2: + version "4.6.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.4.tgz#caa78bbc3a59e6a5c510d35703f6a09877ce45e9" + integrity sha512-9ia/jWHIEbo49HfjrLGfKbZSuWo9iTMwXO+Ca3pRsSpbsMbc7/IU8NKdCZVRRBafVPGnoJeFL76ZOAA84I9fEg== typescript@~5.2.2: version "5.2.2" @@ -28779,11 +28511,6 @@ unbzip2-stream@^1.0.9: buffer "^5.2.1" through "^2.3.8" -undefsafe@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" - integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== - undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" @@ -28939,26 +28666,6 @@ update-browserslist-db@^1.1.1: escalade "^3.2.0" picocolors "^1.1.0" -update-notifier@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" - integrity sha512-ItnICHbeMh9GqUy31hFPrD1kcuZ3rpxDZbf4KUDavXwS0bW5m7SLbDQpGX3UYr072cbrF5hFUs3r5tUsPwjfHw== - dependencies: - boxen "^5.0.0" - chalk "^4.1.0" - configstore "^5.0.1" - has-yarn "^2.1.0" - import-lazy "^2.1.0" - is-ci "^2.0.0" - is-installed-globally "^0.4.0" - is-npm "^5.0.0" - is-yarn-global "^0.3.0" - latest-version "^5.1.0" - pupa "^2.1.1" - semver "^7.3.4" - semver-diff "^3.1.1" - xdg-basedir "^4.0.0" - uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" @@ -28966,13 +28673,6 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -url-parse-lax@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c" - integrity sha1-FrXK/Afb42dsGxmZF3gj1lA6yww= - dependencies: - prepend-http "^2.0.0" - url-parse@^1.5.3: version "1.5.3" resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.3.tgz#71c1303d38fb6639ade183c2992c8cc0686df862" @@ -29864,6 +29564,11 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= +workerpool@^9.2.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-9.2.0.tgz#f74427cbb61234708332ed8ab9cbf56dcb1c4371" + integrity sha512-PKZqBOCo6CYkVOwAxWxQaSF2Fvb5Iv2fCeTP7buyWI2GiynWr46NcXSgK/idoV6e60dgCBfgYc+Un3HMvmqP8w== + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -29995,11 +29700,6 @@ ws@~7.4.2: resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== -xdg-basedir@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" - integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== - xhr@^2.0.1: version "2.6.0" resolved "https://registry.yarnpkg.com/xhr/-/xhr-2.6.0.tgz#b69d4395e792b4173d6b7df077f0fc5e4e2b249d" From 9223f8510aac3f5d054232f80428c8691df2f248 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 6 Feb 2025 00:15:27 +0200 Subject: [PATCH 6/9] use workerpool pkg --- .../src/compiler/DataSchemaCompiler.js | 51 ++++++------- .../LightweightNodeCubeDictionary.ts | 6 +- .../transpilers/LightweightSymbolResolver.ts | 8 +- .../compiler/transpilers/transpiler_worker.ts | 75 ++++++++++--------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index a0dc2ba1e0e10..bed85c8395f51 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -7,6 +7,7 @@ import { parse } from '@babel/parser'; import babelGenerator from '@babel/generator'; import babelTraverse from '@babel/traverse'; import R from 'ramda'; +import workerpool from 'workerpool'; import { getEnv, isNativeSupported } from '@cubejs-backend/shared'; import { UserError } from './UserError'; @@ -43,6 +44,7 @@ export class DataSchemaCompiler { this.yamlCompiler = options.yamlCompiler; this.yamlCompiler.dataSchemaCompiler = this; this.pythonContext = null; + this.workerPool = null; } compileObjects(compileServices, objects, errorsReport) { @@ -92,6 +94,10 @@ export class DataSchemaCompiler { const errorsReport = new ErrorReporter(null, [], this.errorReport); this.errorsReport = errorsReport; + if (getEnv('workerThreadsTranspilation')) { + this.workerPool = workerpool.pool(path.join(__dirname, 'transpilers/transpiler_worker')); + } + const transpile = async () => { let cubeNames; let cubeSymbolsNames; @@ -99,6 +105,8 @@ export class DataSchemaCompiler { if (getEnv('workerThreadsTranspilation')) { cubeNames = Object.keys(this.cubeDictionary.byId); // We need only cubes and all its member names for transpiling. + // Cubes doesn't change during transpiling, but are changed during compilation phase, + // so we can prepare them once for every phase. // Communication between main and worker threads uses // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm) // which doesn't allow passing any function objects, so we need to sanitize the symbols. @@ -133,7 +141,12 @@ export class DataSchemaCompiler { .then(() => compilePhase({ cubeCompilers: this.cubeCompilers, contextCompilers: this.contextCompilers, - })); + })) + .then(() => { + if (this.workerPool) { + this.workerPool.terminate(); + } + }); } compile() { @@ -174,34 +187,18 @@ export class DataSchemaCompiler { } } - async transpileJsFile(file, errorsReport, options) { + async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbolsNames }) { try { if (getEnv('workerThreadsTranspilation')) { - const res = await new Promise((resolve, reject) => { - const { cubeNames, cubeSymbolsNames } = options; - - const worker = new Worker( - path.join(__dirname, 'transpilers/transpiler_worker'), - { workerData: { cubeNames, cubeSymbolsNames } }, - ); - - const data = { - fileName: file.fileName, - content: file.content, - transpilers: this.transpilers.map(t => t.constructor.name), - }; - - worker.postMessage(data); - - worker.on('message', resolve); - worker.on('error', reject); - worker.on('exit', (code) => { - if (code !== 0) { - reject(new Error(`Worker thread exited with code ${code}`)); - } - }); - }); - + const data = { + fileName: file.fileName, + content: file.content, + transpilers: this.transpilers.map(t => t.constructor.name), + cubeNames, + cubeSymbolsNames, + }; + + const res = await this.workerPool.exec('transpile', [data]); errorsReport.addErrors(res.errors); errorsReport.addWarnings(res.warnings); diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts index c65eb0ffd7c3e..159dbb8377a7a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightNodeCubeDictionary.ts @@ -1,10 +1,14 @@ import { TranspilerCubeResolver } from './transpiler.interface'; export class LightweightNodeCubeDictionary implements TranspilerCubeResolver { - public constructor(private readonly cubeNames: string[] = []) { + public constructor(private cubeNames: string[] = []) { } public resolveCube(name: string): boolean { return this.cubeNames.includes(name); } + + public setCubeNames(cubeNames: string[]): void { + this.cubeNames = cubeNames; + } } diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts index 8b0619ee99abc..1041a6190c6eb 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightSymbolResolver.ts @@ -1,8 +1,14 @@ import { TranspilerSymbolResolver } from './transpiler.interface'; import { CONTEXT_SYMBOLS, CURRENT_CUBE_CONSTANTS } from '../CubeSymbols'; +type CubeSymbols = Record>; + export class LightweightSymbolResolver implements TranspilerSymbolResolver { - public constructor(private readonly symbols: any) { + public constructor(private symbols: CubeSymbols = {}) { + } + + public setSymbols(symbols: CubeSymbols) { + this.symbols = symbols; } public isCurrentCube(name): boolean { diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index 31fea4ac65b25..0d5d87ffc9f46 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -1,5 +1,4 @@ -import { parentPort, workerData } from 'worker_threads'; - +import workerpool from 'workerpool'; import { parse } from '@babel/parser'; import babelGenerator from '@babel/generator'; import babelTraverse from '@babel/traverse'; @@ -12,14 +11,16 @@ import { ErrorReporter } from '../ErrorReporter'; import { LightweightSymbolResolver } from './LightweightSymbolResolver'; import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; -type FileContent = { +type TransferContent = { fileName: string; content: string; transpilers: string[]; + cubeNames: string[]; + cubeSymbolsNames: Record>; }; -const cubeDictionary = new LightweightNodeCubeDictionary(workerData.cubeNames); -const cubeSymbols = new LightweightSymbolResolver(workerData.cubeSymbolsNames); +const cubeDictionary = new LightweightNodeCubeDictionary(); +const cubeSymbols = new LightweightSymbolResolver(); const errorsReport = new ErrorReporter(null, []); const transpilers = { @@ -29,34 +30,38 @@ const transpilers = { CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols), }; -if (parentPort) { - parentPort.on('message', (file: FileContent) => { - const ast = parse( - file.content, - { - sourceFilename: file.fileName, - sourceType: 'module', - plugins: ['objectRestSpread'] - }, - ); - - file.transpilers.forEach(transpilerName => { - if (transpilers[transpilerName]) { - errorsReport.inFile(file); - babelTraverse(ast, transpilers[transpilerName].traverseObject(errorsReport)); - errorsReport.exitFile(); - } else { - throw new Error(`Transpiler ${transpilerName} not supported`); - } - }); - - const content = babelGenerator(ast, {}, file.content).code; - - // @ts-ignore - parentPort.postMessage({ - content, - errors: errorsReport.getErrors(), - warnings: errorsReport.getWarnings() - }); +const transpile = (data: TransferContent) => { + cubeDictionary.setCubeNames(data.cubeNames); + cubeSymbols.setSymbols(data.cubeSymbolsNames); + + const ast = parse( + data.content, + { + sourceFilename: data.fileName, + sourceType: 'module', + plugins: ['objectRestSpread'] + }, + ); + + data.transpilers.forEach(transpilerName => { + if (transpilers[transpilerName]) { + errorsReport.inFile(data); + babelTraverse(ast, transpilers[transpilerName].traverseObject(errorsReport)); + errorsReport.exitFile(); + } else { + throw new Error(`Transpiler ${transpilerName} not supported`); + } }); -} + + const content = babelGenerator(ast, {}, data.content).code; + + return { + content, + errors: errorsReport.getErrors(), + warnings: errorsReport.getWarnings() + }; +}; + +workerpool.worker({ + transpile, +}); From 1e7e2b5735c1065839fc6d0759b21b372a3223d9 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 6 Feb 2025 01:02:45 +0200 Subject: [PATCH 7/9] add max workers cfg --- packages/cubejs-backend-shared/src/env.ts | 5 +++- .../src/compiler/DataSchemaCompiler.js | 24 ++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index a6d1541234dc2..cc5417eac04ab 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -196,9 +196,12 @@ const variables: Record any> = { nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR') .default('false') .asBoolStrict(), - workerThreadsTranspilation: () => get('CUBEJS_WORKER_THREADS_TRANSPILATION') + transpilationWorkerThreads: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS') .default('false') .asBoolStrict(), + transpilationWorkerThreadsCount: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS_COUNT') + .default('0') + .asInt(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index bed85c8395f51..2856995ba2d3f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -94,15 +94,19 @@ export class DataSchemaCompiler { const errorsReport = new ErrorReporter(null, [], this.errorReport); this.errorsReport = errorsReport; - if (getEnv('workerThreadsTranspilation')) { - this.workerPool = workerpool.pool(path.join(__dirname, 'transpilers/transpiler_worker')); + if (getEnv('transpilationWorkerThreads')) { + const wc = getEnv('transpilationWorkerThreadsCount'); + this.workerPool = workerpool.pool( + path.join(__dirname, 'transpilers/transpiler_worker'), + wc > 0 ? { maxWorkers: wc } : undefined, + ); } const transpile = async () => { let cubeNames; let cubeSymbolsNames; - if (getEnv('workerThreadsTranspilation')) { + if (getEnv('transpilationWorkerThreads')) { cubeNames = Object.keys(this.cubeDictionary.byId); // We need only cubes and all its member names for transpiling. // Cubes doesn't change during transpiling, but are changed during compilation phase, @@ -119,16 +123,8 @@ export class DataSchemaCompiler { ), ); } - // const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }))); - const results = []; - for (const f of toCompile) { - const result = await this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }); - if (result) { - results.push(result); - } - } - // return results.filter(f => !!f); - return results; + const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames }))); + return results.filter(f => !!f); }; const compilePhase = async (compilers) => this.compileCubeFiles(compilers, await transpile(), errorsReport); @@ -189,7 +185,7 @@ export class DataSchemaCompiler { async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbolsNames }) { try { - if (getEnv('workerThreadsTranspilation')) { + if (getEnv('transpilationWorkerThreads')) { const data = { fileName: file.fileName, content: file.content, From f1509f7906f9aff018249cab328683346eeca567 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 6 Feb 2025 01:08:29 +0200 Subject: [PATCH 8/9] =?UTF-8?q?update=20env=20in=C2=A0ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5eb44dd36e41b..8a1a89da73a23 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -58,7 +58,7 @@ jobs: fail-fast: false env: - CUBEJS_WORKER_THREADS_TRANSPILATION: ${{ matrix.transpile-worker-threads }} + CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }} steps: - id: get-tag-out run: echo "$OUT" From 29db95ed18a63937c3b17f941d43021b5ea63e47 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 6 Feb 2025 01:11:14 +0200 Subject: [PATCH 9/9] remove obsolete stuff --- .../cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 2856995ba2d3f..cd087329b9dfd 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -1,7 +1,6 @@ import vm from 'vm'; import fs from 'fs'; import path from 'path'; -import { Worker } from 'worker_threads'; import syntaxCheck from 'syntax-error'; import { parse } from '@babel/parser'; import babelGenerator from '@babel/generator';