From 4adcace475efe702086eb6372ed80f277cdc5e56 Mon Sep 17 00:00:00 2001 From: zxypro1 <1018995004@qq.com> Date: Wed, 24 Jul 2024 14:37:30 +0800 Subject: [PATCH] feat: preview api support Signed-off-by: zxypro1 <1018995004@qq.com> --- packages/engine/package.json | 2 +- packages/parse-spec/package.json | 5 +- packages/parse-spec/src/index.ts | 1 + packages/parse-spec/src/utils/index.ts | 360 +++++++++++++++++++++++++ pnpm-lock.yaml | 3 + 5 files changed, 368 insertions(+), 3 deletions(-) diff --git a/packages/engine/package.json b/packages/engine/package.json index d69c87b8..0c25d087 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -1,6 +1,6 @@ { "name": "@serverless-devs/engine", - "version": "0.1.4-beta.8", + "version": "0.1.4-beta.9", "description": "a engine lib for serverless-devs", "main": "lib/index.js", "scripts": { diff --git a/packages/parse-spec/package.json b/packages/parse-spec/package.json index b416fd21..3ec331c5 100644 --- a/packages/parse-spec/package.json +++ b/packages/parse-spec/package.json @@ -1,6 +1,6 @@ { "name": "@serverless-devs/parse-spec", - "version": "0.0.28-beta.7", + "version": "0.0.28-beta.8", "description": "a parse yaml spec lib for serverless-devs", "main": "lib/index.js", "scripts": { @@ -30,7 +30,8 @@ "dotenv-expand": "^10.0.0", "extend2": "^1.0.1", "fs-extra": "^11.1.0", - "lodash": "^4.17.21" + "lodash": "^4.17.21", + "js-yaml": "^4.1.0" }, "devDependencies": { "@types/lodash": "^4.14.195" diff --git a/packages/parse-spec/src/index.ts b/packages/parse-spec/src/index.ts index 935bad59..4cc7aec4 100644 --- a/packages/parse-spec/src/index.ts +++ b/packages/parse-spec/src/index.ts @@ -1,6 +1,7 @@ export { default as getInputs } from './get-inputs'; export * from './types'; export * from './contants'; +export { ParseSpecForContent } from './utils'; import * as utils from '@serverless-devs/utils'; import fs from 'fs-extra'; import path from 'path'; diff --git a/packages/parse-spec/src/utils/index.ts b/packages/parse-spec/src/utils/index.ts index ad88b701..fd6fd383 100644 --- a/packages/parse-spec/src/utils/index.ts +++ b/packages/parse-spec/src/utils/index.ts @@ -3,6 +3,7 @@ import * as utils from '@serverless-devs/utils'; import Credential from '@serverless-devs/credential'; import { get } from 'lodash'; import { ETrackerType } from '@serverless-devs/utils'; +import jsYaml from 'js-yaml'; export function getDefaultYamlPath() { const spath = utils.getYamlPath('s'); @@ -29,3 +30,362 @@ export async function getCredential(access: string | undefined, logger: any) { return {}; } } + +/** + * Parse-spec class for the use of apis + * @author neil.zxy + * @date 2024-07-23 17:44 + */ +import fs from 'fs-extra'; +import dotenv from 'dotenv'; +import { expand } from 'dotenv-expand'; +import Order from '../order'; +import ParseContent from '../parse-content'; +import { each, filter, find, has, includes, isArray, isEmpty, isString, keys, map, set, split } from 'lodash'; +import { ISpec, IYaml, IActionType, IActionLevel, IStep, IRecord } from '../types'; +import { ENVIRONMENT_FILE_NAME, ENVIRONMENT_FILE_PATH, ENVIRONMENT_KEY, REGX } from '../contants'; +import assert from 'assert'; +import { DevsError } from '@serverless-devs/utils'; +const extend2 = require('extend2'); + +interface IOptions { + argv?: string[]; + logger?: any; +} + +export class ParseSpecForContent { + private yaml = {} as IYaml; + private record = {} as IRecord; + constructor(yamlContent: string = '', private options: IOptions = {}) { + this.options.argv = this.options.argv || process.argv.slice(2); + this.options.logger = this.options.logger || console; + this.yaml.path = ''; + this.yaml.content = jsYaml.load(yamlContent) || {}; + } + private async doYamlinit() { + await this.doExtend(); + this.doEnvironment(); + this.yaml.access = get(this.yaml.content, 'access'); + const projectKey = this.yaml.use3x ? 'resources' : 'services'; + const projects = get(this.yaml.content, projectKey, {}); + this.yaml.projectNames = keys(get(this.yaml.content, projectKey, {})); + this.yaml.vars = get(this.yaml.content, 'vars', {}); + this.yaml.flow = get(this.yaml.content, 'flow', {}); + this.yaml.useFlow = false; + this.yaml.appName = get(this.yaml.content, 'name'); + + // 兼容2.0: 加入项目的.env环境变量 + for (const i of this.yaml.projectNames) { + const code = get(projects, `${i}.props.code`); + if (code && isString(code)) { + const codePath = utils.getAbsolutePath(get(projects, `${i}.props.code`, '')); + expand(dotenv.config({ path: path.join(codePath, '.env') })); + } + } + + expand(dotenv.config({ path: path.join(path.dirname(this.yaml.path), '.env') })); + } + private async doExtend() { + // this.yaml = { path: '' } + // this.yaml.content = utils.getYamlContent(this.yaml.path); + this.yaml.extend = get(this.yaml.content, 'extend'); + this.yaml.useExtend = isExtendMode(this.yaml.extend, path.dirname(this.yaml.path)); + if (this.yaml.useExtend) { + // if useExtend, 则直接解析前后内容 + const extendPath = utils.getAbsolutePath(this.yaml.extend, path.dirname(this.yaml.path)); + expand(dotenv.config({ path: path.join(extendPath, '.env') })); + const extendContent = utils.getYamlContent(extendPath); + this.yaml.use3x = String(get(this.yaml.content, 'edition', get(extendContent, 'edition'))) === '3.0.0'; + // 1.x 不做extend动作 + if (!this.yaml.use3x) return; + const { resources: extendResource, ...extendRest } = extendContent; + const { resources: currentResource, ...currentRest } = this.yaml.content; + const tempRest = extend2(true, {}, extendRest, currentRest); + const base = await new ParseContent({ ...extendContent, ...tempRest }, this.getParsedContentOptions(extendPath)).start(); + const current = await new ParseContent({ ...this.yaml.content, ...tempRest }, this.getParsedContentOptions(this.yaml.path)).start(); + this.yaml.content = extend2(true, {}, get(base, 'content'), get(current, 'content')); + return; + } + this.yaml.use3x = String(get(this.yaml.content, 'edition')) === '3.0.0'; + } + /** + * # 指定--env时： + - s.yaml使用extend，则报错 + - 如果s.yaml声明了env yaml，则使用指定的env.yaml, 否则使用默认的env.yaml + - 如果env.yaml不存在，则报错 + - 指定的env找不到，则报错 + - s deploy --env test + + # 不指定--env时 + ## s.yaml使用extend + - s.yaml声明了env yaml, 则报错，没声明env yaml，则不使用多环境 + - s deploy + ## s.yaml未使用extend + - s.yaml声明了env yaml，如果env.yaml不存在，则报错 + - 使用指定env.yaml的默认环境，如果没有指定默认环境，则报错 + - 如果指定的默认环境找不到，则报错。 + - s deploy + */ + private doEnvironment() { + if (!this.yaml.use3x) return; + const envInfo = this.record.env ? this.doEnvWithSpecify() : this.doEnvWithNotSpecify(); + // 不使用多环境, 则直接返回 + if (isEmpty(envInfo)) return; + const { project, environment } = envInfo as { project: string; environment: Record }; + set(environment, 'overlays.resources.region', get(environment, 'region')); + set(environment, '__project', project); + this.yaml.environment = environment; + } + // not specify --env + private doEnvWithNotSpecify() { + if (this.yaml.useExtend) { + if (has(this.yaml.content, ENVIRONMENT_KEY)) { + throw new DevsError('Environment and extend is conflict', { + trackerType: ETrackerType.parseException, + }); + } + } + // default-env.json is not exist + if (!fs.existsSync(ENVIRONMENT_FILE_PATH)) { + return {}; + } + // 若存在环境变量，默认项目为devsProject + const devsProject = process.env.ALIYUN_DEVS_REMOTE_PROJECT_NAME; + const project = devsProject ? devsProject : get(this.yaml.content, 'name'); + const defaultEnvContent = require(ENVIRONMENT_FILE_PATH); + const defaultEnv = get(find(defaultEnvContent, { project: project }), 'default'); + // project is not found in default-env.json + if (isEmpty(defaultEnv)) { + return {}; + } + const envPath: string = utils.getAbsolutePath(get(this.yaml.content, ENVIRONMENT_KEY) || 'env.yaml', path.dirname(this.yaml.path)); + const envYamlContent = utils.getYamlContent(envPath); + if (isEmpty(envYamlContent)) { + this.options.logger.warnOnce(`Environment file [${envPath}] is not found, run without environment.`); + return {}; + } + const { environments } = envYamlContent; + const environment = find(environments, item => item.name === defaultEnv); + // default env is not found in env.yaml + if (isEmpty(environment)) { + this.options.logger.warnOnce(`Default env [${defaultEnv}] is not found, run without environment.`); + return {}; + } + return { project, environment }; + } + // specify --env + private doEnvWithSpecify() { + // env and extend is conflict + if (this.yaml.useExtend) { + // TODO: @封崇 + throw new DevsError('Environment and extend is conflict', { + trackerType: ETrackerType.parseException, + }); + } + const envPath: string = utils.getAbsolutePath(get(this.yaml.content, ENVIRONMENT_KEY, ENVIRONMENT_FILE_NAME), path.dirname(this.yaml.path)); + const envYamlContent = utils.getYamlContent(envPath); + // env file is not exist + if (isEmpty(envYamlContent)) { + this.options.logger.warnOnce(`Environment file [${envPath}] is not found, run without environment.`); + return {}; + } + const { environments } = envYamlContent; + // 若存在环境变量，默认项目为devsProject + const devsProject = process.env.ALIYUN_DEVS_REMOTE_PROJECT_NAME; + const project = devsProject ? devsProject : get(this.yaml.content, 'name'); + const environment = find(environments, item => item.name === this.record.env); + // env name is not found + if (isEmpty(environment)) { + this.options.logger.warnOnce(`Env [${this.record.env}] was not found, run without environment.`); + return {}; + } + return { project, environment }; + } + private getParsedContentOptions(basePath: string) { + return { + logger: this.options.logger, + basePath, + projectName: this.record.projectName, + access: this.record.access, + environment: this.yaml.environment, + }; + } + async start(): Promise { + // 第一次尝试解析参数，比如全局access给 extend 用 + // 将命令行参数更新给 this.record + this.parseArgv(); + // 处理继承、多环境后, 将 s.yaml 文件信息传入 this.yaml + await this.doYamlinit(); + // 再次解析参数，比如projectNames + this.parseArgv(); + if (!this.yaml.use3x) return this.v1(); + const { steps, content, originSteps, allSteps } = await new ParseContent(this.yaml.content, this.getParsedContentOptions(this.yaml.path)).start(); + const services = get(this.yaml.content, 'services', {}); + if (isEmpty(steps) && !isEmpty(services)) { + this.options.logger.tips('Check https://docs.serverless-devs.com/user-guide/spec/ for more details. Use the \'s cli fc3 s2tos3\' command for automatic YAML transformation.'); + throw new DevsError(`Keyword 'services' has been replaced by 'resources' in 3.0.0 YAML.`, { + trackerType: ETrackerType.parseException, + }); + } + // steps 存放每个FC组件/函数的 yaml 配置 ([content.resource] => steps) + // content 为 yaml 已解析的整体完整信息 + // originSteps 为 steps 的未解析版 + // 获取到真实值后，重新赋值 + this.yaml.content = content; + this.yaml.vars = get(this.yaml.content, 'vars', {}); + const actions = get(this.yaml.content, 'actions', {}); + this.yaml.actions = this.parseActions(actions); + const result = { + steps: this.record.projectName ? steps : this.doFlow(steps, originSteps), + yaml: this.yaml, + allSteps: allSteps, + ...this.record, + }; + return result; + } + // 简单兼容v1版本，不需要做魔法变量解析 + private v1(): ISpec { + const steps = []; + const services = get(this.yaml.content, 'services', {}); + for (const project in services) { + const element = services[project]; + steps.push({ + ...element, + projectName: project, + }); + } + return { + steps: this.record.projectName ? filter(steps, item => item.projectName === this.record.projectName) : steps, + yaml: this.yaml, + ...this.record, + }; + } + private parseArgv() { + const argv = utils.parseArgv(this.options.argv as string[]); + const { _ } = argv; + this.record.access = get(argv, 'access'); + this.record.version = get(argv, 'version'); + this.record.help = get(argv, 'help'); + this.record.output = get(argv, 'output'); + this.record.skipActions = get(argv, 'skip-actions'); + this.record.debug = get(argv, 'debug'); + this.record.env = get(argv, 'env'); + if (includes(this.yaml.projectNames, _[0])) { + this.record.projectName = _[0]; + this.record.command = _[1]; + return; + } + this.record.command = _[0]; + } + private doFlow(steps: IStep[], originSteps: IStep[]) { + const newSteps: IStep[] = []; + const flowObj = find(this.yaml.flow, (item, key) => this.matchFlow(key)); + const orderInstance = new Order(originSteps).start(); + const { steps: orderSteps, dependencies, useOrder } = orderInstance.sort(steps); + if (!flowObj) return orderSteps; + const projectOrder = {} as Record; + const fn = (projects: string[] = [], index: number) => { + assert(isArray(projects), `flow ${this.record.command} data format is invalid.`); + for (const project of projects) { + const step = find(steps, item => item.projectName === project); + assert(step, `Resource ${project} is not found. Please check the content of flow.`); + newSteps.push({ ...step, flowId: index }); + projectOrder[step.projectName] = index; + } + }; + each(flowObj, fn); + this.yaml.useFlow = true; + // 指定flow后，如果存在依赖关系，校验是否可以正常执行 + if (useOrder) { + for (const p1 in dependencies) { + const ele = dependencies[p1]; + for (const p2 in ele) { + if (projectOrder[p2] >= projectOrder[p1]) { + throw new Error(`flow is invalid, ${p2} must be executed before ${p1}`); + } + } + } + } + return newSteps; + } + private matchFlow(flow: string) { + const useMagic = REGX.test(flow); + if (useMagic) { + const compile = require('@serverless-devs/art-template/lib/devs-compile'); + return compile(flow, { command: this.record.command }); + } + return flow === this.record.command; + } + parseActions(actions: Record = {}, level: string = IActionLevel.GLOBAL) { + const actionList = []; + for (const action in actions) { + const element = actions[action]; + if (!isArray(element)) { + throw new DevsError(`${level} action ${action} is invalid, it must be array`, { + trackerType: ETrackerType.parseException, + }); + } + const actionInfo = this.matchAction(action); + if (actionInfo.validate) { + actionList.push( + ...map(element, item => { + if (item[IActionType.RUN]) { + const { run, ...rest } = item; + return { + ...rest, + value: run, + path: utils.getAbsolutePath(get(item, 'path', './'), path.dirname(this.yaml.path)), + actionType: IActionType.RUN, + hookType: actionInfo.type, + level, + projectName: this.record.projectName, + }; + } + if (item[IActionType.PLUGIN]) { + const { plugin, ...rest } = item; + const value = utils.getAbsolutePath(plugin, path.dirname(this.yaml.path)); + return { + ...rest, + value: fs.existsSync(value) ? value : plugin, + actionType: IActionType.PLUGIN, + hookType: actionInfo.type, + level, + projectName: this.record.projectName, + }; + } + if (item[IActionType.COMPONENT]) { + const { component, ...rest } = item; + return { + ...rest, + value: component, + actionType: IActionType.COMPONENT, + hookType: actionInfo.type, + level, + projectName: this.record.projectName, + }; + } + }), + ); + } + } + return actionList; + } + private matchAction(action: string) { + const useMagic = REGX.test(action); + if (useMagic) { + const compile = require('@serverless-devs/art-template/lib/devs-compile'); + const newAction = compile(action, { command: this.record.command }); + const [type, command] = split(newAction, '-'); + return { + validate: command === 'true', + type, + }; + } + const [type, command] = split(action, '-'); + return { + validate: command === this.record.command, + type, + }; + } +} + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7d97023..06404ada 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -336,6 +336,9 @@ importers: fs-extra: specifier: ^11.1.0 version: 11.1.1 + js-yaml: + specifier: ^4.1.0 + version: 4.1.0 lodash: specifier: ^4.17.21 version: 4.17.21