feat: implement child-pipeline (#1392)

src/argv.ts

@@ -26,7 +26,7 @@ export class Argv {
         "variablesFile": ".gitlab-ci-local-variables.yml",
     };
 
-    private map: Map<string, any> = new Map<string, any>();
+    map: Map<string, any> = new Map<string, any>();
     private writeStreams: WriteStreams | undefined;
 
     private async fallbackCwd (args: any) {
@@ -276,4 +276,8 @@ export class Argv {
     get maximumIncludes (): number {
         return this.map.get("maximumIncludes") ?? 150; // https://docs.gitlab.com/ee/administration/settings/continuous_integration.html#maximum-includes
     }
+
+    get childPipelineDepth (): number {
+        return this.map.get("childPipelineDepth");
+    }
 }

src/commander.ts

@@ -133,9 +133,12 @@ export class Commander {
 
         if (preScripts.successful.length !== 0) {
             preScripts.successful.sort((a, b) => stages.indexOf(a.stage) - stages.indexOf(b.stage));
-            preScripts.successful.forEach(({coveragePercent, name, prettyDuration}) => {
+            preScripts.successful.forEach(({argv, coveragePercent, name, prettyDuration}) => {
+                let prefix = "";
+                if (argv.childPipelineDepth > 0) prefix = `[${argv.variable.GCL_TRIGGERER}] -> `;
+
                 const namePad = name.padEnd(jobNamePad);
-                writeStreams.stdout(chalk`{black.bgGreenBright  PASS }${renderDuration(prettyDuration)} {blueBright ${namePad}}`);
+                writeStreams.stdout(chalk`{black.bgGreenBright  PASS }${renderDuration(prettyDuration)} {blueBright ${prefix}${namePad}}`);
                 if (coveragePercent) {
                     writeStreams.stdout(chalk` ${coveragePercent}% {grey coverage}`);
                 }

src/handler.ts

@@ -21,8 +21,9 @@ const generateGitIgnore = (cwd: string, stateDir: string) => {
     }
 };
 
-export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[] = []) {
-    const argv = await Argv.build(args, writeStreams);
+export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[] = [], childPipelineDepth = 0) {
+    assert(childPipelineDepth <= 2, "Parent and child pipelines have a maximum depth of two levels of child pipelines.");
+    const argv = await Argv.build({...args, childPipelineDepth: childPipelineDepth}, writeStreams);
     const cwd = argv.cwd;
     const stateDir = argv.stateDir;
     const file = argv.file;
@@ -87,13 +88,13 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
     } else {
         generateGitIgnore(cwd, stateDir);
         const time = process.hrtime();
-        await fs.remove(`${cwd}/${stateDir}/artifacts`);
+        if (childPipelineDepth == 0) await fs.remove(`${cwd}/${stateDir}/artifacts`);
         await state.incrementPipelineIid(cwd, stateDir);
         const pipelineIid = await state.getPipelineIid(cwd, stateDir);
         parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
         await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker");
         await Commander.runPipeline(argv, parser, writeStreams);
-        writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
+        if (childPipelineDepth == 0) writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
     }
     writeStreams.flush();
 

src/job.ts

@@ -12,6 +12,11 @@ import {Argv} from "./argv.js";
 import execa from "execa";
 import {CICDVariable} from "./variables-from-files.js";
 import {GitlabRunnerCPUsPresetValue, GitlabRunnerMemoryPresetValue, GitlabRunnerPresetValues} from "./gitlab-preset.js";
+import {handler} from "./handler.js";
+import * as yaml from "js-yaml";
+import {Parser} from "./parser.js";
+import globby from "globby";
+import {validateIncludeLocal} from "./parser-includes.js";
 
 const CI_PROJECT_DIR = "/gcl-builds";
 const GCL_SHELL_PROMPT_PLACEHOLDER = "<gclShellPromptPlaceholder>";
@@ -105,6 +110,8 @@ export class Job {
     };
 
     private readonly _globalVariables: {[key: string]: string} = {};
+
+    readonly variablesForDownstreamPipeline: {[key: string]: string} = {};
     private readonly _variables: {[key: string]: string} = {};
     private _dotenvVariables: {[key: string]: string} = {};
     private _prescriptsExitCode: number | null = null;
@@ -310,12 +317,14 @@ export class Job {
     }
 
     get formattedJobName () {
+        let prefix = "";
+        if (this.argv.childPipelineDepth > 0) prefix = "\t".repeat(this.argv.childPipelineDepth) + `[${this.argv.variable.GCL_TRIGGERER}] -> `;
         const timestampPrefix = this.argv.showTimestamps
             ? `[${dateFormatter.format(new Date())} ${this.prettyDuration.padStart(7)}] `
             : "";
 
         // [16:33:19 1.37 min] my-job     > hello world
-        return chalk`${timestampPrefix}{blueBright ${this.name.padEnd(this.jobNamePad)}}`;
+        return chalk`${timestampPrefix}{blueBright ${prefix}${this.name.padEnd(this.jobNamePad)}}`;
     }
 
     get safeJobName () {
@@ -433,6 +442,24 @@ export class Job {
         return this.jobData["trigger"];
     }
 
+    async startTriggerPipeline () {
+        this.writeStreams.memoStdout(chalk`{bgYellowBright  WARN } downstream pipeline is experimental in gitlab-ci-local\n`);
+        await this.fetchTriggerInclude();
+        const variablesForDownstreamPipeline = Object.entries({...this.globalVariables, ...this.jobData.variables}).map(([key, value]) => `${key}=${value}`);
+
+        const gclTriggerer = this.argv.variable["GCL_TRIGGERER"]
+            ? `${this.argv.variable["GCL_TRIGGERER"]} -> ${this.name}`
+            : this.name;
+        await handler({
+            ...Object.fromEntries(this.argv.map),
+            file: `${this.argv.stateDir}/includes/triggers/${this.name}.yml`,
+            variable: [
+                chalk`GCL_TRIGGERER=${gclTriggerer}`,
+                "CI_PIPELINE_SOURCE=parent_pipeline",
+            ].concat(variablesForDownstreamPipeline),
+        }, this.writeStreams, [], this.argv.childPipelineDepth + 1);
+    }
+
     get preScriptsExitCode () {
         return this._prescriptsExitCode;
     }
@@ -470,6 +497,12 @@ export class Job {
     }
 
     async start (): Promise<void> {
+        if (this.trigger) {
+            await this.startTriggerPipeline();
+            this._prescriptsExitCode = 0; // NOTE: so that `this.finished` will implicitly be set to true
+            return;
+        }
+
         this._running = true;
 
         const argv = this.argv;
@@ -1368,6 +1401,45 @@ export class Job {
             }));
         }
     }
+
+    private async fetchTriggerInclude () {
+        const {cwd, stateDir} = this.argv;
+
+        fs.mkdirpSync(`${cwd}/${stateDir}/includes/triggers`);
+
+        let contents: any = {};
+
+        for (const include of this.jobData.trigger?.include ?? []) {
+            if (include["local"]) {
+                validateIncludeLocal(include["local"]);
+                const files = await globby(include["local"].replace(/^\//, ""), {dot: true, cwd});
+                if (files.length == 0) {
+                    throw new AssertionError({message: `Local include file \`${include["local"]}\` specified in \`.${this.name}\` cannot be found!`});
+                }
+
+                for (const file of files) {
+                    const content = await Parser.loadYaml(`${cwd}/${file}`, {});
+                    contents = {
+                        ...contents,
+                        ...content,
+                    };
+                }
+            } else if (include["artifact"]) {
+                const content = await Parser.loadYaml(`${cwd}/${stateDir}/artifacts/${include["job"]}/${include["artifact"]}`, {});
+                contents = {
+                    ...contents,
+                    ...content,
+                };
+            } else if (include["project"]) {
+                this.writeStreams.memoStdout(chalk`{bgYellowBright  WARN } \`{blueBright ${this.name}.trigger.include.*.project}\` will be no-op. It is currently not implemented\n`);
+
+            } else if (include["template"]) {
+                this.writeStreams.memoStdout(chalk`{bgYellowBright  WARN } \`{blueBright ${this.name}.trigger.include.*.template}\` will be no-op. It is currently not implemented\n`);
+            }
+        }
+        const target = `${cwd}/${stateDir}/includes/triggers/${this.name}.yml`;
+        fs.writeFileSync(target, yaml.dump(contents));
+    }
 }
 
 export async function cleanupJobResources (jobs?: Iterable<Job>) {

src/parser-includes.ts

@@ -27,6 +27,19 @@ export class ParserIncludes {
         this.count = 0;
     }
 
+    private static normalizeTriggerInclude (gitlabData: any, opts: ParserIncludesInitOptions) {
+        const {writeStreams} = opts;
+        for (const [jobName, jobData] of Object.entries<any>(gitlabData ?? {})) {
+            if (typeof jobData.trigger?.include === "string") {
+                jobData.trigger.include = [{
+                    local: jobData.trigger.include,
+                } ];
+            } else if (jobData.trigger?.project) {
+                writeStreams.memoStdout(chalk`{bgYellowBright  WARN } The job: \`{blueBright ${jobName}}\` will be no-op. Multi-project pipeline is not supported by gitlab-ci-local\n`);
+            }
+        }
+    }
+
     static async init (gitlabData: any, opts: ParserIncludesInitOptions): Promise<any[]> {
         this.count++;
         assert(
@@ -39,6 +52,7 @@ export class ParserIncludes {
 
         const include = this.expandInclude(gitlabData?.include, opts.variables);
 
+        this.normalizeTriggerInclude(gitlabData, opts);
         // Find files to fetch from remote and place in .gitlab-ci-local/includes
         for (const value of include) {
             if (value["rules"]) {
@@ -268,7 +282,7 @@ export class ParserIncludes {
     }
 }
 
-function validateIncludeLocal (filePath: string) {
+export function validateIncludeLocal (filePath: string) {
     assert(!filePath.startsWith("./"), `\`${filePath}\` for include:local is invalid. Gitlab does not support relative path (ie. cannot start with \`./\`).`);
     assert(!filePath.includes(".."), `\`${filePath}\` for include:local is invalid. Gitlab does not support directory traversal.`);
 }

src/parser.ts

@@ -70,7 +70,7 @@ export class Parser {
         const pathToExpandedGitLabCi = path.join(argv.cwd, argv.stateDir, "expanded-gitlab-ci.yml");
         fs.mkdirpSync(path.join(argv.cwd, argv.stateDir));
         fs.writeFileSync(pathToExpandedGitLabCi, yaml.dump(parser.gitlabData));
-        writeStreams.stderr(chalk`{grey parsing and downloads finished in ${prettyHrtime(parsingTime)}.}\n`);
+        if (argv.childPipelineDepth == 0) writeStreams.stderr(chalk`{grey parsing and downloads finished in ${prettyHrtime(parsingTime)}.}\n`);
 
         for (const warning of warnings) {
             writeStreams.stderr(chalk`{yellow ${warning}}\n`);
@@ -83,7 +83,7 @@ export class Parser {
                 pathToExpandedGitLabCi,
                 gitLabCiConfig: parser.gitlabData,
             });
-            writeStreams.stderr(chalk`{grey json schema validated in ${prettyHrtime(process.hrtime(time))}}\n`);
+            if (argv.childPipelineDepth == 0) writeStreams.stderr(chalk`{grey json schema validated in ${prettyHrtime(process.hrtime(time))}}\n`);
         }
         return parser;
     }

tests/test-cases/parent-child/.gitignore

@@ -0,0 +1 @@
+generated-config.yml

tests/test-cases/parent-child/.gitlab-ci-1.yml

@@ -0,0 +1,10 @@
+---
+variables:
+  GLOBAL_VAR: i am global var
+
+parent:
+  variables:
+    PARENT_JOB_VAR: i am parent job var
+  trigger:
+    include:
+      - local: child-pipeline.yml

tests/test-cases/parent-child/.gitlab-ci-2.yml

@@ -0,0 +1,12 @@
+---
+variables:
+  GLOBAL_VAR: i am global var
+
+parent:
+  variables:
+    PARENT_JOB_VAR: i am parent job var
+  inherit:
+    variables: false
+  trigger:
+    include:
+      - local: child-pipeline.yml

tests/test-cases/parent-child/.gitlab-ci-3.yml

@@ -0,0 +1,16 @@
+---
+variables:
+  GLOBAL_VAR_1: i am global var 1
+  GLOBAL_VAR_2: i am global var 2
+  GLOBAL_VAR_3: i am global var 3
+
+parent:
+  variables:
+    PARENT_JOB_VAR: i am parent job var
+  inherit:
+    variables:
+      - GLOBAL_VAR_1
+      - GLOBAL_VAR_3
+  trigger:
+    include:
+      - local: child-pipeline.yml

tests/test-cases/parent-child/.gitlab-ci-4.yml

@@ -0,0 +1,10 @@
+---
+variables:
+  GLOBAL_VAR: i am global var
+
+parent:
+  variables:
+    JOB_VAR: i have a higher precedence
+  trigger:
+    include:
+      - local: child-pipeline.yml

tests/test-cases/parent-child/.gitlab-ci-5.yml

@@ -0,0 +1,22 @@
+---
+generate-config:
+  image: busybox
+  stage: build
+  script:
+    - |
+      cat > generated-config.yml << EOF
+      ---
+      child:
+        image: busybox
+        script:
+        - echo i am generated
+      EOF
+  artifacts:
+    paths:
+      - generated-config.yml
+
+dynamic-pipeline:
+  trigger:
+    include:
+      - artifact: generated-config.yml
+        job: generate-config

tests/test-cases/parent-child/.gitlab-ci-6.yml

@@ -0,0 +1,5 @@
+---
+parent:
+  trigger:
+    include:
+      - local: nested-child-1.yml

tests/test-cases/parent-child/.gitlab-ci-7.yml

@@ -0,0 +1,29 @@
+---
+generate-config:
+  image: busybox
+  stage: build
+  script:
+    - sleep 3
+    - |
+      cat > generated-config.yml << EOF
+      ---
+      child:
+        image: busybox
+        script:
+        - echo i am generated
+      EOF
+  artifacts:
+    paths:
+      - generated-config.yml
+
+dynamic-pipeline-1:
+  trigger:
+    include:
+      - artifact: generated-config.yml
+        job: generate-config
+
+dynamic-pipeline-2:
+  trigger:
+    include:
+      - artifact: generated-config.yml
+        job: generate-config

tests/test-cases/parent-child/child-pipeline.yml

@@ -0,0 +1,7 @@
+---
+child:
+  image: busybox
+  variables:
+    JOB_VAR: "i am job var"
+  script:
+    - env | sort | grep VAR