refactor(migrations): framework to build batchable migrations (angula…

…r#57396)

Introduces a migration framework to build batchable migrations that
can run in Large Scale mode against e.g. all of Google, using workers.

This is the original signal input migration infrastructure extracted
into a more generic framework that we can use for writing additional
ones for output, signal queries etc, while making sure those are not
scoped to a single `ts.Program` that limits them to per-directory
execution in very large projects (e.g. G3).

The migration will be updated to use this, and in 1P we will add
helpers to easily integrate such migrations into a Go-based pipeline
runner.

PR Close angular#57396

packages/core/schematics/utils/tsurge/BUILD.bazel

@@ -0,0 +1,20 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "tsurge",
+    srcs = glob(["**/*.ts"]),
+    visibility = [
+        "//packages/core/schematics/utils/tsurge/test:__pkg__",
+    ],
+    deps = [
+        "//packages/compiler-cli",
+        "//packages/compiler-cli/src/ngtsc/core",
+        "//packages/compiler-cli/src/ngtsc/core:api",
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "//packages/compiler-cli/src/ngtsc/file_system/testing",
+        "//packages/compiler-cli/src/ngtsc/shims",
+        "@npm//@types/node",
+        "@npm//magic-string",
+        "@npm//typescript",
+    ],
+)

packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts

@@ -0,0 +1,60 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as fs from 'fs';
+import * as readline from 'readline';
+import {TsurgeMigration} from '../migration';
+
+/**
+ * Integrating a `Tsurge` migration requires the "merging" of all
+ * compilation unit data into a single "global migration data".
+ *
+ * This is achieved in a Beam pipeline by having a pipeline stage that
+ * takes all compilation unit worker data and writing it into a single
+ * buffer, delimited by new lines (`\n`).
+ *
+ * This "merged bytes files", containing all unit data, one per line, can
+ * then be parsed by this function and fed into the migration merge logic.
+ *
+ * @returns All compilation unit data for the migration.
+ */
+export function readCompilationUnitBlob<UnitData, GlobalData>(
+  _migrationForTypeSafety: TsurgeMigration<UnitData, GlobalData>,
+  mergedUnitDataByteAbsFilePath: string,
+): Promise<UnitData[]> {
+  return new Promise((resolve, reject) => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream(mergedUnitDataByteAbsFilePath, 'utf8'),
+      crlfDelay: Infinity,
+    });
+
+    const unitData: UnitData[] = [];
+    let failed = false;
+    rl.on('line', (line) => {
+      const trimmedLine = line.trim();
+      if (trimmedLine === '') {
+        return;
+      }
+
+      try {
+        const parsed = JSON.parse(trimmedLine) as UnitData;
+        unitData.push(parsed);
+      } catch (e) {
+        failed = true;
+        reject(new Error(`Could not parse data line: ${e} — ${trimmedLine}`));
+        rl.close();
+      }
+    });
+
+    rl.on('close', async () => {
+      if (!failed) {
+        resolve(unitData);
+      }
+    });
+  });
+}

packages/core/schematics/utils/tsurge/executors/analyze_exec.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {TsurgeMigration} from '../migration';
+import {Serializable} from '../helpers/serializable';
+
+/**
+ * Executes the analyze phase of the given migration against
+ * the specified TypeScript project.
+ *
+ * @returns the serializable migration unit data.
+ */
+export async function executeAnalyzePhase<UnitData, GlobalData>(
+  migration: TsurgeMigration<UnitData, GlobalData>,
+  tsconfigAbsolutePath: string,
+): Promise<Serializable<UnitData>> {
+  const baseInfo = migration.createProgram(tsconfigAbsolutePath);
+  const info = migration.prepareProgram(baseInfo);
+
+  return await migration.analyze(info);
+}

packages/core/schematics/utils/tsurge/executors/merge_exec.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Serializable} from '../helpers/serializable';
+import {TsurgeMigration} from '../migration';
+
+/**
+ * Executes the merge phase for the given migration against
+ * the given set of analysis unit data.
+ *
+ * @returns the serializable migration global data.
+ */
+export async function executeMergePhase<UnitData, GlobalData>(
+  migration: TsurgeMigration<UnitData, GlobalData>,
+  units: UnitData[],
+): Promise<Serializable<GlobalData>> {
+  return await migration.merge(units);
+}

packages/core/schematics/utils/tsurge/executors/migrate_exec.ts

@@ -0,0 +1,30 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {TsurgeMigration} from '../migration';
+import {Replacement} from '../replacement';
+
+/**
+ * Executes the migrate phase of the given migration against
+ * the specified TypeScript project.
+ *
+ * This requires the global migration data, computed by the
+ * analysis and merge phases of the migration.
+ *
+ * @returns a list of text replacements to apply to disk.
+ */
+export async function executeMigratePhase<UnitData, GlobalData>(
+  migration: TsurgeMigration<UnitData, GlobalData>,
+  globalMetadata: GlobalData,
+  tsconfigAbsolutePath: string,
+): Promise<Replacement[]> {
+  const baseInfo = migration.createProgram(tsconfigAbsolutePath);
+  const info = migration.prepareProgram(baseInfo);
+
+  return await migration.migrate(globalMetadata, info);
+}

packages/core/schematics/utils/tsurge/helpers/group_replacements.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {AbsoluteFsPath} from '../../../../../compiler-cli/src/ngtsc/file_system';
+import {Replacement, TextUpdate} from '../replacement';
+
+/**
+ * Groups the given replacements per file path.
+ *
+ * This allows for simple execution of the replacements
+ * against a given file. E.g. via {@link applyTextUpdates}.
+ */
+export function groupReplacementsByFile(
+  replacements: Replacement[],
+): Map<AbsoluteFsPath, TextUpdate[]> {
+  const result = new Map<AbsoluteFsPath, TextUpdate[]>();
+  for (const {absoluteFilePath, update} of replacements) {
+    if (!result.has(absoluteFilePath)) {
+      result.set(absoluteFilePath, []);
+    }
+    result.get(absoluteFilePath)!.push(update);
+  }
+  return result;
+}

packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts

@@ -0,0 +1,64 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {readConfiguration} from '../../../../../compiler-cli/src/perform_compile';
+import {NgCompilerOptions} from '../../../../../compiler-cli/src/ngtsc/core/api';
+import {
+  FileSystem,
+  NgtscCompilerHost,
+  NodeJSFileSystem,
+  setFileSystem,
+} from '../../../../../compiler-cli/src/ngtsc/file_system';
+import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program';
+import {BaseProgramInfo} from '../program_info';
+
+/**
+ * Parses the configuration of the given TypeScript project and creates
+ * an instance of the Angular compiler for for the project.
+ */
+export function createNgtscProgram(
+  absoluteTsconfigPath: string,
+  fs?: FileSystem,
+  optionOverrides: NgCompilerOptions = {},
+): BaseProgramInfo<NgtscProgram> {
+  if (fs === undefined) {
+    fs = new NodeJSFileSystem();
+    setFileSystem(fs);
+  }
+
+  const tsconfig = readConfiguration(absoluteTsconfigPath, {}, fs);
+
+  if (tsconfig.errors.length > 0) {
+    throw new Error(
+      `Tsconfig could not be parsed or is invalid:\n\n` +
+        `${tsconfig.errors.map((e) => e.messageText)}`,
+    );
+  }
+
+  const tsHost = new NgtscCompilerHost(fs, tsconfig.options);
+  const ngtscProgram = new NgtscProgram(
+    tsconfig.rootNames,
+    {
+      ...tsconfig.options,
+      // Migrations commonly make use of TCB information.
+      _enableTemplateTypeChecker: true,
+      // Avoid checking libraries to speed up migrations.
+      skipLibCheck: true,
+      skipDefaultLibCheck: true,
+      // Additional override options.
+      ...optionOverrides,
+    },
+    tsHost,
+  );
+
+  return {
+    program: ngtscProgram,
+    userOptions: tsconfig.options,
+    tsconfigAbsolutePath: absoluteTsconfigPath,
+  };
+}

packages/core/schematics/utils/tsurge/helpers/serializable.ts

@@ -0,0 +1,15 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/** Branded type indicating that the given data `T` is serializable. */
+export type Serializable<T> = T & {__serializable: true};
+
+/** Confirms that the given data `T` is serializable. */
+export function confirmAsSerializable<T>(data: T): Serializable<T> {
+  return data as Serializable<T>;
+}

packages/core/schematics/utils/tsurge/helpers/unique_id.ts

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/**
+ * Helper type for creating unique branded IDs.
+ *
+ * Unique IDs are a fundamental piece for a `Tsurge` migration because
+ * they allow for serializable analysis data between the stages.
+ *
+ * This is important to e.g. uniquely identify an Angular input across
+ * compilation units, so that shared global data can be built via
+ * the `merge` phase.
+ *
+ * E.g. a unique ID for an input may be the project-relative file path,
+ * in combination with the name of its owning class, plus the field name.
+ */
+export type UniqueID<Name> = string & {__branded: Name};

packages/core/schematics/utils/tsurge/migration.ts

@@ -0,0 +1,98 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {FileSystem} from '../../../../compiler-cli/src/ngtsc/file_system';
+import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program';
+import assert from 'assert';
+import path from 'path';
+import ts from 'typescript';
+import {isShim} from '../../../../compiler-cli/src/ngtsc/shims';
+import {createNgtscProgram} from './helpers/ngtsc_program';
+import {Serializable} from './helpers/serializable';
+import {Replacement} from './replacement';
+import {BaseProgramInfo, ProgramInfo} from './program_info';
+
+/**
+ * Class defining a `Tsurge` migration.
+ *
+ * A tsurge migration is split into three stages:
+ *    - analyze phase
+ *    - merge phase
+ *    - migrate phase
+ *
+ * The motivation for such split is that migrations may be executed
+ * on individual workers, e.g. via go/tsunami or a Beam pipeline. The
+ * individual workers are never seeing the full project, e.g. Google3.
+ *
+ * The analysis phases can operate on smaller TS project units, and later
+ * the expect the isolated unit data to be merged into some sort of global
+ * metadata via the `merge` phase. For example, every analyze worker may
+ * contribute to a list of TS references that are later combined.
+ *
+ * The migrate phase can then compute actual file updates for all individual
+ * compilation units, leveraging the global metadata to e.g. see if there are
+ * any references from other compilation units that may be problematic and prevent
+ * migration of a given file.
+ *
+ * More details can be found in the design doc for signal input migration,
+ * or in the testing examples.
+ *
+ * TODO: Link design doc.
+ */
+export abstract class TsurgeMigration<
+  UnitAnalysisMetadata,
+  CombinedGlobalMetadata,
+  TsProgramType extends ts.Program | NgtscProgram = NgtscProgram,
+  FullProgramInfo extends ProgramInfo<TsProgramType> = ProgramInfo<TsProgramType>,
+> {
+  // By default, ngtsc programs are being created.
+  createProgram(tsconfigAbsPath: string, fs?: FileSystem): BaseProgramInfo<TsProgramType> {
+    return createNgtscProgram(tsconfigAbsPath, fs) as BaseProgramInfo<TsProgramType>;
+  }
+
+  // Optional function to prepare the base `ProgramInfo` even further,
+  // for the analyze and migrate phases. E.g. determining source files.
+  prepareProgram(info: BaseProgramInfo<TsProgramType>): FullProgramInfo {
+    assert(info.program instanceof NgtscProgram);
+
+    const userProgram = info.program.getTsProgram();
+    const fullProgramSourceFiles = userProgram.getSourceFiles();
+    const sourceFiles = fullProgramSourceFiles.filter(
+      (f) =>
+        !f.isDeclarationFile &&
+        // Note `isShim` will work for the initial program, but for TCB programs, the shims are no longer annotated.
+        !isShim(f) &&
+        !f.fileName.endsWith('.ngtypecheck.ts'),
+    );
+
+    const basePath = path.dirname(info.tsconfigAbsolutePath);
+    const projectDirAbsPath = info.userOptions.rootDir ?? basePath;
+
+    return {
+      ...info,
+      sourceFiles,
+      fullProgramSourceFiles,
+      projectDirAbsPath,
+    } as FullProgramInfo;
+  }
+
+  /** Analyzes the given TypeScript project and returns serializable compilation unit data. */
+  abstract analyze(program: FullProgramInfo): Promise<Serializable<UnitAnalysisMetadata>>;
+
+  /** Merges all compilation unit data from previous analysis phases into a global metadata. */
+  abstract merge(units: UnitAnalysisMetadata[]): Promise<Serializable<CombinedGlobalMetadata>>;
+
+  /**
+   * Computes migration updates for the given TypeScript project, leveraging the global
+   * metadata built up from all analyzed projects and their merged "unit data".
+   */
+  abstract migrate(
+    globalMetadata: CombinedGlobalMetadata,
+    program: FullProgramInfo,
+  ): Promise<Replacement[]>;
+}