From 368f36dab3abda69a06c603d833a600d3e501cbe Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 14 Aug 2024 15:48:47 +0000 Subject: [PATCH] refactor(migrations): framework to build batchable migrations (#57396) Introduces a migration framework to build batchable migrations that can run in Large Scale mode against e.g. all of Google, using workers. This is the original signal input migration infrastructure extracted into a more generic framework that we can use for writing additional ones for output, signal queries etc, while making sure those are not scoped to a single `ts.Program` that limits them to per-directory execution in very large projects (e.g. G3). The migration will be updated to use this, and in 1P we will add helpers to easily integrate such migrations into a Go-based pipeline runner. PR Close #57396 --- .../core/schematics/utils/tsurge/BUILD.bazel | 20 ++++ .../tsurge/beam_pipeline/read_units_blob.ts | 60 ++++++++++ .../utils/tsurge/executors/analyze_exec.ts | 26 ++++ .../utils/tsurge/executors/merge_exec.ts | 23 ++++ .../utils/tsurge/executors/migrate_exec.ts | 30 +++++ .../tsurge/helpers/group_replacements.ts | 29 +++++ .../utils/tsurge/helpers/ngtsc_program.ts | 64 ++++++++++ .../utils/tsurge/helpers/serializable.ts | 15 +++ .../utils/tsurge/helpers/unique_id.ts | 22 ++++ .../core/schematics/utils/tsurge/migration.ts | 98 ++++++++++++++++ .../schematics/utils/tsurge/program_info.ts | 36 ++++++ .../schematics/utils/tsurge/replacement.ts | 40 +++++++ .../schematics/utils/tsurge/test/BUILD.bazel | 39 ++++++ .../utils/tsurge/test/output_helpers.ts | 103 ++++++++++++++++ .../tsurge/test/output_migration.spec.ts | 70 +++++++++++ .../utils/tsurge/test/output_migration.ts | 111 ++++++++++++++++++ .../schematics/utils/tsurge/testing/index.ts | 72 ++++++++++++ 17 files changed, 858 insertions(+) create mode 100644 packages/core/schematics/utils/tsurge/BUILD.bazel create mode 100644 packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts create mode 100644 packages/core/schematics/utils/tsurge/executors/analyze_exec.ts create mode 100644 packages/core/schematics/utils/tsurge/executors/merge_exec.ts create mode 100644 packages/core/schematics/utils/tsurge/executors/migrate_exec.ts create mode 100644 packages/core/schematics/utils/tsurge/helpers/group_replacements.ts create mode 100644 packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts create mode 100644 packages/core/schematics/utils/tsurge/helpers/serializable.ts create mode 100644 packages/core/schematics/utils/tsurge/helpers/unique_id.ts create mode 100644 packages/core/schematics/utils/tsurge/migration.ts create mode 100644 packages/core/schematics/utils/tsurge/program_info.ts create mode 100644 packages/core/schematics/utils/tsurge/replacement.ts create mode 100644 packages/core/schematics/utils/tsurge/test/BUILD.bazel create mode 100644 packages/core/schematics/utils/tsurge/test/output_helpers.ts create mode 100644 packages/core/schematics/utils/tsurge/test/output_migration.spec.ts create mode 100644 packages/core/schematics/utils/tsurge/test/output_migration.ts create mode 100644 packages/core/schematics/utils/tsurge/testing/index.ts diff --git a/packages/core/schematics/utils/tsurge/BUILD.bazel b/packages/core/schematics/utils/tsurge/BUILD.bazel new file mode 100644 index 0000000000000..53d46a4ccf500 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/BUILD.bazel @@ -0,0 +1,20 @@ +load("//tools:defaults.bzl", "ts_library") + +ts_library( + name = "tsurge", + srcs = glob(["**/*.ts"]), + visibility = [ + "//packages/core/schematics/utils/tsurge/test:__pkg__", + ], + deps = [ + "//packages/compiler-cli", + "//packages/compiler-cli/src/ngtsc/core", + "//packages/compiler-cli/src/ngtsc/core:api", + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/compiler-cli/src/ngtsc/shims", + "@npm//@types/node", + "@npm//magic-string", + "@npm//typescript", + ], +) diff --git a/packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts b/packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts new file mode 100644 index 0000000000000..06ae1b4a76e41 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import * as fs from 'fs'; +import * as readline from 'readline'; +import {TsurgeMigration} from '../migration'; + +/** + * Integrating a `Tsurge` migration requires the "merging" of all + * compilation unit data into a single "global migration data". + * + * This is achieved in a Beam pipeline by having a pipeline stage that + * takes all compilation unit worker data and writing it into a single + * buffer, delimited by new lines (`\n`). + * + * This "merged bytes files", containing all unit data, one per line, can + * then be parsed by this function and fed into the migration merge logic. + * + * @returns All compilation unit data for the migration. + */ +export function readCompilationUnitBlob( + _migrationForTypeSafety: TsurgeMigration, + mergedUnitDataByteAbsFilePath: string, +): Promise { + return new Promise((resolve, reject) => { + const rl = readline.createInterface({ + input: fs.createReadStream(mergedUnitDataByteAbsFilePath, 'utf8'), + crlfDelay: Infinity, + }); + + const unitData: UnitData[] = []; + let failed = false; + rl.on('line', (line) => { + const trimmedLine = line.trim(); + if (trimmedLine === '') { + return; + } + + try { + const parsed = JSON.parse(trimmedLine) as UnitData; + unitData.push(parsed); + } catch (e) { + failed = true; + reject(new Error(`Could not parse data line: ${e} — ${trimmedLine}`)); + rl.close(); + } + }); + + rl.on('close', async () => { + if (!failed) { + resolve(unitData); + } + }); + }); +} diff --git a/packages/core/schematics/utils/tsurge/executors/analyze_exec.ts b/packages/core/schematics/utils/tsurge/executors/analyze_exec.ts new file mode 100644 index 0000000000000..59714aff2327c --- /dev/null +++ b/packages/core/schematics/utils/tsurge/executors/analyze_exec.ts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {TsurgeMigration} from '../migration'; +import {Serializable} from '../helpers/serializable'; + +/** + * Executes the analyze phase of the given migration against + * the specified TypeScript project. + * + * @returns the serializable migration unit data. + */ +export async function executeAnalyzePhase( + migration: TsurgeMigration, + tsconfigAbsolutePath: string, +): Promise> { + const baseInfo = migration.createProgram(tsconfigAbsolutePath); + const info = migration.prepareProgram(baseInfo); + + return await migration.analyze(info); +} diff --git a/packages/core/schematics/utils/tsurge/executors/merge_exec.ts b/packages/core/schematics/utils/tsurge/executors/merge_exec.ts new file mode 100644 index 0000000000000..7dbe5e037d070 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/executors/merge_exec.ts @@ -0,0 +1,23 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Serializable} from '../helpers/serializable'; +import {TsurgeMigration} from '../migration'; + +/** + * Executes the merge phase for the given migration against + * the given set of analysis unit data. + * + * @returns the serializable migration global data. + */ +export async function executeMergePhase( + migration: TsurgeMigration, + units: UnitData[], +): Promise> { + return await migration.merge(units); +} diff --git a/packages/core/schematics/utils/tsurge/executors/migrate_exec.ts b/packages/core/schematics/utils/tsurge/executors/migrate_exec.ts new file mode 100644 index 0000000000000..48c9f44133072 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/executors/migrate_exec.ts @@ -0,0 +1,30 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {TsurgeMigration} from '../migration'; +import {Replacement} from '../replacement'; + +/** + * Executes the migrate phase of the given migration against + * the specified TypeScript project. + * + * This requires the global migration data, computed by the + * analysis and merge phases of the migration. + * + * @returns a list of text replacements to apply to disk. + */ +export async function executeMigratePhase( + migration: TsurgeMigration, + globalMetadata: GlobalData, + tsconfigAbsolutePath: string, +): Promise { + const baseInfo = migration.createProgram(tsconfigAbsolutePath); + const info = migration.prepareProgram(baseInfo); + + return await migration.migrate(globalMetadata, info); +} diff --git a/packages/core/schematics/utils/tsurge/helpers/group_replacements.ts b/packages/core/schematics/utils/tsurge/helpers/group_replacements.ts new file mode 100644 index 0000000000000..005b568cdbae5 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/helpers/group_replacements.ts @@ -0,0 +1,29 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {AbsoluteFsPath} from '../../../../../compiler-cli/src/ngtsc/file_system'; +import {Replacement, TextUpdate} from '../replacement'; + +/** + * Groups the given replacements per file path. + * + * This allows for simple execution of the replacements + * against a given file. E.g. via {@link applyTextUpdates}. + */ +export function groupReplacementsByFile( + replacements: Replacement[], +): Map { + const result = new Map(); + for (const {absoluteFilePath, update} of replacements) { + if (!result.has(absoluteFilePath)) { + result.set(absoluteFilePath, []); + } + result.get(absoluteFilePath)!.push(update); + } + return result; +} diff --git a/packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts b/packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts new file mode 100644 index 0000000000000..b6b02fae6337e --- /dev/null +++ b/packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts @@ -0,0 +1,64 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {readConfiguration} from '../../../../../compiler-cli/src/perform_compile'; +import {NgCompilerOptions} from '../../../../../compiler-cli/src/ngtsc/core/api'; +import { + FileSystem, + NgtscCompilerHost, + NodeJSFileSystem, + setFileSystem, +} from '../../../../../compiler-cli/src/ngtsc/file_system'; +import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program'; +import {BaseProgramInfo} from '../program_info'; + +/** + * Parses the configuration of the given TypeScript project and creates + * an instance of the Angular compiler for for the project. + */ +export function createNgtscProgram( + absoluteTsconfigPath: string, + fs?: FileSystem, + optionOverrides: NgCompilerOptions = {}, +): BaseProgramInfo { + if (fs === undefined) { + fs = new NodeJSFileSystem(); + setFileSystem(fs); + } + + const tsconfig = readConfiguration(absoluteTsconfigPath, {}, fs); + + if (tsconfig.errors.length > 0) { + throw new Error( + `Tsconfig could not be parsed or is invalid:\n\n` + + `${tsconfig.errors.map((e) => e.messageText)}`, + ); + } + + const tsHost = new NgtscCompilerHost(fs, tsconfig.options); + const ngtscProgram = new NgtscProgram( + tsconfig.rootNames, + { + ...tsconfig.options, + // Migrations commonly make use of TCB information. + _enableTemplateTypeChecker: true, + // Avoid checking libraries to speed up migrations. + skipLibCheck: true, + skipDefaultLibCheck: true, + // Additional override options. + ...optionOverrides, + }, + tsHost, + ); + + return { + program: ngtscProgram, + userOptions: tsconfig.options, + tsconfigAbsolutePath: absoluteTsconfigPath, + }; +} diff --git a/packages/core/schematics/utils/tsurge/helpers/serializable.ts b/packages/core/schematics/utils/tsurge/helpers/serializable.ts new file mode 100644 index 0000000000000..9e0d94e0f0f80 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/helpers/serializable.ts @@ -0,0 +1,15 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** Branded type indicating that the given data `T` is serializable. */ +export type Serializable = T & {__serializable: true}; + +/** Confirms that the given data `T` is serializable. */ +export function confirmAsSerializable(data: T): Serializable { + return data as Serializable; +} diff --git a/packages/core/schematics/utils/tsurge/helpers/unique_id.ts b/packages/core/schematics/utils/tsurge/helpers/unique_id.ts new file mode 100644 index 0000000000000..b64a295eed2c6 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/helpers/unique_id.ts @@ -0,0 +1,22 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** + * Helper type for creating unique branded IDs. + * + * Unique IDs are a fundamental piece for a `Tsurge` migration because + * they allow for serializable analysis data between the stages. + * + * This is important to e.g. uniquely identify an Angular input across + * compilation units, so that shared global data can be built via + * the `merge` phase. + * + * E.g. a unique ID for an input may be the project-relative file path, + * in combination with the name of its owning class, plus the field name. + */ +export type UniqueID = string & {__branded: Name}; diff --git a/packages/core/schematics/utils/tsurge/migration.ts b/packages/core/schematics/utils/tsurge/migration.ts new file mode 100644 index 0000000000000..5ef2cdc546550 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/migration.ts @@ -0,0 +1,98 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {FileSystem} from '../../../../compiler-cli/src/ngtsc/file_system'; +import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program'; +import assert from 'assert'; +import path from 'path'; +import ts from 'typescript'; +import {isShim} from '../../../../compiler-cli/src/ngtsc/shims'; +import {createNgtscProgram} from './helpers/ngtsc_program'; +import {Serializable} from './helpers/serializable'; +import {Replacement} from './replacement'; +import {BaseProgramInfo, ProgramInfo} from './program_info'; + +/** + * Class defining a `Tsurge` migration. + * + * A tsurge migration is split into three stages: + * - analyze phase + * - merge phase + * - migrate phase + * + * The motivation for such split is that migrations may be executed + * on individual workers, e.g. via go/tsunami or a Beam pipeline. The + * individual workers are never seeing the full project, e.g. Google3. + * + * The analysis phases can operate on smaller TS project units, and later + * the expect the isolated unit data to be merged into some sort of global + * metadata via the `merge` phase. For example, every analyze worker may + * contribute to a list of TS references that are later combined. + * + * The migrate phase can then compute actual file updates for all individual + * compilation units, leveraging the global metadata to e.g. see if there are + * any references from other compilation units that may be problematic and prevent + * migration of a given file. + * + * More details can be found in the design doc for signal input migration, + * or in the testing examples. + * + * TODO: Link design doc. + */ +export abstract class TsurgeMigration< + UnitAnalysisMetadata, + CombinedGlobalMetadata, + TsProgramType extends ts.Program | NgtscProgram = NgtscProgram, + FullProgramInfo extends ProgramInfo = ProgramInfo, +> { + // By default, ngtsc programs are being created. + createProgram(tsconfigAbsPath: string, fs?: FileSystem): BaseProgramInfo { + return createNgtscProgram(tsconfigAbsPath, fs) as BaseProgramInfo; + } + + // Optional function to prepare the base `ProgramInfo` even further, + // for the analyze and migrate phases. E.g. determining source files. + prepareProgram(info: BaseProgramInfo): FullProgramInfo { + assert(info.program instanceof NgtscProgram); + + const userProgram = info.program.getTsProgram(); + const fullProgramSourceFiles = userProgram.getSourceFiles(); + const sourceFiles = fullProgramSourceFiles.filter( + (f) => + !f.isDeclarationFile && + // Note `isShim` will work for the initial program, but for TCB programs, the shims are no longer annotated. + !isShim(f) && + !f.fileName.endsWith('.ngtypecheck.ts'), + ); + + const basePath = path.dirname(info.tsconfigAbsolutePath); + const projectDirAbsPath = info.userOptions.rootDir ?? basePath; + + return { + ...info, + sourceFiles, + fullProgramSourceFiles, + projectDirAbsPath, + } as FullProgramInfo; + } + + /** Analyzes the given TypeScript project and returns serializable compilation unit data. */ + abstract analyze(program: FullProgramInfo): Promise>; + + /** Merges all compilation unit data from previous analysis phases into a global metadata. */ + abstract merge(units: UnitAnalysisMetadata[]): Promise>; + + /** + * Computes migration updates for the given TypeScript project, leveraging the global + * metadata built up from all analyzed projects and their merged "unit data". + */ + abstract migrate( + globalMetadata: CombinedGlobalMetadata, + program: FullProgramInfo, + ): Promise; +} diff --git a/packages/core/schematics/utils/tsurge/program_info.ts b/packages/core/schematics/utils/tsurge/program_info.ts new file mode 100644 index 0000000000000..2a16e4817c967 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/program_info.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program'; +import {NgCompilerOptions} from '../../../../compiler-cli/src/ngtsc/core/api'; + +import ts from 'typescript'; + +/** + * Base information for a TypeScript project, including an instantiated + * TypeScript program. Base information may be extended by user-overridden + * migration preparation methods to extend the stages with more data. + */ +export interface BaseProgramInfo { + program: T; + userOptions: NgCompilerOptions; + tsconfigAbsolutePath: string; +} + +/** + * Full program information for a TypeScript project. This is the default "extension" + * of the {@link BaseProgramInfo} with additional commonly accessed information. + * + * A different interface may be used as full program information, configured via a + * {@link TsurgeMigration.prepareProgram} override. + */ +export interface ProgramInfo extends BaseProgramInfo { + sourceFiles: ts.SourceFile[]; + fullProgramSourceFiles: ts.SourceFile[]; + projectDirAbsPath: string; +} diff --git a/packages/core/schematics/utils/tsurge/replacement.ts b/packages/core/schematics/utils/tsurge/replacement.ts new file mode 100644 index 0000000000000..8f1235d5ec6ec --- /dev/null +++ b/packages/core/schematics/utils/tsurge/replacement.ts @@ -0,0 +1,40 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {AbsoluteFsPath} from '../../../../compiler-cli/src/ngtsc/file_system'; + +import MagicString from 'magic-string'; + +/** A text replacement for the given file. */ +export class Replacement { + constructor( + public absoluteFilePath: AbsoluteFsPath, + public update: TextUpdate, + ) {} +} + +/** An isolated text update that may be applied to a file. */ +export class TextUpdate { + constructor( + public data: { + position: number; + end: number; + toInsert: string; + }, + ) {} +} + +/** Helper that applies updates to the given text. */ +export function applyTextUpdates(input: string, updates: TextUpdate[]): string { + const res = new MagicString(input); + for (const update of updates) { + res.remove(update.data.position, update.data.end); + res.appendLeft(update.data.position, update.data.toInsert); + } + return res.toString(); +} diff --git a/packages/core/schematics/utils/tsurge/test/BUILD.bazel b/packages/core/schematics/utils/tsurge/test/BUILD.bazel new file mode 100644 index 0000000000000..c6dd7b23f7c10 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/test/BUILD.bazel @@ -0,0 +1,39 @@ +load("//tools:defaults.bzl", "jasmine_node_test", "ts_library") + +ts_library( + name = "migration_lib", + srcs = glob( + ["**/*.ts"], + exclude = ["*.spec.ts"], + ), + deps = [ + "//packages/compiler-cli", + "//packages/compiler-cli/src/ngtsc/annotations", + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/imports", + "//packages/compiler-cli/src/ngtsc/metadata", + "//packages/compiler-cli/src/ngtsc/reflection", + "//packages/core/schematics/utils/tsurge", + "@npm//@types/node", + "@npm//typescript", + ], +) + +ts_library( + name = "test_lib", + testonly = True, + srcs = glob( + ["**/*.spec.ts"], + ), + deps = [ + ":migration_lib", + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/core/schematics/utils/tsurge", + ], +) + +jasmine_node_test( + name = "test", + deps = [":test_lib"], +) diff --git a/packages/core/schematics/utils/tsurge/test/output_helpers.ts b/packages/core/schematics/utils/tsurge/test/output_helpers.ts new file mode 100644 index 0000000000000..f0e90afdf8f77 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/test/output_helpers.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import path from 'path'; +import {UniqueID} from '../helpers/unique_id'; +import ts from 'typescript'; +import {ProgramInfo} from '../program_info'; +import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program'; +import {DtsMetadataReader} from '../../../../../compiler-cli/src/ngtsc/metadata'; +import {ClassDeclaration, ReflectionHost} from '../../../../../compiler-cli/src/ngtsc/reflection'; +import {Reference} from '../../../../../compiler-cli/src/ngtsc/imports'; +import {getAngularDecorators} from '../../../../../compiler-cli/src/ngtsc/annotations'; + +export type OutputID = UniqueID<'output-node'>; + +export function getIdOfOutput(projectDirAbsPath: string, prop: ts.PropertyDeclaration): OutputID { + const fileId = path.relative(projectDirAbsPath, prop.getSourceFile().fileName); + return `${fileId}@@${prop.parent.name ?? 'unknown-class'}@@${prop.name.getText()}` as OutputID; +} + +export function findOutputDeclarationsAndReferences( + {sourceFiles, projectDirAbsPath}: ProgramInfo, + checker: ts.TypeChecker, + reflector: ReflectionHost, + dtsReader: DtsMetadataReader, +) { + const sourceOutputs = new Map(); + const problematicReferencedOutputs = new Set(); + + for (const sf of sourceFiles) { + const visitor = (node: ts.Node) => { + // Detect output declarations. + if ( + ts.isPropertyDeclaration(node) && + node.initializer !== undefined && + ts.isNewExpression(node.initializer) && + ts.isIdentifier(node.initializer.expression) && + node.initializer.expression.text === 'EventEmitter' + ) { + sourceOutputs.set(getIdOfOutput(projectDirAbsPath, node), node); + } + + // Detect problematic output references. + if ( + ts.isPropertyAccessExpression(node) && + ts.isIdentifier(node.name) && + node.name.text === 'pipe' + ) { + const targetSymbol = checker.getSymbolAtLocation(node.expression); + if ( + targetSymbol !== undefined && + targetSymbol.valueDeclaration !== undefined && + ts.isPropertyDeclaration(targetSymbol.valueDeclaration) && + isOutputDeclaration(targetSymbol.valueDeclaration, reflector, dtsReader) + ) { + // Mark output to indicate a seen problematic usage. + problematicReferencedOutputs.add( + getIdOfOutput(projectDirAbsPath, targetSymbol.valueDeclaration), + ); + } + } + + ts.forEachChild(node, visitor); + }; + + ts.forEachChild(sf, visitor); + } + + return {sourceOutputs, problematicReferencedOutputs}; +} + +function isOutputDeclaration( + node: ts.PropertyDeclaration, + reflector: ReflectionHost, + dtsReader: DtsMetadataReader, +): boolean { + // `.d.ts` file, so we check the `static ecmp` metadata on the `declare class`. + if (node.getSourceFile().isDeclarationFile) { + if ( + !ts.isIdentifier(node.name) || + !ts.isClassDeclaration(node.parent) || + node.parent.name === undefined + ) { + return false; + } + + const ref = new Reference(node.parent as ClassDeclaration); + const directiveMeta = dtsReader.getDirectiveMetadata(ref); + return !!directiveMeta?.outputs.getByClassPropertyName(node.name.text); + } + + // `.ts` file, so we check for the `@Output()` decorator. + const decorators = reflector.getDecoratorsOfDeclaration(node); + const ngDecorators = + decorators !== null ? getAngularDecorators(decorators, ['Output'], /* isCore */ false) : []; + + return ngDecorators.length > 0; +} diff --git a/packages/core/schematics/utils/tsurge/test/output_migration.spec.ts b/packages/core/schematics/utils/tsurge/test/output_migration.spec.ts new file mode 100644 index 0000000000000..e0bd2c0ee576e --- /dev/null +++ b/packages/core/schematics/utils/tsurge/test/output_migration.spec.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {absoluteFrom} from '../../../../../compiler-cli/src/ngtsc/file_system'; +import {initMockFileSystem} from '../../../../../compiler-cli/src/ngtsc/file_system/testing'; +import {runTsurgeMigration} from '../testing'; +import {OutputMigration} from './output_migration'; + +describe('output migration', () => { + beforeEach(() => { + initMockFileSystem('Native'); + }); + + it('should work', async () => { + const migration = new OutputMigration(); + const fs = await runTsurgeMigration(migration, [ + { + name: absoluteFrom('/app.component.ts'), + isProgramRootFile: true, + contents: ` + import {Output, Component, EventEmitter} from '@angular/core'; + + @Component() + class AppComponent { + @Output() clicked = new EventEmitter(); + } + `, + }, + ]); + + expect(fs.readFile(absoluteFrom('/app.component.ts'))).toContain( + '// TODO: Actual migration logic', + ); + }); + + it('should not migrate if there is a problematic usage', async () => { + const migration = new OutputMigration(); + const fs = await runTsurgeMigration(migration, [ + { + name: absoluteFrom('/app.component.ts'), + isProgramRootFile: true, + contents: ` + import {Output, Component, EventEmitter} from '@angular/core'; + + @Component() + export class AppComponent { + @Output() clicked = new EventEmitter(); + } + `, + }, + { + name: absoluteFrom('/other.component.ts'), + isProgramRootFile: true, + contents: ` + import {AppComponent} from './app.component'; + + const cmp: AppComponent = null!; + cmp.clicked.pipe().subscribe(); + `, + }, + ]); + + expect(fs.readFile(absoluteFrom('/app.component.ts'))).not.toContain('TODO'); + }); +}); diff --git a/packages/core/schematics/utils/tsurge/test/output_migration.ts b/packages/core/schematics/utils/tsurge/test/output_migration.ts new file mode 100644 index 0000000000000..38c70ff8a0d9b --- /dev/null +++ b/packages/core/schematics/utils/tsurge/test/output_migration.ts @@ -0,0 +1,111 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program'; +import {absoluteFromSourceFile} from '../../../../../compiler-cli/src/ngtsc/file_system'; +import {TypeScriptReflectionHost} from '../../../../../compiler-cli/src/ngtsc/reflection'; +import {DtsMetadataReader} from '../../../../../compiler-cli/src/ngtsc/metadata'; +import {confirmAsSerializable} from '../helpers/serializable'; +import {TsurgeMigration} from '../migration'; +import {Replacement, TextUpdate} from '../replacement'; +import {findOutputDeclarationsAndReferences, OutputID} from './output_helpers'; +import {ProgramInfo} from '../program_info'; + +type AnalysisUnit = {[id: OutputID]: {seenProblematicUsage: boolean}}; +type GlobalMetadata = {[id: OutputID]: {canBeMigrated: boolean}}; + +/** + * A `Tsurge` migration that can migrate instances of `@Output()` to + * the new `output()` API. + * + * Note that this is simply a testing construct for now, to verify the migration + * framework works as expected. This is **not a full migration**, but rather an example. + */ +export class OutputMigration extends TsurgeMigration { + override async analyze(info: ProgramInfo) { + const program = info.program.getTsProgram(); + const typeChecker = program.getTypeChecker(); + const reflector = new TypeScriptReflectionHost(typeChecker, false); + const dtsReader = new DtsMetadataReader(typeChecker, reflector); + + const {sourceOutputs, problematicReferencedOutputs} = findOutputDeclarationsAndReferences( + info, + typeChecker, + reflector, + dtsReader, + ); + + const discoveredOutputs: AnalysisUnit = {}; + for (const id of sourceOutputs.keys()) { + discoveredOutputs[id] = {seenProblematicUsage: false}; + } + for (const id of problematicReferencedOutputs) { + discoveredOutputs[id] = {seenProblematicUsage: true}; + } + + // Here we confirm it as serializable.. + return confirmAsSerializable(discoveredOutputs); + } + + override async merge(data: AnalysisUnit[]) { + const merged: GlobalMetadata = {}; + + // Merge information from all compilation units. Mark + // outputs that cannot be migrated due to seen problematic usages. + for (const unit of data) { + for (const [idStr, info] of Object.entries(unit)) { + const id = idStr as OutputID; + const existing = merged[id]; + + if (existing === undefined) { + merged[id] = {canBeMigrated: info.seenProblematicUsage === false}; + } else if (existing.canBeMigrated && info.seenProblematicUsage) { + merged[id].canBeMigrated = false; + } + } + } + + // merge units into global metadata. + return confirmAsSerializable(merged); + } + + override async migrate(globalAnalysisData: GlobalMetadata, info: ProgramInfo) { + const program = info.program.getTsProgram(); + const typeChecker = program.getTypeChecker(); + const reflector = new TypeScriptReflectionHost(typeChecker, false); + const dtsReader = new DtsMetadataReader(typeChecker, reflector); + + const {sourceOutputs} = findOutputDeclarationsAndReferences( + info, + typeChecker, + reflector, + dtsReader, + ); + const replacements: Replacement[] = []; + + for (const [id, node] of sourceOutputs.entries()) { + // Output cannot be migrated as per global analysis metadata; skip. + if (globalAnalysisData[id].canBeMigrated === false) { + continue; + } + + replacements.push( + new Replacement( + absoluteFromSourceFile(node.getSourceFile()), + new TextUpdate({ + position: node.getStart(), + end: node.getStart(), + toInsert: `// TODO: Actual migration logic\n`, + }), + ), + ); + } + + return replacements; + } +} diff --git a/packages/core/schematics/utils/tsurge/testing/index.ts b/packages/core/schematics/utils/tsurge/testing/index.ts new file mode 100644 index 0000000000000..1e1c8337e096d --- /dev/null +++ b/packages/core/schematics/utils/tsurge/testing/index.ts @@ -0,0 +1,72 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {TsurgeMigration} from '../migration'; +import { + initMockFileSystem, + MockFileSystem, +} from '../../../../../compiler-cli/src/ngtsc/file_system/testing'; +import { + absoluteFrom, + AbsoluteFsPath, + getFileSystem, +} from '../../../../../compiler-cli/src/ngtsc/file_system'; +import {groupReplacementsByFile} from '../helpers/group_replacements'; +import {applyTextUpdates} from '../replacement'; + +/** + * Runs the given migration against a fake set of files, emulating + * migration of a real TypeScript Angular project. + * + * Note: This helper does not execute the migration in batch mode, where + * e.g. the migration runs per single file and merges the unit data. + * + * TODO: Add helper/solution to test batch execution, like with Tsunami. + * + * @returns a mock file system with the applied replacements of the migration. + */ +export async function runTsurgeMigration( + migration: TsurgeMigration, + files: {name: AbsoluteFsPath; contents: string; isProgramRootFile?: boolean}[], +): Promise { + const mockFs = getFileSystem(); + if (!(mockFs instanceof MockFileSystem)) { + throw new Error('Expected a mock file system for `runTsurgeMigration`.'); + } + + for (const file of files) { + mockFs.ensureDir(mockFs.dirname(file.name)); + mockFs.writeFile(file.name, file.contents); + } + + const rootFiles = files.filter((f) => f.isProgramRootFile).map((f) => f.name); + + mockFs.writeFile( + absoluteFrom('/tsconfig.json'), + JSON.stringify({ + compilerOptions: { + rootDir: '/', + }, + files: rootFiles, + }), + ); + + const baseInfo = migration.createProgram('/tsconfig.json', mockFs); + const info = migration.prepareProgram(baseInfo); + + const unitData = await migration.analyze(info); + const merged = await migration.merge([unitData]); + const replacements = await migration.migrate(merged, info); + const updates = groupReplacementsByFile(replacements); + + for (const [filePath, changes] of updates.entries()) { + mockFs.writeFile(filePath, applyTextUpdates(mockFs.readFile(filePath), changes)); + } + + return mockFs; +}