From 311f4680423ce5ce3870b4b6f65329fbd80ff56c Mon Sep 17 00:00:00 2001 From: Andrii Rublov Date: Wed, 17 Apr 2024 20:50:12 +0200 Subject: [PATCH] SDK: Refactor compiler bundle producing, add docs - Add docs about MSBuild SDK - Add info about SDK tests - Use NUKE in tests docs - Rename compiler pack -> compiler bundle - Introduce CI-friendly single pack and publish compiler bundle targets --- .../workflows/perform-common-steps/action.yml | 2 +- .../workflows/run-build-and-unit-tests.yml | 12 +-- .nuke/build.schema.json | 14 ++- Cesium.Sdk/Sdk/Sdk.props | 2 +- Cesium.sln | 1 + README.md | 1 + build/Build.Sdk.cs | 99 ++++++++++--------- build/Build.Tests.cs | 2 +- build/Build.cs | 8 +- docs/msbuild-sdk.md | 64 ++++++++++++ docs/tests.md | 49 +++++++-- 11 files changed, 184 insertions(+), 70 deletions(-) create mode 100644 docs/msbuild-sdk.md diff --git a/.github/workflows/perform-common-steps/action.yml b/.github/workflows/perform-common-steps/action.yml index 3f36ba61..696bac90 100644 --- a/.github/workflows/perform-common-steps/action.yml +++ b/.github/workflows/perform-common-steps/action.yml @@ -20,4 +20,4 @@ runs: - name: 🔄 Restore Nuget Packages 🔄 shell: bash - run: dotnet restore + run: dotnet nuke RestoreAll diff --git a/.github/workflows/run-build-and-unit-tests.yml b/.github/workflows/run-build-and-unit-tests.yml index 90c39bd7..d4cc8566 100644 --- a/.github/workflows/run-build-and-unit-tests.yml +++ b/.github/workflows/run-build-and-unit-tests.yml @@ -32,16 +32,16 @@ jobs: - name: 🛠 Build Solution 🛠 shell: bash - run: dotnet build + run: dotnet nuke CompileAll - - name: 🚚 Publish Compiler Packs 🚚 + - name: 🚚 Publish Compiler Bundle 🚚 shell: bash - run: dotnet nuke PublishCompilerPacks + run: dotnet nuke PublishCompilerBundle - - name: 📦 Pack Compiler Packs 📦 + - name: 📦 Pack Compiler Bundle 📦 shell: bash - run: dotnet nuke PackCompilerPacks + run: dotnet nuke PackCompilerBundle - name: ✅ Run Unit Tests ✅ shell: bash - run: dotnet nuke TestAll --configuration Debug + run: dotnet nuke TestAll diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index e8ff42e1..ffb7d189 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -8,7 +8,7 @@ "properties": { "Configuration": { "type": "string", - "description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)", + "description": "Configuration to build - Default is 'Debug' or 'Release'", "enum": [ "Debug", "Release" @@ -83,9 +83,11 @@ "Clean", "CompileAll", "ForceClear", - "PackCompilerPacks", + "PackAllCompilerBundles", + "PackCompilerBundle", "PackSdk", - "PublishCompilerPacks", + "PublishAllCompilerBundles", + "PublishCompilerBundle", "RestoreAll", "TestAll", "TestCodeGen", @@ -114,9 +116,11 @@ "Clean", "CompileAll", "ForceClear", - "PackCompilerPacks", + "PackAllCompilerBundles", + "PackCompilerBundle", "PackSdk", - "PublishCompilerPacks", + "PublishAllCompilerBundles", + "PublishCompilerBundle", "RestoreAll", "TestAll", "TestCodeGen", diff --git a/Cesium.Sdk/Sdk/Sdk.props b/Cesium.Sdk/Sdk/Sdk.props index 0b681ff1..e8e643e6 100644 --- a/Cesium.Sdk/Sdk/Sdk.props +++ b/Cesium.Sdk/Sdk/Sdk.props @@ -9,7 +9,7 @@ false false - Cesium.Compiler.Pack.$(NETCoreSdkRuntimeIdentifier) + Cesium.Compiler.Bundle.$(NETCoreSdkRuntimeIdentifier) 0.0.1 false diff --git a/Cesium.sln b/Cesium.sln index ae5b74e1..25e1872e 100644 --- a/Cesium.sln +++ b/Cesium.sln @@ -61,6 +61,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{986C6A13-2 docs\tests.md = docs\tests.md docs\type-system.md = docs\type-system.md docs\design-notes.md = docs\design-notes.md + docs\msbuild-sdk.md = docs\msbuild-sdk.md EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cesium.Preprocessor", "Cesium.Preprocessor\Cesium.Preprocessor.csproj", "{0CDF730D-2A2A-437F-B27F-2BB04770C709}" diff --git a/README.md b/README.md index deb212e4..eda2cbc2 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Documentation - [Contributor Guide][docs.contributing] - [Cesium Tests][docs.tests] - [Cesium Type System][docs.type-system] +- [Cesium SDK][docs.msbuild-sdk] - [Architecture Sets][docs.architecture-sets] - [CLI-Related Language Extensions][docs.language-extensions] - [Built-in Functions][docs.builtins] diff --git a/build/Build.Sdk.cs b/build/Build.Sdk.cs index cdd93615..c158a46b 100644 --- a/build/Build.Sdk.cs +++ b/build/Build.Sdk.cs @@ -11,9 +11,9 @@ public partial class Build { - const string _compilerPackPackagePrefix = "Cesium.Compiler.Pack"; + const string _compilerPackPackagePrefix = "Cesium.Compiler.Bundle"; - Target PublishCompilerPacks => _ => _ + Target PublishAllCompilerBundles => _ => _ .Executes(() => { var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject(); @@ -22,39 +22,18 @@ public partial class Build Log.Information( $"Runtime identifiers defined in {Solution.Cesium_Compiler.Name}: {string.Join(", ", runtimeIds)}"); - if (!string.IsNullOrEmpty(RuntimeId)) - { - Log.Information($"Executing only {RuntimeId} because it was specified explicitly."); - PublishCompiler(RuntimeId); - return; - } - foreach (var runtimeId in runtimeIds) PublishCompiler(runtimeId); + }); - void PublishCompiler(string runtimeId) - { - if (!SkipCaches && !NeedPublishCompilerPack(compilerProject, runtimeId)) - { - Log.Information($"Skipping {runtimeId} because it was already published. Use '--skip-caches true' to re-publish."); - return; - } - - Log.Information($"Publishing for {runtimeId}, AOT {(PublishAot ? "enabled" : "disabled")}..."); - DotNetPublish(o => o - .SetConfiguration(Configuration) - .SetProject(compilerProject.ProjectFileLocation.File) - .SetRuntime(runtimeId) - .SetSelfContained(true) - .SetPublishTrimmed(PublishAot) - .SetPublishSingleFile(PublishAot) - .SetProperty("PublishAot", PublishAot) - .SetOutput(GetCompilerRuntimePublishFolder(compilerProject, runtimeId))); - } + Target PublishCompilerBundle => _ => _ + .Executes(() => + { + PublishCompiler(EffectiveRuntimeId); }); - Target PackCompilerPacks => _ => _ - .DependsOn(PublishCompilerPacks) + Target PackAllCompilerBundles => _ => _ + .DependsOn(PublishAllCompilerBundles) .Executes(() => { var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject(); @@ -63,27 +42,15 @@ void PublishCompiler(string runtimeId) Log.Information( $"Runtime identifiers defined in {Solution.Cesium_Compiler.Name}: {string.Join(", ", runtimeIds)}"); - if (!string.IsNullOrEmpty(RuntimeId)) - { - Log.Information($"Executing only {RuntimeId} because it was specified explicitly."); - PackCompiler(RuntimeId); - return; - } - foreach (var runtimeId in runtimeIds) PackCompiler(runtimeId); + }); - void PackCompiler(string runtimeId) - { - if (!SkipCaches && !NeedPackageCompilerPack(compilerProject, runtimeId)) - { - Log.Information($"Skipping {runtimeId} because it was already packed. Use '--skip-caches true' to re-pack."); - return; - } - - Log.Information($"Packing compiler for {runtimeId}..."); - EmitCompilerPack(runtimeId, compilerProject); - } + Target PackCompilerBundle => _ => _ + .DependsOn(PublishCompilerBundle) + .Executes(() => + { + PackCompiler(EffectiveRuntimeId); }); Target PackSdk => _ => _ @@ -145,6 +112,42 @@ IEnumerable GetPhysicalFiles(string publishDirectory, IEnumerable< } } + void PublishCompiler(string runtimeId) + { + var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject(); + + if (!SkipCaches && !NeedPublishCompilerPack(compilerProject, runtimeId)) + { + Log.Information($"Skipping {runtimeId} because it was already published. Use '--skip-caches true' to re-publish."); + return; + } + + Log.Information($"Publishing for {runtimeId}, AOT {(PublishAot ? "enabled" : "disabled")}..."); + DotNetPublish(o => o + .SetConfiguration(Configuration) + .SetProject(compilerProject.ProjectFileLocation.File) + .SetRuntime(runtimeId) + .SetSelfContained(true) + .SetPublishTrimmed(PublishAot) + .SetPublishSingleFile(PublishAot) + .SetProperty("PublishAot", PublishAot) + .SetOutput(GetCompilerRuntimePublishFolder(compilerProject, runtimeId))); + } + + void PackCompiler(string runtimeId) + { + var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject(); + + if (!SkipCaches && !NeedPackageCompilerPack(compilerProject, runtimeId)) + { + Log.Information($"Skipping {runtimeId} because it was already packed. Use '--skip-caches true' to re-pack."); + return; + } + + Log.Information($"Packing compiler for {runtimeId}..."); + EmitCompilerPack(runtimeId, compilerProject); + } + string GetCompilerRuntimePublishFolder(Project compilerProject, string runtimeId) => Path.Combine( compilerProject.GetProperty("ArtifactsPath").EvaluatedValue, diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs index 0a8a5747..b31bde9a 100644 --- a/build/Build.Tests.cs +++ b/build/Build.Tests.cs @@ -21,7 +21,7 @@ partial class Build .Executes(() => ExecuteTests(Solution.Cesium_Runtime_Tests)); Target TestSdk => _ => _ - .DependsOn(PackCompilerPacks) + .DependsOn(PackCompilerBundle) .DependsOn(PackSdk) .Executes(() => ExecuteTests(Solution.Cesium_Sdk_Tests)); diff --git a/build/Build.cs b/build/Build.cs index 571627a2..303ba6f8 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -12,8 +12,8 @@ public static int Main() return Execute(x => x.CompileAll); } - [Parameter("Configuration to build - Default is 'Debug' (local) or 'Release' (server)")] - readonly Configuration Configuration = IsLocalBuild ? Configuration.Debug : Configuration.Release; + [Parameter("Configuration to build - Default is 'Debug' or 'Release'")] + readonly Configuration Configuration = Configuration.Debug; [Parameter("If set to true, ignores all cached build results. Default: false")] readonly bool SkipCaches = false; @@ -24,6 +24,10 @@ public static int Main() [Parameter("If set, only executes targets for a specified runtime identifier. Provided RID must be included in property of Cesium.Compiler project.")] readonly string RuntimeId = string.Empty; + string EffectiveRuntimeId => !string.IsNullOrEmpty(RuntimeId) + ? RuntimeId + : Solution.Cesium_Compiler.GetProperty("DefaultAppHostRuntimeIdentifier") ?? string.Empty; + [Parameter("If set to true, publishes compiler packs in AOT mode.")] readonly bool PublishAot = false; diff --git a/docs/msbuild-sdk.md b/docs/msbuild-sdk.md new file mode 100644 index 00000000..f5fce3bb --- /dev/null +++ b/docs/msbuild-sdk.md @@ -0,0 +1,64 @@ +Cesium MSBuild Project SDK +-------------------------- + +Cesium provides it's own project SDK that could be used to simplify building of Cesium programs and libraries. + +Cesium MSBuild SDK inherits default behavior from a `Microsoft.NET.Sdk` SDK and tries to integrate with it the same way as C# does. + +Cesium MSBuild SDK only supports SDK-style projects. + +> Note: Some of the common MSBuild properties and items those are not stated in this document could be also used in Cesium project files. Not all of them are tested so something may not work as expected. + +### Source files +Source files are defined with `` items, very similar to other .NET languages: +```xml + + + + +``` +> Note: In the current SDK implementation, compile units should be defined explicitly, in opposite to C# `` items. + +### References + +#### Packages +Not supported yet. + +#### Projects +Not supported yet. + +#### Assemblies +Not supported yet. + +### Preprocessor directives +`` property is directly mapped to a list of preprocessor items. So, you could define such constants in .csproj: +```xml + + $(DefineConstants);FOO;BAR + +``` + +And then use it in your .c code: +```c++ +#ifdef FOO +int foo() { return 0; } +#endif + +#ifdef BAR +int bar() { return 1; } +#endif +``` + +### Output files +Output assembly and additional artifacts will be placed in `bin` folder. Depending on the target framework, output type and a platform you're compiling on, `.runtimeconfig.json` and `.deps.json` files will also be generated. + +### Properties +- `SkipCesiumCompilerInstallation`: if set to `true`, doesn't automatically install a compiler bundle package. In that case it should be explicitly provided by `CesiumCompilerPackageName` and `CesiumCompilerPackageVersion` properties. Default: `false` +- `SkipCesiumRuntimeInstallation`: if set to `true`, doesn't automatically install a `Cesium.Runtime` package. In that case it should be explicitly installed. Default: `false` +- `CesiumCompilerPackageName`: an optional platform-specific compiler bundle package name. Should be specified if `SkipCesiumCompilerInstallation` set to `true`. Default: `Cesium.Compiler.Pack.{RID}` +- `CesiumCompilerPackageName`: an optional platform-specific compiler bundle package version. Should be specified if `SkipCesiumCompilerInstallation` set to `true`. Default: Cesium SDK version +- `CesiumCompilerPath`: an optional path to compiler executable. Use this property to specify a path to the compiler not coming from a compiler package. +- `CesiumCoreLibAssemblyPath`: an optional path to .NET runtime assembly: `System.Runtime` or `mscorlib`, depending on the target framework. + +### Items +- `Compile`: a C source file to be included into compiler execution command diff --git a/docs/tests.md b/docs/tests.md index f27e486d..30781b70 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -3,16 +3,32 @@ Cesium Tests Being a compiler, Cesium requires a complicated test suite checking every feature. -There are two kinds of tests in Cesium: unit tests (directly calling various internal APIs in the compiler) and integration tests (interacting with the compiler executable and comparing the resulting programs' behavior with programs compiled by other compilers). +There are three kinds of tests in Cesium: unit tests (directly calling various internal APIs in the compiler), integration tests (interacting with the compiler executable and comparing the resulting programs' behavior with programs compiled by other compilers) and SDK tests (testing integration with MSBuild via MSBuild project SDK). -Unit Tests ----------- -Unit tests in Cesium are normal .NET tests, so they are runnable by the following shell command: -s +Running Tests +------------- +To run all tests from solution, make sure to restore locally installed tools: +```console +dotnet tool restore +``` + +Then, run `TestAll` target using NUKE: ```console -$ dotnet test +dotnet nuke TestAll ``` +You could also execute test from specific corresponding test projects: +- `dotnet nuke TestParser` +- `dotnet nuke TestCompiler` +- `dotnet nuke TestCodeGen` +- `dotnet nuke TestRuntime` +- `dotnet nuke TestIntegration` +- `dotnet nuke TestSdk` + +Unit Tests +---------- +Unit tests in Cesium are normal .NET tests. + There are two kinds of unit tests: a few of "the real" unit tests (e.g. `Cesium.Parser.Tests.LexerTests.IdentifierTests`) and a set of [characterization tests][wiki.characterization-tests]. The real unit tests verify certain facts using assertions of the Xunit testing framework, but their usage in the compiler is small. The characterization tests, on the other hand, invoke parts of the compiler on various sources, and then dump the results (e.g. a full parse tree of a code fragment, or a whole compiled assembly). The characterization tests comprise the biggest part of the compiler test suite, and it helps us to thoroughly verify the most aspects of the compiler behavior. @@ -53,3 +69,24 @@ There are two categories of integration tests: .NET interop tests and compiler v [wiki.characterization-tests]: https://en.wikipedia.org/wiki/Characterization_test +SDK Tests +--------- +#### How SDK tests work +SDK tests check correctness of integration with MSBuild. They are focused on build output, including artifacts existence asserting. + +SDK tests (and Cesium Project SDK itself) require compiler bundle to be built and packed. A compiler bundle is a special platform-specific NuGet package containing a published compiler executable with dependencies. It is not intended to be used as a runtime dependency and only used while building project. + +Compiler packing is done by 2 NUKE targets: +- `PublishCompilerBundle`: a target that make platform-specific `dotnet publish` of compiler bundle to corresponding artifacts' folder. +- `PackCompilerBundle`: a target that wraps a published compiler bundle into a NuGet package which is then used by SDK to deliver compiler to user's project + +Both targets are called automatically when `TestSdk` target is invoked. + +SDK itself should also be built to be used in test projects. This is done by dependent target `PackSdk` which produces `Cesium.Sdk` NuGet package, suitable as Project SDK. + +Having all necessary dependencies, SDK tests are invoking `dotnet build` CLI with test projects, representing different layouts and configurations. Test result is determined by MSBuild output and artifacts presence. + +#### Adding new tests +Adding new tests is quite straightforward. +1. Add a test project if needed to the `TestProjects` directory. All items from that folder will be automatically included into temporary test execution directory. +2. Write a test with the new test project in use. Look for the examples at `CesiumCompileTests.cs`.