SDK: Refactor compiler bundle producing, add docs

- Add docs about MSBuild SDK - Add info about SDK tests - Use NUKE in tests docs - Rename compiler pack -> compiler bundle - Introduce CI-friendly single pack and publish compiler bundle targets
ForNeVeR · Apr 17, 2024 · 311f468 · 311f468
1 parent 7363fac
commit 311f468
Show file tree

Hide file tree

Showing 11 changed files with 184 additions and 70 deletions.
diff --git a/.github/workflows/perform-common-steps/action.yml b/.github/workflows/perform-common-steps/action.yml
@@ -20,4 +20,4 @@ runs:
 
     - name: 🔄 Restore Nuget Packages 🔄
       shell: bash
-      run: dotnet restore
+      run: dotnet nuke RestoreAll
diff --git a/.github/workflows/run-build-and-unit-tests.yml b/.github/workflows/run-build-and-unit-tests.yml
@@ -32,16 +32,16 @@ jobs:
 
       - name: 🛠 Build Solution 🛠
         shell: bash
-        run: dotnet build
+        run: dotnet nuke CompileAll
 
-      - name: 🚚 Publish Compiler Packs 🚚
+      - name: 🚚 Publish Compiler Bundle 🚚
         shell: bash
-        run: dotnet nuke PublishCompilerPacks
+        run: dotnet nuke PublishCompilerBundle
 
-      - name: 📦 Pack Compiler Packs 📦
+      - name: 📦 Pack Compiler Bundle 📦
         shell: bash
-        run: dotnet nuke PackCompilerPacks
+        run: dotnet nuke PackCompilerBundle
 
       - name: ✅ Run Unit Tests ✅
         shell: bash
-        run: dotnet nuke TestAll --configuration Debug
+        run: dotnet nuke TestAll
diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json
@@ -8,7 +8,7 @@
       "properties": {
         "Configuration": {
           "type": "string",
-          "description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)",
+          "description": "Configuration to build - Default is 'Debug' or 'Release'",
           "enum": [
             "Debug",
             "Release"
@@ -83,9 +83,11 @@
               "Clean",
               "CompileAll",
               "ForceClear",
-              "PackCompilerPacks",
+              "PackAllCompilerBundles",
+              "PackCompilerBundle",
               "PackSdk",
-              "PublishCompilerPacks",
+              "PublishAllCompilerBundles",
+              "PublishCompilerBundle",
               "RestoreAll",
               "TestAll",
               "TestCodeGen",
@@ -114,9 +116,11 @@
               "Clean",
               "CompileAll",
               "ForceClear",
-              "PackCompilerPacks",
+              "PackAllCompilerBundles",
+              "PackCompilerBundle",
               "PackSdk",
-              "PublishCompilerPacks",
+              "PublishAllCompilerBundles",
+              "PublishCompilerBundle",
               "RestoreAll",
               "TestAll",
               "TestCodeGen",

diff --git a/Cesium.Sdk/Sdk/Sdk.props b/Cesium.Sdk/Sdk/Sdk.props
@@ -9,7 +9,7 @@
     <PropertyGroup Label="DefaultProperties">
         <SkipCesiumCompilerInstallation Condition="$(SkipCesiumCompilerInstallation) == ''">false</SkipCesiumCompilerInstallation>
         <SkipCesiumRuntimeInstallation Condition="$(SkipCesiumRuntimeInstallation) == ''">false</SkipCesiumRuntimeInstallation>
-        <CesiumCompilerPackageName Condition="$(CesiumCompilerPackageName) == ''">Cesium.Compiler.Pack.$(NETCoreSdkRuntimeIdentifier)</CesiumCompilerPackageName>
+        <CesiumCompilerPackageName Condition="$(CesiumCompilerPackageName) == ''">Cesium.Compiler.Bundle.$(NETCoreSdkRuntimeIdentifier)</CesiumCompilerPackageName>
         <CesiumCompilerPackageVersion Condition="$(CesiumCompilerPackageVersionPrefix) == ''">0.0.1</CesiumCompilerPackageVersion>
         <ProduceReferenceAssembly>false</ProduceReferenceAssembly>
     </PropertyGroup>

diff --git a/Cesium.sln b/Cesium.sln
@@ -61,6 +61,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{986C6A13-2
 		docs\tests.md = docs\tests.md
 		docs\type-system.md = docs\type-system.md
 		docs\design-notes.md = docs\design-notes.md
+		docs\msbuild-sdk.md = docs\msbuild-sdk.md
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cesium.Preprocessor", "Cesium.Preprocessor\Cesium.Preprocessor.csproj", "{0CDF730D-2A2A-437F-B27F-2BB04770C709}"

diff --git a/README.md b/README.md
@@ -38,6 +38,7 @@ Documentation
 - [Contributor Guide][docs.contributing]
 - [Cesium Tests][docs.tests]
 - [Cesium Type System][docs.type-system]
+- [Cesium SDK][docs.msbuild-sdk]
 - [Architecture Sets][docs.architecture-sets]
 - [CLI-Related Language Extensions][docs.language-extensions]
 - [Built-in Functions][docs.builtins]

diff --git a/build/Build.Sdk.cs b/build/Build.Sdk.cs
@@ -11,9 +11,9 @@
 
 public partial class Build
 {
-    const string _compilerPackPackagePrefix = "Cesium.Compiler.Pack";
+    const string _compilerPackPackagePrefix = "Cesium.Compiler.Bundle";
 
-    Target PublishCompilerPacks => _ => _
+    Target PublishAllCompilerBundles => _ => _
         .Executes(() =>
         {
             var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject();
@@ -22,39 +22,18 @@ public partial class Build
             Log.Information(
                 $"Runtime identifiers defined in {Solution.Cesium_Compiler.Name}: {string.Join(", ", runtimeIds)}");
 
-            if (!string.IsNullOrEmpty(RuntimeId))
-            {
-                Log.Information($"Executing only {RuntimeId} because it was specified explicitly.");
-                PublishCompiler(RuntimeId);
-                return;
-            }
-
             foreach (var runtimeId in runtimeIds)
                 PublishCompiler(runtimeId);
+        });
 
-            void PublishCompiler(string runtimeId)
-            {
-                if (!SkipCaches && !NeedPublishCompilerPack(compilerProject, runtimeId))
-                {
-                    Log.Information($"Skipping {runtimeId} because it was already published. Use '--skip-caches true' to re-publish.");
-                    return;
-                }
-
-                Log.Information($"Publishing for {runtimeId}, AOT {(PublishAot ? "enabled" : "disabled")}...");
-                DotNetPublish(o => o
-                    .SetConfiguration(Configuration)
-                    .SetProject(compilerProject.ProjectFileLocation.File)
-                    .SetRuntime(runtimeId)
-                    .SetSelfContained(true)
-                    .SetPublishTrimmed(PublishAot)
-                    .SetPublishSingleFile(PublishAot)
-                    .SetProperty("PublishAot", PublishAot)
-                    .SetOutput(GetCompilerRuntimePublishFolder(compilerProject, runtimeId)));
-            }
+    Target PublishCompilerBundle => _ => _
+        .Executes(() =>
+        {
+            PublishCompiler(EffectiveRuntimeId);
         });
 
-    Target PackCompilerPacks => _ => _
-        .DependsOn(PublishCompilerPacks)
+    Target PackAllCompilerBundles => _ => _
+        .DependsOn(PublishAllCompilerBundles)
         .Executes(() =>
         {
             var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject();
@@ -63,27 +42,15 @@ void PublishCompiler(string runtimeId)
             Log.Information(
                 $"Runtime identifiers defined in {Solution.Cesium_Compiler.Name}: {string.Join(", ", runtimeIds)}");
 
-            if (!string.IsNullOrEmpty(RuntimeId))
-            {
-                Log.Information($"Executing only {RuntimeId} because it was specified explicitly.");
-                PackCompiler(RuntimeId);
-                return;
-            }
-
             foreach (var runtimeId in runtimeIds)
                 PackCompiler(runtimeId);
+        });
 
-            void PackCompiler(string runtimeId)
-            {
-                if (!SkipCaches && !NeedPackageCompilerPack(compilerProject, runtimeId))
-                {
-                    Log.Information($"Skipping {runtimeId} because it was already packed. Use '--skip-caches true' to re-pack.");
-                    return;
-                }
-
-                Log.Information($"Packing compiler for {runtimeId}...");
-                EmitCompilerPack(runtimeId, compilerProject);
-            }
+    Target PackCompilerBundle => _ => _
+        .DependsOn(PublishCompilerBundle)
+        .Executes(() =>
+        {
+            PackCompiler(EffectiveRuntimeId);
         });
 
     Target PackSdk => _ => _
@@ -145,6 +112,42 @@ IEnumerable<IPackageFile> GetPhysicalFiles(string publishDirectory, IEnumerable<
         }
     }
 
+    void PublishCompiler(string runtimeId)
+    {
+        var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject();
+
+        if (!SkipCaches && !NeedPublishCompilerPack(compilerProject, runtimeId))
+        {
+            Log.Information($"Skipping {runtimeId} because it was already published. Use '--skip-caches true' to re-publish.");
+            return;
+        }
+
+        Log.Information($"Publishing for {runtimeId}, AOT {(PublishAot ? "enabled" : "disabled")}...");
+        DotNetPublish(o => o
+            .SetConfiguration(Configuration)
+            .SetProject(compilerProject.ProjectFileLocation.File)
+            .SetRuntime(runtimeId)
+            .SetSelfContained(true)
+            .SetPublishTrimmed(PublishAot)
+            .SetPublishSingleFile(PublishAot)
+            .SetProperty("PublishAot", PublishAot)
+            .SetOutput(GetCompilerRuntimePublishFolder(compilerProject, runtimeId)));
+    }
+
+    void PackCompiler(string runtimeId)
+    {
+        var compilerProject = Solution.Cesium_Compiler.GetMSBuildProject();
+
+        if (!SkipCaches && !NeedPackageCompilerPack(compilerProject, runtimeId))
+        {
+            Log.Information($"Skipping {runtimeId} because it was already packed. Use '--skip-caches true' to re-pack.");
+            return;
+        }
+
+        Log.Information($"Packing compiler for {runtimeId}...");
+        EmitCompilerPack(runtimeId, compilerProject);
+    }
+
     string GetCompilerRuntimePublishFolder(Project compilerProject, string runtimeId) =>
         Path.Combine(
             compilerProject.GetProperty("ArtifactsPath").EvaluatedValue,

diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs
@@ -21,7 +21,7 @@ partial class Build
         .Executes(() => ExecuteTests(Solution.Cesium_Runtime_Tests));
 
     Target TestSdk => _ => _
-        .DependsOn(PackCompilerPacks)
+        .DependsOn(PackCompilerBundle)
         .DependsOn(PackSdk)
         .Executes(() => ExecuteTests(Solution.Cesium_Sdk_Tests));
 

diff --git a/build/Build.cs b/build/Build.cs
@@ -12,8 +12,8 @@ public static int Main()
         return Execute<Build>(x => x.CompileAll);
     }
 
-    [Parameter("Configuration to build - Default is 'Debug' (local) or 'Release' (server)")]
-    readonly Configuration Configuration = IsLocalBuild ? Configuration.Debug : Configuration.Release;
+    [Parameter("Configuration to build - Default is 'Debug' or 'Release'")]
+    readonly Configuration Configuration = Configuration.Debug;
 
     [Parameter("If set to true, ignores all cached build results. Default: false")]
     readonly bool SkipCaches = false;
@@ -24,6 +24,10 @@ public static int Main()
     [Parameter("If set, only executes targets for a specified runtime identifier. Provided RID must be included in <RuntimeIdentifiers> property of Cesium.Compiler project.")]
     readonly string RuntimeId = string.Empty;
 
+    string EffectiveRuntimeId => !string.IsNullOrEmpty(RuntimeId)
+        ? RuntimeId
+        : Solution.Cesium_Compiler.GetProperty("DefaultAppHostRuntimeIdentifier") ?? string.Empty;
+
     [Parameter("If set to true, publishes compiler packs in AOT mode.")]
     readonly bool PublishAot = false;
 

diff --git a/docs/msbuild-sdk.md b/docs/msbuild-sdk.md
@@ -0,0 +1,64 @@
+Cesium MSBuild Project SDK
+--------------------------
+
+Cesium provides it's own project SDK that could be used to simplify building of Cesium programs and libraries.
+
+Cesium MSBuild SDK inherits default behavior from a `Microsoft.NET.Sdk` SDK and tries to integrate with it the same way as C# does.
+
+Cesium MSBuild SDK only supports SDK-style projects.
+
+> Note: Some of the common MSBuild properties and items those are not stated in this document could be also used in Cesium project files. Not all of them are tested so something may not work as expected.
+
+### Source files
+Source files are defined with `<Compile>` items, very similar to other .NET languages:
+```xml
+<ItemGroup>
+    <Compile Include="hello.c" />
+    <Compile Include="example.c" />
+</ItemGroup>
+```
+> Note: In the current SDK implementation, compile units should be defined explicitly, in opposite to C# `<Compile>` items.
+
+### References
+
+#### Packages
+Not supported yet.
+
+#### Projects
+Not supported yet.
+
+#### Assemblies
+Not supported yet.
+
+### Preprocessor directives
+`<DefineConstants>` property is directly mapped to a list of preprocessor items. So, you could define such constants in .csproj:
+```xml
+<PropertyGroup>
+    <DefineConstants>$(DefineConstants);FOO;BAR</DefineConstants>
+</PropertyGroup>
+```
+
+And then use it in your .c code:
+```c++
+#ifdef FOO
+int foo() { return 0; }
+#endif
+
+#ifdef BAR
+int bar() { return 1; }
+#endif
+```
+
+### Output files
+Output assembly and additional artifacts will be placed in `bin` folder. Depending on the target framework, output type and a platform you're compiling on, `.runtimeconfig.json` and `.deps.json` files will also be generated.
+
+### Properties
+- `SkipCesiumCompilerInstallation`: if set to `true`, doesn't automatically install a compiler bundle package. In that case it should be explicitly provided by `CesiumCompilerPackageName` and `CesiumCompilerPackageVersion` properties. Default: `false`
+- `SkipCesiumRuntimeInstallation`: if set to `true`, doesn't automatically install a `Cesium.Runtime` package. In that case it should be explicitly installed. Default: `false`
+- `CesiumCompilerPackageName`: an optional platform-specific compiler bundle package name. Should be specified if `SkipCesiumCompilerInstallation` set to `true`. Default: `Cesium.Compiler.Pack.{RID}`
+- `CesiumCompilerPackageName`: an optional platform-specific compiler bundle package version. Should be specified if `SkipCesiumCompilerInstallation` set to `true`. Default: Cesium SDK version
+- `CesiumCompilerPath`: an optional path to compiler executable. Use this property to specify a path to the compiler not coming from a compiler package.
+- `CesiumCoreLibAssemblyPath`: an optional path to .NET runtime assembly: `System.Runtime` or `mscorlib`, depending on the target framework.
+
+### Items
+- `Compile`: a C source file to be included into compiler execution command
diff --git a/docs/tests.md b/docs/tests.md
@@ -3,16 +3,32 @@ Cesium Tests
 
 Being a compiler, Cesium requires a complicated test suite checking every feature.
 
-There are two kinds of tests in Cesium: unit tests (directly calling various internal APIs in the compiler) and integration tests (interacting with the compiler executable and comparing the resulting programs' behavior with programs compiled by other compilers).
+There are three kinds of tests in Cesium: unit tests (directly calling various internal APIs in the compiler), integration tests (interacting with the compiler executable and comparing the resulting programs' behavior with programs compiled by other compilers) and SDK tests (testing integration with MSBuild via MSBuild project SDK).
 
-Unit Tests
-----------
-Unit tests in Cesium are normal .NET tests, so they are runnable by the following shell command:
-s
+Running Tests
+-------------
+To run all tests from solution, make sure to restore locally installed tools:
+```console
+dotnet tool restore
+```
+
+Then, run `TestAll` target using NUKE:
 ```console
-$ dotnet test
+dotnet nuke TestAll
 ```
 
+You could also execute test from specific corresponding test projects:
+- `dotnet nuke TestParser`
+- `dotnet nuke TestCompiler`
+- `dotnet nuke TestCodeGen`
+- `dotnet nuke TestRuntime`
+- `dotnet nuke TestIntegration`
+- `dotnet nuke TestSdk`
+
+Unit Tests
+----------
+Unit tests in Cesium are normal .NET tests.
+
 There are two kinds of unit tests: a few of "the real" unit tests (e.g. `Cesium.Parser.Tests.LexerTests.IdentifierTests`) and a set of [characterization tests][wiki.characterization-tests]. The real unit tests verify certain facts using assertions of the Xunit testing framework, but their usage in the compiler is small. The characterization tests, on the other hand, invoke parts of the compiler on various sources, and then dump the results (e.g. a full parse tree of a code fragment, or a whole compiled assembly).
 
 The characterization tests comprise the biggest part of the compiler test suite, and it helps us to thoroughly verify the most aspects of the compiler behavior.
@@ -53,3 +69,24 @@ There are two categories of integration tests: .NET interop tests and compiler v
 
 [wiki.characterization-tests]: https://en.wikipedia.org/wiki/Characterization_test
 
+SDK Tests
+---------
+#### How SDK tests work
+SDK tests check correctness of integration with MSBuild. They are focused on build output, including artifacts existence asserting.
+
+SDK tests (and Cesium Project SDK itself) require compiler bundle to be built and packed. A compiler bundle is a special platform-specific NuGet package containing a published compiler executable with dependencies. It is not intended to be used as a runtime dependency and only used while building project.
+
+Compiler packing is done by 2 NUKE targets:
+- `PublishCompilerBundle`: a target that make platform-specific `dotnet publish` of compiler bundle to corresponding artifacts' folder.
+- `PackCompilerBundle`: a target that wraps a published compiler bundle into a NuGet package which is then used by SDK to deliver compiler to user's project
+
+Both targets are called automatically when `TestSdk` target is invoked.
+
+SDK itself should also be built to be used in test projects. This is done by dependent target `PackSdk` which produces `Cesium.Sdk` NuGet package, suitable as Project SDK.
+
+Having all necessary dependencies, SDK tests are invoking `dotnet build` CLI with test projects, representing different layouts and configurations. Test result is determined by MSBuild output and artifacts presence.
+
+#### Adding new tests
+Adding new tests is quite straightforward.
+1. Add a test project if needed to the `TestProjects` directory. All items from that folder will be automatically included into temporary test execution directory.
+2. Write a test with the new test project in use. Look for the examples at `CesiumCompileTests.cs`.