Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace reflection only load context with shared metadataLoadContext for publishing packages. Solution 1. #14576

Merged
merged 21 commits into from
Nov 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/DynamoCoreWpf/DynamoCoreWpf.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@
<PackageReference Include="SharpDX.Direct3D9" Version="4.2.0" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="SharpDX.Mathematics" Version="4.2.0" />
<PackageReference Include="System.Reflection.MetadataLoadContext" Version="6.0.0" />
<Reference Include="Dynamo.Microsoft.Xaml.Behaviors">
<HintPath>..\..\extern\Microsoft.Xaml.Behaviors\$(TargetFramework)\Dynamo.Microsoft.Xaml.Behaviors.dll</HintPath>
</Reference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using String = System.String;
using NotificationObject = Dynamo.Core.NotificationObject;
using Prism.Commands;
using System.Runtime.InteropServices;

namespace Dynamo.PackageManager
{
Expand Down Expand Up @@ -738,6 +739,22 @@ public string CurrentWarningMessage
}
}

private static MetadataLoadContext sharedMetaDataLoadContext = null;
/// <summary>
/// A shared MetaDataLoadContext that is used for assembly inspection during package publishing.
/// This member is shared so the behavior is similar to the ReflectionOnlyLoadContext this is replacing.
/// TODO - eventually it would be good to move to separate publish load contexts that are cleaned up at the appropriate time(?).
/// </summary>
private static MetadataLoadContext SharedPublishLoadContext
{
get
{
sharedMetaDataLoadContext ??= InitSharedPublishLoadContext();
return sharedMetaDataLoadContext;
}
}


#endregion

internal PublishPackageViewModel()
Expand Down Expand Up @@ -865,11 +882,11 @@ private void ThisPropertyChanged(object sender, PropertyChangedEventArgs e)
}
}

public static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoViewModel, Package l)
public static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoViewModel, Package pkg)
{
var defs = new List<CustomNodeDefinition>();

foreach (var x in l.LoadedCustomNodes)
foreach (var x in pkg.LoadedCustomNodes)
{
CustomNodeDefinition def;
if (dynamoViewModel.Model.CustomNodeManager.TryGetFunctionDefinition(
Expand All @@ -881,44 +898,43 @@ public static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoVie
}
}

var vm = new PublishPackageViewModel(dynamoViewModel)
var pkgViewModel = new PublishPackageViewModel(dynamoViewModel)
{
Group = l.Group,
Description = l.Description,
Keywords = l.Keywords != null ? String.Join(" ", l.Keywords) : "",
Group = pkg.Group,
Description = pkg.Description,
Keywords = pkg.Keywords != null ? String.Join(" ", pkg.Keywords) : "",
CustomNodeDefinitions = defs,
Name = l.Name,
RepositoryUrl = l.RepositoryUrl ?? "",
SiteUrl = l.SiteUrl ?? "",
Package = l,
License = l.License,
SelectedHosts = l.HostDependencies as List<string>,
CopyrightHolder = l.CopyrightHolder,
CopyrightYear = l.CopyrightYear
Name = pkg.Name,
RepositoryUrl = pkg.RepositoryUrl ?? "",
SiteUrl = pkg.SiteUrl ?? "",
Package = pkg,
License = pkg.License,
SelectedHosts = pkg.HostDependencies as List<string>,
CopyrightHolder = pkg.CopyrightHolder,
CopyrightYear = pkg.CopyrightYear
};

// add additional files
l.EnumerateAdditionalFiles();
foreach (var file in l.AdditionalFiles)
pkg.EnumerateAdditionalFiles();
foreach (var file in pkg.AdditionalFiles)
{
vm.AdditionalFiles.Add(file.Model.FullName);
pkgViewModel.AdditionalFiles.Add(file.Model.FullName);
}

var nodeLibraryNames = l.Header.node_libraries;
var nodeLibraryNames = pkg.Header.node_libraries;

var assembliesLoadedTwice = new List<string>();
// load assemblies into reflection only context
foreach (var file in l.EnumerateAssemblyFilesInBinDirectory())
foreach (var file in pkg.EnumerateAssemblyFilesInPackage())
{
Assembly assem;
var result = PackageLoader.TryReflectionOnlyLoadFrom(file, out assem);
var result = PackageLoader.TryMetaDataContextLoad(file, SharedPublishLoadContext, out assem);

switch (result)
{
case AssemblyLoadingState.Success:
{
var isNodeLibrary = nodeLibraryNames == null || nodeLibraryNames.Contains(assem.FullName);
vm.Assemblies.Add(new PackageAssembly()
pkgViewModel.Assemblies.Add(new PackageAssembly()
{
IsNodeLibrary = isNodeLibrary,
Assembly = assem
Expand All @@ -928,7 +944,7 @@ public static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoVie
case AssemblyLoadingState.NotManagedAssembly:
{
// if it's not a .NET assembly, we load it as an additional file
vm.AdditionalFiles.Add(file);
pkgViewModel.AdditionalFiles.Add(file);
break;
}
case AssemblyLoadingState.AlreadyLoaded:
Expand All @@ -940,24 +956,24 @@ public static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoVie
}

//after dependencies are loaded refresh package contents
vm.RefreshPackageContents();
vm.UpdateDependencies();
pkgViewModel.RefreshPackageContents();
pkgViewModel.UpdateDependencies();

if (assembliesLoadedTwice.Any())
{
vm.UploadState = PackageUploadHandle.State.Error;
vm.ErrorString = Resources.OneAssemblyWasLoadedSeveralTimesErrorMessage + string.Join("\n", assembliesLoadedTwice);
pkgViewModel.UploadState = PackageUploadHandle.State.Error;
pkgViewModel.ErrorString = Resources.OneAssemblyWasLoadedSeveralTimesErrorMessage + string.Join("\n", assembliesLoadedTwice);
}

if (l.VersionName == null) return vm;
if (pkg.VersionName == null) return pkgViewModel;

var parts = l.VersionName.Split('.');
if (parts.Count() != 3) return vm;
var parts = pkg.VersionName.Split('.');
if (parts.Count() != 3) return pkgViewModel;

vm.MajorVersion = parts[0];
vm.MinorVersion = parts[1];
vm.BuildVersion = parts[2];
return vm;
pkgViewModel.MajorVersion = parts[0];
pkgViewModel.MinorVersion = parts[1];
pkgViewModel.BuildVersion = parts[2];
return pkgViewModel;

}

Expand Down Expand Up @@ -1868,5 +1884,14 @@ internal void EnableInvalidNameWarningState(string warningMessage)
CurrentWarningMessage = warningMessage;
IsWarningEnabled = true;
}

private static MetadataLoadContext InitSharedPublishLoadContext()
{
// Retrieve the location of the assembly and the referenced assemblies used by the domain
var runtimeAssemblies = Directory.GetFiles(RuntimeEnvironment.GetRuntimeDirectory(), "*.dll");
// Create PathAssemblyResolver that can resolve assemblies using the created list.
var resolver = new PathAssemblyResolver(runtimeAssemblies);
return new MetadataLoadContext(resolver);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we no longer doing this:

//and all the package assemblies.
            foreach (var assemblyFile in pkg.EnumerateAssemblyFilesInPackage())
            {
                assemblyPaths.Add(assemblyFile);
            }

            // Create PathAssemblyResolver that can resolve assemblies using the created list.
            var resolver = new PathAssemblyResolver(assemblyPaths);
            var mlc = new MetadataLoadContext(resolver);

Copy link
Member Author

@mjkkirschner mjkkirschner Nov 9, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I found that it does not appear to be necessary to add the assemblies to the MLC as we pass the full path to the file we want to load.

Additionally, we only need the dependencies loaded if want to inspect those types - but we don't do much with the assemblies as far as I can tell except for accessing the location and assembly name.

In an MLC the dependencies are not required to load the assembly AFAIK.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not see any immediate need to use those types...but as soon as we do ...we will have exceptions thrown. THese assemblies are not exposed publicly right ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no, and in addition this would have caused the same issue with ReflectionOnlyLoadContext.

Copy link
Member Author

@mjkkirschner mjkkirschner Nov 9, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually, the PublicPackageViewModel.Assemblies property is public and so is the class. These classes are only used in the UI layer though AFAIK and are not easily accessible from anywhere but our views, ie not from node or extensions without clients hacking around.

Copy link
Contributor

@aparajit-pratap aparajit-pratap Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, I'm a bit confused after reading this, just trying to understand it better. The docs seem to say that the resolver that is passed to the mlc resolves assemblies that are dependencies of the assembly being loaded (by the mlc). I can understand that this should include runtime assemblies, which most likely the assembly being loaded (for inspection) has a dependency on but won't it also have dependencies most likely on the other helper assemblies that are in the same package folder itself for the package that's being published? Or are you saying that since the assembly being loaded and the other assemblies are in the same location, that isn't required? Maybe I'm missing something.

Copy link
Member Author

@mjkkirschner mjkkirschner Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, the binary we are loading very likely will have dependencies on other binaries in the package - but it doesn't matter. It does not matter because the MLC won't fail to load a binary simply because a dependency is not loaded.

It only becomes an issue when you try to inspect a type and that type has a dependency on an assembly the resolver cannot resolve.

But in our case, as far as I can tell, we don't do any inspection, we just look at the assembly name and location... we're just using the MLC to do some sanity checking that the assembly is a real managed assembly.

ReflectionLoadContext also behaved like this, except instead of a pathresolver, you had to implement the ReflectionOnlyAssemblyResolve which we did not do.

}
}
}
1 change: 1 addition & 0 deletions src/DynamoPackages/DynamoPackages.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
<PackageReference Include="Greg" Version="3.0.0.2955" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="RestSharp" Version="109.0.1" />
<PackageReference Include="System.Reflection.MetadataLoadContext" Version="6.0.0" />
</ItemGroup>
<ItemGroup>
<Compile Update="Properties\Resources.Designer.cs">
Expand Down
3 changes: 2 additions & 1 deletion src/DynamoPackages/Package.cs
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,8 @@ public void EnumerateAdditionalFiles()
AdditionalFiles.AddRange(nonDyfDllFiles);
}

public IEnumerable<string> EnumerateAssemblyFilesInBinDirectory()
//TODO can we make this internal?
public IEnumerable<string> EnumerateAssemblyFilesInPackage()
{
if (String.IsNullOrEmpty(RootDirectory) || !Directory.Exists(RootDirectory))
return new List<string>();
Expand Down
44 changes: 27 additions & 17 deletions src/DynamoPackages/PackageLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -743,28 +743,38 @@ private void CheckPackageNodeLibraryCertificates(string packageDirectoryPath, Pa
}

/// <summary>
/// Attempt to load a managed assembly in to ReflectionOnlyLoadFrom context.
/// Attempt to load a managed assembly in to MetaDataLoad context.
/// </summary>
/// <param name="filename">The filename of a DLL</param>
/// <param name="mlc">The MetaDataLoadContext to load the package assemblies into for inspection.</param>
/// <param name="assem">out Assembly - the passed value does not matter and will only be set if loading succeeds</param>
/// <returns>Returns Success if success, NotManagedAssembly if BadImageFormatException, AlreadyLoaded if FileLoadException</returns>
internal static AssemblyLoadingState TryReflectionOnlyLoadFrom(string filename, out Assembly assem)
internal static AssemblyLoadingState TryMetaDataContextLoad(string filename,MetadataLoadContext mlc, out Assembly assem)
{
try
{
assem = Assembly.ReflectionOnlyLoadFrom(filename);
return AssemblyLoadingState.Success;
}
catch (BadImageFormatException)
{
assem = null;
return AssemblyLoadingState.NotManagedAssembly;
}
catch (FileLoadException)
{
assem = null;
return AssemblyLoadingState.AlreadyLoaded;
}
try
{
var mlcAssemblies = mlc.GetAssemblies();
assem = mlc.LoadFromAssemblyPath(filename);
var mlcAssemblies2 = mlc.GetAssemblies();
//if loading the assembly did not actually add a new assembly to the MLC
//then we've loaded it already, and our current behavior is to
//disable publish when a package contains the same assembly twice.
if (mlcAssemblies2.Count() == mlcAssemblies.Count())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this the best way to check if the assembly has been loaded successfully ?
assem = mlc.LoadFromAssemblyPath(filename);
assem != null is not good enough ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not checking if the assembly has been loaded successfully, I am checking if it had been loaded already.

{
throw new FileLoadException(filename);
}
return AssemblyLoadingState.Success;
}
catch (BadImageFormatException)
{
assem = null;
return AssemblyLoadingState.NotManagedAssembly;
}
catch (FileLoadException)
{
assem = null;
return AssemblyLoadingState.AlreadyLoaded;
}
}

/// <summary>
Expand Down
12 changes: 6 additions & 6 deletions test/DynamoCoreWpfTests/PackagePathTests.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@


using System.IO;
using System.Linq;
using System.Reflection;
Expand Down Expand Up @@ -233,17 +233,17 @@ public void InstalledPackagesContainsCorrectNumberOfPackages()
vm.packageLoader.PackagesLoaded += pkgsLoadedDelegate;

vm.packageLoader.LoadAll(vm.loadPackageParams);
Assert.AreEqual(19, vm.packageLoader.LocalPackages.Count());
Assert.AreEqual(20, vm.packageLoader.LocalPackages.Count());
Assert.AreEqual(true, packagesLoaded);

var installedPackagesViewModel = new InstalledPackagesViewModel(ViewModel, vm.packageLoader);
Assert.AreEqual(19, installedPackagesViewModel.LocalPackages.Count);
Assert.AreEqual(20, installedPackagesViewModel.LocalPackages.Count);

var installedPackagesView = new Dynamo.Wpf.Controls.InstalledPackagesControl();
installedPackagesView.DataContext = installedPackagesViewModel;
DispatcherUtil.DoEvents();

Assert.AreEqual(19, installedPackagesView.SearchResultsListBox.Items.Count);
Assert.AreEqual(20, installedPackagesView.SearchResultsListBox.Items.Count);
Assert.AreEqual(2, installedPackagesView.Filters.Items.Count);

vm.packageLoader.PackagesLoaded -= libraryLoader.LoadPackages;
Expand All @@ -267,7 +267,7 @@ public void RemoveAddPackagePathChangesInstalledPackageState()
// Load packages in package path.
vm.packageLoader.LoadAll(vm.loadPackageParams);

Assert.AreEqual(19, vm.packageLoader.LocalPackages.Count());
Assert.AreEqual(20, vm.packageLoader.LocalPackages.Count());
// Remove package path.
vm.DeletePathCommand.Execute(0);

Expand Down Expand Up @@ -323,7 +323,7 @@ public void EnableCustomPackagePathsLoadsPackagesOnClosingPreferences()
vm.SaveSettingCommand.Execute(null);

// packages are expected to load from 'PackagesDirectory' above when toggle is turned off
Assert.AreEqual(19, vm.packageLoader.LocalPackages.Count());
Assert.AreEqual(20, vm.packageLoader.LocalPackages.Count());

vm.packageLoader.PackagesLoaded -= libraryLoader.LoadPackages;
vm.packageLoader.RequestLoadNodeLibrary -= libraryLoader.LoadLibraryAndSuppressZTSearchImport;
Expand Down
34 changes: 31 additions & 3 deletions test/DynamoCoreWpfTests/PublishPackageViewModelTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using Dynamo;
using Dynamo.Graph.Nodes.CustomNodes;
using Dynamo.Graph.Workspaces;
Expand All @@ -17,7 +18,7 @@ namespace DynamoCoreWpfTests
public class PublishPackageViewModelTests: DynamoViewModelUnitTest
{

[Test, Category("Failure")]
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this seems to be passing... this test is unrelated to this task so if it fails on CI I will just revert this.

[Test]
public void AddingDyfRaisesCanExecuteChangeOnDelegateCommand()
{

Expand Down Expand Up @@ -99,7 +100,34 @@ public void CanPublishLateInitializedJsonCustomNode()
}


[Test, Category("Failure")]
[Test]
public void NewPackageDoesNotThrow_NativeBinaryIsAddedAsAdditionalFile_NotBinary()
Copy link
Contributor

@aparajit-pratap aparajit-pratap Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does the dependencies field in the pkg.json being empty indicate that the native assembly is not added as a dependent binary but as an additional file?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the dependencies field points to other packages

{
string packagesDirectory = Path.Combine(TestDirectory, "pkgs");

var pathManager = new Mock<Dynamo.Interfaces.IPathManager>();
pathManager.SetupGet(x => x.PackagesDirectories).Returns(() => new List<string> { packagesDirectory });

var loader = new PackageLoader(pathManager.Object);
loader.LoadAll(new LoadPackageParams
{
Preferences = ViewModel.Model.PreferenceSettings
});

PublishPackageViewModel vm = null;
var package = loader.LocalPackages.FirstOrDefault(x => x.Name == "package with native assembly");
Assert.DoesNotThrow(() =>
{
vm = PublishPackageViewModel.FromLocalPackage(ViewModel, package);
});

Assert.AreEqual(1, vm.AdditionalFiles.Count);
Assert.AreEqual(0, vm.Assemblies.Count);

Assert.AreEqual(PackageUploadHandle.State.Ready, vm.UploadState);
}

[Test]
public void NewPackageVersionUpload_DoesNotThrowExceptionWhenDLLIsLoadedSeveralTimes()
{
string packagesDirectory = Path.Combine(TestDirectory, "pkgs");
Expand All @@ -123,7 +151,7 @@ public void NewPackageVersionUpload_DoesNotThrowExceptionWhenDLLIsLoadedSeveralT
Assert.AreEqual(PackageUploadHandle.State.Error, vm.UploadState);
}

[Test, Category("Failure")]
[Test]
public void NewPackageVersionUpload_CanAddAndRemoveFiles()
{
string packagesDirectory = Path.Combine(TestDirectory, "pkgs");
Expand Down
Loading