From 7395b999de9480d94af00056b475c6f159934fb5 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 18 Dec 2024 17:10:35 -0800 Subject: [PATCH 01/20] Initial implementation Still requires tests --- .../Manifest/GrainProperties.cs | 5 + .../Hosting/DefaultSiloServices.cs | 7 +- .../SiloMetadata/ISiloMetadataCache.cs | 6 ++ .../SiloMetadata/ISiloMetadataClient.cs | 9 ++ .../SiloMetadata/ISiloMetadataGrainService.cs | 11 +++ .../SiloMetadata/SiloMetadaCache.cs | 96 +++++++++++++++++++ .../SiloMetadata/SiloMetadata.cs | 17 ++++ .../SiloMetadata/SiloMetadataClient.cs | 16 ++++ .../SiloMetadata/SiloMetadataGrainService.cs | 22 +++++ .../SiloMetadataHostingExtensions.cs | 90 +++++++++++++++++ .../Filtering/IPlacementFilterDirector.cs | 9 ++ .../Filtering/PlacementFilterAttribute.cs | 24 +++++ .../PlacementFilterDirectorResolver.cs | 12 +++ .../Filtering/PlacementFilterExtensions.cs | 23 +++++ .../Filtering/PlacementFilterStrategy.cs | 52 ++++++++++ .../PlacementFilterStrategyResolver.cs | 63 ++++++++++++ ...redSiloMetadataPlacementFilterAttribute.cs | 7 ++ ...rredSiloMetadataPlacementFilterDirector.cs | 73 ++++++++++++++ ...rredSiloMetadataPlacementFilterStrategy.cs | 26 +++++ .../RequiredSiloMetadataFilterDirector.cs | 59 ++++++++++++ ...redSiloMetadataPlacementFilterAttribute.cs | 7 ++ ...iredSiloMetadataPlacementFilterStrategy.cs | 26 +++++ .../Placement/PlacementService.cs | 25 ++++- 23 files changed, 682 insertions(+), 3 deletions(-) create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs diff --git a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs index 41181c248d..4f994e01f2 100644 --- a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs +++ b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs @@ -68,6 +68,11 @@ public static class WellKnownGrainTypeProperties /// public const string PlacementStrategy = "placement-strategy"; + /// + /// The name of the placement strategy for grains of this type. + /// + public const string PlacementFilter = "placement-filter"; + /// /// The directory policy for grains of this type. /// diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 67f98907cd..bba9fb3276 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -43,8 +43,7 @@ using Orleans.Serialization.Internal; using Orleans.Core; using Orleans.Placement.Repartitioning; -using Orleans.GrainDirectory; -using Orleans.Runtime.Hosting; +using Orleans.Runtime.Placement.Filtering; namespace Orleans.Hosting { @@ -206,6 +205,10 @@ internal static void AddDefaultServices(ISiloBuilder builder) // Configure the default placement strategy. services.TryAddSingleton(); + // Placement filters + services.AddSingleton(); + services.AddSingleton(); + // Placement directors services.AddPlacementDirector(); services.AddPlacementDirector(); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs new file mode 100644 index 0000000000..a15cb67a36 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -0,0 +1,6 @@ +namespace Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +public interface ISiloMetadataCache +{ + SiloMetadata GetMetadata(SiloAddress siloAddress); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs new file mode 100644 index 0000000000..eb622eaaa4 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; +using Orleans.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public interface ISiloMetadataClient : IGrainServiceClient +{ + Task GetSiloMetadata(SiloAddress siloAddress); +} diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs new file mode 100644 index 0000000000..97e3c7e600 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; +using Orleans.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] +public interface ISiloMetadataGrainService : IGrainService +{ + [Alias("GetSiloMetadata")] + Task GetSiloMetadata(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs new file mode 100644 index 0000000000..afdcb39c95 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Internal; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +internal class SiloMetadataCache( + ISiloMetadataClient siloMetadataClient, + MembershipTableManager membershipTableManager, + ILogger logger) + : ISiloMetadataCache, ILifecycleParticipant, IDisposable +{ + private readonly ConcurrentDictionary _metadata = new(); + private readonly CancellationTokenSource _cts = new(); + + void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) + { + var tasks = new List(1); + var cancellation = new CancellationTokenSource(); + Task OnRuntimeInitializeStart(CancellationToken _) + { + tasks.Add(Task.Run(() => this.ProcessMembershipUpdates(cancellation.Token))); + return Task.CompletedTask; + } + + async Task OnRuntimeInitializeStop(CancellationToken ct) + { + cancellation.Cancel(throwOnFirstException: false); + var shutdownGracePeriod = Task.WhenAll(Task.Delay(ClusterMembershipOptions.ClusteringShutdownGracePeriod), ct.WhenCancelled()); + await Task.WhenAny(shutdownGracePeriod, Task.WhenAll(tasks)); + } + + lifecycle.Subscribe( + nameof(ClusterMembershipService), + ServiceLifecycleStage.RuntimeInitialize, + OnRuntimeInitializeStart, + OnRuntimeInitializeStop); + } + + + private async Task ProcessMembershipUpdates(CancellationToken ct) + { + try + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates"); + await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) + { + // Add entries for members that aren't already in the cache + foreach (var membershipEntry in update.Entries) + { + if (!_metadata.ContainsKey(membershipEntry.Key)) + { + try + { + var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key); + _metadata.TryAdd(membershipEntry.Key, metadata); + } + catch(Exception exception) + { + logger.LogError(exception, "Error fetching metadata for silo {Silo}", membershipEntry.Key); + } + } + } + + // Remove entries for members that are no longer in the table + foreach (var silo in _metadata.Keys.ToList()) + { + if (!update.Entries.ContainsKey(silo)) + { + _metadata.TryRemove(silo, out _); + } + } + } + } + catch (Exception exception) + { + logger.LogError(exception, "Error processing membership updates"); + } + finally + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Stopping membership update processor"); + } + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; + + public void SetMetadata(SiloAddress siloAddress, SiloMetadata metadata) => _metadata.TryAdd(siloAddress, metadata); + + public void Dispose() => _cts.Cancel(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs new file mode 100644 index 0000000000..a8adf8397c --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[GenerateSerializer] +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.SiloMetadata")] +public record SiloMetadata +{ + public static SiloMetadata Empty { get; } = new SiloMetadata(); + + [Id(0)] + public ImmutableDictionary Metadata { get; private set; } = ImmutableDictionary.Empty; + + internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.AddRange(metadata); + internal void AddMetadata(string key, string value) => Metadata = Metadata.Add(key, value); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs new file mode 100644 index 0000000000..377bf30288 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -0,0 +1,16 @@ +using System; +using System.Threading.Tasks; +using Orleans.Runtime.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataClient(IServiceProvider serviceProvider) + : GrainServiceClient(serviceProvider), ISiloMetadataClient +{ + public async Task GetSiloMetadata(SiloAddress siloAddress) + { + var grainService = GetGrainService(siloAddress); + var metadata = await grainService.GetSiloMetadata(); + return metadata; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs new file mode 100644 index 0000000000..336b527458 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -0,0 +1,22 @@ +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService +{ + private readonly SiloMetadata _siloMetadata; + + public SiloMetadataGrainService(IOptions siloMetadata) : base() + { + _siloMetadata = siloMetadata.Value; + } + + public SiloMetadataGrainService(IOptions siloMetadata, GrainId grainId, Silo silo, ILoggerFactory loggerFactory) : base(grainId, silo, loggerFactory) + { + _siloMetadata = siloMetadata.Value; + } + + public Task GetSiloMetadata() => Task.FromResult(_siloMetadata); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs new file mode 100644 index 0000000000..3457bd0984 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration.Internal; +using Orleans.Hosting; +using Orleans.Runtime.Placement.Filtering; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public static class SiloMetadataHostingExtensions +{ + + /// + /// Configure silo metadata from the builder configuration. + /// + /// Silo builder + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder) => builder.UseSiloMetadata(builder.Configuration); + + /// + /// Configure silo metadata from configuration. + /// + /// Silo builder + /// Configuration to pull from + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) + { + + var metadataConfigSection = builder.Configuration.GetSection("ORLEANS").GetSection("METADATA"); + + return builder.UseSiloMetadata(metadataConfigSection); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Configuration section to pull from + /// + /// Get the ORLEANS__METADATA section from config section + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfigurationSection configurationSection) + { + var dictionary = configurationSection.Get>(); + + return builder.UseSiloMetadata(dictionary ?? new Dictionary()); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Metadata to add + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary metadata) + { + builder.ConfigureServices(services => + { + services + .AddOptionsWithValidateOnStart() + .Configure(m => + { + m.AddMetadata(metadata); + }); + + services.AddGrainService(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, SiloMetadataCache>(); + services.AddSingleton(); + // Placement filters + services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); + }); + return builder; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs new file mode 100644 index 0000000000..23b2437771 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Orleans.Runtime.Placement.Filtering; + +public interface IPlacementFilterDirector +{ + IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, + IEnumerable silos); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs new file mode 100644 index 0000000000..b67e23fd10 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Base for all placement filter marker attributes. +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public abstract class PlacementFilterAttribute : Attribute, IGrainPropertiesProviderAttribute +{ + public PlacementFilterStrategy PlacementFilterStrategy { get; private set; } + + protected PlacementFilterAttribute(PlacementFilterStrategy placement) + { + ArgumentNullException.ThrowIfNull(placement); + PlacementFilterStrategy = placement; + } + + /// + public virtual void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + => PlacementFilterStrategy?.PopulateGrainProperties(services, grainClass, grainType, properties); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs new file mode 100644 index 0000000000..8401f5eac7 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -0,0 +1,12 @@ +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterDirectorResolver(IServiceProvider services) +{ + public IPlacementFilterDirector GetFilterDirector(PlacementFilterStrategy placementFilterStrategy) => services.GetRequiredKeyedService(placementFilterStrategy.GetType()); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs new file mode 100644 index 0000000000..faf9367f27 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs @@ -0,0 +1,23 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace Orleans.Runtime.Placement.Filtering; + +public static class PlacementFilterExtensions +{ + /// + /// Configures a for filtering candidate grain placements. + /// + /// The placement filter. + /// The placement filter director. + /// The service collection. + /// The lifetime of the placement strategy. + /// The service collection. + public static void AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) + where TFilter : PlacementFilterStrategy + where TDirector : class, IPlacementFilterDirector + { + services.Add(ServiceDescriptor.DescribeKeyed(typeof(PlacementFilterStrategy), typeof(TFilter).Name, typeof(TFilter), strategyLifetime)); + services.AddKeyedSingleton(typeof(TFilter)); + } + +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs new file mode 100644 index 0000000000..614b42ec99 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public abstract class PlacementFilterStrategy +{ + /// + /// Initializes an instance of this type using the provided grain properties. + /// + /// + /// The grain properties. + /// + public virtual void Initialize(GrainProperties properties) + { + } + + /// + /// Populates grain properties to specify the preferred placement strategy. + /// + /// The service provider. + /// The grain class. + /// The grain type. + /// The grain properties which will be populated by this method call. + public void PopulateGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + { + var typeName = GetType().Name; + if (properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var existingValue)) + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = $"{existingValue},{typeName}"; + } + else + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; + } + + foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) + { + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{additionalGrainProperty.Key}"] = additionalGrainProperty.Value; + } + } + + protected string GetPlacementFilterGrainProperty(string key, GrainProperties properties) + { + var typeName = GetType().Name; + return properties.Properties.TryGetValue($"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{key}", out var value) ? value : null; + } + + protected virtual IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, IReadOnlyDictionary existingProperties) + => Array.Empty>(); +} diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs new file mode 100644 index 0000000000..cb36817056 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterStrategyResolver +{ + private readonly ConcurrentDictionary _resolvedFilters = new(); + private readonly Func _getFiltersInternal; + private readonly GrainPropertiesResolver _grainPropertiesResolver; + private readonly IServiceProvider _services; + + /// + /// Create a instance. + /// + public PlacementFilterStrategyResolver( + IServiceProvider services, + GrainPropertiesResolver grainPropertiesResolver) + { + _services = services; + _getFiltersInternal = GetPlacementFilterStrategyInternal; + _grainPropertiesResolver = grainPropertiesResolver; + } + + /// + /// Gets the placement filter strategy associated with the provided grain type. + /// + public PlacementFilterStrategy[] GetPlacementFilterStrategies(GrainType grainType) => _resolvedFilters.GetOrAdd(grainType, _getFiltersInternal); + + private PlacementFilterStrategy[] GetPlacementFilterStrategyInternal(GrainType grainType) + { + _grainPropertiesResolver.TryGetGrainProperties(grainType, out var properties); + + if (properties is not null + && properties.Properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var placementFilterIds) + && !string.IsNullOrWhiteSpace(placementFilterIds)) + { + var filterList = new List(); + foreach (var filterId in placementFilterIds.Split(",")) + { + var filter = _services.GetKeyedService(filterId); + if (filter is not null) + { + filter.Initialize(properties); + filterList.Add(filter); + } + else + { + throw new KeyNotFoundException($"Could not resolve placement filter strategy {filterId} for grain type {grainType}. Ensure that dependencies for that filter have been configured in the Container. This is often through a .Use* extension method provided by the implementation."); + } + } + return filterList.ToArray(); + } + + return []; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..1c2183fcf1 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,7 @@ +using System; + +namespace Orleans.Runtime.Placement.Filtering; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class PreferredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) + : PlacementFilterAttribute(new PreferredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..e6d08aa060 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class PreferredSiloMetadataPlacementFilterDirector( + ILocalSiloDetails localSiloDetails, + ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var orderedMetadataKeys = (filterStrategy as PreferredSiloMetadataPlacementFilterStrategy)?.OrderedMetadataKeys ?? []; + var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + + if (localSiloMetadata.Count == 0) + { + // yield return all silos if no metadata keys are configured + foreach (var silo in silos) + { + yield return silo; + } + } + else + { + // return the list of silos that match the most metadata keys. The first key in the list is the least important. + // This means that the last key in the list is the most important. + // If no silos match any metadata keys, return the original list of silos. + + var siloList = silos.ToList(); + var maxScore = 0; + var siloScores = new int[siloList.Count]; + for (var i = 0; i < siloList.Count; i++) + { + var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) + { + if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && + localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && + siloMetadataValue == localSiloMetadataValue) + { + var newScore = siloScores[i]++; + maxScore = Math.Max(maxScore, newScore); + } + else + { + break; + } + } + } + + if (maxScore == 0) + { + // yield return all silos if no silos match any metadata keys + foreach (var silo in siloList) + { + yield return silo; + } + } + + // return the list of silos that match the most metadata keys + for (var i = 0; i < siloScores.Length; i++) + { + if (siloScores[i] == maxScore) + { + yield return siloList[i]; + } + } + } + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..6316fd0c0b --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public class PreferredSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys) : PlacementFilterStrategy +{ + public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; + + public PreferredSiloMetadataPlacementFilterStrategy() : this([]) + { + } + + public override void Initialize(GrainProperties properties) + { + base.Initialize(properties); + OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs new file mode 100644 index 0000000000..ba1fd242bd --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; + +namespace Orleans.Runtime.Placement.Filtering; + +internal class RequiredSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var metadataKeys = (filterStrategy as RequiredSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; + + // yield return all silos if no silos match any metadata keys + if (metadataKeys.Length == 0) + { + foreach (var silo in silos) + { + yield return silo; + } + } + else + { + var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); + + foreach (var silo in silos) + { + var remoteMetadata = siloMetadataCache.GetMetadata(silo); + if(DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys)) + { + yield return silo; + } + } + } + } + + private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) + { + for (var i = 0; i < metadataKeys.Length; i++) + { + if(localMetadata[i] != siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i])) + { + return false; + } + } + + return true; + } + private static string[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) + { + var result = new string[metadataKeys.Length]; + for (var i = 0; i < metadataKeys.Length; i++) + { + result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); + } + return result; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..180a409c46 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,7 @@ +using System; + +namespace Orleans.Runtime.Placement.Filtering; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class RequiredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) + : PlacementFilterAttribute(new RequiredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..ac79c94080 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public class RequiredSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +{ + public string[] MetadataKeys { get; private set; } = metadataKeys; + + public RequiredSiloMetadataPlacementFilterStrategy() : this([]) + { + } + + public override void Initialize(GrainProperties properties) + { + base.Initialize(properties); + MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("metadata-keys", String.Join(",", MetadataKeys)); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index cd17709a05..4688ccafeb 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -9,6 +9,7 @@ using Orleans.Configuration; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Internal; +using Orleans.Runtime.Placement.Filtering; using Orleans.Runtime.Versions; namespace Orleans.Runtime.Placement @@ -28,6 +29,8 @@ internal class PlacementService : IPlacementContext private readonly ISiloStatusOracle _siloStatusOracle; private readonly bool _assumeHomogeneousSilosForTesting; private readonly PlacementWorker[] _workers; + private readonly PlacementFilterStrategyResolver _filterStrategyResolver; + private readonly PlacementFilterDirectorResolver _placementFilterDirectoryResolver; /// /// Create a instance. @@ -41,11 +44,15 @@ public PlacementService( GrainVersionManifest grainInterfaceVersions, CachedVersionSelectorManager versionSelectorManager, PlacementDirectorResolver directorResolver, - PlacementStrategyResolver strategyResolver) + PlacementStrategyResolver strategyResolver, + PlacementFilterStrategyResolver filterStrategyResolver, + PlacementFilterDirectorResolver placementFilterDirectoryResolver) { LocalSilo = localSiloDetails.SiloAddress; _strategyResolver = strategyResolver; _directorResolver = directorResolver; + _filterStrategyResolver = filterStrategyResolver; + _placementFilterDirectoryResolver = placementFilterDirectoryResolver; _logger = logger; _grainLocator = grainLocator; _grainInterfaceVersions = grainInterfaceVersions; @@ -117,6 +124,22 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) : _grainInterfaceVersions.GetSupportedSilos(grainType).Result; var compatibleSilos = silos.Intersect(AllActiveSilos).ToArray(); + + + var filters = _filterStrategyResolver.GetPlacementFilterStrategies(grainType); + if (filters.Length > 0) + { + IEnumerable filteredSilos = compatibleSilos; + foreach (var placementFilter in filters) + { + var director = _placementFilterDirectoryResolver.GetFilterDirector(placementFilter); + filteredSilos = director.Filter(placementFilter, target, filteredSilos); + } + + compatibleSilos = filteredSilos.ToArray(); + } + + if (compatibleSilos.Length == 0) { var allWithType = _grainInterfaceVersions.GetSupportedSilos(grainType).Result; From 8aa83aa158fb8d67aa317430af0c6bb2e13c2a57 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:16:32 -0800 Subject: [PATCH 02/20] Remove dead silos from metadata cache --- .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index afdcb39c95..d7bdf011ed 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -52,7 +52,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) { // Add entries for members that aren't already in the cache - foreach (var membershipEntry in update.Entries) + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status != SiloStatus.Dead)) { if (!_metadata.ContainsKey(membershipEntry.Key)) { @@ -68,6 +68,12 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } + // Add entries for members that aren't already in the cache + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status == SiloStatus.Dead)) + { + _metadata.TryRemove(membershipEntry.Key, out _); + } + // Remove entries for members that are no longer in the table foreach (var silo in _metadata.Keys.ToList()) { From d65153c8a1a02eb172967a628738a4a9f7ed570d Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:16:54 -0800 Subject: [PATCH 03/20] Allow overwriting of metadata values --- .../MembershipService/SiloMetadata/SiloMetadata.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs index a8adf8397c..395aa9d3df 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -12,6 +12,6 @@ public record SiloMetadata [Id(0)] public ImmutableDictionary Metadata { get; private set; } = ImmutableDictionary.Empty; - internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.AddRange(metadata); - internal void AddMetadata(string key, string value) => Metadata = Metadata.Add(key, value); + internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.SetItems(metadata); + internal void AddMetadata(string key, string value) => Metadata = Metadata.SetItem(key, value); } \ No newline at end of file From 3ee780aa543bae38c3cac9e32ba22bfc9ae9b5c4 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:17:40 -0800 Subject: [PATCH 04/20] Adding Silo Metadata tests --- .../General/SiloMetadataTests.cs | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 test/TesterInternal/General/SiloMetadataTests.cs diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/General/SiloMetadataTests.cs new file mode 100644 index 0000000000..32540ee52d --- /dev/null +++ b/test/TesterInternal/General/SiloMetadataTests.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("SiloMetadata")] +public class SiloMetadataTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"host.id", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_NewSilosHaveMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_RemovedSiloHasNoMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + + var second = HostedCluster.Silos.Skip(1).First(); + var metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.NotEmpty(metadata.Metadata); + + await HostedCluster.StopSiloAsync(second); + metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_BadSiloAddressHasNoMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var first = this.HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } + + private void AssertAllSiloMetadataMatchesOnAllSilos() + { + var exampleSiloMetadata = new Dictionary(); + var first = this.HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + foreach (var otherSilo in this.HostedCluster.Silos) + { + var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.True(metadata.Metadata.ContainsKey("host.id")); + exampleSiloMetadata.Add(otherSilo.SiloAddress, metadata); + } + foreach (var hostedClusterSilo in this.HostedCluster.Silos.Skip(1)) + { + var sp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var siloMetadataCache = sp.GetRequiredService(); + var remoteMetadata = new Dictionary(); + foreach (var otherSilo in this.HostedCluster.Silos) + { + var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.True(metadata.Metadata.ContainsKey("host.id")); + remoteMetadata.Add(otherSilo.SiloAddress, metadata); + } + //Assert that the two dictionaries have the same keys and the values for those keys are the same + Assert.Equal(exampleSiloMetadata.Count, remoteMetadata.Count); + foreach (var kvp in exampleSiloMetadata) + { + Assert.Equal(kvp.Value.Metadata.Count, remoteMetadata[kvp.Key].Metadata.Count); + foreach (var kvp2 in kvp.Value.Metadata) + { + Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), $"Key '{kvp2.Key}' not found in actual dictionary."); + Assert.Equal(kvp2.Value, value); + } + } + } + } +} \ No newline at end of file From 47899c16cea4014976c65ba918801ad3b71109a3 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 20:19:49 -0800 Subject: [PATCH 05/20] Tests for Grain Placement Filter --- .../General/GrainPlacementFilterTests.cs | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 test/TesterInternal/General/GrainPlacementFilterTests.cs diff --git a/test/TesterInternal/General/GrainPlacementFilterTests.cs b/test/TesterInternal/General/GrainPlacementFilterTests.cs new file mode 100644 index 0000000000..79d59ed2ac --- /dev/null +++ b/test/TesterInternal/General/GrainPlacementFilterTests.cs @@ -0,0 +1,78 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("Placement"), TestCategory("Filters")] +public class GrainPlacementFilterTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.ConfigureServices(services => + { + services.AddPlacementFilter(ServiceLifetime.Singleton); + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = this.Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_FilterIsTriggered() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var triggered = false; + var task = Task.Run(async () => + { + triggered = await TestPlacementFilterDirector.Triggered.WaitAsync(TimeSpan.FromSeconds(1)); + }); + var localOnlyGrain = Client.GetGrain(0); + await localOnlyGrain.Ping(); + await task; + Assert.True(triggered); + } +} + +[TestPlacementFilter] +public class TestFilteredGrain : Grain, ITestFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +public class TestPlacementFilterAttribute() : PlacementFilterAttribute(new TestPlacementFilterStrategy()); + +public class TestPlacementFilterStrategy : PlacementFilterStrategy; + +public class TestPlacementFilterDirector() : IPlacementFilterDirector +{ + public static SemaphoreSlim Triggered { get; } = new(0); + + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + Triggered.Release(1); + return silos; + } +} From 9f4f60d4b5ebbcc5c39144cf80d50a5e7076f63b Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 11:44:05 -0800 Subject: [PATCH 06/20] Tests for silo metadata placement filters --- .../SiloMetadataHostingExtensions.cs | 4 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 15 ++ ...atchSiloMetadataPlacementFilterDirector.cs | 78 ++++++ ...tchSiloMetadataPlacementFilterStrategy.cs} | 14 +- ...redSiloMetadataPlacementFilterAttribute.cs | 7 - ...rredSiloMetadataPlacementFilterDirector.cs | 73 ------ ...equiredMatchSiloMetadataFilterDirector.cs} | 30 +-- ...tchSiloMetadataPlacementFilterAttribute.cs | 13 + ...tchSiloMetadataPlacementFilterStrategy.cs} | 4 +- ...redSiloMetadataPlacementFilterAttribute.cs | 7 - .../SiloMetadataPlacementFilterTests.cs | 247 ++++++++++++++++++ 11 files changed, 380 insertions(+), 112 deletions(-) create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs rename src/Orleans.Runtime/Placement/Filtering/{PreferredSiloMetadataPlacementFilterStrategy.cs => PreferredMatchSiloMetadataPlacementFilterStrategy.cs} (52%) delete mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs delete mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs rename src/Orleans.Runtime/Placement/Filtering/{RequiredSiloMetadataFilterDirector.cs => RequiredMatchSiloMetadataFilterDirector.cs} (58%) create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs rename src/Orleans.Runtime/Placement/Filtering/{RequiredSiloMetadataPlacementFilterStrategy.cs => RequiredMatchSiloMetadataPlacementFilterStrategy.cs} (80%) delete mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs create mode 100644 test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 3457bd0984..7afef23762 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -82,8 +82,8 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary services.AddFromExisting, SiloMetadataCache>(); services.AddSingleton(); // Placement filters - services.AddPlacementFilter(ServiceLifetime.Transient); - services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); }); return builder; } diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..f80955e855 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify the preferred match silo metadata placement filter that preferentially filters down to silos where the metadata matches the local (calling) silo metadata. +/// +/// Ordered set of metadata keys to try to match. The earlier entries are considered less important and will be dropped to find a less-specific match if sufficient more-specific matches do not have enough results. +/// The minimum desired candidates to filter. This is to balance meeting the metadata preferences with not overloading a single or small set of silos with activations. Set this to 1 if you only want the best matches, even if there's only one silo that is currently the best match. +/// Example: If keys ["first","second"] are specified, then it will attempt to return only silos where both keys match the local silo's metadata values. If there are not sufficient silos matching both, then it will also include silos matching only the second key. Finally, if there are still fewer than minCandidates results then it will include all silos. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2) + : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..b847ffbf08 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class PreferredMatchSiloMetadataPlacementFilterDirector( + ILocalSiloDetails localSiloDetails, + ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var preferredMatchSiloMetadataPlacementFilterStrategy = (filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy); + var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1; + var orderedMetadataKeys = preferredMatchSiloMetadataPlacementFilterStrategy?.OrderedMetadataKeys ?? []; + + var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + + if (localSiloMetadata.Count == 0) + { + return silos; + } + + var siloList = silos.ToList(); + if (siloList.Count <= minCandidates) + { + return siloList; + } + + // return the list of silos that match the most metadata keys. The first key in the list is the least important. + // This means that the last key in the list is the most important. + // If no silos match any metadata keys, return the original list of silos. + var maxScore = 0; + var siloScores = new int[siloList.Count]; + var scoreCounts = new int[orderedMetadataKeys.Length+1]; + for (var i = 0; i < siloList.Count; i++) + { + var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + var siloScore = 0; + for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) + { + if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && + localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && + siloMetadataValue == localSiloMetadataValue) + { + siloScore = ++siloScores[i]; + maxScore = Math.Max(maxScore, siloScore); + } + else + { + break; + } + } + scoreCounts[siloScore]++; + } + + if (maxScore == 0) + { + return siloList; + } + + var candidateCount = 0; + var scoreCutOff = orderedMetadataKeys.Length; + for (var i = scoreCounts.Length-1; i >= 0; i--) + { + candidateCount += scoreCounts[i]; + if (candidateCount >= minCandidates) + { + scoreCutOff = i; + break; + } + } + + return siloList.Where((_, i) => siloScores[i] >= scoreCutOff); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs similarity index 52% rename from src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs rename to src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 6316fd0c0b..74c551b5c8 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,11 +4,13 @@ namespace Orleans.Runtime.Placement.Filtering; -public class PreferredSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys) : PlacementFilterStrategy +public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates) + : PlacementFilterStrategy { public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; + public int MinCandidates { get; set; } = minCandidates; - public PreferredSiloMetadataPlacementFilterStrategy() : this([]) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1) { } @@ -16,11 +18,19 @@ public override void Initialize(GrainProperties properties) { base.Initialize(properties); OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); + if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) + { + throw new ArgumentException("Invalid min-candidates property value."); + } + + MinCandidates = parsedMinCandidates; } protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, IReadOnlyDictionary existingProperties) { yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); + yield return new KeyValuePair("min-candidates", MinCandidates.ToString()); } } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs deleted file mode 100644 index 1c2183fcf1..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System; - -namespace Orleans.Runtime.Placement.Filtering; - -[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] -public class PreferredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) - : PlacementFilterAttribute(new PreferredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs deleted file mode 100644 index e6d08aa060..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Orleans.Runtime.MembershipService.SiloMetadata; -#nullable enable -namespace Orleans.Runtime.Placement.Filtering; - -internal class PreferredSiloMetadataPlacementFilterDirector( - ILocalSiloDetails localSiloDetails, - ISiloMetadataCache siloMetadataCache) - : IPlacementFilterDirector -{ - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) - { - var orderedMetadataKeys = (filterStrategy as PreferredSiloMetadataPlacementFilterStrategy)?.OrderedMetadataKeys ?? []; - var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; - - if (localSiloMetadata.Count == 0) - { - // yield return all silos if no metadata keys are configured - foreach (var silo in silos) - { - yield return silo; - } - } - else - { - // return the list of silos that match the most metadata keys. The first key in the list is the least important. - // This means that the last key in the list is the most important. - // If no silos match any metadata keys, return the original list of silos. - - var siloList = silos.ToList(); - var maxScore = 0; - var siloScores = new int[siloList.Count]; - for (var i = 0; i < siloList.Count; i++) - { - var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; - for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) - { - if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && - localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && - siloMetadataValue == localSiloMetadataValue) - { - var newScore = siloScores[i]++; - maxScore = Math.Max(maxScore, newScore); - } - else - { - break; - } - } - } - - if (maxScore == 0) - { - // yield return all silos if no silos match any metadata keys - foreach (var silo in siloList) - { - yield return silo; - } - } - - // return the list of silos that match the most metadata keys - for (var i = 0; i < siloScores.Length; i++) - { - if (siloScores[i] == maxScore) - { - yield return siloList[i]; - } - } - } - } -} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs similarity index 58% rename from src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs index ba1fd242bd..4b5a3f85f4 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs @@ -4,35 +4,27 @@ namespace Orleans.Runtime.Placement.Filtering; -internal class RequiredSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) +internal class RequiredMatchSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) { - var metadataKeys = (filterStrategy as RequiredSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; + var metadataKeys = (filterStrategy as RequiredMatchSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; // yield return all silos if no silos match any metadata keys if (metadataKeys.Length == 0) { - foreach (var silo in silos) - { - yield return silo; - } + return silos; } - else + + var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); + + return silos.Where(silo => { - var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); - var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); - - foreach (var silo in silos) - { - var remoteMetadata = siloMetadataCache.GetMetadata(silo); - if(DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys)) - { - yield return silo; - } - } - } + var remoteMetadata = siloMetadataCache.GetMetadata(silo); + return DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys); + }); } private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..2bdda2645f --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,13 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify that a silo must have a specific metadata key-value pair matching the local (calling) silo to be considered for placement. +/// +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys) + : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs similarity index 80% rename from src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index ac79c94080..fc0ca730fe 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,11 +4,11 @@ namespace Orleans.Runtime.Placement.Filtering; -public class RequiredSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy { public string[] MetadataKeys { get; private set; } = metadataKeys; - public RequiredSiloMetadataPlacementFilterStrategy() : this([]) + public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([]) { } diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs deleted file mode 100644 index 180a409c46..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System; - -namespace Orleans.Runtime.Placement.Filtering; - -[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] -public class RequiredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) - : PlacementFilterAttribute(new RequiredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs new file mode 100644 index 0000000000..8a3604a2c6 --- /dev/null +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -0,0 +1,247 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement.Filtering; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("Placement"), TestCategory("Filters")] +public class SiloMetadataPlacementFilterTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"first", "1"}, + {"second", "2"}, + {"third", "3"}, + {"unique", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = this.Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_RequiredFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo because it is the only one that matches. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// If no metadata key is defined then it should fall back to matching all silos + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } +} + +public interface IUniqueRequiredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[RequiredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class UniqueRequiredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IUniqueRequiredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} +public interface IPreferredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"], 1)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + + +public interface IPreferredMatchMin2FilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMinTwoFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMin2FilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["not.there"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchNoMetadataFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchNoMetadataFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +public interface IPreferredMatchNoMetadataFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} \ No newline at end of file From d715fdd5e99ce63a2ef4ab4dd51b5b37cd897268 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 11:45:07 -0800 Subject: [PATCH 07/20] Adding test category --- test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs index 8a3604a2c6..7b765c7b52 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -8,7 +8,7 @@ namespace UnitTests.General; -[TestCategory("Placement"), TestCategory("Filters")] +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] public class SiloMetadataPlacementFilterTests : TestClusterPerTest { protected override void ConfigureTestCluster(TestClusterBuilder builder) From 814a3bab3e4de811698d1a3f94ea538b8119bb73 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 12:21:47 -0800 Subject: [PATCH 08/20] Testing loading silo metadata from config --- .../SiloMetadataHostingExtensions.cs | 6 +- .../SiloMetadataPlacementFilterTests.cs | 1 - .../General/SiloMetadataTests.cs | 92 ++++++++++++++++--- 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 7afef23762..1530d0316e 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -28,15 +28,15 @@ public static class SiloMetadataHostingExtensions /// Silo builder /// Configuration to pull from /// - /// Get the ORLEANS__METADATA section from config + /// Get the Orleans:Metadata section from config /// Key/value pairs in configuration as a will look like this as environment variables: - /// ORLEANS__METADATA__key1=value1 + /// Orleans:Metadata:key1=value1 /// /// public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) { - var metadataConfigSection = builder.Configuration.GetSection("ORLEANS").GetSection("METADATA"); + var metadataConfigSection = configuration.GetSection("Orleans").GetSection("Metadata"); return builder.UseSiloMetadata(metadataConfigSection); } diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs index 7b765c7b52..15fcce09a1 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.Runtime.Placement.Filtering; using Orleans.TestingHost; diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/General/SiloMetadataTests.cs index 32540ee52d..fbd847c5de 100644 --- a/test/TesterInternal/General/SiloMetadataTests.cs +++ b/test/TesterInternal/General/SiloMetadataTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.TestingHost; @@ -6,6 +7,59 @@ namespace UnitTests.General; + +[TestCategory("SiloMetadata")] +public class SiloMetadataConfigTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public static readonly List> Metadata = + [ + new("Orleans:Metadata:first", "1"), + new("Orleans:Metadata:second", "2"), + new("Orleans:Metadata:third", "3") + ]; + + public void Configure(ISiloBuilder hostBuilder) + { + var config = new ConfigurationBuilder() + .AddInMemoryCollection(Metadata) + .Build(); + hostBuilder.UseSiloMetadata(config); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_HasConfiguredValues() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(first.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.Equal(SiloConfigurator.Metadata.Count, metadata.Metadata.Count); + foreach (var kv in SiloConfigurator.Metadata) + { + Assert.Equal(kv.Value, metadata.Metadata[kv.Key.Split(':').Last()]); + } + } +} + [TestCategory("SiloMetadata")] public class SiloMetadataTests : TestClusterPerTest { @@ -29,7 +83,7 @@ public void Configure(ISiloBuilder hostBuilder) public async Task SiloMetadata_CanBeSetAndRead() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] @@ -37,14 +91,14 @@ public async Task SiloMetadata_NewSilosHaveMetadata() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_RemovedSiloHasNoMetadata() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); @@ -71,34 +125,45 @@ public async Task SiloMetadata_BadSiloAddressHasNoMetadata() Assert.NotNull(metadata); Assert.Empty(metadata.Metadata); } +} - private void AssertAllSiloMetadataMatchesOnAllSilos() +public static class SiloMetadataTestExtensions +{ + public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hostedCluster, string[] expectedKeys) { var exampleSiloMetadata = new Dictionary(); - var first = this.HostedCluster.Silos.First(); - var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var first = hostedCluster.Silos.First(); + var firstSp = hostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); - foreach (var otherSilo in this.HostedCluster.Silos) + foreach (var otherSilo in hostedCluster.Silos) { var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); - Assert.True(metadata.Metadata.ContainsKey("host.id")); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } exampleSiloMetadata.Add(otherSilo.SiloAddress, metadata); } - foreach (var hostedClusterSilo in this.HostedCluster.Silos.Skip(1)) + + foreach (var hostedClusterSilo in hostedCluster.Silos.Skip(1)) { - var sp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var sp = hostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var siloMetadataCache = sp.GetRequiredService(); var remoteMetadata = new Dictionary(); - foreach (var otherSilo in this.HostedCluster.Silos) + foreach (var otherSilo in hostedCluster.Silos) { var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); - Assert.True(metadata.Metadata.ContainsKey("host.id")); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } remoteMetadata.Add(otherSilo.SiloAddress, metadata); } + //Assert that the two dictionaries have the same keys and the values for those keys are the same Assert.Equal(exampleSiloMetadata.Count, remoteMetadata.Count); foreach (var kvp in exampleSiloMetadata) @@ -106,7 +171,8 @@ private void AssertAllSiloMetadataMatchesOnAllSilos() Assert.Equal(kvp.Value.Metadata.Count, remoteMetadata[kvp.Key].Metadata.Count); foreach (var kvp2 in kvp.Value.Metadata) { - Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), $"Key '{kvp2.Key}' not found in actual dictionary."); + Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), + $"Key '{kvp2.Key}' not found in actual dictionary."); Assert.Equal(kvp2.Value, value); } } From d969a8564d300b2070c4ca6e924e8abcaa3984e9 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 13:41:07 -0800 Subject: [PATCH 09/20] Required and Preferred match filtering unit tests --- .../SiloMetadataHostingExtensions.cs | 2 +- ...tchSiloMetadataPlacementFilterDirector.cs} | 2 +- .../GrainPlacementFilterTests.cs | 6 +- ...iloMetadataPlacementFilterDirectorTests.cs | 176 ++++++++++++++++++ ...iloMetadataPlacementFilterDirectorTests.cs | 141 ++++++++++++++ .../SiloMetadataPlacementFilterTests.cs | 40 ++-- .../TestLocalSiloDetails.cs | 19 ++ .../TestSiloMetadataCache.cs | 15 ++ .../SiloMetadataTests.cs | 16 +- test/TesterInternal/TesterInternal.csproj | 2 +- 10 files changed, 385 insertions(+), 34 deletions(-) rename src/Orleans.Runtime/Placement/Filtering/{RequiredMatchSiloMetadataFilterDirector.cs => RequiredMatchSiloMetadataPlacementFilterDirector.cs} (92%) rename test/TesterInternal/{General => PlacementFilterTests}/GrainPlacementFilterTests.cs (92%) create mode 100644 test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs create mode 100644 test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs rename test/TesterInternal/{General => PlacementFilterTests}/SiloMetadataPlacementFilterTests.cs (90%) create mode 100644 test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs create mode 100644 test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs rename test/TesterInternal/{General => SiloMetadataTests}/SiloMetadataTests.cs (93%) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 1530d0316e..200577e263 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -83,7 +83,7 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary services.AddSingleton(); // Placement filters services.AddPlacementFilter(ServiceLifetime.Transient); - services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); }); return builder; } diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs similarity index 92% rename from src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index 4b5a3f85f4..ffbc4782f6 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -4,7 +4,7 @@ namespace Orleans.Runtime.Placement.Filtering; -internal class RequiredMatchSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) +internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) diff --git a/test/TesterInternal/General/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs similarity index 92% rename from test/TesterInternal/General/GrainPlacementFilterTests.cs rename to test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index 79d59ed2ac..9f34e11e9b 100644 --- a/test/TesterInternal/General/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters")] public class GrainPlacementFilterTests : TestClusterPerTest @@ -29,8 +29,8 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = this.Client.GetGrain(0); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..9274d75f96 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,176 @@ +using System.Net; +using Orleans.Metadata; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class PreferredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + { testLocalSiloAddress, SiloMetadata.Empty } + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void FiltersToAllWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match")] + [InlineData(2, 3, "no.match")] + [InlineData( 1, 1, "one.match")] + [InlineData( 2, 3, "one.match")] + [InlineData( 1, 2, "two.match")] + [InlineData( 2, 2, "two.match")] + [InlineData( 3, 3, "two.match")] + [InlineData( 1, 3, "all.match")] + [InlineData( 2, 3, "all.match")] + + public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string key) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "nomatch"); + otherSiloMetadata3.AddMetadata("two.match", "nomatch"); + otherSiloMetadata3.AddMetadata("no.match", "nomatch"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "two.match")] + [InlineData(2, 2, "no.match", "one.match", "two.match")] + [InlineData(3, 3, "no.match", "one.match", "two.match")] + [InlineData(1, 3, "all.match", "no.match")] + [InlineData(1, 1, "one.match", "all.match")] + [InlineData(2, 3, "one.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "all.match")] + [InlineData(2, 3, "no.match", "one.match", "all.match")] + + public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, params string[] keys) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "not.match"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "not.match"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "not.match"); + otherSiloMetadata3.AddMetadata("two.match", "not.match"); + otherSiloMetadata3.AddMetadata("no.match", "not.match"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..f63afed282 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class RequiredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenDifferentValue() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "local"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "other"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSilosWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("metadata.key", "same"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(2, result.Count); + } + +} \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs similarity index 90% rename from test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs rename to test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs index 15fcce09a1..c19aff7d82 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] public class SiloMetadataPlacementFilterTests : TestClusterPerTest @@ -32,8 +32,8 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = this.Client.GetGrain(0); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } @@ -45,11 +45,11 @@ public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_RequiredFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -69,11 +69,11 @@ public async Task PlacementFilter_RequiredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -93,8 +93,8 @@ public async Task PlacementFilter_PreferredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -102,7 +102,7 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -127,10 +127,10 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -138,7 +138,7 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -163,10 +163,10 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -174,7 +174,7 @@ public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); diff --git a/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs new file mode 100644 index 0000000000..849480f0e9 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs @@ -0,0 +1,19 @@ +namespace UnitTests.PlacementFilterTests; + +internal class TestLocalSiloDetails : ILocalSiloDetails +{ + public TestLocalSiloDetails(string name, string clusterId, string dnsHostName, SiloAddress siloAddress, SiloAddress gatewayAddress) + { + Name = name; + ClusterId = clusterId; + DnsHostName = dnsHostName; + SiloAddress = siloAddress; + GatewayAddress = gatewayAddress; + } + + public string Name { get; } + public string ClusterId { get; } + public string DnsHostName { get; } + public SiloAddress SiloAddress { get; } + public SiloAddress GatewayAddress { get; } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs new file mode 100644 index 0000000000..a6ab78dda4 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs @@ -0,0 +1,15 @@ +using Orleans.Runtime.MembershipService.SiloMetadata; + +namespace UnitTests.PlacementFilterTests; + +internal class TestSiloMetadataCache : ISiloMetadataCache +{ + private readonly Dictionary _metadata; + + public TestSiloMetadataCache(Dictionary metadata) + { + _metadata = metadata; + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? new SiloMetadata(); +} \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs similarity index 93% rename from test/TesterInternal/General/SiloMetadataTests.cs rename to test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs index fbd847c5de..30cfc94ec3 100644 --- a/test/TesterInternal/General/SiloMetadataTests.cs +++ b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.SiloMetadataTests; [TestCategory("SiloMetadata")] @@ -37,14 +37,14 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task SiloMetadata_CanBeSetAndRead() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_HasConfiguredValues() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); @@ -82,14 +82,14 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task SiloMetadata_CanBeSetAndRead() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_NewSilosHaveMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } @@ -97,7 +97,7 @@ public async Task SiloMetadata_NewSilosHaveMetadata() [Fact, TestCategory("Functional")] public async Task SiloMetadata_RemovedSiloHasNoMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); @@ -117,8 +117,8 @@ public async Task SiloMetadata_RemovedSiloHasNoMetadata() [Fact, TestCategory("Functional")] public async Task SiloMetadata_BadSiloAddressHasNoMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var first = this.HostedCluster.Silos.First(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); diff --git a/test/TesterInternal/TesterInternal.csproj b/test/TesterInternal/TesterInternal.csproj index ceefb25f91..0adc91be07 100644 --- a/test/TesterInternal/TesterInternal.csproj +++ b/test/TesterInternal/TesterInternal.csproj @@ -1,4 +1,4 @@ - + UnitTests true From 9b44825c160a2c13e5a9e1863a34be828b34750e Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 13:52:10 -0800 Subject: [PATCH 10/20] Testing multiple metadata keys in attribute --- .../SiloMetadataPlacementFilterTests.cs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs index c19aff7d82..e0fca917bb 100644 --- a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -120,6 +120,40 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() } } + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + /// /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. /// @@ -232,6 +266,19 @@ public class PreferredMatchMinTwoFilteredGrain(ILocalSiloDetails localSiloDetail public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); } +public interface IPreferredMatchMultipleFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique", "other"], 2)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMultipleFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMultipleFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + #pragma warning disable ORLEANSEXP004 [PreferredMatchSiloMetadataPlacementFilter(["not.there"])] #pragma warning restore ORLEANSEXP004 From 49e12bdb3c7c314f897f842a39cbb3a014e78efe Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Tue, 7 Jan 2025 15:38:02 -0800 Subject: [PATCH 11/20] Ordering support for filters --- .../Filtering/PlacementFilterStrategy.cs | 25 ++- .../PlacementFilterStrategyResolver.cs | 10 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 4 +- ...atchSiloMetadataPlacementFilterStrategy.cs | 9 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 4 +- ...atchSiloMetadataPlacementFilterStrategy.cs | 8 +- .../GrainPlacementFilterTests.cs | 195 +++++++++++++++++- ...iloMetadataPlacementFilterDirectorTests.cs | 6 +- ...iloMetadataPlacementFilterDirectorTests.cs | 8 +- 9 files changed, 244 insertions(+), 25 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs index 614b42ec99..28ffaf202d 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs @@ -6,14 +6,35 @@ namespace Orleans.Runtime.Placement.Filtering; public abstract class PlacementFilterStrategy { + public int Order { get; private set; } + + protected PlacementFilterStrategy(int order) + { + Order = order; + } + /// /// Initializes an instance of this type using the provided grain properties. /// /// /// The grain properties. /// - public virtual void Initialize(GrainProperties properties) + public void Initialize(GrainProperties properties) { + var orderProperty = GetPlacementFilterGrainProperty("order", properties); + if (!int.TryParse(orderProperty, out var parsedOrder)) + { + throw new ArgumentException("Invalid order property value."); + } + + Order = parsedOrder; + + AdditionalInitialize(properties); + } + + public virtual void AdditionalInitialize(GrainProperties properties) + { + } /// @@ -35,6 +56,8 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass, properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; } + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(); + foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) { properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{additionalGrainProperty.Key}"] = additionalGrainProperty.Value; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index cb36817056..97c9aa3705 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.DependencyInjection; using Orleans.Metadata; @@ -55,7 +56,14 @@ private PlacementFilterStrategy[] GetPlacementFilterStrategyInternal(GrainType g throw new KeyNotFoundException($"Could not resolve placement filter strategy {filterId} for grain type {grainType}. Ensure that dependencies for that filter have been configured in the Container. This is often through a .Use* extension method provided by the implementation."); } } - return filterList.ToArray(); + + var orderedFilters = filterList.OrderBy(f => f.Order).ToArray(); + // check that the order is unique + if (orderedFilters.Select(f => f.Order).Distinct().Count() != orderedFilters.Length) + { + throw new InvalidOperationException($"Placement filters for grain type {grainType} have duplicate order values. Order values must be specified if more than one filter is applied and must be unique."); + } + return orderedFilters; } return []; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index f80955e855..8d5afcfce2 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -11,5 +11,5 @@ namespace Orleans.Runtime.Placement.Filtering; /// Example: If keys ["first","second"] are specified, then it will attempt to return only silos where both keys match the local silo's metadata values. If there are not sufficient silos matching both, then it will also include silos matching only the second key. Finally, if there are still fewer than minCandidates results then it will include all silos. [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] [Experimental("ORLEANSEXP004")] -public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2) - : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates)); \ No newline at end of file +public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2, int order = 0) + : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 74c551b5c8..772076c7c9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,19 +4,18 @@ namespace Orleans.Runtime.Placement.Filtering; -public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates) - : PlacementFilterStrategy +public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates, int order) + : PlacementFilterStrategy(order) { public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; public int MinCandidates { get; set; } = minCandidates; - public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1, 0) { } - public override void Initialize(GrainProperties properties) + public override void AdditionalInitialize(GrainProperties properties) { - base.Initialize(properties); OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 2bdda2645f..8db4f0f675 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -9,5 +9,5 @@ namespace Orleans.Runtime.Placement.Filtering; /// [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] [Experimental("ORLEANSEXP004")] -public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys) - : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys)); \ No newline at end of file +public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys, int order = 0) + : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index fc0ca730fe..db3a8fac73 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,17 +4,17 @@ namespace Orleans.Runtime.Placement.Filtering; -public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys, int order) + : PlacementFilterStrategy(order) { public string[] MetadataKeys { get; private set; } = metadataKeys; - public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([]) + public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([], 0) { } - public override void Initialize(GrainProperties properties) + public override void AdditionalInitialize(GrainProperties properties) { - base.Initialize(properties); MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); } diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index 9f34e11e9b..c38730c0d0 100644 --- a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Filtering; @@ -10,6 +11,9 @@ namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters")] public class GrainPlacementFilterTests : TestClusterPerTest { + public static Dictionary> FilterScratchpad = new(); + private static Random random = new(); + protected override void ConfigureTestCluster(TestClusterBuilder builder) { builder.AddSiloBuilderConfigurator(); @@ -22,10 +26,13 @@ public void Configure(ISiloBuilder hostBuilder) hostBuilder.ConfigureServices(services => { services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); }); } } + [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { @@ -49,9 +56,79 @@ public async Task PlacementFilter_FilterIsTriggered() await task; Assert.True(triggered); } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_DuplicateOrder() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await Assert.ThrowsAsync(async () => + { + await testGrain.Ping(); + }); + } } -[TestPlacementFilter] +[TestPlacementFilter(order: 1)] public class TestFilteredGrain : Grain, ITestFilteredGrain { public Task Ping() => Task.CompletedTask; @@ -62,9 +139,14 @@ public interface ITestFilteredGrain : IGrainWithIntegerKey Task Ping(); } -public class TestPlacementFilterAttribute() : PlacementFilterAttribute(new TestPlacementFilterStrategy()); +public class TestPlacementFilterAttribute(int order) : PlacementFilterAttribute(new TestPlacementFilterStrategy(order)); -public class TestPlacementFilterStrategy : PlacementFilterStrategy; +public class TestPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public TestPlacementFilterStrategy() : this(0) + { + } +} public class TestPlacementFilterDirector() : IPlacementFilterDirector { @@ -76,3 +158,110 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P return silos; } } + + + +public class OrderAPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderAPlacementFilterStrategy(order)); + +public class OrderAPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public OrderAPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderAPlacementFilterDirector : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("A"); + return silos; + } +} + + +public class OrderBPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderBPlacementFilterStrategy(order)); + +public class OrderBPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + + public OrderBPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderBPlacementFilterDirector() : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("B"); + return silos; + } +} + +[OrderAPlacementFilter(order: 1)] +[OrderBPlacementFilter(order: 2)] +public class TestAB12FilteredGrain : Grain, ITestAB12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderAPlacementFilter(order: 2)] +[OrderBPlacementFilter(order: 1)] +public class TestAB21FilteredGrain : Grain, ITestAB21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderBPlacementFilter(order: 1)] +[OrderAPlacementFilter(order: 2)] +public class TestBA12FilteredGrain : Grain, ITestBA12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 1)] +public class TestBA121FilteredGrain : Grain, ITestBA21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 2)] +public class TestDuplicateOrderFilteredGrain : Grain, ITestDuplicateOrderFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestDuplicateOrderFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs index 9274d75f96..369829fb7b 100644 --- a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -60,7 +60,7 @@ public void FiltersToAllWhenNoEntry() {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1, 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -114,7 +114,7 @@ public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates, 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); @@ -168,7 +168,7 @@ public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, para {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates, 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs index f63afed282..19d03626c9 100644 --- a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -59,7 +59,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -82,7 +82,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -105,7 +105,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMa {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -132,7 +132,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSi {testOtherSiloAddress2, otherSiloMetadata2}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); Assert.NotEmpty(result); Assert.Equal(2, result.Count); From 6aa01d5c1039ef1a5d8d3f9cd8cc200591caca6e Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 08:44:52 -0800 Subject: [PATCH 12/20] Moving reusable types to Core projects Orleans.Core.Abstractions: - PlacementFilterAttribute - PlacementFilterStrategy Orelans.Core: - IPlacementFilterDirector - PlacementFilterExtensions --- .../Placement}/PlacementFilterAttribute.cs | 3 ++- .../Placement}/PlacementFilterStrategy.cs | 3 ++- .../Placement}/IPlacementFilterDirector.cs | 4 +++- .../Placement}/PlacementFilterExtensions.cs | 2 +- .../SiloMetadata/SiloMetadataHostingExtensions.cs | 1 + .../Placement/Filtering/PlacementFilterDirectorResolver.cs | 1 + .../Placement/Filtering/PlacementFilterStrategyResolver.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterDirector.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterStrategy.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterDirector.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterStrategy.cs | 1 + .../PlacementFilterTests/GrainPlacementFilterTests.cs | 3 +-- 14 files changed, 18 insertions(+), 6 deletions(-) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core.Abstractions/Placement}/PlacementFilterAttribute.cs (94%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core.Abstractions/Placement}/PlacementFilterStrategy.cs (98%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core/Placement}/IPlacementFilterDirector.cs (72%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core/Placement}/PlacementFilterExtensions.cs (95%) diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs similarity index 94% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs rename to src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs index b67e23fd10..9cb03476db 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Runtime; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; /// /// Base for all placement filter marker attributes. diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs similarity index 98% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs rename to src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs index 28ffaf202d..1ec787803b 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Runtime; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public abstract class PlacementFilterStrategy { diff --git a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs similarity index 72% rename from src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs rename to src/Orleans.Core/Placement/IPlacementFilterDirector.cs index 23b2437771..18c9486ed5 100644 --- a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -1,6 +1,8 @@ using System.Collections.Generic; +using Orleans.Runtime; +using Orleans.Runtime.Placement; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public interface IPlacementFilterDirector { diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs similarity index 95% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs rename to src/Orleans.Core/Placement/PlacementFilterExtensions.cs index faf9367f27..05805ed15f 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -1,6 +1,6 @@ using Microsoft.Extensions.DependencyInjection; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public static class PlacementFilterExtensions { diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 200577e263..cb3c8f5d3b 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Configuration.Internal; using Orleans.Hosting; +using Orleans.Placement; using Orleans.Runtime.Placement.Filtering; namespace Orleans.Runtime.MembershipService.SiloMetadata; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs index 8401f5eac7..167daff151 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -1,5 +1,6 @@ using System; using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index 97c9aa3705..0c462b1fdd 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -4,6 +4,7 @@ using System.Linq; using Microsoft.Extensions.DependencyInjection; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index 8d5afcfce2..b8c6ca356e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index b847ffbf08..24a9ba7c1e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 772076c7c9..acef5ecb16 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 8db4f0f675..3c929dd297 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index ffbc4782f6..df3e579fbc 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index db3a8fac73..413f218af9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index c38730c0d0..ce80d6ae5b 100644 --- a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -1,7 +1,6 @@ -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; using Orleans.Runtime.Placement; -using Orleans.Runtime.Placement.Filtering; using Orleans.TestingHost; using TestExtensions; using Xunit; From 27ddb5ed223351edfa195e678be63dd30d4abed0 Mon Sep 17 00:00:00 2001 From: rkargMsft <164392675+rkargMsft@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:55:22 -0800 Subject: [PATCH 13/20] Correct comment Co-authored-by: dmorganMsft --- .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index d7bdf011ed..bc594fb928 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -68,7 +68,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } - // Add entries for members that aren't already in the cache + // Remove entries for members that are now dead foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status == SiloStatus.Dead)) { _metadata.TryRemove(membershipEntry.Key, out _); From c6905be5565df0d756979a74da883a50be4612d9 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 13:57:13 -0800 Subject: [PATCH 14/20] Correcting documentation --- .../SiloMetadata/SiloMetadataHostingExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index cb3c8f5d3b..02a0bb8cb9 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -29,9 +29,9 @@ public static class SiloMetadataHostingExtensions /// Silo builder /// Configuration to pull from /// - /// Get the Orleans:Metadata section from config + /// Get the ORLEANS__METADATA section from config /// Key/value pairs in configuration as a will look like this as environment variables: - /// Orleans:Metadata:key1=value1 + /// ORLEANS__METADATA__key1=value1 /// /// public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) From 0bc5dedb6d3e5d2c88f6cf746557303cb44d8525 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 13:59:52 -0800 Subject: [PATCH 15/20] Allowing chaining extension method --- src/Orleans.Core/Placement/PlacementFilterExtensions.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs index 05805ed15f..087d81fbb0 100644 --- a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -12,12 +12,14 @@ public static class PlacementFilterExtensions /// The service collection. /// The lifetime of the placement strategy. /// The service collection. - public static void AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) + public static IServiceCollection AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) where TFilter : PlacementFilterStrategy where TDirector : class, IPlacementFilterDirector { services.Add(ServiceDescriptor.DescribeKeyed(typeof(PlacementFilterStrategy), typeof(TFilter).Name, typeof(TFilter), strategyLifetime)); services.AddKeyedSingleton(typeof(TFilter)); + + return services; } } \ No newline at end of file From 3c8a9626f073e5747dd35e40e17664a4874d9725 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 14:54:13 -0800 Subject: [PATCH 16/20] #nullable enable for new files --- .../Placement/PlacementFilterAttribute.cs | 1 + .../Placement/PlacementFilterStrategy.cs | 3 ++- src/Orleans.Core/Placement/IPlacementFilterDirector.cs | 1 + .../Placement/PlacementFilterExtensions.cs | 1 + .../SiloMetadata/ISiloMetadataCache.cs | 3 ++- .../SiloMetadata/ISiloMetadataClient.cs | 1 + .../SiloMetadata/ISiloMetadataGrainService.cs | 1 + .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 3 ++- .../MembershipService/SiloMetadata/SiloMetadata.cs | 1 + .../SiloMetadata/SiloMetadataClient.cs | 1 + .../SiloMetadata/SiloMetadataGrainService.cs | 1 + .../SiloMetadata/SiloMetadataHostingExtensions.cs | 1 + .../Filtering/PlacementFilterDirectorResolver.cs | 1 + .../Filtering/PlacementFilterStrategyResolver.cs | 1 + ...eferredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + ...referredMatchSiloMetadataPlacementFilterDirector.cs | 1 + ...referredMatchSiloMetadataPlacementFilterStrategy.cs | 10 ++++++++-- ...equiredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + ...RequiredMatchSiloMetadataPlacementFilterDirector.cs | 7 ++++--- ...RequiredMatchSiloMetadataPlacementFilterStrategy.cs | 8 +++++++- 20 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs index 9cb03476db..0203d6439b 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Runtime; +#nullable enable namespace Orleans.Placement; /// diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs index 1ec787803b..eede6af61b 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Runtime; +#nullable enable namespace Orleans.Placement; public abstract class PlacementFilterStrategy @@ -65,7 +66,7 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass, } } - protected string GetPlacementFilterGrainProperty(string key, GrainProperties properties) + protected string? GetPlacementFilterGrainProperty(string key, GrainProperties properties) { var typeName = GetType().Name; return properties.Properties.TryGetValue($"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{key}", out var value) ? value : null; diff --git a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs index 18c9486ed5..fe9a225a0e 100644 --- a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -2,6 +2,7 @@ using Orleans.Runtime; using Orleans.Runtime.Placement; +#nullable enable namespace Orleans.Placement; public interface IPlacementFilterDirector diff --git a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs index 087d81fbb0..7592656607 100644 --- a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; +#nullable enable namespace Orleans.Placement; public static class PlacementFilterExtensions diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs index a15cb67a36..96f5d5bbfb 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -1,5 +1,6 @@ -namespace Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + public interface ISiloMetadataCache { SiloMetadata GetMetadata(SiloAddress siloAddress); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs index eb622eaaa4..e2ab060bea 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Orleans.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public interface ISiloMetadataClient : IGrainServiceClient diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs index 97e3c7e600..06121ae40c 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Orleans.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; [Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index bc594fb928..8483c570a1 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -8,8 +8,9 @@ using Orleans.Configuration; using Orleans.Internal; -namespace Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + internal class SiloMetadataCache( ISiloMetadataClient siloMetadataClient, MembershipTableManager membershipTableManager, diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs index 395aa9d3df..278a7aa76a 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Collections.Immutable; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; [GenerateSerializer] diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs index 377bf30288..5d507bde72 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using Orleans.Runtime.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public class SiloMetadataClient(IServiceProvider serviceProvider) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs index 336b527458..8e60bff08d 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 02a0bb8cb9..a2adf33f55 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -6,6 +6,7 @@ using Orleans.Placement; using Orleans.Runtime.Placement.Filtering; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public static class SiloMetadataHostingExtensions diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs index 167daff151..f6ef6cba0a 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index 0c462b1fdd..41524681cf 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -6,6 +6,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index b8c6ca356e..bde94878c9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index 24a9ba7c1e..7b92a41c0e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -3,6 +3,7 @@ using System.Linq; using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; + #nullable enable namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index acef5ecb16..e6a160adef 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates, int order) @@ -11,13 +12,18 @@ public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedM public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; public int MinCandidates { get; set; } = minCandidates; - public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1, 0) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 2, 0) { } public override void AdditionalInitialize(GrainProperties properties) { - OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid ordered-metadata-keys property value."); + } + OrderedMetadataKeys = placementFilterGrainProperty.Split(","); var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) { diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 3c929dd297..ec317ad2c6 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index df3e579fbc..91ed8cbac7 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -3,6 +3,7 @@ using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) @@ -28,7 +29,7 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P }); } - private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) + private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) { for (var i = 0; i < metadataKeys.Length; i++) { @@ -40,9 +41,9 @@ private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloM return true; } - private static string[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) + private static string?[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) { - var result = new string[metadataKeys.Length]; + var result = new string?[metadataKeys.Length]; for (var i = 0; i < metadataKeys.Length; i++) { result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index 413f218af9..c2c19a995b 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys, int order) @@ -16,7 +17,12 @@ public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([], 0) public override void AdditionalInitialize(GrainProperties properties) { - MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid metadata-keys property value."); + } + MetadataKeys = placementFilterGrainProperty.Split(","); } protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, From 005d8c6413ac2c72ded4b8784490ac1a45351f7d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Feb 2025 11:04:36 -0800 Subject: [PATCH 17/20] Review suggestions --- .../Placement/PlacementFilterAttribute.cs | 3 + .../Placement/PlacementFilterStrategy.cs | 7 +- src/Orleans.Core/Runtime/Constants.cs | 8 +- .../DistributedGrainDirectory.cs | 2 +- .../GrainDirectory/GrainDirectoryPartition.cs | 4 +- .../SiloMetadata/ISiloMetadataCache.cs | 3 +- .../SiloMetadata/ISiloMetadataClient.cs | 3 +- ...ervice.cs => ISiloMetadataSystemTarget.cs} | 5 +- .../SiloMetadata/SiloMetadaCache.cs | 38 +++--- .../SiloMetadata/SiloMetadataClient.cs | 9 +- .../SiloMetadata/SiloMetadataGrainService.cs | 31 +++-- .../SiloMetadataHostingExtensions.cs | 12 +- ...atchSiloMetadataPlacementFilterDirector.cs | 6 +- ...atchSiloMetadataPlacementFilterStrategy.cs | 4 +- ...atchSiloMetadataPlacementFilterDirector.cs | 9 +- .../Placement/PlacementService.cs | 2 - src/Orleans.TestingHost/InProcTestCluster.cs | 78 +++++++++-- .../SiloMetadataPlacementFilterTests.cs | 75 +++++----- .../TestSiloMetadataCache.cs | 2 +- .../SiloMetadataTests/SiloMetadataTests.cs | 129 ++++++++---------- 20 files changed, 246 insertions(+), 184 deletions(-) rename src/Orleans.Runtime/MembershipService/SiloMetadata/{ISiloMetadataGrainService.cs => ISiloMetadataSystemTarget.cs} (70%) diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs index 0203d6439b..e4dadbd435 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -12,6 +12,9 @@ namespace Orleans.Placement; [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public abstract class PlacementFilterAttribute : Attribute, IGrainPropertiesProviderAttribute { + /// + /// Gets the placement filter strategy. + /// public PlacementFilterStrategy PlacementFilterStrategy { get; private set; } protected PlacementFilterAttribute(PlacementFilterStrategy placement) diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs index eede6af61b..e695a67e4e 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -1,11 +1,15 @@ using System; using System.Collections.Generic; +using System.Globalization; using Orleans.Metadata; using Orleans.Runtime; #nullable enable namespace Orleans.Placement; +/// +/// Represents a strategy for filtering silos which a grain can be placed on. +/// public abstract class PlacementFilterStrategy { public int Order { get; private set; } @@ -36,7 +40,6 @@ public void Initialize(GrainProperties properties) public virtual void AdditionalInitialize(GrainProperties properties) { - } /// @@ -58,7 +61,7 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass, properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; } - properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(); + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(CultureInfo.InvariantCulture); foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) { diff --git a/src/Orleans.Core/Runtime/Constants.cs b/src/Orleans.Core/Runtime/Constants.cs index 1d6f2f9153..580914678f 100644 --- a/src/Orleans.Core/Runtime/Constants.cs +++ b/src/Orleans.Core/Runtime/Constants.cs @@ -12,6 +12,7 @@ internal static class Constants public static readonly GrainType DirectoryCacheValidatorType = SystemTargetGrainId.CreateGrainType("dir.cache-validator"); public static readonly GrainType ClientDirectoryType = SystemTargetGrainId.CreateGrainType("dir.client"); public static readonly GrainType SiloControlType = SystemTargetGrainId.CreateGrainType("silo-control"); + public static readonly GrainType SiloMetadataType = SystemTargetGrainId.CreateGrainType("silo-metadata"); public static readonly GrainType CatalogType = SystemTargetGrainId.CreateGrainType("catalog"); public static readonly GrainType MembershipServiceType = SystemTargetGrainId.CreateGrainType("clustering"); public static readonly GrainType SystemMembershipTableType = SystemTargetGrainId.CreateGrainType("clustering.dev"); @@ -27,8 +28,8 @@ internal static class Constants public static readonly GrainType ActivationMigratorType = SystemTargetGrainId.CreateGrainType("migrator"); public static readonly GrainType ActivationRepartitionerType = SystemTargetGrainId.CreateGrainType("repartitioner"); public static readonly GrainType ActivationRebalancerMonitorType = SystemTargetGrainId.CreateGrainType("rebalancer-monitor"); - public static readonly GrainType GrainDirectoryPartition = SystemTargetGrainId.CreateGrainType("dir.grain.part"); - public static readonly GrainType GrainDirectory = SystemTargetGrainId.CreateGrainType("dir.grain"); + public static readonly GrainType GrainDirectoryPartitionType = SystemTargetGrainId.CreateGrainType("dir.grain.part"); + public static readonly GrainType GrainDirectoryType = SystemTargetGrainId.CreateGrainType("dir.grain"); public static readonly GrainId SiloDirectConnectionId = GrainId.Create( GrainType.Create(GrainTypePrefix.SystemPrefix + "silo"), @@ -41,6 +42,7 @@ internal static class Constants {DirectoryServiceType, "DirectoryService"}, {DirectoryCacheValidatorType, "DirectoryCacheValidator"}, {SiloControlType, "SiloControl"}, + {SiloMetadataType, "SiloMetadata"}, {ClientDirectoryType, "ClientDirectory"}, {CatalogType,"Catalog"}, {MembershipServiceType,"MembershipService"}, @@ -57,7 +59,7 @@ internal static class Constants {ActivationMigratorType, "ActivationMigrator"}, {ActivationRepartitionerType, "ActivationRepartitioner"}, {ActivationRebalancerMonitorType, "ActivationRebalancerMonitor"}, - {GrainDirectory, "GrainDirectory"}, + {GrainDirectoryType, "GrainDirectory"}, }.ToFrozenDictionary(); public static string SystemTargetName(GrainType id) => SingletonSystemTargetNames.TryGetValue(id, out var name) ? name : id.ToString(); diff --git a/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs b/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs index a78ca9bdf8..8aa53a61de 100644 --- a/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs @@ -84,7 +84,7 @@ public DistributedGrainDirectory( ILocalSiloDetails localSiloDetails, ILoggerFactory loggerFactory, IServiceProvider serviceProvider, - IInternalGrainFactory grainFactory) : base(Constants.GrainDirectory, localSiloDetails.SiloAddress, loggerFactory) + IInternalGrainFactory grainFactory) : base(Constants.GrainDirectoryType, localSiloDetails.SiloAddress, loggerFactory) { _serviceProvider = serviceProvider; _membershipService = membershipService; diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs index cb8b81de52..dfbd02cb93 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs @@ -29,7 +29,7 @@ internal sealed partial class GrainDirectoryPartition( IInternalGrainFactory grainFactory) : SystemTarget(CreateGrainId(localSiloDetails.SiloAddress, partitionIndex), localSiloDetails.SiloAddress, loggerFactory), IGrainDirectoryPartition, IGrainDirectoryTestHooks { - internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartition, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture)); + internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartitionType, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture)); private readonly Dictionary _directory = []; private readonly int _partitionIndex = partitionIndex; private readonly DistributedGrainDirectory _owner = owner; @@ -665,7 +665,7 @@ private async IAsyncEnumerable> GetRegisteredActivations(Dire async Task> GetRegisteredActivationsFromClusterMember(MembershipVersion version, RingRange range, SiloAddress siloAddress, bool isValidation) { var stopwatch = ValueStopwatch.StartNew(); - var client = _grainFactory.GetSystemTarget(Constants.GrainDirectory, siloAddress); + var client = _grainFactory.GetSystemTarget(Constants.GrainDirectoryType, siloAddress); var result = await InvokeOnClusterMember( siloAddress, async () => diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs index 96f5d5bbfb..7d3bee25f8 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -1,7 +1,8 @@ #nullable enable + namespace Orleans.Runtime.MembershipService.SiloMetadata; public interface ISiloMetadataCache { - SiloMetadata GetMetadata(SiloAddress siloAddress); + SiloMetadata GetSiloMetadata(SiloAddress siloAddress); } \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs index e2ab060bea..bc16473bd6 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -1,10 +1,9 @@ using System.Threading.Tasks; -using Orleans.Services; #nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; -public interface ISiloMetadataClient : IGrainServiceClient +internal interface ISiloMetadataClient { Task GetSiloMetadata(SiloAddress siloAddress); } diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataSystemTarget.cs similarity index 70% rename from src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs rename to src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataSystemTarget.cs index 06121ae40c..49ea39f13c 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataSystemTarget.cs @@ -1,11 +1,10 @@ using System.Threading.Tasks; -using Orleans.Services; #nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; -[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] -public interface ISiloMetadataGrainService : IGrainService +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataSystemTarget")] +internal interface ISiloMetadataSystemTarget : ISystemTarget { [Alias("GetSiloMetadata")] Task GetSiloMetadata(); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index 8483c570a1..823d063eb4 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -5,8 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Orleans.Configuration; -using Orleans.Internal; #nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; @@ -22,44 +20,44 @@ internal class SiloMetadataCache( void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) { - var tasks = new List(1); - var cancellation = new CancellationTokenSource(); - Task OnRuntimeInitializeStart(CancellationToken _) + Task? task = null; + Task OnStart(CancellationToken _) { - tasks.Add(Task.Run(() => this.ProcessMembershipUpdates(cancellation.Token))); + task = Task.Run(() => this.ProcessMembershipUpdates(_cts.Token)); return Task.CompletedTask; } - async Task OnRuntimeInitializeStop(CancellationToken ct) + async Task OnStop(CancellationToken ct) { - cancellation.Cancel(throwOnFirstException: false); - var shutdownGracePeriod = Task.WhenAll(Task.Delay(ClusterMembershipOptions.ClusteringShutdownGracePeriod), ct.WhenCancelled()); - await Task.WhenAny(shutdownGracePeriod, Task.WhenAll(tasks)); + await _cts.CancelAsync().ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + if (task is not null) + { + await task.WaitAsync(ct).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } } lifecycle.Subscribe( nameof(ClusterMembershipService), - ServiceLifecycleStage.RuntimeInitialize, - OnRuntimeInitializeStart, - OnRuntimeInitializeStop); + ServiceLifecycleStage.RuntimeServices, + OnStart, + OnStop); } - private async Task ProcessMembershipUpdates(CancellationToken ct) { try { - if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates"); + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates."); await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) { // Add entries for members that aren't already in the cache - foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status != SiloStatus.Dead)) + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status is SiloStatus.Active or SiloStatus.Joining)) { if (!_metadata.ContainsKey(membershipEntry.Key)) { try { - var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key); + var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key).WaitAsync(ct); _metadata.TryAdd(membershipEntry.Key, metadata); } catch(Exception exception) @@ -85,6 +83,10 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + // Ignore and continue shutting down. + } catch (Exception exception) { logger.LogError(exception, "Error processing membership updates"); @@ -95,7 +97,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } - public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; + public SiloMetadata GetSiloMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; public void SetMetadata(SiloAddress siloAddress, SiloMetadata metadata) => _metadata.TryAdd(siloAddress, metadata); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs index 5d507bde72..bb5aae6d77 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -1,17 +1,14 @@ -using System; using System.Threading.Tasks; -using Orleans.Runtime.Services; #nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; -public class SiloMetadataClient(IServiceProvider serviceProvider) - : GrainServiceClient(serviceProvider), ISiloMetadataClient +internal sealed class SiloMetadataClient(IInternalGrainFactory grainFactory) : ISiloMetadataClient { public async Task GetSiloMetadata(SiloAddress siloAddress) { - var grainService = GetGrainService(siloAddress); - var metadata = await grainService.GetSiloMetadata(); + var metadataSystemTarget = grainFactory.GetSystemTarget(Constants.SiloMetadataType, siloAddress); + var metadata = await metadataSystemTarget.GetSiloMetadata(); return metadata; } } \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs index 8e60bff08d..b4fbb1e009 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -1,23 +1,34 @@ +using System; +using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; #nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; -public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService +internal sealed class SiloMetadataSystemTarget( + IOptions siloMetadata, + ILocalSiloDetails localSiloDetails, + ILoggerFactory loggerFactory, + IServiceProvider serviceProvider) + : SystemTarget(Constants.SiloMetadataType, localSiloDetails.SiloAddress, loggerFactory), ISiloMetadataSystemTarget, ILifecycleParticipant { - private readonly SiloMetadata _siloMetadata; + private readonly SiloMetadata _siloMetadata = siloMetadata.Value; - public SiloMetadataGrainService(IOptions siloMetadata) : base() - { - _siloMetadata = siloMetadata.Value; - } + public Task GetSiloMetadata() => Task.FromResult(_siloMetadata); - public SiloMetadataGrainService(IOptions siloMetadata, GrainId grainId, Silo silo, ILoggerFactory loggerFactory) : base(grainId, silo, loggerFactory) + void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) { - _siloMetadata = siloMetadata.Value; - } + lifecycle.Subscribe(nameof(SiloMetadataSystemTarget), ServiceLifecycleStage.RuntimeInitialize, OnRuntimeInitializeStart, OnRuntimeInitializeStop); - public Task GetSiloMetadata() => Task.FromResult(_siloMetadata); + Task OnRuntimeInitializeStart(CancellationToken token) + { + serviceProvider.GetRequiredService().RegisterSystemTarget(this); + return Task.CompletedTask; + } + + Task OnRuntimeInitializeStop(CancellationToken token) => Task.CompletedTask; + } } \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index a2adf33f55..3451b5197d 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -11,7 +11,6 @@ namespace Orleans.Runtime.MembershipService.SiloMetadata; public static class SiloMetadataHostingExtensions { - /// /// Configure silo metadata from the builder configuration. /// @@ -58,7 +57,7 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfigura { var dictionary = configurationSection.Get>(); - return builder.UseSiloMetadata(dictionary ?? new Dictionary()); + return builder.UseSiloMetadata(dictionary ?? []); } /// @@ -73,16 +72,15 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary { services .AddOptionsWithValidateOnStart() - .Configure(m => - { - m.AddMetadata(metadata); - }); + .Configure(m => m.AddMetadata(metadata)); - services.AddGrainService(); + services.AddSingleton(); + services.AddFromExisting, SiloMetadataSystemTarget>(); services.AddSingleton(); services.AddFromExisting(); services.AddFromExisting, SiloMetadataCache>(); services.AddSingleton(); + // Placement filters services.AddPlacementFilter(ServiceLifetime.Transient); services.AddPlacementFilter(ServiceLifetime.Transient); diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index 7b92a41c0e..4de43de466 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -14,11 +14,11 @@ internal class PreferredMatchSiloMetadataPlacementFilterDirector( { public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) { - var preferredMatchSiloMetadataPlacementFilterStrategy = (filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy); + var preferredMatchSiloMetadataPlacementFilterStrategy = filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy; var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1; var orderedMetadataKeys = preferredMatchSiloMetadataPlacementFilterStrategy?.OrderedMetadataKeys ?? []; - var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + var localSiloMetadata = siloMetadataCache.GetSiloMetadata(localSiloDetails.SiloAddress).Metadata; if (localSiloMetadata.Count == 0) { @@ -39,7 +39,7 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P var scoreCounts = new int[orderedMetadataKeys.Length+1]; for (var i = 0; i < siloList.Count; i++) { - var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + var siloMetadata = siloMetadataCache.GetSiloMetadata(siloList[i]).Metadata; var siloScore = 0; for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) { diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index e6a160adef..cca9e15b32 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using Orleans.Metadata; using Orleans.Placement; @@ -23,6 +24,7 @@ public override void AdditionalInitialize(GrainProperties properties) { throw new ArgumentException("Invalid ordered-metadata-keys property value."); } + OrderedMetadataKeys = placementFilterGrainProperty.Split(","); var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) @@ -37,6 +39,6 @@ protected override IEnumerable> GetAdditionalGrainP IReadOnlyDictionary existingProperties) { yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); - yield return new KeyValuePair("min-candidates", MinCandidates.ToString()); + yield return new KeyValuePair("min-candidates", MinCandidates.ToString(CultureInfo.InvariantCulture)); } } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index 91ed8cbac7..e4ace3c379 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -19,12 +19,12 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P return silos; } - var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localMetadata = siloMetadataCache.GetSiloMetadata(localSiloDetails.SiloAddress); var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); return silos.Where(silo => { - var remoteMetadata = siloMetadataCache.GetMetadata(silo); + var remoteMetadata = siloMetadataCache.GetSiloMetadata(silo); return DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys); }); } @@ -33,7 +33,7 @@ private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata silo { for (var i = 0; i < metadataKeys.Length; i++) { - if(localMetadata[i] != siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i])) + if (localMetadata[i] != siloMetadata.Metadata.GetValueOrDefault(metadataKeys[i])) { return false; } @@ -46,8 +46,9 @@ private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata silo var result = new string?[metadataKeys.Length]; for (var i = 0; i < metadataKeys.Length; i++) { - result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); + result[i] = siloMetadata.Metadata.GetValueOrDefault(metadataKeys[i]); } + return result; } } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index 4688ccafeb..5de219c36c 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -125,7 +125,6 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) var compatibleSilos = silos.Intersect(AllActiveSilos).ToArray(); - var filters = _filterStrategyResolver.GetPlacementFilterStrategies(grainType); if (filters.Length > 0) { @@ -139,7 +138,6 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) compatibleSilos = filteredSilos.ToArray(); } - if (compatibleSilos.Length == 0) { var allWithType = _grainInterfaceVersions.GetSupportedSilos(grainType).Result; diff --git a/src/Orleans.TestingHost/InProcTestCluster.cs b/src/Orleans.TestingHost/InProcTestCluster.cs index ec2c87310f..69b05dc1c1 100644 --- a/src/Orleans.TestingHost/InProcTestCluster.cs +++ b/src/Orleans.TestingHost/InProcTestCluster.cs @@ -10,7 +10,6 @@ using Orleans.Runtime; using Orleans.TestingHost.Utils; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Memory; using Orleans.Configuration; using Microsoft.Extensions.Options; using Microsoft.Extensions.Hosting; @@ -272,6 +271,7 @@ public static TimeSpan GetLivenessStabilizationTime(ClusterMembershipOptions clu { stabilizationTime += TestingUtils.Multiply(clusterMembershipOptions.TableRefreshTimeout, 2); } + return stabilizationTime; } @@ -279,7 +279,26 @@ public static TimeSpan GetLivenessStabilizationTime(ClusterMembershipOptions clu /// Start an additional silo, so that it joins the existing cluster. /// /// SiloHandle for the newly started silo. - public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort = false) + public InProcessSiloHandle StartAdditionalSilo() + { + return StartAdditionalSiloAsync().GetAwaiter().GetResult(); + } + + /// + /// Start an additional silo, so that it joins the existing cluster. + /// + /// SiloHandle for the newly started silo. + public async Task StartAdditionalSiloAsync() + { + return (await StartSilosAsync(1)).Single(); + } + + /// + /// Start an additional silo, so that it joins the existing cluster. + /// + /// SiloHandle for the newly started silo. + [Obsolete("Use overload which does not have a 'startAdditionalSiloOnNewPort' parameter.")] + public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort) { return StartAdditionalSiloAsync(startAdditionalSiloOnNewPort).GetAwaiter().GetResult(); } @@ -288,9 +307,9 @@ public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort /// Start an additional silo, so that it joins the existing cluster. /// /// SiloHandle for the newly started silo. - public async Task StartAdditionalSiloAsync(bool startAdditionalSiloOnNewPort = false) + public async Task StartAdditionalSiloAsync(bool startAdditionalSiloOnNewPort) { - return (await StartSilosAsync(1, startAdditionalSiloOnNewPort)).Single(); + return (await StartSilosAsync(1)).Single(); } /// @@ -299,13 +318,24 @@ public async Task StartAdditionalSiloAsync(bool startAdditi /// Number of silos to start. /// /// List of SiloHandles for the newly started silos. - public async Task> StartSilosAsync(int silosToStart, bool startAdditionalSiloOnNewPort = false) + [Obsolete("Use overload which does not have a 'startAdditionalSiloOnNewPort' parameter.")] + public async Task> StartSilosAsync(int silosToStart, bool startAdditionalSiloOnNewPort) + { + return await StartSilosAsync(silosToStart); + } + + /// + /// Start a number of additional silo, so that they join the existing cluster. + /// + /// Number of silos to start. + /// List of SiloHandles for the newly started silos. + public async Task> StartSilosAsync(int silosToStart) { var instances = new List(); if (silosToStart > 0) { var siloStartTasks = Enumerable.Range(_startedInstances, silosToStart) - .Select(instanceNumber => Task.Run(() => StartSiloAsync((short)instanceNumber, Options, startSiloOnNewPort: startAdditionalSiloOnNewPort))).ToArray(); + .Select(instanceNumber => Task.Run(() => StartSiloAsync((short)instanceNumber, Options))).ToArray(); try { @@ -622,6 +652,19 @@ public async Task CreateSiloAsync(InProcessTestSiloSpecific }; } + /// + /// Start a new silo in the target cluster + /// + /// The InProcessTestCluster in which the silo should be deployed + /// The instance number to deploy + /// The options to use. + /// A handle to the silo deployed + public static async Task StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions) + { + if (cluster == null) throw new ArgumentNullException(nameof(cluster)); + return await cluster.StartSiloAsync(instanceNumber, clusterOptions); + } + /// /// Start a new silo in the target cluster /// @@ -631,7 +674,8 @@ public async Task CreateSiloAsync(InProcessTestSiloSpecific /// Configuration overrides. /// Whether we start this silo on a new port, instead of the default one /// A handle to the silo deployed - public static async Task StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList configurationOverrides = null, bool startSiloOnNewPort = false) + [Obsolete("Use the overload which does not have a 'startSiloOnNewPort' parameter.")] + public static async Task StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList configurationOverrides, bool startSiloOnNewPort) { if (cluster == null) throw new ArgumentNullException(nameof(cluster)); return await cluster.StartSiloAsync(instanceNumber, clusterOptions, configurationOverrides, startSiloOnNewPort); @@ -642,18 +686,30 @@ public static async Task StartSiloAsync(InProcessTestCluste /// /// The instance number to deploy /// The options to use. - /// Configuration overrides. - /// Whether we start this silo on a new port, instead of the default one /// A handle to the deployed silo. - public async Task StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList configurationOverrides = null, bool startSiloOnNewPort = false) + public async Task StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions) { - var siloOptions = InProcessTestSiloSpecificOptions.Create(this, clusterOptions, instanceNumber, startSiloOnNewPort); + var siloOptions = InProcessTestSiloSpecificOptions.Create(this, clusterOptions, instanceNumber, assignNewPort: true); var handle = await CreateSiloAsync(siloOptions); handle.InstanceNumber = (short)instanceNumber; Interlocked.Increment(ref _startedInstances); return handle; } + /// + /// Starts a new silo. + /// + /// The instance number to deploy + /// The options to use. + /// Configuration overrides. + /// Whether we start this silo on a new port, instead of the default one + /// A handle to the deployed silo. + [Obsolete("Use the overload which does not have a 'startSiloOnNewPort' parameter.")] + public async Task StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList configurationOverrides, bool startSiloOnNewPort) + { + return await StartSiloAsync(instanceNumber, clusterOptions); + } + private async Task StopSiloAsync(InProcessSiloHandle instance, bool stopGracefully) { try diff --git a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs index e0fca917bb..8e323a48c4 100644 --- a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -2,38 +2,45 @@ using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.Runtime.Placement.Filtering; using Orleans.TestingHost; -using TestExtensions; using Xunit; namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] -public class SiloMetadataPlacementFilterTests : TestClusterPerTest +public class SiloMetadataPlacementFilterTests(SiloMetadataPlacementFilterTests.Fixture fixture) : IClassFixture { - protected override void ConfigureTestCluster(TestClusterBuilder builder) + public class Fixture : IAsyncLifetime { - builder.AddSiloBuilderConfigurator(); - } + public InProcessTestCluster Cluster { get; private set; } + public async Task DisposeAsync() + { + if (Cluster is { } cluster) + { + await cluster.DisposeAsync(); + } + } - private class SiloConfigurator : ISiloConfigurator - { - public void Configure(ISiloBuilder hostBuilder) + public async Task InitializeAsync() { - hostBuilder.UseSiloMetadata(new Dictionary + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSilo((options, siloBuilder) => siloBuilder.UseSiloMetadata(new Dictionary { {"first", "1"}, {"second", "2"}, {"third", "3"}, {"unique", Guid.NewGuid().ToString()} - }); + })); + + Cluster = builder.Build(); + await Cluster.DeployAsync(); + await Cluster.WaitForLivenessToStabilizeAsync(); } } [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = Client.GetGrain(0); + var managementGrain = fixture.Cluster.Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } @@ -45,14 +52,13 @@ public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_RequiredFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); @@ -69,14 +75,13 @@ public async Task PlacementFilter_RequiredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); @@ -93,19 +98,19 @@ public async Task PlacementFilter_PreferredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { var dict = new Dictionary(); - foreach (var clusterSilo in HostedCluster.Silos) + foreach (var clusterSilo in fixture.Cluster.Silos) { dict[clusterSilo.SiloAddress] = 0; } + for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); @@ -127,19 +132,19 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { var dict = new Dictionary(); - foreach (var clusterSilo in HostedCluster.Silos) + foreach (var clusterSilo in fixture.Cluster.Silos) { dict[clusterSilo.SiloAddress] = 0; } + for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); @@ -161,21 +166,18 @@ public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - await HostedCluster.StartAdditionalSiloAsync(); - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { var dict = new Dictionary(); - foreach (var clusterSilo in HostedCluster.Silos) + foreach (var clusterSilo in fixture.Cluster.Silos) { dict[clusterSilo.SiloAddress] = 0; } for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); @@ -197,21 +199,18 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - await HostedCluster.StartAdditionalSiloAsync(); - await HostedCluster.WaitForLivenessToStabilizeAsync(); var id = 0; - foreach (var hostedClusterSilo in HostedCluster.Silos) + foreach (var hostedClusterSilo in fixture.Cluster.Silos) { var dict = new Dictionary(); - foreach (var clusterSilo in HostedCluster.Silos) + foreach (var clusterSilo in fixture.Cluster.Silos) { dict[clusterSilo.SiloAddress] = 0; } for (var i = 0; i < 50; i++) { ++id; - var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var managementGrain = firstSiloMetadataCache.GetGrain(id); var hostingSilo = await managementGrain.GetHostingSilo(); diff --git a/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs index a6ab78dda4..5f520e4aad 100644 --- a/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs +++ b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs @@ -11,5 +11,5 @@ public TestSiloMetadataCache(Dictionary metadata) _metadata = metadata; } - public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? new SiloMetadata(); + public SiloMetadata GetSiloMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; } \ No newline at end of file diff --git a/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs index 30cfc94ec3..906fb5e5fb 100644 --- a/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs +++ b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs @@ -2,126 +2,117 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.TestingHost; -using TestExtensions; using Xunit; namespace UnitTests.SiloMetadataTests; - [TestCategory("SiloMetadata")] -public class SiloMetadataConfigTests : TestClusterPerTest +public class SiloMetadataTests(SiloMetadataTests.Fixture fixture) : IClassFixture { - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloConfigurator : ISiloConfigurator - { - public static readonly List> Metadata = + private static readonly List> Metadata = [ new("Orleans:Metadata:first", "1"), new("Orleans:Metadata:second", "2"), new("Orleans:Metadata:third", "3") ]; - public void Configure(ISiloBuilder hostBuilder) + public class Fixture : IAsyncLifetime + { + public InProcessTestCluster Cluster { get; private set; } + public async Task DisposeAsync() + { + if (Cluster is { } cluster) + { + await cluster.DisposeAsync(); + } + } + + public async Task InitializeAsync() { - var config = new ConfigurationBuilder() - .AddInMemoryCollection(Metadata) - .Build(); - hostBuilder.UseSiloMetadata(config); + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSiloHost((options, hostBuilder) => + { + hostBuilder.Configuration.AddInMemoryCollection(Metadata); + }); + + builder.ConfigureSilo((options, siloBuilder) => + { + siloBuilder + .UseSiloMetadata() + .UseSiloMetadata(new Dictionary + { + {"host.id", Guid.NewGuid().ToString()} + }); + }); + + Cluster = builder.Build(); + await Cluster.DeployAsync(); + await Cluster.WaitForLivenessToStabilizeAsync(); } } [Fact, TestCategory("Functional")] - public async Task SiloMetadata_CanBeSetAndRead() + public void SiloMetadata_FromConfiguration_CanBeSetAndRead() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); + fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); } [Fact, TestCategory("Functional")] - public async Task SiloMetadata_HasConfiguredValues() + public void SiloMetadata_HasConfiguredValues() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - - var first = HostedCluster.Silos.First(); - var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var first = fixture.Cluster.Silos.First(); + var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); - var metadata = firstSiloMetadataCache.GetMetadata(first.SiloAddress); + var metadata = firstSiloMetadataCache.GetSiloMetadata(first.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); - Assert.Equal(SiloConfigurator.Metadata.Count, metadata.Metadata.Count); - foreach (var kv in SiloConfigurator.Metadata) + Assert.True(metadata.Metadata.Count >= Metadata.Count); + foreach (var kv in Metadata) { Assert.Equal(kv.Value, metadata.Metadata[kv.Key.Split(':').Last()]); } } -} - -[TestCategory("SiloMetadata")] -public class SiloMetadataTests : TestClusterPerTest -{ - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloConfigurator : ISiloConfigurator - { - public void Configure(ISiloBuilder hostBuilder) - { - hostBuilder.UseSiloMetadata(new Dictionary - { - {"host.id", Guid.NewGuid().ToString()} - }); - } - } [Fact, TestCategory("Functional")] - public async Task SiloMetadata_CanBeSetAndRead() + public void SiloMetadata_CanBeSetAndRead() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_NewSilosHaveMetadata() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - await HostedCluster.StartAdditionalSiloAsync(); - HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + await fixture.Cluster.StartAdditionalSiloAsync(); + await fixture.Cluster.WaitForLivenessToStabilizeAsync(); + fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_RemovedSiloHasNoMetadata() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); - var first = HostedCluster.Silos.First(); - var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + var first = fixture.Cluster.Silos.First(); + var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); - var second = HostedCluster.Silos.Skip(1).First(); - var metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + var second = fixture.Cluster.Silos.Skip(1).First(); + var metadata = firstSiloMetadataCache.GetSiloMetadata(second.SiloAddress); Assert.NotNull(metadata); Assert.NotEmpty(metadata.Metadata); - await HostedCluster.StopSiloAsync(second); - metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + await fixture.Cluster.StopSiloAsync(second); + metadata = firstSiloMetadataCache.GetSiloMetadata(second.SiloAddress); Assert.NotNull(metadata); Assert.Empty(metadata.Metadata); } [Fact, TestCategory("Functional")] - public async Task SiloMetadata_BadSiloAddressHasNoMetadata() + public void SiloMetadata_BadSiloAddressHasNoMetadata() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var first = HostedCluster.Silos.First(); - var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var first = fixture.Cluster.Silos.First(); + var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); - var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); + var metadata = firstSiloMetadataCache.GetSiloMetadata(SiloAddress.Zero); Assert.NotNull(metadata); Assert.Empty(metadata.Metadata); } @@ -129,7 +120,7 @@ public async Task SiloMetadata_BadSiloAddressHasNoMetadata() public static class SiloMetadataTestExtensions { - public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hostedCluster, string[] expectedKeys) + public static void AssertAllSiloMetadataMatchesOnAllSilos(this InProcessTestCluster hostedCluster, string[] expectedKeys) { var exampleSiloMetadata = new Dictionary(); var first = hostedCluster.Silos.First(); @@ -137,7 +128,7 @@ public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hoste var firstSiloMetadataCache = firstSp.GetRequiredService(); foreach (var otherSilo in hostedCluster.Silos) { - var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); + var metadata = firstSiloMetadataCache.GetSiloMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); foreach (var expectedKey in expectedKeys) @@ -154,7 +145,7 @@ public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hoste var remoteMetadata = new Dictionary(); foreach (var otherSilo in hostedCluster.Silos) { - var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); + var metadata = siloMetadataCache.GetSiloMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); foreach (var expectedKey in expectedKeys) From 9aae9d06bfa1eca027e4a6f713725298fa3231d4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Feb 2025 11:28:20 -0800 Subject: [PATCH 18/20] Fixups --- test/Extensions/AWSUtils.Tests/Streaming/SQSStreamTests.cs | 2 +- .../Extensions/TesterAdoNet/Streaming/AdoNetStreamingTests.cs | 2 +- .../StreamingTests/MultipleStreamsTestRunner.cs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Extensions/AWSUtils.Tests/Streaming/SQSStreamTests.cs b/test/Extensions/AWSUtils.Tests/Streaming/SQSStreamTests.cs index d97930abfe..b0b6983bcf 100644 --- a/test/Extensions/AWSUtils.Tests/Streaming/SQSStreamTests.cs +++ b/test/Extensions/AWSUtils.Tests/Streaming/SQSStreamTests.cs @@ -196,7 +196,7 @@ public async Task SQS_17_MultipleStreams_1J_ManyProducerGrainsManyConsumerGrains { var multiRunner = new MultipleStreamsTestRunner(this.InternalClient, SQS_STREAM_PROVIDER_NAME, 17, false); await multiRunner.StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains( - this.HostedCluster.StartAdditionalSilo); + () => HostedCluster.StartAdditionalSilo()); } } } diff --git a/test/Extensions/TesterAdoNet/Streaming/AdoNetStreamingTests.cs b/test/Extensions/TesterAdoNet/Streaming/AdoNetStreamingTests.cs index 16dd6b95a9..f2dc808e4e 100644 --- a/test/Extensions/TesterAdoNet/Streaming/AdoNetStreamingTests.cs +++ b/test/Extensions/TesterAdoNet/Streaming/AdoNetStreamingTests.cs @@ -169,6 +169,6 @@ public async Task AdoNet_17_MultipleStreams_1J_ManyProducerGrainsManyConsumerGra { var multiRunner = new MultipleStreamsTestRunner(InternalClient, AdoNetStreamProviderName, 17, false); - await multiRunner.StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains(HostedCluster.StartAdditionalSilo); + await multiRunner.StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains(() => HostedCluster.StartAdditionalSilo()); } } \ No newline at end of file diff --git a/test/TesterInternal/StreamingTests/MultipleStreamsTestRunner.cs b/test/TesterInternal/StreamingTests/MultipleStreamsTestRunner.cs index 84281b17bd..557ef259b8 100644 --- a/test/TesterInternal/StreamingTests/MultipleStreamsTestRunner.cs +++ b/test/TesterInternal/StreamingTests/MultipleStreamsTestRunner.cs @@ -31,7 +31,7 @@ private void Heading(string testName) logger.LogInformation("\n\n************************ {StreamProviderName}_{TestNumber}_{TestName} ********************************* \n\n", streamProviderName, testNumber, testName); } - public async Task StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains(Func startSiloFunc = null, Action stopSiloFunc = null) + public async Task StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains(Func startSiloFunc = null, Action stopSiloFunc = null) { Heading("MultipleStreams_ManyDifferent_ManyProducerGrainsManyConsumerGrains"); List runners = new List(); @@ -50,7 +50,7 @@ public async Task StreamTest_MultipleStreams_ManyDifferent_ManyProducerGrainsMan SiloHandle silo = null; if (startSiloFunc != null) { - silo = startSiloFunc(false); + silo = startSiloFunc(); } foreach (var runner in runners) From ec8a4078145e15bc06d4141afe0311aae1f13c1e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Feb 2025 11:53:17 -0800 Subject: [PATCH 19/20] Add PlacementFilterContext --- .../Placement/IPlacementFilterDirector.cs | 5 +- .../Placement/PlacementFilterContext.cs | 6 ++ ...atchSiloMetadataPlacementFilterDirector.cs | 2 +- ...atchSiloMetadataPlacementFilterDirector.cs | 2 +- .../Placement/PlacementService.cs | 4 +- .../GrainPlacementFilterTests.cs | 78 +++++++++---------- ...iloMetadataPlacementFilterDirectorTests.cs | 9 ++- ...iloMetadataPlacementFilterDirectorTests.cs | 11 ++- 8 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 src/Orleans.Core/Placement/PlacementFilterContext.cs diff --git a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs index fe9a225a0e..d02be4de3c 100644 --- a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -7,6 +7,5 @@ namespace Orleans.Placement; public interface IPlacementFilterDirector { - IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, - IEnumerable silos); -} \ No newline at end of file + IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos); +} diff --git a/src/Orleans.Core/Placement/PlacementFilterContext.cs b/src/Orleans.Core/Placement/PlacementFilterContext.cs new file mode 100644 index 0000000000..b2a7942c5c --- /dev/null +++ b/src/Orleans.Core/Placement/PlacementFilterContext.cs @@ -0,0 +1,6 @@ +using Orleans.Runtime; + +#nullable enable +namespace Orleans.Placement; + +public readonly record struct PlacementFilterContext(GrainType GrainType, GrainInterfaceType InterfaceType, ushort InterfaceVersion); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index 4de43de466..6463001665 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -12,7 +12,7 @@ internal class PreferredMatchSiloMetadataPlacementFilterDirector( ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos) { var preferredMatchSiloMetadataPlacementFilterStrategy = filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy; var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index e4ace3c379..f7aadcd346 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -9,7 +9,7 @@ namespace Orleans.Runtime.Placement.Filtering; internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos) { var metadataKeys = (filterStrategy as RequiredMatchSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index 5de219c36c..4b667249e7 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; +using Orleans.Placement; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Internal; using Orleans.Runtime.Placement.Filtering; @@ -129,10 +130,11 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) if (filters.Length > 0) { IEnumerable filteredSilos = compatibleSilos; + var context = new PlacementFilterContext(target.GrainIdentity.Type, target.InterfaceType, target.InterfaceVersion); foreach (var placementFilter in filters) { var director = _placementFilterDirectoryResolver.GetFilterDirector(placementFilter); - filteredSilos = director.Filter(placementFilter, target, filteredSilos); + filteredSilos = director.Filter(placementFilter, context, filteredSilos); } compatibleSilos = filteredSilos.ToArray(); diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index ce80d6ae5b..c3cb89393a 100644 --- a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -8,35 +8,42 @@ namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters")] -public class GrainPlacementFilterTests : TestClusterPerTest +public class GrainPlacementFilterTests(GrainPlacementFilterTests.Fixture fixture) : IClassFixture { - public static Dictionary> FilterScratchpad = new(); + public static Dictionary> FilterScratchpad = []; private static Random random = new(); - protected override void ConfigureTestCluster(TestClusterBuilder builder) + public class Fixture : IAsyncLifetime { - builder.AddSiloBuilderConfigurator(); - } + public InProcessTestCluster Cluster { get; private set; } + public async Task DisposeAsync() + { + if (Cluster is { } cluster) + { + await cluster.DisposeAsync(); + } + } - private class SiloConfigurator : ISiloConfigurator - { - public void Configure(ISiloBuilder hostBuilder) + public async Task InitializeAsync() { - hostBuilder.ConfigureServices(services => + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSilo((options, siloBuilder) => { - services.AddPlacementFilter(ServiceLifetime.Singleton); - services.AddPlacementFilter(ServiceLifetime.Singleton); - services.AddPlacementFilter(ServiceLifetime.Singleton); + siloBuilder.Services.AddPlacementFilter(ServiceLifetime.Singleton); + siloBuilder.Services.AddPlacementFilter(ServiceLifetime.Singleton); + siloBuilder.Services.AddPlacementFilter(ServiceLifetime.Singleton); }); + + Cluster = builder.Build(); + await Cluster.DeployAsync(); + await Cluster.WaitForLivenessToStabilizeAsync(); } } - [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = Client.GetGrain(0); + var managementGrain = fixture.Cluster.Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } @@ -44,13 +51,12 @@ public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_FilterIsTriggered() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); var triggered = false; var task = Task.Run(async () => { triggered = await TestPlacementFilterDirector.Triggered.WaitAsync(TimeSpan.FromSeconds(1)); }); - var localOnlyGrain = Client.GetGrain(0); + var localOnlyGrain = fixture.Cluster.Client.GetGrain(0); await localOnlyGrain.Ping(); await task; Assert.True(triggered); @@ -59,12 +65,10 @@ public async Task PlacementFilter_FilterIsTriggered() [Fact, TestCategory("Functional")] public async Task PlacementFilter_OrderAB12() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var primaryKey = random.Next(); - var testGrain = Client.GetGrain(primaryKey); + var testGrain = fixture.Cluster.Client.GetGrain(primaryKey); await testGrain.Ping(); - var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().Type.ToString()); Assert.Equal(2, list.Count); Assert.Equal("A", list[0]); Assert.Equal("B", list[1]); @@ -73,12 +77,10 @@ public async Task PlacementFilter_OrderAB12() [Fact, TestCategory("Functional")] public async Task PlacementFilter_OrderAB21() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var primaryKey = random.Next(); - var testGrain = Client.GetGrain(primaryKey); + var testGrain = fixture.Cluster.Client.GetGrain(primaryKey); await testGrain.Ping(); - var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().Type.ToString()); Assert.Equal(2, list.Count); Assert.Equal("B", list[0]); Assert.Equal("A", list[1]); @@ -87,12 +89,10 @@ public async Task PlacementFilter_OrderAB21() [Fact, TestCategory("Functional")] public async Task PlacementFilter_OrderBA12() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var primaryKey = random.Next(); - var testGrain = Client.GetGrain(primaryKey); + var testGrain = fixture.Cluster.Client.GetGrain(primaryKey); await testGrain.Ping(); - var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().Type.ToString()); Assert.Equal(2, list.Count); Assert.Equal("B", list[0]); Assert.Equal("A", list[1]); @@ -101,13 +101,11 @@ public async Task PlacementFilter_OrderBA12() [Fact, TestCategory("Functional")] public async Task PlacementFilter_OrderBA21() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var primaryKey = random.Next(); - var testGrain = Client.GetGrain(primaryKey); + var testGrain = fixture.Cluster.Client.GetGrain(primaryKey); await testGrain.Ping(); - var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().Type.ToString()); Assert.Equal(2, list.Count); Assert.Equal("A", list[0]); Assert.Equal("B", list[1]); @@ -116,10 +114,8 @@ public async Task PlacementFilter_OrderBA21() [Fact, TestCategory("Functional")] public async Task PlacementFilter_DuplicateOrder() { - await HostedCluster.WaitForLivenessToStabilizeAsync(); - var primaryKey = random.Next(); - var testGrain = Client.GetGrain(primaryKey); + var testGrain = fixture.Cluster.Client.GetGrain(primaryKey); await Assert.ThrowsAsync(async () => { await testGrain.Ping(); @@ -151,7 +147,7 @@ public class TestPlacementFilterDirector() : IPlacementFilterDirector { public static SemaphoreSlim Triggered { get; } = new(0); - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos) { Triggered.Release(1); return silos; @@ -171,10 +167,10 @@ public OrderAPlacementFilterStrategy() : this(0) public class OrderAPlacementFilterDirector : IPlacementFilterDirector { - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos) { var dict = GrainPlacementFilterTests.FilterScratchpad; - var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + var list = dict.GetValueOrAddNew(context.GrainType.ToString()); list.Add("A"); return silos; } @@ -193,10 +189,10 @@ public OrderBPlacementFilterStrategy() : this(0) public class OrderBPlacementFilterDirector() : IPlacementFilterDirector { - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementFilterContext context, IEnumerable silos) { var dict = GrainPlacementFilterTests.FilterScratchpad; - var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + var list = dict.GetValueOrAddNew(context.GrainType.ToString()); list.Add("B"); return silos; } diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs index 369829fb7b..b15eb92154 100644 --- a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -1,5 +1,6 @@ using System.Net; using Orleans.Metadata; +using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Filtering; @@ -37,7 +38,7 @@ public void CanBeCalled() { {testLocalSiloAddress, SiloMetadata.Empty} })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(), default, new List() { testLocalSiloAddress } ).ToList(); Assert.NotNull(result); @@ -60,7 +61,7 @@ public void FiltersToAllWhenNoEntry() {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1, 0), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1, 0), default, new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -114,7 +115,7 @@ public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates, 0), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates, 0), default, new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); @@ -168,7 +169,7 @@ public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, para {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates, 0), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates, 0), default, new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs index 19d03626c9..6fbb6433de 100644 --- a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -1,6 +1,5 @@ using System.Net; using Orleans.Runtime.MembershipService.SiloMetadata; -using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Filtering; using Xunit; @@ -36,7 +35,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCalled() { {testLocalSiloAddress, SiloMetadata.Empty} })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(), default, new List() { testLocalSiloAddress } ).ToList(); Assert.NotNull(result); @@ -59,7 +58,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), default, new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -82,7 +81,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), default, new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -105,7 +104,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMa {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), default, new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -132,7 +131,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSi {testOtherSiloAddress2, otherSiloMetadata2}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), default, new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); Assert.NotEmpty(result); Assert.Equal(2, result.Count); From ac45d1ef6fd69c3e926bc044d9c7ecd088956e10 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Feb 2025 08:23:18 -0800 Subject: [PATCH 20/20] rename file --- .../{SiloMetadataGrainService.cs => SiloMetadataSystemTarget.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Orleans.Runtime/MembershipService/SiloMetadata/{SiloMetadataGrainService.cs => SiloMetadataSystemTarget.cs} (100%) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataSystemTarget.cs similarity index 100% rename from src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs rename to src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataSystemTarget.cs