Skip to content

Commit

Permalink
feat: orleans k8s refact
Browse files Browse the repository at this point in the history
  • Loading branch information
@mixhsnhd
mixhsnhd committed Dec 25, 2024
1 parent c4d6ee7 commit d20ee5f
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 61 deletions.
35 changes: 29 additions & 6 deletions src/EoaServer.HttpApi.Host/appsettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
"CorsOrigins": "https://*.EoaServer.com,http://localhost:4200"
},
"ConnectionStrings": {
"Default": "mongodb://admin:[email protected]:27017,portkey-eoa-mongodb-1.portkey-eoa-mongodb:27017,portkey-eoa-mongodb-2.portkey-eoa-mongodb:27017/?retryWrites=false&maxPoolSize=555"
"Default": "mongodb://127.0.0.1:27017/EoaServer?authSource=admin&retryWrites=false"
},
"Redis": {
"Configuration": "portkey-eoa-redis"
"Configuration": "127.0.0.1"
},
"AuthServer": {
"Authority": "http://portkey-eoa-authserver-svc:8080",
"Authority": "http://127.0.0.1:8080",
"RequireHttpsMetadata": "false",
"SwaggerClientId": "EoaServer_App"
},
Expand All @@ -23,9 +23,32 @@
}
}
},
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Default": "Warning",
"System": "Warning",
"Microsoft": "Warning",
"Microsoft.AspNetCore.Hosting.Diagnostics": "Information"
}
},
"Properties": {
"Application": "EoaHttpApi.Service",
"Environment": "Testnet"
},
"WriteTo": [
{
"Name": "Console",
"Args": {
"formatter": "Serilog.Formatting.Compact.RenderedCompactJsonFormatter, Serilog.Formatting.Compact"
}
}
]
},
"ElasticUris": {
"Uris": [
"portkey-eoa-es-http:9200"
"http://127.0.0.1:9200"
]
},
"IndexSetting": {
Expand All @@ -36,7 +59,7 @@
"Kafka": {
"Connections": {
"Default": {
"BootstrapServers": "portkey-eoa-kafka-cluster-kafka-bootstrap:9092"
"BootstrapServers": "localhost:9092"
}
},
"Producer": {
Expand Down Expand Up @@ -65,7 +88,7 @@
"Orleans": {
"ClusterId": "EoaServerSiloCluster",
"ServiceId": "EoaServerOrleansBasicService",
"MongoDBClient": "mongodb://admin:[email protected]:27017,portkey-eoa-mongodb-1.portkey-eoa-mongodb:27017,portkey-eoa-mongodb-2.portkey-eoa-mongodb:27017/?retryWrites=false&maxPoolSize=555",
"MongoDBClient": "mongodb://localhost:27017/?maxPoolSize=555",
"DataBase": "EoaServerOrleansDB"
},
"AElfScanOptions": {
Expand Down
189 changes: 140 additions & 49 deletions src/EoaServer.Silo/Extensions/OrleansHostExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
using Orleans.Hosting;
using Orleans.Providers.MongoDB.Configuration;
using Orleans.Statistics;
using Serilog;

namespace EoaServer.Silo.Extensions;

public static class OrleansHostExtensions
{
public static IHostBuilder UseOrleans(this IHostBuilder hostBuilder)
public static IHostBuilder UseOrleans(this IHostBuilder hostBuilder)
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
Expand All @@ -24,61 +25,151 @@ public static IHostBuilder UseOrleans(this IHostBuilder hostBuilder)
throw new ArgumentNullException(nameof(configSection), "The OrleansServer node is missing");
return hostBuilder.UseOrleans(siloBuilder =>
{
//Configure OrleansSnapshot
siloBuilder
.ConfigureEndpoints(advertisedIP:IPAddress.Parse(configSection.GetValue<string>("AdvertisedIP")),siloPort: configSection.GetValue<int>("SiloPort"), gatewayPort: configSection.GetValue<int>("GatewayPort"), listenOnAnyHostAddress: true)
.UseMongoDBClient(configSection.GetValue<string>("MongoDBClient"))
.UseMongoDBClustering(options =>
if (configSection.GetValue<bool>("IsRunningInKubernetes"))
{
Log.Warning("==Use kubernetes hosting...");
UseKubernetesHostClustering(siloBuilder, configSection);
Log.Warning("==Use kubernetes hosting end...");
}
else
{
Log.Warning("==Use docker hosting...");
UseDockerHostClustering(siloBuilder, configSection);
Log.Warning("==Use docker hosting end...");
}
});
}

private static void UseKubernetesHostClustering(ISiloBuilder siloBuilder, IConfigurationSection configSection)
{
Log.Warning("==Configuration");
Log.Warning("== POD_IP: {0}", Environment.GetEnvironmentVariable("POD_IP"));
Log.Warning("== SiloPort: {0}", configSection.GetValue<int>("SiloPort"));
Log.Warning("== GatewayPort: {0}", configSection.GetValue<int>("GatewayPort"));
Log.Warning("== DatabaseName: {0}", configSection.GetValue<string>("DataBase"));
Log.Warning("== ClusterId: {0}", Environment.GetEnvironmentVariable("ORLEANS_CLUSTER_ID"));
Log.Warning("== ServiceId: {0}", Environment.GetEnvironmentVariable("ORLEANS_SERVICE_ID"));
Log.Warning("==Configuration");

//Configure OrleansSnapshot
siloBuilder
.ConfigureEndpoints(
advertisedIP: IPAddress.Parse(Environment.GetEnvironmentVariable("POD_IP") ?? string.Empty),
siloPort: configSection.GetValue<int>("SiloPort"),
gatewayPort: configSection.GetValue<int>("GatewayPort"), listenOnAnyHostAddress: true)
.UseMongoDBClient(configSection.GetValue<string>("MongoDBClient"))
.UseMongoDBClustering(options =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.Strategy = MongoDBMembershipStrategy.SingleDocument;
})
.Configure<JsonGrainStateSerializerOptions>(options => options.ConfigureJsonSerializerSettings =
settings =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.Strategy = MongoDBMembershipStrategy.SingleDocument;
settings.NullValueHandling = NullValueHandling.Include;
settings.ObjectCreationHandling = ObjectCreationHandling.Replace;
settings.DefaultValueHandling = DefaultValueHandling.Populate;
})
.Configure<JsonGrainStateSerializerOptions>(options => options.ConfigureJsonSerializerSettings =
settings =>
{
settings.NullValueHandling = NullValueHandling.Include;
settings.ObjectCreationHandling = ObjectCreationHandling.Replace;
settings.DefaultValueHandling = DefaultValueHandling.Populate;
})
.AddMongoDBGrainStorage("Default",(MongoDBGrainStorageOptions op) =>
.AddMongoDBGrainStorage("Default", (MongoDBGrainStorageOptions op) =>
{
op.CollectionPrefix = "GrainStorage";
op.DatabaseName = configSection.GetValue<string>("DataBase");

var grainIdPrefix = configSection
.GetSection("GrainSpecificIdPrefix").GetChildren().ToDictionary(o => o.Key.ToLower(), o => o.Value);
op.KeyGenerator = id =>
{
op.CollectionPrefix = "GrainStorage";
op.DatabaseName = configSection.GetValue<string>("DataBase");

var grainIdPrefix = configSection
.GetSection("GrainSpecificIdPrefix").GetChildren().ToDictionary(o => o.Key.ToLower(), o => o.Value);
op.KeyGenerator = id =>
var grainType = id.Type.ToString();
if (grainIdPrefix.TryGetValue(grainType, out var prefix))
{
var grainType = id.Type.ToString();
if (grainIdPrefix.TryGetValue(grainType, out var prefix))
{
return prefix.StartsWith("GrainReference=000000") ? $"{prefix}+{id.Key}" : prefix;
}
return id.ToString();
};
op.CreateShardKeyForCosmos = configSection.GetValue<bool>("CreateShardKeyForMongoDB", false);
return prefix.StartsWith("GrainReference=000000") ? $"{prefix}+{id.Key}" : prefix;
}

})
.UseMongoDBReminders(options =>
return id.ToString();
};
op.CreateShardKeyForCosmos = configSection.GetValue<bool>("CreateShardKeyForMongoDB", false);
})
.UseMongoDBReminders(options =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.CreateShardKeyForCosmos = false;
})
.Configure<ClusterOptions>(options =>
{
options.ClusterId = Environment.GetEnvironmentVariable("ORLEANS_CLUSTER_ID");
options.ServiceId = Environment.GetEnvironmentVariable("ORLEANS_SERVICE_ID");
})
// .AddMemoryGrainStorage("PubSubStore")
.UseDashboard(options =>
{
options.Username = configSection.GetValue<string>("DashboardUserName");
options.Password = configSection.GetValue<string>("DashboardPassword");
options.Host = "*";
options.Port = configSection.GetValue<int>("DashboardPort");
options.HostSelf = true;
options.CounterUpdateIntervalMs = configSection.GetValue<int>("DashboardCounterUpdateIntervalMs");
})
.ConfigureLogging(logging => { logging.SetMinimumLevel(LogLevel.Debug).AddConsole(); });
}

private static void UseDockerHostClustering(ISiloBuilder siloBuilder, IConfigurationSection configSection)
{
siloBuilder
.ConfigureEndpoints(advertisedIP: IPAddress.Parse(configSection.GetValue<string>("AdvertisedIP")),
siloPort: configSection.GetValue<int>("SiloPort"),
gatewayPort: configSection.GetValue<int>("GatewayPort"), listenOnAnyHostAddress: true)
.UseMongoDBClient(configSection.GetValue<string>("MongoDBClient"))
.UseMongoDBClustering(options =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.Strategy = MongoDBMembershipStrategy.SingleDocument;
})
.Configure<JsonGrainStateSerializerOptions>(options => options.ConfigureJsonSerializerSettings =
settings =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.CreateShardKeyForCosmos = false;
settings.NullValueHandling = NullValueHandling.Include;
settings.ObjectCreationHandling = ObjectCreationHandling.Replace;
settings.DefaultValueHandling = DefaultValueHandling.Populate;
})
.Configure<ClusterOptions>(options =>
.AddMongoDBGrainStorage("Default", (MongoDBGrainStorageOptions op) =>
{
op.CollectionPrefix = "GrainStorage";
op.DatabaseName = configSection.GetValue<string>("DataBase");

var grainIdPrefix = configSection
.GetSection("GrainSpecificIdPrefix").GetChildren().ToDictionary(o => o.Key.ToLower(), o => o.Value);
op.KeyGenerator = id =>
{
options.ClusterId = configSection.GetValue<string>("ClusterId");
options.ServiceId = configSection.GetValue<string>("ServiceId");
})
// .AddMemoryGrainStorage("PubSubStore")
.UseDashboard(options => {
options.Username = configSection.GetValue<string>("DashboardUserName");
options.Password = configSection.GetValue<string>("DashboardPassword");
options.Host = "*";
options.Port = configSection.GetValue<int>("DashboardPort");
options.HostSelf = true;
options.CounterUpdateIntervalMs = configSection.GetValue<int>("DashboardCounterUpdateIntervalMs");
})
.ConfigureLogging(logging => { logging.SetMinimumLevel(LogLevel.Debug).AddConsole(); });
});
var grainType = id.Type.ToString();
if (grainIdPrefix.TryGetValue(grainType, out var prefix))
{
return prefix.StartsWith("GrainReference=000000") ? $"{prefix}+{id.Key}" : prefix;
}

return id.ToString();
};
op.CreateShardKeyForCosmos = configSection.GetValue<bool>("CreateShardKeyForMongoDB", false);
})
.UseMongoDBReminders(options =>
{
options.DatabaseName = configSection.GetValue<string>("DataBase");
options.CreateShardKeyForCosmos = false;
})
.Configure<ClusterOptions>(options =>
{
options.ClusterId = configSection.GetValue<string>("ClusterId");
options.ServiceId = configSection.GetValue<string>("ServiceId");
})
// .AddMemoryGrainStorage("PubSubStore")
.UseDashboard(options =>
{
options.Username = configSection.GetValue<string>("DashboardUserName");
options.Password = configSection.GetValue<string>("DashboardPassword");
options.Host = "*";
options.Port = configSection.GetValue<int>("DashboardPort");
options.HostSelf = true;
options.CounterUpdateIntervalMs = configSection.GetValue<int>("DashboardCounterUpdateIntervalMs");
})
.ConfigureLogging(logging => { logging.SetMinimumLevel(LogLevel.Debug).AddConsole(); });
}
}
35 changes: 29 additions & 6 deletions src/EoaServer.Silo/appsettings.json
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,56 @@
{
"Orleans": {
"IsRunningInKubernetes": false,
"ClusterId": "EoaServerSiloCluster",
"ServiceId": "EoaServerOrleansBasicService",
"AdvertisedIP": "",
"AdvertisedIP": "127.0.0.1",
"GatewayPort" : 20001,
"SiloPort": 10001,
"MongoDBClient": "mongodb://admin:[email protected]:27017,portkey-eoa-mongodb-1.portkey-eoa-mongodb:27017,portkey-eoa-mongodb-2.portkey-eoa-mongodb:27017/?retryWrites=false&maxPoolSize=555",
"MongoDBClient": "mongodb://127.0.0.1:27017/?maxPoolSize=555",
"DataBase": "EoaServerOrleansDB",
"DashboardUserName": "sjn",
"DashboardPassword": "123456",
"DashboardCounterUpdateIntervalMs": 1000,
"DashboardPort": 8088,
"EventStoreConnection": "ConnectTo=tcp://admin:changeit@localhost:1113; HeartBeatTimeout=500",
"ClusterDbConnection": "portkey-eoa-redis:6379",
"ClusterDbConnection": "127.0.0.1:6379",
"ClusterDbNumber": 0,
"GrainStorageDbConnection": "portkey-eoa-redis:6379",
"GrainStorageDbConnection": "127.0.0.1:6379",
"GrainStorageDbNumber": 0
},
"ConnectionStrings": {
"Default": "mongodb://admin:[email protected]:27017,portkey-eoa-mongodb-1.portkey-eoa-mongodb:27017,portkey-eoa-mongodb-2.portkey-eoa-mongodb:27017/?authSource=admin&retryWrites=false"
"Default": "mongodb://127.0.0.1:27017/EoaServer?authSource=admin&retryWrites=false"
},
"ElasticUris": {
"Uris": [
"portkey-eoa-es-http:9200"
"http://127.0.0.1:9200"
]
},
"IndexSetting": {
"NumberOfShards": 5,
"NumberOfReplicas": 1,
"IndexPrefix": "EoaServer"
},
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Default": "Warning",
"System": "Warning",
"Microsoft": "Warning"
}
},
"Properties": {
"Application": "EoaSilo.Service",
"Environment": "Testnet"
},
"WriteTo": [
{
"Name": "Console",
"Args": {
"formatter": "Serilog.Formatting.Compact.RenderedCompactJsonFormatter, Serilog.Formatting.Compact"
}
}
]
}
}

0 comments on commit d20ee5f

Please sign in to comment.