feat: silo observability

CAServer.sln

@@ -71,7 +71,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CAServer.Monitor", "src\CAS
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CAServer.ContractEventHandler.Core.Tests", "test\CAServer.ContractEventHandler.Core.Tests\CAServer.ContractEventHandler.Core.Tests.csproj", "{7AF6AC0F-D8CE-405B-B537-CF9E74B0F591}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CAServer.TelemetryConsumers.Nightingale", "src\CAServer.TelemetryConsumers.Nightingale\CAServer.TelemetryConsumers.Nightingale.csproj", "{F65B38A5-9835-4146-A82F-E7B73D61A28A}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CAServer.Nightingale", "src\CAServer.Nightingale\CAServer.Nightingale.csproj", "{F65B38A5-9835-4146-A82F-E7B73D61A28A}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

src/CAServer.HttpApi.Host/CAServer.HttpApi.Host.csproj

@@ -30,6 +30,7 @@
     <ProjectReference Include="..\CAServer.CoinGeckoApi\CAServer.CoinGeckoApi.csproj" />
     <ProjectReference Include="..\CAServer.MongoDB\CAServer.MongoDB.csproj" />
     <ProjectReference Include="..\CAServer.HttpApi\CAServer.HttpApi.csproj" />
+    <ProjectReference Include="..\CAServer.Nightingale\CAServer.Nightingale.csproj" />
     <ProjectReference Include="..\CAServer.Redis\CAServer.Redis.csproj" />
   </ItemGroup>
 

src/CAServer.HttpApi.Host/CAServerHttpApiHostModule.cs

@@ -8,6 +8,7 @@
 using CAServer.Middleware;
 using CAServer.MongoDB;
 using CAServer.MultiTenancy;
+using CAServer.Nightingale.Orleans.Filters;
 using CAServer.Options;
 using CAServer.Redis;
 using CAServer.ThirdPart.Adaptor;
@@ -215,6 +216,7 @@ private static void ConfigureOrleans(ServiceConfigurationContext context, IConfi
                 .ConfigureApplicationParts(parts =>
                     parts.AddApplicationPart(typeof(CAServerGrainsModule).Assembly).WithReferences())
                 .ConfigureLogging(builder => builder.AddProvider(o.GetService<ILoggerProvider>()))
+                .AddNightingaleMethodFilter()
                 .Build();
         });
     }

src/CAServer.HttpApi.Host/Program.cs

@@ -2,6 +2,7 @@
 using System.Threading.Tasks;
 using CAServer.Commons;
 using CAServer.Hubs;
+using CAServer.Nightingale;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -29,6 +30,7 @@ public static async Task<int> Main(string[] args)
             var hostBuilder = builder.Host.AddAppSettingsSecretsJson()
                 .InitAppConfiguration(false)
                 .UseApolloForConfigureHostBuilder()
+                .UseNightingaleMonitoring()
                 .UseAutofac()
                 .UseSerilog();
 

src/CAServer.TelemetryConsumers.Nightingale/CAServer.TelemetryConsumers.Nightingale.csproj → src/CAServer.Nightingale/CAServer.Nightingale.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net7.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <RootNamespace>Orleans.TelemetryConsumers.Nightingale</RootNamespace>
+        <RootNamespace>CAServer.Nightingale</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>

src/CAServer.Nightingale/HostHelper.cs

@@ -0,0 +1,25 @@
+using System.Net;
+using Serilog;
+
+namespace CAServer.Nightingale;
+
+public static class HostHelper
+{
+    /// <summary>
+    /// note: Failure will return "".
+    /// </summary>
+    /// <returns></returns>
+    public static string GetLocalHostName()
+    {
+        try
+        {
+            return Dns.GetHostName();
+        }
+        catch (Exception e)
+        {
+            Log.Logger.Error(e, "get host name error.");
+        }
+
+        return string.Empty;
+    }
+}

src/CAServer.Nightingale/IN9EClient.cs

@@ -0,0 +1,49 @@
+using Orleans.Runtime;
+
+namespace CAServer.Nightingale;
+
+public interface IN9EClient
+{
+    //IMetricTelemetryConsumer
+    public void TrackMetric(string name, double value, IDictionary<string, string>? properties = null);
+
+    public void TrackMetric(string name, TimeSpan value, IDictionary<string, string> properties = null);
+
+    public void IncrementMetric(string name);
+
+    public void IncrementMetric(string name, double value);
+
+    public void DecrementMetric(string name);
+
+    public void DecrementMetric(string name, double value);
+
+    //ITraceTelemetryConsumer
+    public void TrackTrace(string message);
+
+    public void TrackTrace(string message, Severity severity);
+
+    public void TrackTrace(string message, Severity severity, IDictionary<string, string> properties);
+
+    public void TrackTrace(string message, IDictionary<string, string> properties);
+
+    //IRequestTelemetryConsumer
+    public void TrackRequest(string name, DateTimeOffset startTime, TimeSpan duration, string responseCode,
+        bool success);
+
+    //IDependencyTelemetryConsumer
+    public void TrackDependency(string dependencyName, string commandName, DateTimeOffset startTime, TimeSpan duration,
+        bool success);
+
+    //IExceptionTelemetryConsumer
+    public void TrackException(Exception exception, IDictionary<string, string> properties = null, IDictionary<string, double> metrics = null);
+
+    //IEventTelemetryConsumer
+    public void TrackEvent(string eventName, IDictionary<string, string> properties = null,
+        IDictionary<string, double> metrics = null);
+
+    public bool IsEnabled();
+
+    public void Flush();
+
+    public void Close();
+}

src/CAServer.Nightingale/Logging/N9EClientForLogging.cs

@@ -0,0 +1,209 @@
+using Microsoft.Extensions.Options;
+using Orleans.Runtime;
+using Serilog;
+
+namespace CAServer.Nightingale.Logging;
+
+public class N9EClientForLogging : IN9EClient
+{
+    private readonly ILogger _logger;
+    private readonly IOptionsMonitor<N9EClientForLoggingOptions> _n9EClientForLoggingOptions;
+    private readonly string _hostName;
+
+    public N9EClientForLogging(IServiceProvider serviceProvider,
+        IOptionsMonitor<N9EClientForLoggingOptions> n9EClientForLoggingOptions)
+    {
+        _n9EClientForLoggingOptions = n9EClientForLoggingOptions;
+        _logger = N9EClientLogHelper.CreateLogger(n9EClientForLoggingOptions);
+        _hostName = HostHelper.GetLocalHostName();
+    }
+
+    public void TrackMetric(string name, double value, IDictionary<string, string>? properties = null)
+    {
+        TrackMetric(FormatMetric(name, value, properties));
+    }
+
+    public void TrackMetric(string name, TimeSpan value, IDictionary<string, string> properties = null)
+    {
+        TrackMetric(FormatMetric(name, value, properties));
+    }
+
+    public void IncrementMetric(string name)
+    {
+        TrackMetric(FormatMetric(name, 1, null));
+    }
+
+    public void IncrementMetric(string name, double value)
+    {
+        TrackMetric(FormatMetric(name, value, null));
+    }
+
+    public void DecrementMetric(string name)
+    {
+        TrackMetric(FormatMetric(name, -1, null));
+    }
+
+    public void DecrementMetric(string name, double value)
+    {
+        TrackMetric(FormatMetric(name, value * -1, null));
+    }
+
+    public void TrackTrace(string message)
+    {
+        //Doing nothing
+    }
+
+    public void TrackTrace(string message, Severity severity)
+    {
+        //Doing nothing
+    }
+
+    public void TrackTrace(string message, Severity severity, IDictionary<string, string> properties)
+    {
+        //Doing nothing
+    }
+
+    public void TrackTrace(string message, IDictionary<string, string> properties)
+    {
+        //Doing nothing
+    }
+
+    public void TrackRequest(string name, DateTimeOffset startTime, TimeSpan duration, string responseCode,
+        bool success)
+    {
+        TrackMetric(FormatMetric(name, duration, new Dictionary<string, string>()
+        {
+            { N9EClientConstant.LabelStartTime, startTime.Millisecond.ToString() },
+            { N9EClientConstant.LabelResponseCode, responseCode },
+            { N9EClientConstant.LabelSuccess, success.ToString() }
+        }));
+    }
+
+    public void TrackDependency(string dependencyName, string commandName, DateTimeOffset startTime, TimeSpan duration,
+        bool success)
+    {
+        TrackMetric(FormatMetric(dependencyName, duration, new Dictionary<string, string>()
+        {
+            { N9EClientConstant.LabelCommandName, commandName },
+            { N9EClientConstant.LabelStartTime, startTime.Millisecond.ToString() },
+            { N9EClientConstant.LabelSuccess, success.ToString() }
+        }));
+    }
+
+    public void TrackException(Exception exception, IDictionary<string, string> properties = null,
+        IDictionary<string, double> metrics = null)
+    {
+        if (metrics != null && metrics.Count > 0)
+        {
+            foreach (var metric in metrics)
+            {
+                var dictionary = properties == null
+                    ? new Dictionary<string, string>()
+                    : new Dictionary<string, string>(properties);
+                dictionary.Add(N9EClientConstant.LabelChart, metric.Key);
+                dictionary.Add(N9EClientConstant.LabelMessage, exception.Message.ToString());
+                TrackMetric(FormatMetric(N9EClientConstant.MetricExceptions, metric.Value, dictionary));
+            }
+        }
+        else
+        {
+            var dictionary = properties == null
+                ? new Dictionary<string, string>()
+                : new Dictionary<string, string>(properties);
+            dictionary.Add(N9EClientConstant.LabelMessage, exception.Message.ToString());
+            TrackMetric(FormatMetric(N9EClientConstant.MetricExceptions, 1, dictionary));
+        }
+    }
+
+    public void TrackEvent(string eventName, IDictionary<string, string> properties = null,
+        IDictionary<string, double> metrics = null)
+    {
+        if (metrics != null && metrics.Count > 0)
+        {
+            foreach (var metric in metrics)
+            {
+                var dictionary = properties == null
+                    ? new Dictionary<string, string>()
+                    : new Dictionary<string, string>(properties);
+                dictionary.Add(N9EClientConstant.LabelChart, metric.Key);
+                TrackMetric(FormatMetric(eventName, metric.Value, dictionary));
+            }
+        }
+        else
+        {
+            TrackMetric(FormatMetric(eventName, 1, properties));
+        }
+    }
+
+    public bool IsEnabled()
+    {
+        return !_n9EClientForLoggingOptions.CurrentValue.DisableLogging;
+    }
+
+    public void Flush()
+    {
+    }
+
+    public void Close()
+    {
+    }
+
+    private string FormatMetric(string? name, int value, IDictionary<string, string>? properties)
+    {
+        return $"{FormatMetricName(name)}{{{FormatMetricLabel(properties)}}} {value}";
+    }
+
+    private string FormatMetric(string? name, long value, IDictionary<string, string>? properties)
+    {
+        return $"{FormatMetricName(name)}{{{FormatMetricLabel(properties)}}} {value}";
+    }
+
+    private string FormatMetric(string? name, double value, IDictionary<string, string>? properties)
+    {
+        return $"{FormatMetricName(name)}{{{FormatMetricLabel(properties)}}} {value}";
+    }
+
+    private string FormatMetric(string? name, TimeSpan value, IDictionary<string, string>? properties)
+    {
+        return $"{FormatMetricName(name)}{{{FormatMetricLabel(properties)}}} {value}";
+    }
+
+    private string FormatMetricName(string? name)
+    {
+        if (string.IsNullOrWhiteSpace(name))
+        {
+            return string.Empty;
+        }
+
+        var formatMetricName = name.Replace(".", "_");
+        if (formatMetricName.Length > _n9EClientForLoggingOptions.CurrentValue.MetricNameMaxLength)
+        {
+            formatMetricName = formatMetricName[.._n9EClientForLoggingOptions.CurrentValue.MetricNameMaxLength];
+        }
+
+        return formatMetricName;
+    }
+
+    private string FormatMetricLabel(IDictionary<string, string>? properties)
+    {
+        properties = properties != null && properties.Count > 0
+            ? new Dictionary<string, string>(properties)
+            : new Dictionary<string, string>();
+        properties.Add(N9EClientConstant.LabelHostName, _hostName);
+        properties.Add(N9EClientConstant.LabelTimestamp, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString());
+        var labelStr = string.Empty;
+        properties.Aggregate(labelStr,
+            (current, property) => current + $"{property.Key}=\"{property.Value}\"");
+        return labelStr;
+    }
+
+    private void TrackMetric(string metric)
+    {
+        if (_n9EClientForLoggingOptions.CurrentValue.DisableLogging)
+        {
+            return;
+        }
+
+        _logger.Warning(metric);
+    }
+}

src/CAServer.Nightingale/Logging/N9EClientForLoggingOptions.cs

@@ -0,0 +1,12 @@
+namespace CAServer.Nightingale.Logging;
+
+public class N9EClientForLoggingOptions
+{
+    public bool DisableLogging { get; set; } = false;
+
+    public string LogFilePathFormat { get; set; } = "./telemetry/trace-.log";
+
+    public int LogRetainedFileCountLimit { get; set; } = 5;
+
+    public int MetricNameMaxLength { get; set; } = 100;
+}

src/CAServer.TelemetryConsumers.Nightingale/LogHelper.cs → src/CAServer.Nightingale/Logging/N9EClientLogHelper.cs

@@ -1,25 +1,27 @@
+using Microsoft.Extensions.Options;
 using Serilog;
 using Serilog.Events;
 
-namespace Orleans.TelemetryConsumers.Nightingale;
+namespace CAServer.Nightingale.Logging;
 
-public static class LogHelper
+public static class N9EClientLogHelper
 {
-    public static string FilePath { get; set; } =
-        "./other/n9e-v6.7.3-linux-amd64/docker/compose-bridge/etc-categraf/logs/log-.log";
-    public static ILogger CreateLogger(LogEventLevel logEventLevel)
+    public static ILogger CreateLogger(IOptionsMonitor<N9EClientForLoggingOptions> options)
     {
         return new LoggerConfiguration()
-            .MinimumLevel.Is(logEventLevel)
+            .MinimumLevel.Is(LogEventLevel.Debug)
             .Enrich.FromLogContext()
+#if DEBUG
             .WriteTo.Async(c =>
                 c.Console(
                     outputTemplate:
-                    "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}{Offset:zzz}][{Level:u3}] [{SourceContext}] {Message}{NewLine}"))
+                    "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}{Offset:zzz}][{Level:u3}] {Message}{NewLine}"))
+#endif
             .WriteTo.Async(c =>
                 c.File(
-                    FilePath,
+                    options.CurrentValue.LogFilePathFormat,
                     rollingInterval: RollingInterval.Day,
+                    retainedFileCountLimit: options.CurrentValue.LogRetainedFileCountLimit,
                     outputTemplate:
                     "{Message}{NewLine}"))
             .CreateLogger();