From e8b2f08c300626ab6c9b4bb2c2f23df209c813f2 Mon Sep 17 00:00:00 2001 From: schwartz-concordium <132270889+schwartz-concordium@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:43:18 +0100 Subject: [PATCH 1/2] add metric for main import flow duration --- .../Aggregates/Contract/ContractAggregate.cs | 2 +- .../InitialContractAggregateCatchUpJob.cs | 2 +- .../Contract/Observability/ContractMetrics.cs | 38 +-------------- .../GraphQL/Import/ImportWriteController.cs | 10 ++-- .../Observability/ApplicationMetrics.cs | 47 ++++++++++++++++++- 5 files changed, 56 insertions(+), 43 deletions(-) diff --git a/backend/Application/Aggregates/Contract/ContractAggregate.cs b/backend/Application/Aggregates/Contract/ContractAggregate.cs index 9fe6913b3..fc82c5a66 100644 --- a/backend/Application/Aggregates/Contract/ContractAggregate.cs +++ b/backend/Application/Aggregates/Contract/ContractAggregate.cs @@ -517,7 +517,7 @@ private async Task NodeImportRange(IContractNodeClient client, ulong fromBlockHe for (var height = fromBlockHeight; height <= toBlockHeight; height++) { using var __ = TraceContext.StartActivity(NodeImportJobLoopActivity); - using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.NodeImport); + using var durationMetric = ContractMetrics.CreateContractReadDurationMetric(ImportSource.NodeImport); try { await using var repository = await _repositoryFactory.CreateContractRepositoryAsync(); diff --git a/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs b/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs index 178bcab00..a1c4e2f42 100644 --- a/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs +++ b/backend/Application/Aggregates/Contract/Jobs/InitialContractAggregateCatchUpJob.cs @@ -68,7 +68,7 @@ public async Task BatchImportJob(ulong heightFrom, ulong heightTo, Cancel return await Policies.GetTransientPolicy(GetUniqueIdentifier(), _logger, _contractAggregateOptions.RetryCount, _contractAggregateOptions.RetryDelay) .ExecuteAsync(async () => { - using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.DatabaseImport); + using var durationMetric =ContractMetrics.CreateContractReadDurationMetric(ImportSource.DatabaseImport); try { await using var repository = await _repositoryFactory.CreateContractRepositoryAsync(); diff --git a/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs b/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs index 699ba1be5..a7793812d 100644 --- a/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs +++ b/backend/Application/Aggregates/Contract/Observability/ContractMetrics.cs @@ -1,22 +1,11 @@ -using System.Diagnostics; using Application.Aggregates.Contract.Types; using Application.Observability; using Prometheus; -using static Application.Observability.ApplicationMetrics; namespace Application.Aggregates.Contract.Observability; internal static class ContractMetrics { - private static readonly Histogram ReadDuration = Metrics.CreateHistogram( - "contract_read_duration_seconds", - "Duration of import in seconds", - new HistogramConfiguration - { - LabelNames = new[] { "source", "exception" } - } - ); - private static readonly Counter ImportedTransactionEvents = Metrics.CreateCounter( "contract_imported_transaction_events_total", "Number of transaction event which has been processed and triggered one or more events", new CounterConfiguration @@ -36,31 +25,8 @@ internal static void SetReadHeight(double value, ImportSource source) { ApplicationMetrics.SetReadHeight(value, "contract", source); } - - internal class DurationMetric : IDisposable - { - private string _exceptionName = ""; - private readonly Stopwatch _time; - private readonly ImportSource _source; - - public DurationMetric(ImportSource source) - { - _source = source; - _time = Stopwatch.StartNew(); - } - internal void SetException(Exception ex) - { - _exceptionName = PrettyPrintException(ex); - } - - public void Dispose() - { - var elapsedSeconds = _time.ElapsedMilliseconds / 1_000d; - ReadDuration - .WithLabels(_source.ToStringCached(), _exceptionName) - .Observe(elapsedSeconds); - } - } + internal static ApplicationMetrics.DurationMetric CreateContractReadDurationMetric(ImportSource source) => + new("contract_import", source); } diff --git a/backend/Application/Api/GraphQL/Import/ImportWriteController.cs b/backend/Application/Api/GraphQL/Import/ImportWriteController.cs index 1fa20155d..c253aff40 100644 --- a/backend/Application/Api/GraphQL/Import/ImportWriteController.cs +++ b/backend/Application/Api/GraphQL/Import/ImportWriteController.cs @@ -11,7 +11,6 @@ using Application.Configurations; using Application.Database; using Application.Database.MigrationJobs; -using Application.Entities; using Application.Import; using Application.Jobs; using Application.Observability; @@ -159,7 +158,7 @@ private async Task ReadAndPublishInitialState() private async Task WriteData(BlockDataPayload payload, ConsensusInfo consensusStatus, CancellationToken stoppingToken) { using var counter = _metrics.MeasureDuration(nameof(ImportWriteController), nameof(WriteData)); - + using var durationMetric = new ApplicationMetrics.DurationMetric("main_import", ImportSource.NodeImport); var txScope = CreateTransactionScope(); BlockWriteResult result; @@ -168,7 +167,7 @@ private async Task WriteData(BlockDataPayload payload, Consens var importState = payload switch { GenesisBlockDataPayload genesisPayload => ImportState.CreateGenesisState( - genesisPayload, + genesisPayload, (int)consensusStatus.EpochDuration.TotalMilliseconds ), _ => await _importStateController.GetState() @@ -185,6 +184,11 @@ private async Task WriteData(BlockDataPayload payload, Consens txScope.Complete(); } + catch (Exception exception) + { + durationMetric.SetException(exception); + throw; + } finally { using (_metrics.MeasureDuration(nameof(ImportWriteController), "WriteDataDisposeTx")) diff --git a/backend/Application/Observability/ApplicationMetrics.cs b/backend/Application/Observability/ApplicationMetrics.cs index 3a783aced..6d959cf63 100644 --- a/backend/Application/Observability/ApplicationMetrics.cs +++ b/backend/Application/Observability/ApplicationMetrics.cs @@ -10,6 +10,15 @@ namespace Application.Observability; internal static class ApplicationMetrics { + private static readonly Histogram ProcessDuration = Metrics.CreateHistogram( + "process_duration_seconds", + "Duration of a process in seconds", + new HistogramConfiguration + { + LabelNames = new[] { "process", "source", "exception" } + } + ); + private static readonly Gauge ProcessReadHeight = Metrics.CreateGauge( "import_process_read_height", "Max height read by an import process", @@ -43,6 +52,15 @@ internal static class ApplicationMetrics LabelNames = new[] { "process", "exception" } } ); + + private static void AddProcessDuration(TimeSpan elapsed, string process, ImportSource source, Exception? exception) + { + var exceptionName = exception != null ? PrettyPrintException(exception) : ""; + var elapsedSeconds = elapsed.TotalMilliseconds / 1_000d; + ProcessDuration + .WithLabels(process, source.ToStringCached(), exceptionName) + .Observe(elapsedSeconds); + } internal static void SetReadHeight(double value, string processIdentifier, ImportSource source) { @@ -65,6 +83,31 @@ internal static void IncRetryPolicyExceptions(string process, Exception exceptio .Inc(); } + internal class DurationMetric : IDisposable + { + private Exception? _exception; + private readonly Stopwatch _time; + private readonly ImportSource _source; + private readonly string _process; + + public DurationMetric(string process, ImportSource source) + { + _process = process; + _source = source; + _time = Stopwatch.StartNew(); + } + + internal void SetException(Exception ex) + { + _exception = ex; + } + + public void Dispose() + { + AddProcessDuration(_time.Elapsed, _process, _source, _exception); + } + } + internal class GraphQlDurationMetric : IDisposable { /// @@ -155,8 +198,8 @@ private string CreateOperationDisplayName() } } } - - internal static string PrettyPrintException(Exception ex) + + private static string PrettyPrintException(Exception ex) { var type = ex.GetType(); if (type.GenericTypeArguments.Length == 0) From 31ff020792b46b46d6e247e1d750c544bda26982 Mon Sep 17 00:00:00 2001 From: schwartz-concordium <132270889+schwartz-concordium@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:52:54 +0100 Subject: [PATCH 2/2] bump version --- backend/Application/Application.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Application/Application.csproj b/backend/Application/Application.csproj index b9fff6ae6..4c9291956 100644 --- a/backend/Application/Application.csproj +++ b/backend/Application/Application.csproj @@ -4,7 +4,7 @@ net6.0 enable disable - 1.8.1 + 1.8.2 true true true