Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Main import duration metric #149

Merged
2 commits merged into from
Dec 1, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ private async Task NodeImportRange(IContractNodeClient client, ulong fromBlockHe
for (var height = fromBlockHeight; height <= toBlockHeight; height++)
{
using var __ = TraceContext.StartActivity(NodeImportJobLoopActivity);
using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.NodeImport);
using var durationMetric = ContractMetrics.CreateContractReadDurationMetric(ImportSource.NodeImport);
try
{
await using var repository = await _repositoryFactory.CreateContractRepositoryAsync();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public async Task<ulong> BatchImportJob(ulong heightFrom, ulong heightTo, Cancel
return await Policies.GetTransientPolicy<ulong>(GetUniqueIdentifier(), _logger, _contractAggregateOptions.RetryCount, _contractAggregateOptions.RetryDelay)
.ExecuteAsync(async () =>
{
using var durationMetric = new ContractMetrics.DurationMetric(ImportSource.DatabaseImport);
using var durationMetric =ContractMetrics.CreateContractReadDurationMetric(ImportSource.DatabaseImport);
try
{
await using var repository = await _repositoryFactory.CreateContractRepositoryAsync();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
using System.Diagnostics;
using Application.Aggregates.Contract.Types;
using Application.Observability;
using Prometheus;
using static Application.Observability.ApplicationMetrics;

namespace Application.Aggregates.Contract.Observability;

internal static class ContractMetrics
{
private static readonly Histogram ReadDuration = Metrics.CreateHistogram(
"contract_read_duration_seconds",
"Duration of import in seconds",
new HistogramConfiguration
{
LabelNames = new[] { "source", "exception" }
}
);

private static readonly Counter ImportedTransactionEvents = Metrics.CreateCounter(
"contract_imported_transaction_events_total", "Number of transaction event which has been processed and triggered one or more events",
new CounterConfiguration
Expand All @@ -36,31 +25,8 @@ internal static void SetReadHeight(double value, ImportSource source)
{
ApplicationMetrics.SetReadHeight(value, "contract", source);
}

internal class DurationMetric : IDisposable
{
private string _exceptionName = "";
private readonly Stopwatch _time;
private readonly ImportSource _source;

public DurationMetric(ImportSource source)
{
_source = source;
_time = Stopwatch.StartNew();
}

internal void SetException(Exception ex)
{
_exceptionName = PrettyPrintException(ex);
}

public void Dispose()
{
var elapsedSeconds = _time.ElapsedMilliseconds / 1_000d;
ReadDuration
.WithLabels(_source.ToStringCached(), _exceptionName)
.Observe(elapsedSeconds);
}
}
internal static ApplicationMetrics.DurationMetric CreateContractReadDurationMetric(ImportSource source) =>
new("contract_import", source);
}

10 changes: 7 additions & 3 deletions backend/Application/Api/GraphQL/Import/ImportWriteController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
using Application.Configurations;
using Application.Database;
using Application.Database.MigrationJobs;
using Application.Entities;
using Application.Import;
using Application.Jobs;
using Application.Observability;
Expand Down Expand Up @@ -159,7 +158,7 @@ private async Task ReadAndPublishInitialState()
private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, ConsensusInfo consensusStatus, CancellationToken stoppingToken)
{
using var counter = _metrics.MeasureDuration(nameof(ImportWriteController), nameof(WriteData));

using var durationMetric = new ApplicationMetrics.DurationMetric("main_import", ImportSource.NodeImport);
var txScope = CreateTransactionScope();

BlockWriteResult result;
Expand All @@ -168,7 +167,7 @@ private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, Consens
var importState = payload switch
{
GenesisBlockDataPayload genesisPayload => ImportState.CreateGenesisState(
genesisPayload,
genesisPayload,
(int)consensusStatus.EpochDuration.TotalMilliseconds
),
_ => await _importStateController.GetState()
Expand All @@ -185,6 +184,11 @@ private async Task<BlockWriteResult> WriteData(BlockDataPayload payload, Consens

txScope.Complete();
}
catch (Exception exception)
{
durationMetric.SetException(exception);
throw;
}
finally
{
using (_metrics.MeasureDuration(nameof(ImportWriteController), "WriteDataDisposeTx"))
Expand Down
2 changes: 1 addition & 1 deletion backend/Application/Application.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<TargetFramework>net6.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>disable</ImplicitUsings>
<Version>1.8.1</Version>
<Version>1.8.2</Version>
<IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
<IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
<IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>
Expand Down
47 changes: 45 additions & 2 deletions backend/Application/Observability/ApplicationMetrics.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@ namespace Application.Observability;

internal static class ApplicationMetrics
{
private static readonly Histogram ProcessDuration = Metrics.CreateHistogram(
"process_duration_seconds",
"Duration of a process in seconds",
new HistogramConfiguration
{
LabelNames = new[] { "process", "source", "exception" }
}
);

private static readonly Gauge ProcessReadHeight = Metrics.CreateGauge(
"import_process_read_height",
"Max height read by an import process",
Expand Down Expand Up @@ -43,6 +52,15 @@ internal static class ApplicationMetrics
LabelNames = new[] { "process", "exception" }
}
);

private static void AddProcessDuration(TimeSpan elapsed, string process, ImportSource source, Exception? exception)
{
var exceptionName = exception != null ? PrettyPrintException(exception) : "";
var elapsedSeconds = elapsed.TotalMilliseconds / 1_000d;
ProcessDuration
.WithLabels(process, source.ToStringCached(), exceptionName)
.Observe(elapsedSeconds);
}

internal static void SetReadHeight(double value, string processIdentifier, ImportSource source)
{
Expand All @@ -65,6 +83,31 @@ internal static void IncRetryPolicyExceptions(string process, Exception exceptio
.Inc();
}

internal class DurationMetric : IDisposable
{
private Exception? _exception;
private readonly Stopwatch _time;
private readonly ImportSource _source;
private readonly string _process;

public DurationMetric(string process, ImportSource source)
{
_process = process;
_source = source;
_time = Stopwatch.StartNew();
}

internal void SetException(Exception ex)
{
_exception = ex;
}

public void Dispose()
{
AddProcessDuration(_time.Elapsed, _process, _source, _exception);
}
}

internal class GraphQlDurationMetric : IDisposable
{
/// <summary>
Expand Down Expand Up @@ -155,8 +198,8 @@ private string CreateOperationDisplayName()
}
}
}
internal static string PrettyPrintException(Exception ex)

private static string PrettyPrintException(Exception ex)
{
var type = ex.GetType();
if (type.GenericTypeArguments.Length == 0)
Expand Down
Loading