From 06120564d5aab1071bebc3986b702a3d99d230c0 Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Sat, 25 Jan 2025 15:40:54 +0000 Subject: [PATCH 1/2] draft --- cloud/disk_manager/pkg/app/run.go | 9 ++- cloud/disk_manager/pkg/schema/dataplane.go | 6 +- cloud/tasks/persistence/health.go | 54 +++++++++++++++++ cloud/tasks/persistence/health_test.go | 48 ++++++++++++++++ cloud/tasks/persistence/s3.go | 11 ++-- cloud/tasks/persistence/s3_metrics.go | 24 +++++--- cloud/tasks/persistence/s3_test.go | 67 ++++++++++++++++------ cloud/tasks/persistence/ya.make | 2 + 8 files changed, 188 insertions(+), 33 deletions(-) create mode 100644 cloud/tasks/persistence/health.go create mode 100644 cloud/tasks/persistence/health_test.go diff --git a/cloud/disk_manager/pkg/app/run.go b/cloud/disk_manager/pkg/app/run.go index 75d0272a95..3fc2ce6371 100644 --- a/cloud/disk_manager/pkg/app/run.go +++ b/cloud/disk_manager/pkg/app/run.go @@ -204,10 +204,15 @@ func run( s3Bucket = snapshotConfig.GetS3Bucket() // TODO: remove when s3 will always be initialized. if s3Config != nil { - registry := mon.NewRegistry("s3_client") + s3MetricsRegistry := mon.NewRegistry("s3_client") + healthMetricsRegistry := mon.NewRegistry("health") var err error - s3, err = persistence.NewS3ClientFromConfig(s3Config, registry) + s3, err = persistence.NewS3ClientFromConfig( + s3Config, + s3MetricsRegistry, + healthMetricsRegistry, + ) if err != nil { return err } diff --git a/cloud/disk_manager/pkg/schema/dataplane.go b/cloud/disk_manager/pkg/schema/dataplane.go index 6646ca1d8b..5b58cbdf31 100644 --- a/cloud/disk_manager/pkg/schema/dataplane.go +++ b/cloud/disk_manager/pkg/schema/dataplane.go @@ -48,7 +48,11 @@ func initDataplane( var s3 *persistence.S3Client // TODO: remove when s3 will always be initialized. if s3Config != nil { - s3, err = persistence.NewS3ClientFromConfig(s3Config, metrics.NewEmptyRegistry()) + s3, err = persistence.NewS3ClientFromConfig( + s3Config, + metrics.NewEmptyRegistry(), + metrics.NewEmptyRegistry(), + ) if err != nil { return err } diff --git a/cloud/tasks/persistence/health.go b/cloud/tasks/persistence/health.go new file mode 100644 index 0000000000..4b7b08b57a --- /dev/null +++ b/cloud/tasks/persistence/health.go @@ -0,0 +1,54 @@ +package persistence + +import ( + "time" + + "github.com/ydb-platform/nbs/cloud/tasks/metrics" +) + +//////////////////////////////////////////////////////////////////////////////// + +type healthCheck struct { + queriesCount uint64 + successQueriesCount uint64 + registry metrics.Registry + metricsCollectionInterval time.Duration +} + +func (h *healthCheck) accountQuery(err error) { + h.queriesCount++ + if err == nil { + h.successQueriesCount++ + } +} + +func (h *healthCheck) reportSuccessRate() { + if h.queriesCount == 0 { + h.registry.Gauge("successRate").Set(0) + return + } + + h.registry.Gauge("successRate").Set(float64(h.successQueriesCount) / float64(h.queriesCount)) +} + +func newHealthCheck(componentName string, registry metrics.Registry) *healthCheck { + subRegistry := registry.WithTags(map[string]string{ + "component": componentName, + }) + + h := healthCheck{ + registry: subRegistry, + metricsCollectionInterval: 15 * time.Second, // todo + } + + go func() { + ticker := time.NewTicker(h.metricsCollectionInterval) + defer ticker.Stop() + + for range ticker.C { + h.reportSuccessRate() + } + }() + + return &h +} diff --git a/cloud/tasks/persistence/health_test.go b/cloud/tasks/persistence/health_test.go new file mode 100644 index 0000000000..de59f9798b --- /dev/null +++ b/cloud/tasks/persistence/health_test.go @@ -0,0 +1,48 @@ +package persistence + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/ydb-platform/nbs/cloud/tasks/errors" + "github.com/ydb-platform/nbs/cloud/tasks/metrics/mocks" +) + +//////////////////////////////////////////////////////////////////////////////// + +func TestHealthCheckMetric(t *testing.T) { + registry := mocks.NewRegistryMock() + healthCheck := newHealthCheck("test", registry) + + gaugeSetWg := sync.WaitGroup{} + + gaugeSetWg.Add(1) + registry.GetGauge( + "successRate", + map[string]string{"component": "test"}, + ).On("Set", float64(0)).Once().Run( + func(args mock.Arguments) { + gaugeSetWg.Done() + }, + ) + gaugeSetWg.Wait() + + healthCheck.accountQuery(nil) + healthCheck.accountQuery(nil) + healthCheck.accountQuery(nil) + healthCheck.accountQuery(errors.NewEmptyRetriableError()) + + gaugeSetWg.Add(1) + registry.GetGauge( + "successRate", + map[string]string{"component": "test"}, + ).On("Set", float64(3.0/4.0)).Once().Run( + func(args mock.Arguments) { + gaugeSetWg.Done() + }, + ) + gaugeSetWg.Wait() + + registry.AssertAllExpectations(t) +} diff --git a/cloud/tasks/persistence/s3.go b/cloud/tasks/persistence/s3.go index 1deeacded7..3fab65e9dc 100644 --- a/cloud/tasks/persistence/s3.go +++ b/cloud/tasks/persistence/s3.go @@ -53,11 +53,12 @@ func NewS3Client( region string, credentials S3Credentials, callTimeout time.Duration, - registry metrics.Registry, + s3MetricsRegistry metrics.Registry, + healthMetricsRegistry metrics.Registry, maxRetriableErrorCount uint64, ) (*S3Client, error) { - s3Metrics := newS3Metrics(registry, callTimeout) + s3Metrics := newS3Metrics(callTimeout, s3MetricsRegistry, healthMetricsRegistry) sessionConfig := &aws.Config{ Credentials: aws_credentials.NewStaticCredentials( @@ -90,7 +91,8 @@ func NewS3Client( func NewS3ClientFromConfig( config *persistence_config.S3Config, - registry metrics.Registry, + s3MetricsRegistry metrics.Registry, + healthMetricsRegistry metrics.Registry, ) (*S3Client, error) { credentials, err := NewS3CredentialsFromFile(config.GetCredentialsFilePath()) @@ -111,7 +113,8 @@ func NewS3ClientFromConfig( config.GetRegion(), credentials, callTimeout, - registry, + s3MetricsRegistry, + healthMetricsRegistry, config.GetMaxRetriableErrorCount(), ) } diff --git a/cloud/tasks/persistence/s3_metrics.go b/cloud/tasks/persistence/s3_metrics.go index 645cb4c227..197f622c98 100644 --- a/cloud/tasks/persistence/s3_metrics.go +++ b/cloud/tasks/persistence/s3_metrics.go @@ -24,8 +24,9 @@ func s3CallDurationBuckets() metrics.DurationBuckets { //////////////////////////////////////////////////////////////////////////////// type s3Metrics struct { - registry metrics.Registry - callTimeout time.Duration + callTimeout time.Duration + s3MetricsRegistry metrics.Registry + healthCheck *healthCheck } func (m *s3Metrics) StatCall( @@ -38,18 +39,20 @@ func (m *s3Metrics) StatCall( start := time.Now() return func(err *error) { - subRegistry := m.registry.WithTags(map[string]string{ + subRegistry := m.s3MetricsRegistry.WithTags(map[string]string{ "call": name, }) // Should initialize all counters before using them, to avoid 'no data'. - errorCounter := subRegistry.Counter("errors") successCounter := subRegistry.Counter("success") hangingCounter := subRegistry.Counter("hanging") + errorCounter := subRegistry.Counter("errors") timeoutCounter := subRegistry.Counter("errors/timeout") canceledCounter := subRegistry.Counter("errors/canceled") timeHistogram := subRegistry.DurationHistogram("time", s3CallDurationBuckets()) + m.healthCheck.accountQuery(*err) + if time.Since(start) >= m.callTimeout { logging.Error( ctx, @@ -110,7 +113,7 @@ func (m *s3Metrics) OnRetry(req *request.Request) { req.RetryCount+1, ) - subRegistry := m.registry.WithTags(map[string]string{ + subRegistry := m.s3MetricsRegistry.WithTags(map[string]string{ "call": req.Operation.Name, }) @@ -119,9 +122,14 @@ func (m *s3Metrics) OnRetry(req *request.Request) { retryCounter.Inc() } -func newS3Metrics(registry metrics.Registry, callTimeout time.Duration) *s3Metrics { +func newS3Metrics( + callTimeout time.Duration, + s3MetricsRegistry metrics.Registry, + healthMetricsRegistry metrics.Registry, +) *s3Metrics { return &s3Metrics{ - registry: registry, - callTimeout: callTimeout, + callTimeout: callTimeout, + s3MetricsRegistry: s3MetricsRegistry, + healthCheck: newHealthCheck("s3", healthMetricsRegistry), } } diff --git a/cloud/tasks/persistence/s3_test.go b/cloud/tasks/persistence/s3_test.go index fabb5fff32..4c4697f42d 100644 --- a/cloud/tasks/persistence/s3_test.go +++ b/cloud/tasks/persistence/s3_test.go @@ -17,8 +17,9 @@ const maxRetriableErrorCount = 3 //////////////////////////////////////////////////////////////////////////////// func newS3Client( - metricsRegistry *mocks.RegistryMock, callTimeout time.Duration, + s3MetricsRegistry *mocks.RegistryMock, + healthMetricsRegistry *mocks.RegistryMock, ) (*S3Client, error) { credentials := NewS3Credentials("test", "test") @@ -27,7 +28,8 @@ func newS3Client( "region", credentials, callTimeout, - metricsRegistry, + s3MetricsRegistry, + healthMetricsRegistry, maxRetriableErrorCount, ) } @@ -37,19 +39,20 @@ func newS3Client( func TestS3ShouldSendErrorCanceledMetric(t *testing.T) { ctx, cancel := context.WithCancel(newContext()) - metricsRegistry := mocks.NewRegistryMock() + s3MetricsRegistry := mocks.NewRegistryMock() + healthMetricsRegistry := mocks.NewRegistryMock() - s3, err := newS3Client(metricsRegistry, 10*time.Second /* callTimeout */) + s3, err := newS3Client(10*time.Second /* callTimeout */, s3MetricsRegistry, healthMetricsRegistry) require.NoError(t, err) cancel() - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "errors", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "errors/canceled", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() @@ -57,29 +60,30 @@ func TestS3ShouldSendErrorCanceledMetric(t *testing.T) { err = s3.CreateBucket(ctx, "test") require.True(t, errors.Is(err, errors.NewEmptyRetriableError())) - metricsRegistry.AssertAllExpectations(t) + s3MetricsRegistry.AssertAllExpectations(t) } func TestS3ShouldSendErrorTimeoutMetric(t *testing.T) { ctx, cancel := context.WithCancel(newContext()) defer cancel() - metricsRegistry := mocks.NewRegistryMock() + s3MetricsRegistry := mocks.NewRegistryMock() + healthMetricsRegistry := mocks.NewRegistryMock() - s3, err := newS3Client(metricsRegistry, 0 /* callTimeout */) + s3, err := newS3Client(0 /* callTimeout */, s3MetricsRegistry, healthMetricsRegistry) require.NoError(t, err) - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "errors", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "hanging", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "errors/timeout", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() @@ -87,24 +91,25 @@ func TestS3ShouldSendErrorTimeoutMetric(t *testing.T) { err = s3.CreateBucket(ctx, "test") require.True(t, errors.Is(err, errors.NewEmptyRetriableError())) - metricsRegistry.AssertAllExpectations(t) + s3MetricsRegistry.AssertAllExpectations(t) } func TestS3ShouldRetryRequests(t *testing.T) { ctx, cancel := context.WithCancel(newContext()) defer cancel() - metricsRegistry := mocks.NewRegistryMock() + s3MetricsRegistry := mocks.NewRegistryMock() + healthMetricsRegistry := mocks.NewRegistryMock() - s3, err := newS3Client(metricsRegistry, 10*time.Second /* callTimeout */) + s3, err := newS3Client(10*time.Second /* callTimeout */, s3MetricsRegistry, healthMetricsRegistry) require.NoError(t, err) - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "errors", map[string]string{"call": "CreateBucket"}, ).On("Inc").Once() - metricsRegistry.GetCounter( + s3MetricsRegistry.GetCounter( "retry", map[string]string{"call": "CreateBucket"}, ).On("Inc").Times(maxRetriableErrorCount) @@ -112,5 +117,31 @@ func TestS3ShouldRetryRequests(t *testing.T) { err = s3.CreateBucket(ctx, "test") require.True(t, errors.Is(err, errors.NewEmptyRetriableError())) - metricsRegistry.AssertAllExpectations(t) + s3MetricsRegistry.AssertAllExpectations(t) +} + +func TestS3ShouldSendHealthMetric(t *testing.T) { + ctx, cancel := context.WithCancel(newContext()) + defer cancel() + + s3MetricsRegistry := mocks.NewRegistryMock() + healthMetricsRegistry := mocks.NewRegistryMock() + + s3, err := newS3Client(10*time.Second /* callTimeout */, s3MetricsRegistry, healthMetricsRegistry) + require.NoError(t, err) + + s3MetricsRegistry.GetCounter( + "errors", + map[string]string{"call": "CreateBucket"}, + ).On("Inc").Once() + + s3MetricsRegistry.GetCounter( + "retry", + map[string]string{"call": "CreateBucket"}, + ).On("Inc").Times(maxRetriableErrorCount) + + err = s3.CreateBucket(ctx, "test") + require.True(t, errors.Is(err, errors.NewEmptyRetriableError())) + + s3MetricsRegistry.AssertAllExpectations(t) } diff --git a/cloud/tasks/persistence/ya.make b/cloud/tasks/persistence/ya.make index bde3608b4c..eb6bb1e781 100644 --- a/cloud/tasks/persistence/ya.make +++ b/cloud/tasks/persistence/ya.make @@ -1,6 +1,7 @@ GO_LIBRARY() SRCS( + health.go s3.go s3_metrics.go ydb.go @@ -9,6 +10,7 @@ SRCS( ) GO_TEST_SRCS( + health_test.go s3_test.go ydb_test.go ) From 13ef599f3d479c04034e119bda90c55c20185726 Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Sun, 26 Jan 2025 18:59:00 +0000 Subject: [PATCH 2/2] maybe v2 --- cloud/tasks/persistence/health.go | 31 +++--- cloud/tasks/persistence/health_storage.go | 112 ++++++++++++++++++++++ cloud/tasks/persistence/health_test.go | 43 ++++++++- cloud/tasks/persistence/s3_metrics.go | 6 +- cloud/tasks/persistence/ya.make | 6 +- 5 files changed, 177 insertions(+), 21 deletions(-) create mode 100644 cloud/tasks/persistence/health_storage.go diff --git a/cloud/tasks/persistence/health.go b/cloud/tasks/persistence/health.go index 4b7b08b57a..804763cc65 100644 --- a/cloud/tasks/persistence/health.go +++ b/cloud/tasks/persistence/health.go @@ -8,21 +8,15 @@ import ( //////////////////////////////////////////////////////////////////////////////// -type healthCheck struct { +type HealthCheck struct { queriesCount uint64 successQueriesCount uint64 + storage *healthCheckStorage registry metrics.Registry metricsCollectionInterval time.Duration } -func (h *healthCheck) accountQuery(err error) { - h.queriesCount++ - if err == nil { - h.successQueriesCount++ - } -} - -func (h *healthCheck) reportSuccessRate() { +func (h *HealthCheck) reportSuccessRate() { if h.queriesCount == 0 { h.registry.Gauge("successRate").Set(0) return @@ -31,12 +25,27 @@ func (h *healthCheck) reportSuccessRate() { h.registry.Gauge("successRate").Set(float64(h.successQueriesCount) / float64(h.queriesCount)) } -func newHealthCheck(componentName string, registry metrics.Registry) *healthCheck { +//////////////////////////////////////////////////////////////////////////////// + +func (h *HealthCheck) AccountQuery(err error) { + h.queriesCount++ + if err == nil { + h.successQueriesCount++ + } +} + +func NewHealthCheck( + componentName string, + storage *healthCheckStorage, + registry metrics.Registry, +) *HealthCheck { + subRegistry := registry.WithTags(map[string]string{ "component": componentName, }) - h := healthCheck{ + h := HealthCheck{ + storage: storage, registry: subRegistry, metricsCollectionInterval: 15 * time.Second, // todo } diff --git a/cloud/tasks/persistence/health_storage.go b/cloud/tasks/persistence/health_storage.go new file mode 100644 index 0000000000..aa0cd70bf4 --- /dev/null +++ b/cloud/tasks/persistence/health_storage.go @@ -0,0 +1,112 @@ +package persistence + +import ( + "context" + + "github.com/ydb-platform/nbs/cloud/tasks/logging" +) + +// import ( +// "time" +// ) + +//////////////////////////////////////////////////////////////////////////////// + +func healthCheckTableDescription() CreateTableDescription { + return NewCreateTableDescription( + WithColumn("component", Optional(TypeUTF8)), + WithColumn("update_at", Optional(TypeTimestamp)), + WithPrimaryKeyColumn("component", "update_at"), + ) +} + +func CreateYDBTables( + ctx context.Context, + db *YDBClient, + dropUnusedColumns bool, +) error { + + err := db.CreateOrAlterTable( + ctx, + "kek", + "health", + healthCheckTableDescription(), + dropUnusedColumns, + ) + if err != nil { + return err + } + logging.Info(ctx, "Created nodes table") + + return nil +} + +type healthCheckStorage struct { +} + +func NewStorage(db *YDBClient) *healthCheckStorage { + return &healthCheckStorage{} +} + +// type HealthCheck struct { +// componentName string +// Rate float64 +// LastUpdate time.Time +// } + +//////////////////////////////////////////////////////////////////////////////// + +// Returns ydb entity of the node object. +// func (h *HealthCheck) structValue() persistence.Value { +// return persistence.StructValue( +// persistence.StructFieldValue("rate", persistence.DoubleValue(h.Rate)), +// persistence.StructFieldValue("last_update", persistence.DatetimeValueFromTime(h.LastUpdate)), +// ) +// } + +// // Scans single node from the YDB result set. +// func scanHealthCheck(result persistence.Result) (healthCheck HealthCheck, err error) { +// err = result.ScanNamed( +// persistence.OptionalWithDefault("rate", &healthCheck.Rate), +// persistence.OptionalWithDefault("last_update", &healthCheck.LastUpdate), +// ) +// return +// } + +// // Scans all nodes from the YDB result set. +// func scanNodes(ctx context.Context, res persistence.Result) ([]Node, error) { +// var nodes []Node +// for res.NextResultSet(ctx) { +// for res.NextRow() { +// node, err := scanNode(res) +// if err != nil { +// return nil, err +// } +// nodes = append(nodes, node) +// } +// } + +// return nodes, nil +// } + +// Returns node struct definition in YQL. +// func nodeStructTypeString() string { +// return `Struct< +// host: Utf8, +// last_heartbeat: Timestamp, +// inflight_task_count: Uint32>` +// } + +// Returns table description for the table that holds nodes. +// func healthCheckTableDescription() persistence.CreateTableDescription { +// return persistence.NewCreateTableDescription( +// persistence.WithColumn("host", persistence.Optional(persistence.TypeUTF8)), +// persistence.WithColumn("last_heartbeat", persistence.Optional(persistence.TypeTimestamp)), +// persistence.WithColumn("inflight_task_count", persistence.Optional(persistence.TypeUint32)), +// persistence.WithPrimaryKeyColumn("host"), +// ) +// } + +//////////////////////////////////////////////////////////////////////////////// + +// Updates heartbeat timestamp and the current number of inflight tasks. diff --git a/cloud/tasks/persistence/health_test.go b/cloud/tasks/persistence/health_test.go index de59f9798b..616763a0d3 100644 --- a/cloud/tasks/persistence/health_test.go +++ b/cloud/tasks/persistence/health_test.go @@ -1,19 +1,52 @@ package persistence import ( + "context" "sync" "testing" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/ydb-platform/nbs/cloud/tasks/errors" + "github.com/ydb-platform/nbs/cloud/tasks/metrics/empty" "github.com/ydb-platform/nbs/cloud/tasks/metrics/mocks" ) //////////////////////////////////////////////////////////////////////////////// +func newStorage( + t *testing.T, + ctx context.Context, + db *YDBClient, +) *healthCheckStorage { + + err := CreateYDBTables( + ctx, + db, + false, // dropUnusedColums + ) + require.NoError(t, err) + + storage := NewStorage(db) + require.NoError(t, err) + + return storage +} + +//////////////////////////////////////////////////////////////////////////////// + func TestHealthCheckMetric(t *testing.T) { + ctx, cancel := context.WithCancel(newContext()) + defer cancel() + + db, err := newYDB(ctx, empty.NewRegistry()) + require.NoError(t, err) + defer db.Close(ctx) + + storage := newStorage(t, ctx, db) + registry := mocks.NewRegistryMock() - healthCheck := newHealthCheck("test", registry) + healthCheck := NewHealthCheck("test", storage, registry) gaugeSetWg := sync.WaitGroup{} @@ -28,10 +61,10 @@ func TestHealthCheckMetric(t *testing.T) { ) gaugeSetWg.Wait() - healthCheck.accountQuery(nil) - healthCheck.accountQuery(nil) - healthCheck.accountQuery(nil) - healthCheck.accountQuery(errors.NewEmptyRetriableError()) + healthCheck.AccountQuery(nil) + healthCheck.AccountQuery(nil) + healthCheck.AccountQuery(nil) + healthCheck.AccountQuery(errors.NewEmptyRetriableError()) gaugeSetWg.Add(1) registry.GetGauge( diff --git a/cloud/tasks/persistence/s3_metrics.go b/cloud/tasks/persistence/s3_metrics.go index 197f622c98..59830751d8 100644 --- a/cloud/tasks/persistence/s3_metrics.go +++ b/cloud/tasks/persistence/s3_metrics.go @@ -26,7 +26,7 @@ func s3CallDurationBuckets() metrics.DurationBuckets { type s3Metrics struct { callTimeout time.Duration s3MetricsRegistry metrics.Registry - healthCheck *healthCheck + healthCheck *HealthCheck } func (m *s3Metrics) StatCall( @@ -51,7 +51,7 @@ func (m *s3Metrics) StatCall( canceledCounter := subRegistry.Counter("errors/canceled") timeHistogram := subRegistry.DurationHistogram("time", s3CallDurationBuckets()) - m.healthCheck.accountQuery(*err) + m.healthCheck.AccountQuery(*err) if time.Since(start) >= m.callTimeout { logging.Error( @@ -130,6 +130,6 @@ func newS3Metrics( return &s3Metrics{ callTimeout: callTimeout, s3MetricsRegistry: s3MetricsRegistry, - healthCheck: newHealthCheck("s3", healthMetricsRegistry), + healthCheck: NewHealthCheck("s3", healthMetricsRegistry), } } diff --git a/cloud/tasks/persistence/ya.make b/cloud/tasks/persistence/ya.make index eb6bb1e781..16b99e1c34 100644 --- a/cloud/tasks/persistence/ya.make +++ b/cloud/tasks/persistence/ya.make @@ -1,18 +1,20 @@ GO_LIBRARY() SRCS( - health.go s3.go s3_metrics.go ydb.go ydb_logger.go ydb_metrics.go + + health.go + health_storage.go ) GO_TEST_SRCS( - health_test.go s3_test.go ydb_test.go + health_test.go ) END()