services/horizon: Add Stellar-Core sync metrics (#3418)

This commit adds two new metrics connected to Stellar-Core:

* `horizon_ ingest_captive_stellar_core_synced` determines if Captive
Stellar-Core instance started by ingesting server is synced.
* `horizon_stellar_core_synced` determines if Stellar-Core defined by
`--stellar-core-url` is synced.

The reason for adding two separate metrics is that in some deployments
Stellar-Core connected via `--stellar-core-url` and Captive Stellar-Core are
separate instances.

services/horizon/internal/actions/root.go

@@ -10,6 +10,7 @@ import (
 )
 
 type CoreSettings struct {
+	Synced                       bool
 	CurrentProtocolVersion       int32
 	CoreSupportedProtocolVersion int32
 	CoreVersion                  string

services/horizon/internal/app.go

@@ -38,6 +38,7 @@ type coreSettingsStore struct {
 func (c *coreSettingsStore) set(resp *proto.InfoResponse) {
 	c.Lock()
 	defer c.Unlock()
+	c.Synced = resp.IsSynced()
 	c.CoreVersion = resp.Info.Build
 	c.CurrentProtocolVersion = int32(resp.Info.Ledger.Version)
 	c.CoreSupportedProtocolVersion = int32(resp.Info.ProtocolVersion)
@@ -80,6 +81,7 @@ type App struct {
 	dbWaitCountCounter                prometheus.CounterFunc
 	dbWaitDurationCounter             prometheus.CounterFunc
 	coreLatestLedgerCounter           prometheus.CounterFunc
+	coreSynced                        prometheus.GaugeFunc
 }
 
 func (a *App) GetCoreSettings() actions.CoreSettings {

services/horizon/internal/ingest/main.go

@@ -5,6 +5,8 @@ package ingest
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
@@ -116,6 +118,10 @@ type Metrics struct {
 
 	// ProcessorsRunDuration exposes processors run durations.
 	ProcessorsRunDuration *prometheus.CounterVec
+
+	// CaptiveStellarCoreSynced exposes synced status of Captive Stellar-Core.
+	// 1 if sync, 0 if not synced, -1 if unable to connect or HTTP server disabled.
+	CaptiveStellarCoreSynced prometheus.GaugeFunc
 }
 
 type System interface {
@@ -293,6 +299,37 @@ func (s *system) initMetrics() {
 		},
 		[]string{"name"},
 	)
+
+	s.metrics.CaptiveStellarCoreSynced = prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Namespace: "horizon", Subsystem: "ingest", Name: "captive_stellar_core_synced",
+			Help: "1 if sync, 0 if not synced, -1 if unable to connect or HTTP server disabled.",
+		},
+		func() float64 {
+			if !s.config.EnableCaptiveCore || s.config.CaptiveCoreHTTPPort == 0 {
+				return -1
+			}
+
+			client := stellarcore.Client{
+				HTTP: &http.Client{
+					Timeout: 2 * time.Second,
+				},
+				URL: fmt.Sprintf("http://localhost:%d", s.config.CaptiveCoreHTTPPort),
+			}
+
+			info, err := client.Info(s.ctx)
+			if err != nil {
+				log.WithError(err).Error("Cannot connect to Captive Stellar-Core HTTP server")
+				return -1
+			}
+
+			if info.IsSynced() {
+				return 1
+			} else {
+				return 0
+			}
+		},
+	)
 }
 
 func (s *system) Metrics() Metrics {

services/horizon/internal/init.go

@@ -180,6 +180,21 @@ func initDbMetrics(app *App) {
 	)
 	app.prometheusRegistry.MustRegister(app.coreLatestLedgerCounter)
 
+	app.coreSynced = prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Namespace: "horizon", Subsystem: "stellar_core", Name: "synced",
+			Help: "determines if Stellar-Core defined by --stellar-core-url is synced with the network",
+		},
+		func() float64 {
+			if app.coreSettings.Synced {
+				return 1
+			} else {
+				return 0
+			}
+		},
+	)
+	app.prometheusRegistry.MustRegister(app.coreSynced)
+
 	app.dbMaxOpenConnectionsGauge = prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{Namespace: "horizon", Subsystem: "db", Name: "max_open_connections"},
 		func() float64 {
@@ -261,6 +276,7 @@ func initIngestMetrics(app *App) {
 	app.prometheusRegistry.MustRegister(app.ingester.Metrics().StateInvalidGauge)
 	app.prometheusRegistry.MustRegister(app.ingester.Metrics().LedgerStatsCounter)
 	app.prometheusRegistry.MustRegister(app.ingester.Metrics().ProcessorsRunDuration)
+	app.prometheusRegistry.MustRegister(app.ingester.Metrics().CaptiveStellarCoreSynced)
 }
 
 func initTxSubMetrics(app *App) {