From 74500abb620c2a5e41f0ad3568fea0cf8f18dbf7 Mon Sep 17 00:00:00 2001
From: Tim Diekmann <21277928+TimDiekmann@users.noreply.github.com>
Date: Fri, 19 Jan 2024 10:47:55 +0100
Subject: [PATCH] H-1928: Make sure all Rust spans are captured in Sentry
 (#3901)

---
 .env                                        |  2 +-
 apps/hash-graph/libs/api/src/rest/status.rs |  3 ++
 infra/terraform/hash/main.tf                |  4 +++
 libs/@local/tracing/src/lib.rs              |  2 +-
 libs/@local/tracing/src/sentry.rs           | 38 +++++++++++++++++++--
 5 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/.env b/.env
index 7da169130b6..8053192495c 100644
--- a/.env
+++ b/.env
@@ -49,7 +49,7 @@ HASH_TEMPORAL_VISIBILITY_PG_DATABASE=temporal_visibility
 HASH_GRAPH_PG_USER=graph
 HASH_GRAPH_PG_PASSWORD=graph
 HASH_GRAPH_PG_DATABASE=graph
-HASH_GRAPH_LOG_LEVEL=trace,h2=info,tokio_util=debug,tower=info,tonic=debug,hyper=info,tokio_postgres=info,rustls=info
+HASH_GRAPH_LOG_LEVEL=trace,h2=info,tokio_util=debug,tower=info,tonic=debug,hyper=info,tokio_postgres=info,rustls=info,tarpc=warn
 
 HASH_GRAPH_TYPE_FETCHER_HOST=localhost
 HASH_GRAPH_TYPE_FETCHER_PORT=4444
diff --git a/apps/hash-graph/libs/api/src/rest/status.rs b/apps/hash-graph/libs/api/src/rest/status.rs
index 2e03ed4a20a..4ab941a6afb 100644
--- a/apps/hash-graph/libs/api/src/rest/status.rs
+++ b/apps/hash-graph/libs/api/src/rest/status.rs
@@ -30,6 +30,9 @@ where
         .copied()
         .or_else(|| report.request_value::<StatusCode>().next())
         .unwrap_or(StatusCode::Internal);
+    // TODO: Currently, this mostly duplicates the error printed below, when more information is
+    //       added to the `Report` event consider commenting in this line again.
+    // hash_tracing::sentry::capture_report(&report);
     tracing::error!(error = ?report, tags.code = ?status_code.to_http_code());
 
     status_to_response(Status::new(
diff --git a/infra/terraform/hash/main.tf b/infra/terraform/hash/main.tf
index 06def7861e1..2ca354289d2 100644
--- a/infra/terraform/hash/main.tf
+++ b/infra/terraform/hash/main.tf
@@ -233,6 +233,8 @@ module "application" {
       value = sensitive(data.vault_kv_secret_v2.secrets.data["graph_sentry_dsn"])
     },
     { name = "HASH_GRAPH_SENTRY_ENVIRONMENT", secret = false, value = "production" },
+    { name = "HASH_GRAPH_SENTRY_EVENT_FILTER", secret = false, value = "debug" },
+    { name = "HASH_GRAPH_SENTRY_SPAN_FILTER", secret = false, value = "trace" },
   ])
   graph_env_vars = concat(var.hash_graph_env_vars, [
     { name = "HASH_GRAPH_PG_USER", secret = false, value = "graph" },
@@ -252,6 +254,8 @@ module "application" {
       value = sensitive(data.vault_kv_secret_v2.secrets.data["graph_sentry_dsn"])
     },
     { name = "HASH_GRAPH_SENTRY_ENVIRONMENT", secret = false, value = "production" },
+    { name = "HASH_GRAPH_SENTRY_EVENT_FILTER", secret = false, value = "debug" },
+    { name = "HASH_GRAPH_SENTRY_SPAN_FILTER", secret = false, value = "trace" },
   ])
   # The type fetcher uses the same image as the graph right now
   type_fetcher_image = module.graph_ecr
diff --git a/libs/@local/tracing/src/lib.rs b/libs/@local/tracing/src/lib.rs
index 4b134edd8d8..a118c7b6d0f 100644
--- a/libs/@local/tracing/src/lib.rs
+++ b/libs/@local/tracing/src/lib.rs
@@ -70,7 +70,7 @@ pub async fn init_tracing(config: TracingConfig) -> Result<impl Drop + 'static,
     } else {
         None
     };
-    let sentry_layer = sentry::layer();
+    let sentry_layer = sentry::layer(&config.sentry);
 
     tracing_subscriber::registry()
         .with(filter)
diff --git a/libs/@local/tracing/src/sentry.rs b/libs/@local/tracing/src/sentry.rs
index bb135238f35..fe433dac455 100644
--- a/libs/@local/tracing/src/sentry.rs
+++ b/libs/@local/tracing/src/sentry.rs
@@ -15,7 +15,7 @@ use clap::{builder::TypedValueParser, error::ErrorKind, Arg, Command, Error, Par
 use error_stack::Report;
 pub use sentry::release_name;
 use sentry::{
-    integrations::tracing::SentryLayer,
+    integrations::tracing::{EventFilter, SentryLayer},
     protocol::{Event, TemplateInfo},
     types::Dsn,
     ClientInitGuard, Hub, Level,
@@ -71,6 +71,7 @@ impl TypedValueParser for OptionalSentryDsnParser {
 /// Arguments for configuring the logging setup
 #[derive(Debug, Clone)]
 #[cfg_attr(feature = "clap", derive(Parser))]
+#[expect(clippy::struct_field_names, reason = "Used as clap arguments")]
 pub struct SentryConfig {
     // we need to qualify `Option` here, as otherwise `clap` tries to be too smart and only uses
     // the `value_parser` on the internal `sentry::types::Dsn`, failing.
@@ -83,6 +84,25 @@ pub struct SentryConfig {
         default_value_t = SentryEnvironment::default(),
     ))]
     pub sentry_environment: SentryEnvironment,
+
+    /// Enable every parent span's attributes to be sent along with own event's attributes.
+    #[cfg_attr(
+        feature = "clap",
+        arg(long, env = "HASH_GRAPH_SENTRY_ENABLE_SPAN_ATTRIBUTES",)
+    )]
+    pub sentry_enable_span_attributes: bool,
+
+    #[cfg_attr(
+        feature = "clap",
+        arg(long, env = "HASH_GRAPH_SENTRY_SPAN_FILTER", default_value_t = tracing::Level::INFO)
+    )]
+    pub sentry_span_filter: tracing::Level,
+
+    #[cfg_attr(
+        feature = "clap",
+        arg(long, env = "HASH_GRAPH_SENTRY_EVENT_FILTER", default_value_t = tracing::Level::INFO)
+    )]
+    pub sentry_event_filter: tracing::Level,
 }
 
 pub fn init_sentry(
@@ -114,11 +134,23 @@ pub fn init_sentry(
 }
 
 #[must_use]
-pub fn layer<S>() -> SentryLayer<S>
+pub fn layer<S>(config: &SentryConfig) -> SentryLayer<S>
 where
     S: Subscriber + for<'a> LookupSpan<'a>,
 {
-    ::sentry::integrations::tracing::layer().enable_span_attributes()
+    let mut layer = ::sentry::integrations::tracing::layer();
+    if config.sentry_enable_span_attributes {
+        layer = layer.enable_span_attributes();
+    }
+    let span_filter = config.sentry_event_filter;
+    let event_filter = config.sentry_event_filter;
+    layer
+        .span_filter(move |metadata| *metadata.level() <= span_filter)
+        .event_filter(move |metadata| match *metadata.level() {
+            tracing::Level::ERROR => EventFilter::Exception,
+            level if level <= event_filter => EventFilter::Breadcrumb,
+            _ => EventFilter::Ignore,
+        })
 }
 
 fn read_source(location: Location) -> (Vec<String>, Option<String>, Vec<String>) {