Merge branch 'main' into rayz/dogstatsd-mapper

Signed-off-by: Raymond Zhao <[email protected]>

bin/agent-data-plane/Cargo.toml

@@ -25,14 +25,24 @@ saluki-health = { workspace = true }
 saluki-io = { workspace = true }
 saluki-metadata = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 stringtheory = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "signal"] }
+tokio = { workspace = true, features = [
+  "macros",
+  "rt",
+  "rt-multi-thread",
+  "signal",
+] }
 tokio-rustls = { workspace = true }
 tracing = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 tikv-jemalloc-ctl = { workspace = true, features = ["use_std"] }
-tikv-jemallocator = { workspace = true, features = ["background_threads", "unprefixed_malloc_on_supported_platforms", "stats"] }
+tikv-jemallocator = { workspace = true, features = [
+  "background_threads",
+  "unprefixed_malloc_on_supported_platforms",
+  "stats",
+] }
 
 [build-dependencies]
 chrono = { workspace = true }

bin/agent-data-plane/src/components/remapper/mod.rs

@@ -68,10 +68,10 @@ impl MemoryBounds for AgentTelemetryRemapperConfiguration {
         builder
             .minimum()
             // Capture the size of the heap allocation when the component is built.
-            .with_single_value::<AgentTelemetryRemapper>()
+            .with_single_value::<AgentTelemetryRemapper>("component struct")
             // We also allocate the backing storage for the string interner up front, which is used by our context
             // resolver.
-            .with_fixed_amount(self.context_string_interner_bytes.as_u64() as usize);
+            .with_fixed_amount("string interner", self.context_string_interner_bytes.as_u64() as usize);
     }
 }
 

bin/agent-data-plane/src/main.rs

@@ -10,7 +10,7 @@ use std::{
     time::{Duration, Instant},
 };
 
-use memory_accounting::ComponentRegistry;
+use memory_accounting::{ComponentBounds, ComponentRegistry};
 use saluki_app::{api::APIBuilder, logging::LoggingAPIHandler, prelude::*};
 use saluki_components::{
     destinations::{
@@ -128,7 +128,17 @@ async fn run(started: Instant, logging_api_handler: LoggingAPIHandler) -> Result
 
     // Run memory bounds validation to ensure that we can launch the topology with our configured memory limit, if any.
     let bounds_configuration = MemoryBoundsConfiguration::try_from_config(&configuration)?;
-    let memory_limiter = initialize_memory_bounds(bounds_configuration, component_registry)?;
+    let memory_limiter = initialize_memory_bounds(bounds_configuration, &component_registry)?;
+
+    if let Ok(val) = std::env::var("DD_ADP_WRITE_SIZING_GUIDE") {
+        if val != "false" {
+            if let Err(error) = write_sizing_guide(component_registry.as_bounds()) {
+                warn!("Failed to write sizing guide: {}", error);
+            } else {
+                return Ok(());
+            }
+        }
+    }
 
     // Bounds validation succeeded, so now we'll build and spawn the topology.
     let built_topology = blueprint.build().await?;
@@ -262,6 +272,28 @@ async fn create_topology(
     Ok(blueprint)
 }
 
+fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> {
+    use std::{
+        fs::File,
+        io::{BufWriter, Write},
+    };
+
+    let template = include_str!("sizing_guide_template.html");
+    let mut output = BufWriter::new(File::create("sizing_guide.html")?);
+    for line in template.lines() {
+        if line.trim() == "<!-- INSERT GENERATED CONTENT -->" {
+            serde_json::to_writer_pretty(&mut output, &bounds.to_exprs())?;
+        } else {
+            output.write_all(line.as_bytes())?;
+        }
+        output.write_all(b"\n")?;
+    }
+    info!("Wrote sizing guide to sizing_guide.html");
+    output.flush()?;
+
+    Ok(())
+}
+
 async fn spawn_unprivileged_api(
     configuration: &GenericConfiguration, api_builder: APIBuilder,
 ) -> Result<(), GenericError> {