Factor out instrumentation code. Add metrics and logging to otel.

Cargo.lock

@@ -1095,6 +1095,20 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "opentelemetry-appender-tracing"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b"
+dependencies = [
+ "log",
+ "opentelemetry",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "opentelemetry-http"
 version = "0.27.0"
@@ -1149,6 +1163,23 @@ version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
 
+[[package]]
+name = "opentelemetry-stdout"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc8a298402aa5c260be90d10dc54b5a7d4e1025c354848f8e2c976d761351049"
+dependencies = [
+ "async-trait",
+ "chrono",
+ "futures-util",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "ordered-float",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "opentelemetry_sdk"
 version = "0.27.0"
@@ -1171,6 +1202,15 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -1226,9 +1266,12 @@ dependencies = [
  "clap",
  "dropshot",
  "http",
+ "once_cell",
  "opentelemetry",
+ "opentelemetry-appender-tracing",
  "opentelemetry-otlp",
  "opentelemetry-semantic-conventions",
+ "opentelemetry-stdout",
  "opentelemetry_sdk",
  "schemars",
  "serde",
@@ -1237,6 +1280,7 @@ dependencies = [
  "tokio",
  "tracing",
  "tracing-chrome",
+ "tracing-core",
  "tracing-opentelemetry",
  "tracing-slog",
  "tracing-subscriber",
@@ -2094,9 +2138,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.32"
+version = "0.1.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
 dependencies = [
  "once_cell",
  "valuable",

controller/Cargo.toml

@@ -1,5 +1,6 @@
 [package]
 name = "patagia-controller"
+description = "Patagia controller server"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
@@ -23,6 +24,10 @@ opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.27.0", features = ["http-proto", "hyper-client", "opentelemetry-http", "trace"] }
 opentelemetry = "0.27.0"
 opentelemetry-semantic-conventions = "0.27.0"
+tracing-core = "0.1.33"
+opentelemetry-appender-tracing = { version = "0.27.0", features = ["log", "experimental_metadata_attributes"] }
+opentelemetry-stdout = "0.27.0"
+once_cell = "1.20.2"
 
 [[bin]]
 name = "patagia-controller"

controller/src/bin/patagia-controller.rs

@@ -4,27 +4,17 @@ use dropshot::{
     endpoint, ApiDescription, ConfigDropshot, HttpError, HttpResponseOk, RequestContext,
     ServerBuilder,
 };
-use opentelemetry::{trace::TracerProvider as _, KeyValue};
-use opentelemetry_otlp::WithExportConfig;
-use opentelemetry_sdk::{
-    trace::{RandomIdGenerator, Sampler},
-    Resource,
-};
-use opentelemetry_semantic_conventions::{
-    attribute::{SERVICE_NAME, SERVICE_VERSION},
-    SCHEMA_URL,
-};
 use schemars::JsonSchema;
 use serde::Serialize;
 use slog::Drain;
-use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_slog::TracingSlogDrain;
-use tracing_subscriber::prelude::*;
 
 use std::net::SocketAddr;
 use std::str::FromStr;
 use std::sync::Arc;
 
+use patagia_controller::instrumentation;
+
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
 struct Cli {}
@@ -66,44 +56,16 @@ async fn api_version(
 #[tokio::main]
 async fn main() -> Result<()> {
     let _args = Cli::parse();
-    let fmt_layer = tracing_subscriber::fmt::layer();
-
-    let otlp_exporter = opentelemetry_otlp::SpanExporter::builder()
-        .with_tonic()
-        .with_endpoint("https://localhost:4317")
-        .build()
-        .map_err(|e| anyhow!("Error creating OTLP exporter: {:?}", e))?;
-
-    let resource = Resource::from_schema_url(
-        [
-            KeyValue::new(SERVICE_NAME, env!("CARGO_PKG_NAME")),
-            KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
-        ],
-        SCHEMA_URL,
-    );
-
-    let tracer_provider = opentelemetry_sdk::trace::TracerProvider::builder()
-        .with_config(
-            opentelemetry_sdk::trace::Config::default()
-                .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
-                    1.0,
-                ))))
-                .with_id_generator(RandomIdGenerator::default())
-                .with_resource(resource),
-        )
-        .with_batch_exporter(otlp_exporter, opentelemetry_sdk::runtime::Tokio)
-        .build();
-
-    let tracer = tracer_provider.tracer("patagia-controller");
-
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::EnvFilter::from_default_env())
-        .with(fmt_layer)
-        .with(OpenTelemetryLayer::new(tracer))
-        .init();
+    let _tracing = instrumentation::init_tracing_subscriber()?;
 
     tracing::info!("Patagia Controller");
 
+    tracing::debug!("Starting server");
+
+    tracing::error!(name: "my-event-name", target: "my-system", event_id = 20, user_name = "otel", user_email = "otel@opentelemetry.io", message = "This is an example message");
+
+    foo().await;
+
     let mut api = ApiDescription::new();
     api.register(api_version).unwrap();
 
@@ -126,3 +88,15 @@ async fn main() -> Result<()> {
         .await
         .map_err(|e| anyhow!(e))
 }
+
+#[tracing::instrument]
+async fn foo() {
+    tracing::info!(
+        monotonic_counter.foo = 1_u64,
+        key_1 = "bar",
+        key_2 = 10,
+        "handle foo",
+    );
+
+    tracing::info!(histogram.baz = 10, "histogram example",);
+}

controller/src/instrumentation.rs

@@ -0,0 +1,126 @@
+use anyhow::{anyhow, Result};
+use once_cell::sync::Lazy;
+use opentelemetry::{trace::TracerProvider as _, KeyValue};
+use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
+use opentelemetry_sdk::{
+    logs::LoggerProvider,
+    metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider},
+    runtime,
+    trace::{RandomIdGenerator, Sampler, TracerProvider},
+    Resource,
+};
+use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+static RESOURCE: Lazy<Resource> = Lazy::new(|| {
+    Resource::new(vec![
+        KeyValue::new(
+            opentelemetry_semantic_conventions::resource::SERVICE_NAME,
+            env!("CARGO_PKG_NAME"),
+        ),
+        KeyValue::new(
+            opentelemetry_semantic_conventions::resource::SERVICE_VERSION,
+            env!("CARGO_PKG_VERSION"),
+        ),
+    ])
+});
+
+// Construct LoggingProdiver for LoggingLayer
+fn init_logging_provider() -> Result<LoggerProvider> {
+    let exporter = opentelemetry_otlp::LogExporter::builder()
+        .with_tonic()
+        .build()?;
+    let logging_provider: LoggerProvider = LoggerProvider::builder()
+        .with_resource(RESOURCE.clone())
+        .with_batch_exporter(exporter, runtime::Tokio)
+        .build();
+    Ok(logging_provider)
+}
+
+// Construct MeterProvider for MetricsLayer
+fn init_meter_provider() -> Result<SdkMeterProvider> {
+    let exporter = opentelemetry_otlp::MetricExporter::builder()
+        .with_tonic()
+        .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+        .build()
+        .map_err(|e| anyhow!("Error creating OTLP metric exporter: {:?}", e))?;
+
+    let meter_provider = MeterProviderBuilder::default()
+        .with_resource(RESOURCE.clone())
+        .with_reader(
+            PeriodicReader::builder(exporter, runtime::Tokio)
+                .with_interval(std::time::Duration::from_secs(10))
+                .build(),
+        )
+        .build();
+
+    opentelemetry::global::set_meter_provider(meter_provider.clone());
+
+    Ok(meter_provider)
+}
+
+// Construct TracerProvider for OpenTelemetryLayer
+fn init_tracer_provider() -> Result<TracerProvider> {
+    let exporter = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
+        .build()
+        .map_err(|e| anyhow!("Error creating OTLP span exporter: {:?}", e))?;
+
+    let tracer_provider = opentelemetry_sdk::trace::TracerProvider::builder()
+        .with_config(
+            opentelemetry_sdk::trace::Config::default()
+                .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
+                    1.0,
+                ))))
+                .with_id_generator(RandomIdGenerator::default())
+                .with_resource(RESOURCE.clone()),
+        )
+        .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio)
+        .build();
+
+    Ok(tracer_provider)
+}
+
+// Initialize tracing-subscriber and return OtelGuard for opentelemetry-related termination processing
+pub fn init_tracing_subscriber() -> Result<OtelGuard> {
+    let logging_provider = init_logging_provider()?;
+    let meter_provider = init_meter_provider()?;
+    let tracer_provider = init_tracer_provider()?;
+
+    let tracer = tracer_provider.tracer("tracing-otel-subscriber");
+
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::EnvFilter::from_default_env())
+        .with(tracing_subscriber::fmt::layer())
+        .with(MetricsLayer::new(meter_provider.clone()))
+        .with(OpenTelemetryLayer::new(tracer))
+        .with(OpenTelemetryTracingBridge::new(&logging_provider))
+        .init();
+
+    Ok(OtelGuard {
+        logging_provider,
+        meter_provider,
+        tracer_provider,
+    })
+}
+
+pub struct OtelGuard {
+    logging_provider: LoggerProvider,
+    meter_provider: SdkMeterProvider,
+    tracer_provider: TracerProvider,
+}
+
+impl Drop for OtelGuard {
+    fn drop(&mut self) {
+        if let Err(err) = self.tracer_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+        if let Err(err) = self.logging_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+        if let Err(err) = self.meter_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+    }
+}

controller/src/lib.rs

@@ -0,0 +1 @@
+pub mod instrumentation;

justfile

@@ -5,7 +5,7 @@ default:
 	@just --choose
 
 # Run controller
-run-controller $RUST_LOG="debug":
+run-controller $RUST_LOG="debug,h2=info,hyper_util=info,tower=info":
   cargo run --package patagia-controller
 
 # Run controller local development