Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(admin): Return error status code from admin console if tracing is disabled on startup. #5092

Merged
merged 3 commits into from
Jan 30, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 18 additions & 6 deletions crates/iota-node/src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use iota_types::{
error::IotaError,
};
use serde::Deserialize;
use telemetry_subscribers::TracingHandle;
use telemetry_subscribers::{TelemetryError, TracingHandle};
use tokio::sync::oneshot;
use tracing::info;

Expand Down Expand Up @@ -204,6 +204,10 @@ async fn enable_tracing(
response.push(format!("filter will be reset after {:?}", duration));
(StatusCode::OK, response.join("\n"))
}
Err(TelemetryError::TracingDisabled) => {
response.push("tracing is not supported as it was disabled in the node's TelemetryConfig\nignoring request".into());
(StatusCode::FORBIDDEN, response.join("\n"))
}
Err(err) => {
response.push(format!("can't update filter: {:?}", err));
(StatusCode::BAD_REQUEST, response.join("\n"))
Expand All @@ -212,11 +216,19 @@ async fn enable_tracing(
}

async fn reset_tracing(State(state): State<Arc<AppState>>) -> (StatusCode, String) {
state.tracing_handle.reset_trace();
(
StatusCode::OK,
"tracing filter reset to TRACE_FILTER env var".into(),
)
match state.tracing_handle.reset_trace() {
Ok(()) => {
(
StatusCode::OK,
"tracing filter reset to TRACE_FILTER env var".into(),
)
}
Err(TelemetryError::TracingDisabled) => (
StatusCode::FORBIDDEN,
"tracing was disabled in the node's TelemetryConfig. so it is not supported\nignoring request".into(),
),
Err(err) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()),
}
}

async fn get_filter(State(state): State<Arc<AppState>>) -> (StatusCode, String) {
Expand Down
1 change: 1 addition & 0 deletions crates/telemetry-subscribers/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ opentelemetry-proto = "0.7"
opentelemetry_sdk = { version = "0.24", features = ["rt-tokio"] }
prometheus.workspace = true
prost.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["full"] }
tonic.workspace = true
tracing.workspace = true
Expand Down
32 changes: 21 additions & 11 deletions crates/telemetry-subscribers/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use opentelemetry_sdk::{
trace::{BatchSpanProcessor, Sampler, ShouldSample, TracerProvider},
};
use span_latency_prom::PrometheusSpanLatencyLayer;
use thiserror::Error;
use tracing::{Level, error, info, metadata::LevelFilter};
use tracing_appender::non_blocking::{NonBlocking, WorkerGuard};
use tracing_subscriber::{EnvFilter, Layer, Registry, filter, fmt, layer::SubscriberExt, reload};
Expand All @@ -36,6 +37,15 @@ pub mod span_latency_prom;
/// Alias for a type-erased error type.
pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;

#[derive(Debug, Error)]
pub enum TelemetryError {
#[error("OTLP protocol not enabled in the node's configuration")]
TracingDisabled,

#[error("{0}")]
Other(#[from] BoxError),
}

/// Configuration for different logging/tracing options
/// ===
/// - json_log_output: Output JSON logs to stdout only.
Expand Down Expand Up @@ -107,9 +117,8 @@ impl FilterHandle {
}

pub fn get(&self) -> Result<String, BoxError> {
self.0
.with_current(|filter| filter.to_string())
.map_err(Into::into)
let result = self.0.with_current(|filter| filter.to_string())?;
Ok(result)
}
}

Expand Down Expand Up @@ -143,9 +152,9 @@ impl TracingHandle {
&self,
directives: S,
duration: Duration,
) -> Result<(), BoxError> {
) -> Result<(), TelemetryError> {
if let Some(trace) = &self.trace {
let res = trace.update(directives);
trace.update(directives)?;
// after duration is elapsed, reset to the env setting
let trace = trace.clone();
let trace_filter_env = env::var("TRACE_FILTER").unwrap_or_else(|_| "off".to_string());
Expand All @@ -155,23 +164,24 @@ impl TracingHandle {
error!("failed to reset trace filter: {}", e);
}
});
res
Ok(())
} else {
info!("tracing not enabled, ignoring update");
Ok(())
Err(TelemetryError::TracingDisabled)
}
}

pub fn clear_file_output(&self) {
self.file_output.clear_path();
}

pub fn reset_trace(&self) {
pub fn reset_trace(&self) -> Result<(), TelemetryError> {
if let Some(trace) = &self.trace {
let trace_filter_env = env::var("TRACE_FILTER").unwrap_or_else(|_| "off".to_string());
if let Err(e) = trace.update(trace_filter_env) {
error!("failed to reset trace filter: {}", e);
}
trace.update(trace_filter_env).map_err(|e| e.into())
} else {
info!("tracing not enabled, ignoring reset");
Err(TelemetryError::TracingDisabled)
}
}
}
Expand Down
Loading