From 20f0c8c35010b2744479a3375eb9b0593746646a Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Thu, 11 Jul 2024 09:30:24 +0200 Subject: [PATCH 1/5] capture otel events --- .../OtelSentrySpanProcessor.java | 28 +++++++++++++++++++ sentry/api/sentry.api | 4 +++ .../main/java/io/sentry/ExternalOptions.java | 14 ++++++++++ .../main/java/io/sentry/SentryOptions.java | 15 ++++++++++ 4 files changed, 61 insertions(+) diff --git a/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java b/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java index 7439906868..b4e1ef60d2 100644 --- a/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java +++ b/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java @@ -8,6 +8,8 @@ import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.EventData; +import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData; import io.sentry.Baggage; import io.sentry.IScopes; import io.sentry.PropagationContext; @@ -19,7 +21,10 @@ import io.sentry.SentryTraceHeader; import io.sentry.SpanId; import io.sentry.TracesSamplingDecision; +import io.sentry.exception.ExceptionMechanismException; +import io.sentry.protocol.Mechanism; import io.sentry.protocol.SentryId; +import java.util.List; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -135,9 +140,32 @@ public void onEnd(final @NotNull ReadableSpan spanBeingEnded) { final @NotNull SentryDate finishDate = new SentryLongDate(spanBeingEnded.toSpanData().getEndEpochNanos()); sentrySpan.updateEndDate(finishDate); + + final @NotNull IScopes spanScopes = sentrySpan.getScopes(); + if (spanScopes.getOptions().isCaptureOpenTelemetryEvents()) { + final @NotNull List events = spanBeingEnded.toSpanData().getEvents(); + for (EventData event : events) { + if (event instanceof ExceptionEventData) { + final @NotNull ExceptionEventData exceptionEvent = (ExceptionEventData) event; + final @NotNull Throwable exception = exceptionEvent.getException(); + captureException(spanScopes, exception); + } + } + } } } + private void captureException(final @NotNull IScopes scopes, final @NotNull Throwable throwable) { + final Mechanism mechanism = new Mechanism(); + mechanism.setType("OpenTelemetryInstrumentation"); + mechanism.setHandled(true); + // TODO [POTEL] thread might be wrong + final Throwable mechanismException = + new ExceptionMechanismException(mechanism, throwable, Thread.currentThread()); + // TODO [POTEL] event timestamp should be taken from ExceptionEventData + scopes.captureException(mechanismException); + } + @Override public boolean isEndRequired() { return true; diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index 05213c6b91..f9c997790e 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -456,11 +456,13 @@ public final class io/sentry/ExternalOptions { public fun getTracePropagationTargets ()Ljava/util/List; public fun getTracesSampleRate ()Ljava/lang/Double; public fun getTracingOrigins ()Ljava/util/List; + public fun isCaptureOpenTelemetryEvents ()Ljava/lang/Boolean; public fun isEnableBackpressureHandling ()Ljava/lang/Boolean; public fun isEnablePrettySerializationOutput ()Ljava/lang/Boolean; public fun isEnabled ()Ljava/lang/Boolean; public fun isSendDefaultPii ()Ljava/lang/Boolean; public fun isSendModules ()Ljava/lang/Boolean; + public fun setCaptureOpenTelemetryEvents (Ljava/lang/Boolean;)V public fun setCron (Lio/sentry/SentryOptions$Cron;)V public fun setDebug (Ljava/lang/Boolean;)V public fun setDist (Ljava/lang/String;)V @@ -2757,6 +2759,7 @@ public class io/sentry/SentryOptions { public fun isAttachServerName ()Z public fun isAttachStacktrace ()Z public fun isAttachThreads ()Z + public fun isCaptureOpenTelemetryEvents ()Z public fun isDebug ()Z public fun isEnableAppStartProfiling ()Z public fun isEnableAutoSessionTracking ()Z @@ -2794,6 +2797,7 @@ public class io/sentry/SentryOptions { public fun setBeforeSend (Lio/sentry/SentryOptions$BeforeSendCallback;)V public fun setBeforeSendTransaction (Lio/sentry/SentryOptions$BeforeSendTransactionCallback;)V public fun setCacheDirPath (Ljava/lang/String;)V + public fun setCaptureOpenTelemetryEvents (Z)V public fun setConnectionStatusProvider (Lio/sentry/IConnectionStatusProvider;)V public fun setConnectionTimeoutMillis (I)V public fun setCron (Lio/sentry/SentryOptions$Cron;)V diff --git a/sentry/src/main/java/io/sentry/ExternalOptions.java b/sentry/src/main/java/io/sentry/ExternalOptions.java index aa5aa43937..6e812f2c7f 100644 --- a/sentry/src/main/java/io/sentry/ExternalOptions.java +++ b/sentry/src/main/java/io/sentry/ExternalOptions.java @@ -51,6 +51,7 @@ public final class ExternalOptions { private @Nullable Boolean sendModules; private @Nullable Boolean sendDefaultPii; private @Nullable Boolean enableBackpressureHandling; + private @Nullable Boolean captureOpenTelemetryEvents; private @Nullable SentryOptions.Cron cron; @@ -139,6 +140,9 @@ public final class ExternalOptions { options.setEnableBackpressureHandling( propertiesProvider.getBooleanProperty("enable-backpressure-handling")); + options.setCaptureOpenTelemetryEvents( + propertiesProvider.getBooleanProperty("capture-opentelemetry-events")); + for (final String ignoredExceptionType : propertiesProvider.getList("ignored-exceptions-for-type")) { try { @@ -460,4 +464,14 @@ public void setEnableBackpressureHandling(final @Nullable Boolean enableBackpres public void setCron(final @Nullable SentryOptions.Cron cron) { this.cron = cron; } + + @ApiStatus.Experimental + public void setCaptureOpenTelemetryEvents(final @Nullable Boolean captureOpenTelemetryEvents) { + this.captureOpenTelemetryEvents = captureOpenTelemetryEvents; + } + + @ApiStatus.Experimental + public @Nullable Boolean isCaptureOpenTelemetryEvents() { + return captureOpenTelemetryEvents; + } } diff --git a/sentry/src/main/java/io/sentry/SentryOptions.java b/sentry/src/main/java/io/sentry/SentryOptions.java index 7e24e81dce..e743d037f3 100644 --- a/sentry/src/main/java/io/sentry/SentryOptions.java +++ b/sentry/src/main/java/io/sentry/SentryOptions.java @@ -489,6 +489,8 @@ public class SentryOptions { private @NotNull ScopeType defaultScopeType = ScopeType.ISOLATION; + @ApiStatus.Experimental private boolean captureOpenTelemetryEvents = false; + /** * Adds an event processor * @@ -2440,6 +2442,16 @@ public void setDefaultScopeType(final @NotNull ScopeType scopeType) { return defaultScopeType; } + @ApiStatus.Experimental + public void setCaptureOpenTelemetryEvents(final boolean captureOpenTelemetryEvents) { + this.captureOpenTelemetryEvents = captureOpenTelemetryEvents; + } + + @ApiStatus.Experimental + public boolean isCaptureOpenTelemetryEvents() { + return captureOpenTelemetryEvents; + } + /** The BeforeSend callback */ public interface BeforeSendCallback { @@ -2694,6 +2706,9 @@ public void merge(final @NotNull ExternalOptions options) { if (options.isSendDefaultPii() != null) { setSendDefaultPii(options.isSendDefaultPii()); } + if (options.isCaptureOpenTelemetryEvents() != null) { + setCaptureOpenTelemetryEvents(options.isCaptureOpenTelemetryEvents()); + } if (options.getCron() != null) { if (getCron() == null) { From 92a63ae67e1a776b1a4e93e953c1dff0762b0744 Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Fri, 28 Feb 2025 11:50:12 +0100 Subject: [PATCH 2/5] Set trace for captured error; set timestamp; refactor --- .../OtelSentrySpanProcessor.java | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java b/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java index 3730bfb9d5..5aa3ab7633 100644 --- a/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java +++ b/sentry-opentelemetry/sentry-opentelemetry-core/src/main/java/io/sentry/opentelemetry/OtelSentrySpanProcessor.java @@ -11,11 +11,13 @@ import io.opentelemetry.sdk.trace.data.EventData; import io.opentelemetry.sdk.trace.data.ExceptionEventData; import io.sentry.Baggage; +import io.sentry.DateUtils; import io.sentry.IScopes; import io.sentry.PropagationContext; import io.sentry.ScopesAdapter; import io.sentry.Sentry; import io.sentry.SentryDate; +import io.sentry.SentryEvent; import io.sentry.SentryLevel; import io.sentry.SentryLongDate; import io.sentry.SentryTraceHeader; @@ -149,29 +151,43 @@ public void onEnd(final @NotNull ReadableSpan spanBeingEnded) { new SentryLongDate(spanBeingEnded.toSpanData().getEndEpochNanos()); sentrySpan.updateEndDate(finishDate); - final @NotNull IScopes spanScopes = sentrySpan.getScopes(); - if (spanScopes.getOptions().isCaptureOpenTelemetryEvents()) { - final @NotNull List events = spanBeingEnded.toSpanData().getEvents(); - for (EventData event : events) { - if (event instanceof ExceptionEventData) { - final @NotNull ExceptionEventData exceptionEvent = (ExceptionEventData) event; - final @NotNull Throwable exception = exceptionEvent.getException(); - captureException(spanScopes, exception); - } + maybeCaptureSpanEventsAsExceptions(spanBeingEnded, sentrySpan); + } + } + + private void maybeCaptureSpanEventsAsExceptions( + final @NotNull ReadableSpan spanBeingEnded, final @NotNull IOtelSpanWrapper sentrySpan) { + final @NotNull IScopes spanScopes = sentrySpan.getScopes(); + if (spanScopes.getOptions().isCaptureOpenTelemetryEvents()) { + final @NotNull List events = spanBeingEnded.toSpanData().getEvents(); + for (EventData event : events) { + if (event instanceof ExceptionEventData) { + final @NotNull ExceptionEventData exceptionEvent = (ExceptionEventData) event; + captureException(spanScopes, exceptionEvent, sentrySpan); } } } } - private void captureException(final @NotNull IScopes scopes, final @NotNull Throwable throwable) { + private void captureException( + final @NotNull IScopes scopes, + final @NotNull ExceptionEventData exceptionEvent, + final @NotNull IOtelSpanWrapper sentrySpan) { + final @NotNull Throwable exception = exceptionEvent.getException(); final Mechanism mechanism = new Mechanism(); - mechanism.setType("OpenTelemetryInstrumentation"); + mechanism.setType("OpenTelemetrySpanEvent"); mechanism.setHandled(true); - // TODO [POTEL] thread might be wrong + // This is potentially the wrong Thread as it's the current thread meaning the thread where + // the span is being ended on. This may not match the thread where the exception occurred. final Throwable mechanismException = - new ExceptionMechanismException(mechanism, throwable, Thread.currentThread()); - // TODO [POTEL] event timestamp should be taken from ExceptionEventData - scopes.captureException(mechanismException); + new ExceptionMechanismException(mechanism, exception, Thread.currentThread()); + + final SentryEvent event = new SentryEvent(mechanismException); + event.setTimestamp(DateUtils.nanosToDate(exceptionEvent.getEpochNanos())); + event.setLevel(SentryLevel.ERROR); + event.getContexts().setTrace(sentrySpan.getSpanContext()); + + scopes.captureEvent(event); } @Override From 3201de3f82fda10737241eef248fefe959bc423a Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Fri, 28 Feb 2025 11:58:22 +0100 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7c2efd7a..0f180eef92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- Capture OpenTelemetry span events ([#3564](https://github.com/getsentry/sentry-java/pull/3564)) + - OpenTelemetry spans may have exceptions attached to them (`openTelemetrySpan.recordException`). We can now send those to Sentry as errors. + - Set `capture-open-telemetry-events=true` in `sentry.properties` to enable it + - Set `sentry.capture-open-telemetry-events=true` in Springs `application.properties` to enable it + - Set `sentry.captureOpenTelemetryEvents: true` in Springs `application.yml` to enable it + ## 8.3.0 ### Features From 903f700099ed676b7dbb0c803278bafe24446336 Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Fri, 7 Mar 2025 06:37:41 +0100 Subject: [PATCH 4/5] fix external option name --- .../boot/jakarta/SentryAutoConfigurationTest.kt | 2 ++ .../spring/boot/SentryAutoConfigurationTest.kt | 2 ++ .../src/main/java/io/sentry/ExternalOptions.java | 2 +- .../src/test/java/io/sentry/ExternalOptionsTest.kt | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt b/sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt index e5b4a90166..bcc56f3bd9 100644 --- a/sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt +++ b/sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt @@ -182,6 +182,7 @@ class SentryAutoConfigurationTest { "sentry.spotlight-connection-url=http://local.sentry.io:1234", "sentry.force-init=true", "sentry.global-hub-mode=true", + "sentry.capture-open-telemetry-events=true", "sentry.cron.default-checkin-margin=10", "sentry.cron.default-max-runtime=30", "sentry.cron.default-timezone=America/New_York", @@ -222,6 +223,7 @@ class SentryAutoConfigurationTest { assertThat(options.isEnableBackpressureHandling).isEqualTo(false) assertThat(options.isForceInit).isEqualTo(true) assertThat(options.isGlobalHubMode).isEqualTo(true) + assertThat(options.isCaptureOpenTelemetryEvents).isEqualTo(true) assertThat(options.isEnableSpotlight).isEqualTo(true) assertThat(options.spotlightConnectionUrl).isEqualTo("http://local.sentry.io:1234") assertThat(options.cron).isNotNull diff --git a/sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt b/sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt index b3c1effa41..ddaa51a764 100644 --- a/sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt +++ b/sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt @@ -181,6 +181,7 @@ class SentryAutoConfigurationTest { "sentry.spotlight-connection-url=http://local.sentry.io:1234", "sentry.force-init=true", "sentry.global-hub-mode=true", + "sentry.capture-open-telemetry-events=true", "sentry.cron.default-checkin-margin=10", "sentry.cron.default-max-runtime=30", "sentry.cron.default-timezone=America/New_York", @@ -221,6 +222,7 @@ class SentryAutoConfigurationTest { assertThat(options.isEnableBackpressureHandling).isEqualTo(false) assertThat(options.isForceInit).isEqualTo(true) assertThat(options.isGlobalHubMode).isEqualTo(true) + assertThat(options.isCaptureOpenTelemetryEvents).isEqualTo(true) assertThat(options.isEnableSpotlight).isEqualTo(true) assertThat(options.spotlightConnectionUrl).isEqualTo("http://local.sentry.io:1234") assertThat(options.cron).isNotNull diff --git a/sentry/src/main/java/io/sentry/ExternalOptions.java b/sentry/src/main/java/io/sentry/ExternalOptions.java index dbadbb52d2..1590734266 100644 --- a/sentry/src/main/java/io/sentry/ExternalOptions.java +++ b/sentry/src/main/java/io/sentry/ExternalOptions.java @@ -148,7 +148,7 @@ public final class ExternalOptions { options.setGlobalHubMode(propertiesProvider.getBooleanProperty("global-hub-mode")); options.setCaptureOpenTelemetryEvents( - propertiesProvider.getBooleanProperty("capture-opentelemetry-events")); + propertiesProvider.getBooleanProperty("capture-open-telemetry-events")); for (final String ignoredExceptionType : propertiesProvider.getList("ignored-exceptions-for-type")) { diff --git a/sentry/src/test/java/io/sentry/ExternalOptionsTest.kt b/sentry/src/test/java/io/sentry/ExternalOptionsTest.kt index 5bb0e5bae0..dbf0001d1c 100644 --- a/sentry/src/test/java/io/sentry/ExternalOptionsTest.kt +++ b/sentry/src/test/java/io/sentry/ExternalOptionsTest.kt @@ -361,6 +361,20 @@ class ExternalOptionsTest { } } + @Test + fun `creates options with captureOpenTelemetryEvents set to false`() { + withPropertiesFile("capture-open-telemetry-events=false") { options -> + assertTrue(options.isCaptureOpenTelemetryEvents == false) + } + } + + @Test + fun `creates options with captureOpenTelemetryEvents set to true`() { + withPropertiesFile("capture-open-telemetry-events=true") { options -> + assertTrue(options.isCaptureOpenTelemetryEvents == true) + } + } + private fun withPropertiesFile(textLines: List = emptyList(), logger: ILogger = mock(), fn: (ExternalOptions) -> Unit) { // create a sentry.properties file in temporary folder val temporaryFolder = TemporaryFolder() From 629baa81d5527a0fff6d870edc5bb0cdfee8149d Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Fri, 7 Mar 2025 06:38:31 +0100 Subject: [PATCH 5/5] remove duplicate dependency entry --- sentry-opentelemetry/sentry-opentelemetry-core/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/sentry-opentelemetry/sentry-opentelemetry-core/build.gradle.kts b/sentry-opentelemetry/sentry-opentelemetry-core/build.gradle.kts index e2aaec04c7..de2143f01d 100644 --- a/sentry-opentelemetry/sentry-opentelemetry-core/build.gradle.kts +++ b/sentry-opentelemetry/sentry-opentelemetry-core/build.gradle.kts @@ -22,7 +22,6 @@ dependencies { */ compileOnly(projects.sentryOpentelemetry.sentryOpentelemetryBootstrap) - implementation(Config.Libs.OpenTelemetry.otelSdk) implementation(Config.Libs.OpenTelemetry.otelSdk) compileOnly(Config.Libs.OpenTelemetry.otelSemconv) compileOnly(Config.Libs.OpenTelemetry.otelSemconvIncubating)