add Exemplar support for OpenTelemetry tracing

Author: fstab

.circleci/config.yml

@@ -1,18 +1,18 @@
 version: 2
+machine: true
 jobs:
   build:
+    machine:
+      image: ubuntu-2004:202010-01
     working_directory: ~/circleci-java
-    docker:
-      - image: circleci/openjdk:8-jdk-stretch
     steps:
       - checkout
-      - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed
-          key: circleci-demo-java-spring-{{ checksum "pom.xml" }}
-      - run: mvn dependency:go-offline # gets the project dependencies
+      - restore_cache:
+          key: maven-dependencies-{{ checksum "pom.xml" }}
+      - run: ./mvnw clean verify
       - save_cache:
           paths:
             - ~/.m2
-          key: circleci-demo-java-spring-{{ checksum "pom.xml" }}
-      - run: mvn package
+          key: maven-dependencies-{{ checksum "pom.xml" }}
 orbs:
   prometheus: prometheus/prometheus@0.11.0

.gitignore

@@ -12,4 +12,4 @@ target/
 simpleclient_pushgateway/mockserver.log
 simpleclient_pushgateway/mockserver_request.log
 nb-configuration.xml
-
+dependency-reduced-pom.xml

.mvn/wrapper/MavenWrapperDownloader.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    private static final String WRAPPER_VERSION = "0.5.6";
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+        + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if(mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if(mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if(!outputFile.getParentFile().exists()) {
+            if(!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+            String username = System.getenv("MVNW_USERNAME");
+            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+            Authenticator.setDefault(new Authenticator() {
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(username, password);
+                }
+            });
+        }
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

.mvn/wrapper/maven-wrapper.jar

@@ -1 +1,2 @@
-distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,80 @@
+# OpenTelemetry Exemplars
+
+The `DefaultExemplarSampler` will provide OpenTelemetry exemplars if the [opentelemetry-java](https://github.com/open-telemetry/opentelemetry-java) library version 0.16.0 or higher is found. When `client_java` 0.11.0 was released, the current [opentelemetry-java](https://github.com/open-telemetry/opentelemetry-java) version was 1.2.0.
+
+## Running the Example
+
+If you want to see this in action, you can run the example from the `ExemplarsClientJavaIT`:
+
+```
+./mvnw package
+cd integration_tests/exemplars_otel_agent/target/
+curl -LO https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.2.0/opentelemetry-javaagent-all.jar
+java -Dotel.traces.exporter=logging -Dotel.metrics.exporter=none -javaagent:./opentelemetry-javaagent-all.jar -jar ./sample-rest-application.jar
+```
+
+Now you have a Spring REST service running on [http://localhost:8080/hello](http://localhost:8080/hello) that is instrumented with the OpenTelemetry Java agent.
+
+Run a request to generate an OpenTelemetry trace
+
+```
+curl http://localhost:8080/hello
+```
+
+In order to get metrics in [OpenMetrics](http://openmetrics.io) format, run
+
+```
+curl -H 'Accept: application/openmetrics-text; version=1.0.0; charset=utf-8' http://localhost:8080/metrics
+```
+
+You should see metrics with Exemplars, for example in the `request_duration_histogram` metric:
+
+```
+request_duration_histogram_bucket{path="/god-of-fire",le="0.004"} 4.0 # {trace_id="043cd631811e373e4180a678c06b128e",span_id="cd122e457d2ca5b0"} 0.0033 1618261159.027
+```
+
+Note that this is an example application for demonstration, so durations don't represent real durations, and some example metrics might not make sense in the real world.
+
+## Disabling OpenTelemetry Exemplars
+
+If you use OpenTelemetry tracing but do not want Exemplars, you can disable OpenTelemetries in multiple ways.
+
+### Disabling OpenTelemetry Exemplars in Code
+
+The default exemplar sampler can be disabled via the `ExemplarConfig` API. This is described in [README.md], as this is not specific to OpenTelemetry.
+
+### Disabling OpenTelemetry Exemplars at Compile Time
+
+If you don't want to change code, but still build an application that uses OpenTelemetry but does not provide OpenTelemetry exemplars,
+you can exclude the corresponding dependencies in your `pom.xml`:
+
+```xml
+<dependencies>
+  <dependency>
+    <groupId>io.prometheus</groupId>
+    <artifactId>simpleclient</artifactId>
+    <version>0.11.0</version>
+    <exclusions>
+      <!-- The following will disable OpenTelemetry exemplars when your application uses OpenTelemetry directly -->
+      <exclusion>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient_tracer_otel</artifactId>
+      </exclusion>
+      <!-- The following will disable OpenTelemetry exemplars when your application uses the OpenTelemetry Java agent -->
+      <exclusion>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient_tracer_otel_agent</artifactId>
+      </exclusion>
+    </exclusions>
+  </dependency>
+</dependencies>
+```
+
+### Disable OpenTelemetry Exemplars at Runtime
+
+If your application uses OpenTelemetry tracing, but you want to disable OpenTelemetry at runtime without changing code,
+start your application with the `io.prometheus.otelExemplars` system property:
+
+```
+java -Dio.prometheus.otelExemplars=inactive -jar my-application.jar
+```

OTEL_EXEMPLARS.md

@@ -17,6 +17,11 @@ Table of Contents
      * [Histogram](#histogram)
      * [Labels](#labels)
      * [Registering Metrics](#registering-metrics)
+  * [Exemplars](#exemplars)
+     * [Global Exemplar Samplers](#global-exemplar-samplers)
+     * [Per Metric Exemplar Samplers](#per-metric-exemplar-samplers)
+     * [Per Observation Exemplars](#per-observation-exemplars)
+     * [Built-in Support for Tracing Systems](#built-in-support-for-tracing-systems)
   * [Included Collectors](#included-collectors)
      * [Logging](#logging)
      * [Caches](#caches)
@@ -291,6 +296,123 @@ class YourClass {
 }
 ```
 
+## Exemplars
+
+Exemplars are a feature of the [OpenMetrics](http://openmetrics.io) format that allows applications to link metrics to example traces.
+Exemplars are supported since `client_java` version 0.11.0. Exemplars are supported for `Counter` and `Histogram` metrics.
+
+### Global Exemplar Samplers
+
+An `ExemplarSampler` is used implicitly under the hood to add exemplars to your metrics. The idea is that you get exemplars without changing your code.
+
+The `DefaultExemplarSampler` comes with built-in support for [OpenTelemetry tracing](https://github.com/open-telemetry/opentelemetry-java), see  [built-in support for tracing systems](#built-in-support-for-tracing-systems) below.
+
+You can disable the default exemplar samplers globally with:
+
+```java
+ExemplarConfig.disableExemplars();
+```
+
+You can set your own custom implementation of `ExemplarSampler` as a global default like this:
+
+```java
+ExemplarSampler myExemplarSampler = new MyExemplarSampler();
+ExemplarConfig.setCounterExemplarSampler(myExemplarSampler);
+ExemplarConfig.setHistogramExemplarSampler(myExemplarSampler);
+```
+
+### Per Metric Exemplar Samplers
+
+The metric builders for `Counter` and `Histogram` have methods for setting the exemplar sampler for that individual metric. This takes precedence over the global setting in `ExemplarConfig`.
+
+The following calls enable the default exemplar sampler for individual metrics. This is useful if you disabled the exemplar sampler globally with `ExemplarConfig.disableExemplars()`.
+
+```java
+Counter myCounter = Counter.build()
+    .name("number_of_events_total")
+    .help("help")
+    .withExemplars()
+    ...
+    .register();
+```
+
+```java
+Histogram myHistogram = Histogram.build()
+    .name("my_latency")
+    .help("help")
+    .withExemplars()
+    ...
+    .register();
+```
+
+The following calls disable exemplars for individual metrics:
+
+```java
+Counter myCounter = Counter.build()
+    .name("number_of_events_total")
+    .help("help")
+    .withoutExemplars()
+    ...
+    .register();
+```
+
+```java
+Histogram myHistogram = Histogram.build()
+    .name("my_latency")
+    .help("help")
+    .withoutExemplars()
+    ...
+    .register();
+```
+
+The following calls enable exemplars and set a custom `MyExemplarSampler` for individual metrics:
+
+```java
+// CounterExemplarSampler
+Counter myCounter = Counter.build()
+    .name("number_of_events_total")
+    .help("help")
+    .withExemplarSampler(new MyExemplarSampler())
+    ...
+    .register();
+```
+
+```java
+// HistogramExemplarSampler
+Histogram myHistogram = Histogram.build()
+    .name("my_latency")
+    .help("help")
+    .withExemplarSampler(new MyExemplarSampler())
+    ...
+    .register();
+```
+
+### Per Observation Exemplars
+
+You can explicitly provide an exemplar for an individual observation. This takes precedence over the exemplar sampler configured with the metric.
+
+The following call will increment a counter, and create an exemplar with the specified `span_id` and `trace_id` labels:
+
+```java
+myCounter.incWithExemplar("span_id", "abcdef", "trace_id", "123456");
+```
+
+The following call will observe a value of `0.12` in a histogram, and create an exemplar with the specified `span_id` and `trace_id` labels:
+
+```java
+myHistogram.observeWithExemplar(0.12, "span_id", "abcdef", "trace_id", "123456");
+```
+
+All methods for observing and incrementing values have `...withExemplar` equivalents. There are versions taking the exemplar labels as a `String...` as shown in the example, as well as versions taking the exemplar labels as a `Map<String, String>`.
+
+### Built-in Support for Tracing Systems
+
+The `DefaultExemplarSampler` detects if a tracing library is found on startup, and provides exemplars for that tracing library by default. Currently, only [OpenTelemetry tracing](https://github.com/open-telemetry/opentelemetry-java) is supported.
+If you are a tracing vendor, feel free to open a PR and add support for your tracing library.
+
+Documentation of the individual tracer integrations:
+
+* [OTEL_EXEMPLARS.md]: OpenTelemetry
 
 ## Included Collectors