add jmxfetch telemetry in code (#467)

* failing attempt

* removed logic for adding jmx_tele into a init conf

* temp debug logs

* added logic for in-code check configuration

* updated some formatting

* simplified process

* fixed some formating

* changed names

* removed tags and added changelog

Author: cmetz100

CHANGELOG.md

@@ -2,6 +2,7 @@ Changelog
 =========
 # Next / TBD
 
+* [FEATURE] Adds a configurable jmxfetch telemetry check to improve jmxfetch observability [#467][]
 * [FEATURE] Added an option to enable removal of extra quotation marks during tag extraction from Java management beans' parameters/attributes [#469][]
 
 # 0.47.10 / 2023-08-10
@@ -748,9 +749,10 @@ Changelog
 [#432]: https://github.com/DataDog/jmxfetch/issues/432
 [#436]: https://github.com/DataDog/jmxfetch/issues/436
 [#437]: https://github.com/DataDog/jmxfetch/issues/437
+[#467]: https://github.com/DataDog/jmxfetch/issues/467
 [#457]: https://github.com/DataDog/jmxfetch/issues/457
 [#449]: https://github.com/DataDog/jmxfetch/issues/449
-[#469]: https://github.com/DataDog/jmxfetch/pull/469
+[#469]: https://github.com/DataDog/jmxfetch/issues/469
 [@alz]: https://github.com/alz
 [@aoking]: https://github.com/aoking
 [@arrawatia]: https://github.com/arrawatia

src/main/java/org/datadog/jmxfetch/App.java

@@ -28,6 +28,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
@@ -899,6 +900,15 @@ public void init(final boolean forceNewConnection) {
             }
         }
 
+        // Enables jmxfetch telemetry if there are other checks active and it's been enabled
+        if (appConfig.getJmxfetchTelemetry() && newInstances.size() >= 1) {
+            log.info("Adding jmxfetch telemetry check");
+            final Instance instance = instantiate(getTelemetryInstanceConfig(),
+                        getTelemetryInitConfig(), "jmxfetch_telemetry_check",
+                        this.appConfig);
+            newInstances.add(instance);
+        }
+
         final List<InstanceTask<Void>> instanceInitTasks =
                 new ArrayList<>(newInstances.size());
         for (Instance instance : newInstances) {
@@ -947,6 +957,33 @@ public TaskStatusHandler invoke(
         }
     }
 
+    private Map<String,Object> getTelemetryInitConfig() {
+        Map<String,Object> config = new HashMap<String,Object>();
+        config.put("is_jmx",true);
+        return config;
+    }
+
+    private Map<String,Object> getTelemetryInstanceConfig() {
+        Map<String,Object> config = new HashMap<String,Object>();
+        config.put("name","jmxfetch_telemetry_instance");
+        config.put("collect_default_jvm_metrics",true);
+        config.put("new_gc_metrics",true);
+        config.put("process_name_regex",".*org.datadog.jmxfetch.App.*");
+
+        List<Object> conf = new ArrayList<Object>();
+        Map<String,Object> confMap = new HashMap<String,Object>();
+        Map<String,Object> includeMap = new HashMap<String,Object>();
+        includeMap.put("domain","jmxfetch");
+        confMap.put("include", includeMap);
+        conf.add(confMap);
+        config.put("conf",conf);
+
+        List<String> tags = new ArrayList<String>();
+        config.put("tags", tags);
+
+        return config;
+    }
+
     static TaskStatusHandler processRecoveryResults(
             final Instance instance,
             final Future<Void> future,

src/main/java/org/datadog/jmxfetch/AppConfig.java

@@ -122,6 +122,12 @@ public class AppConfig {
             required = false)
     private boolean statsdTelemetry;
 
+    @Parameter(
+            names = {"--jmxfetch_telemetry", "-jt"},
+            description = "Enable additional jmxfetch telemetry reporting",
+            required = false)
+    private boolean jmxfetchTelemetry;
+
     @Parameter(
             names = {"--statsd_queue_size", "-sq"},
             description = "Maximum number of unprocessed messages in the StatsD client queue.",
@@ -411,6 +417,10 @@ public boolean getStatsdTelemetry() {
         return statsdTelemetry;
     }
 
+    public boolean getJmxfetchTelemetry() {
+        return jmxfetchTelemetry;
+    }
+
     public int getStatsdQueueSize() {
         return statsdQueueSize;
     }