Add default labels and env labels to jvm metrics

[burmanm]

Rewrites all the JVM metrics to include our own set of labels.

Fixes #415

[burmanm]

Example output:

# HELP jvm_memory_pool_bytes_init Initial bytes of a given JVM memory pool.
# TYPE jvm_memory_pool_bytes_init gauge
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="CodeHeap 'non-nmethods'",} 2555904.0
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="Metaspace",} 0.0
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="CodeHeap 'profiled nmethods'",} 2555904.0
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="Compressed Class Space",} 0.0
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Eden Space",} 5.0331648E7
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Old Gen",} 4.86539264E8
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Survivor Space",} 0.0
jvm_memory_pool_bytes_init{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="CodeHeap 'non-profiled nmethods'",} 2555904.0
# HELP jvm_memory_pool_collection_used_bytes Used bytes after last collection of a given JVM memory pool.
# TYPE jvm_memory_pool_collection_used_bytes gauge
jvm_memory_pool_collection_used_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Eden Space",} 0.0
jvm_memory_pool_collection_used_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Old Gen",} 0.0
jvm_memory_pool_collection_used_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Survivor Space",} 1.6777216E7
# HELP jvm_memory_pool_collection_committed_bytes Committed after last collection bytes of a given JVM memory pool.
# TYPE jvm_memory_pool_collection_committed_bytes gauge
jvm_memory_pool_collection_committed_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Eden Space",} 3.01989888E8
jvm_memory_pool_collection_committed_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Old Gen",} 0.0
jvm_memory_pool_collection_committed_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Survivor Space",} 1.6777216E7
# HELP jvm_memory_pool_collection_max_bytes Max bytes after last collection of a given JVM memory pool.
# TYPE jvm_memory_pool_collection_max_bytes gauge
jvm_memory_pool_collection_max_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Eden Space",} -1.0
jvm_memory_pool_collection_max_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Old Gen",} 5.36870912E8
jvm_memory_pool_collection_max_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Survivor Space",} -1.0
# HELP jvm_memory_pool_collection_init_bytes Initial after last collection bytes of a given JVM memory pool.
# TYPE jvm_memory_pool_collection_init_bytes gauge
jvm_memory_pool_collection_init_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Eden Space",} 5.0331648E7
jvm_memory_pool_collection_init_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Old Gen",} 4.86539264E8
jvm_memory_pool_collection_init_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="G1 Survivor Space",} 0.0
# HELP jvm_memory_pool_allocated_bytes_total Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously.
# TYPE jvm_memory_pool_allocated_bytes_total counter
# HELP jvm_buffer_pool_used_bytes Used bytes of a given JVM buffer pool.
# TYPE jvm_buffer_pool_used_bytes gauge
jvm_buffer_pool_used_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="mapped",} 6.7122384E7
jvm_buffer_pool_used_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="direct",} 4.5160769E7
# HELP jvm_buffer_pool_capacity_bytes Bytes capacity of a given JVM buffer pool.
# TYPE jvm_buffer_pool_capacity_bytes gauge
jvm_buffer_pool_capacity_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="mapped",} 6.7122384E7
jvm_buffer_pool_capacity_bytes{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="direct",} 4.5160767E7
# HELP jvm_buffer_pool_used_buffers Used buffers of a given JVM buffer pool.
# TYPE jvm_buffer_pool_used_buffers gauge
jvm_buffer_pool_used_buffers{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="mapped",} 36.0
jvm_buffer_pool_used_buffers{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",pool="direct",} 70.0
# HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds.
# TYPE jvm_gc_collection_seconds summary
jvm_gc_collection_seconds_count{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",gc="G1 Young Generation",} 12.0
jvm_gc_collection_seconds_sum{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",gc="G1 Young Generation",} 0.105
jvm_gc_collection_seconds_count{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",gc="G1 Old Generation",} 0.0
jvm_gc_collection_seconds_sum{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",gc="G1 Old Generation",} 0.0
# HELP jvm_threads_current Current thread count of a JVM
# TYPE jvm_threads_current gauge
jvm_threads_current{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 46.0
# HELP jvm_threads_daemon Daemon thread count of a JVM
# TYPE jvm_threads_daemon gauge
jvm_threads_daemon{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 37.0
# HELP jvm_threads_peak Peak thread count of a JVM
# TYPE jvm_threads_peak gauge
jvm_threads_peak{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 46.0
# HELP jvm_threads_started_total Started thread count of a JVM
# TYPE jvm_threads_started_total counter
# HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers
# TYPE jvm_threads_deadlocked gauge
jvm_threads_deadlocked{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 0.0
# HELP jvm_threads_deadlocked_monitor Cycles of JVM-threads that are in deadlock waiting to acquire object monitors
# TYPE jvm_threads_deadlocked_monitor gauge
jvm_threads_deadlocked_monitor{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 0.0
# HELP jvm_threads_state Current count of threads by state
# TYPE jvm_threads_state gauge
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="NEW",} 0.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="TERMINATED",} 0.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="RUNNABLE",} 12.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="BLOCKED",} 0.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="WAITING",} 13.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="TIMED_WAITING",} 21.0
jvm_threads_state{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",state="UNKNOWN",} 0.0
# HELP jvm_classes_currently_loaded The number of classes that are currently loaded in the JVM
# TYPE jvm_classes_currently_loaded gauge
jvm_classes_currently_loaded{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",} 11753.0
# HELP jvm_classes_loaded_total The total number of classes that have been loaded since the JVM has started execution
# TYPE jvm_classes_loaded_total counter
# HELP jvm_classes_unloaded_total The total number of classes that have been unloaded since the JVM has started execution
# TYPE jvm_classes_unloaded_total counter
# HELP jvm_info VM version info
# TYPE jvm_info gauge
jvm_info{host="6a86675e-f184-4916-bf95-7a4a4df47fcc",instance="10.244.3.3",cluster="cluster2",datacenter="dc2",rack="r1",pod_name="cluster2-dc2-r1-sts-0",node_name="kind-worker6",runtime="OpenJDK Runtime Environment",vendor="Red Hat, Inc.",version="11.0.21+9-LTS",} 1.0

[adejanovski]

Thanks for the quick turnaround. Works great!