Merge branch 'main' of github.com:open-telemetry/opentelemetry-collec…

…tor-contrib into feat/snmpreceiver-resource-attribute-updates

.chloggen/pkg-stanza-encoding.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: deprecation
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Deprecate encoding related elements of helper pacakge, in favor of new decoder package
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [26019]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: Includes the following deprecations |
+  - Decoder
+  - NewDecoder
+  - LookupEncoding
+  - IsNop
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.chloggen/pkg-stanza-tokenize.yaml

@@ -0,0 +1,40 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: deprecation
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Deprecate tokenization related elements of helper pacakge, in favor of new tokenize package
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [25914]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: Includes the following deprecations |
+  - Flusher
+  - FlusherConfig
+  - NewFlusherConfig
+  - Multiline
+  - MultilineConfig
+  - NewMultilineConfig
+  - NewLineStartSplitFunc
+  - NewLineEndSplitFunc
+  - NewNewlineSplitFunc
+  - Splitter
+  - SplitterConfig
+  - NewSplitterConfig
+  - SplitNone
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.chloggen/prometheus_translate_ucum.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: receiver/prometheus
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: translate units from prometheus to UCUM
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [23208]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user, api]

exporter/coralogixexporter/README.md

@@ -103,7 +103,7 @@ Learn more about [AWS PrivateLink in the documentation page](https://coralogix.c
 ### Application and SubSystem attributes
 
 v0.62.0 release of OpenTelemetry Collector allows you to map Application name and Subsystem name to Resource attributes. 
-You need to set `application_name_attributes` and `subsystem_name_attributes` fields with a list of potential Resource attributes for the AppName and Subsystem values. The first not-empty Resource attribute is going to be used.
+You need to set `application_name_attributes` and `subsystem_name_attributes` fields with a list of potential Resource attributes for the AppName and Subsystem values. The first not-empty Resource attribute is going to be used. If multiple resource attributes are available, **the order of the attributes in the list determines their priority.**
 
 ### Kubernetes attributes
 
@@ -114,16 +114,17 @@ exporters:
   coralogix:
     domain: "coralogix.com"
     application_name_attributes:
-      - "service.namespace"
       - "k8s.namespace.name" 
+      - "service.namespace"
     subsystem_name_attributes:
-      - "service.name"
+      - "k8s.job.name"
       - "k8s.deployment.name"
       - "k8s.statefulset.name"
       - "k8s.daemonset.name"
       - "k8s.cronjob.name"
-      - "k8s.job.name"
-      - "k8s.container.name"
+      - "k8s.pod.name"
+      - "k8s.node.name"
+      - "service.name"
 ```
 ### Host Attributes
 

exporter/coralogixexporter/config_test.go

@@ -19,6 +19,7 @@ import (
 	"go.opentelemetry.io/collector/confmap/confmaptest"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.opentelemetry.io/collector/exporter/exportertest"
+	"go.opentelemetry.io/collector/pdata/pcommon"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/coralogixexporter/internal/metadata"
 )
@@ -124,8 +125,8 @@ func TestLoadConfig(t *testing.T) {
 					WaitForReady:    false,
 					BalancerName:    "",
 				},
-				AppNameAttributes:   []string{"service.namespace"},
-				SubSystemAttributes: []string{"service.name"},
+				AppNameAttributes:   []string{"service.namespace", "k8s.namespace.name"},
+				SubSystemAttributes: []string{"service.name", "k8s.deployment.name", "k8s.statefulset.name", "k8s.daemonset.name", "k8s.cronjob.name", "k8s.job.name", "k8s.container.name"},
 				GRPCClientSettings: configgrpc.GRPCClientSettings{
 					Endpoint: "https://",
 					TLSSetting: configtls.TLSClientSetting{
@@ -270,3 +271,28 @@ func TestEndpoindsAndDomainWithAllExporters(t *testing.T) {
 	require.NotNil(t, le, "failed to create logs exporter")
 	require.NoError(t, le.start(context.Background(), componenttest.NewNopHost()))
 }
+
+func TestGetMetadataFromResource(t *testing.T) {
+	r1 := pcommon.NewResource()
+	r1.Attributes().PutStr("k8s.node.name", "node-test")
+	r1.Attributes().PutStr("k8s.container.name", "container-test")
+	r1.Attributes().PutStr("k8s.deployment.name", "deployment-test")
+	r1.Attributes().PutStr("k8s.namespace.name", "namespace-test")
+
+	r2 := pcommon.NewResource()
+	r2.Attributes().PutStr("k8s.node.name", "node-test")
+	r2.Attributes().PutStr("k8s.namespace.name", "namespace-test")
+
+	c := &Config{
+		AppNameAttributes:   []string{"k8s.container.name", "k8s.deployment.name", "k8s.node.name"},
+		SubSystemAttributes: []string{"k8s.namespace.name", "k8s.node.name"},
+	}
+
+	appName, subSystemName := c.getMetadataFromResource(r1)
+	assert.Equal(t, "container-test", appName)
+	assert.Equal(t, "namespace-test", subSystemName)
+
+	appName, subSystemName = c.getMetadataFromResource(r2)
+	assert.Equal(t, "node-test", appName)
+	assert.Equal(t, "namespace-test", subSystemName)
+}

exporter/coralogixexporter/testdata/config.yaml

@@ -24,8 +24,15 @@ coralogix/all:
     endpoint: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
   application_name_attributes:
     - "service.namespace"
+    - "k8s.namespace.name" 
   subsystem_name_attributes:
     - "service.name"
+    - "k8s.deployment.name"
+    - "k8s.statefulset.name"
+    - "k8s.daemonset.name"
+    - "k8s.cronjob.name"
+    - "k8s.job.name"
+    - "k8s.container.name"
   private_key: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
   application_name: "APP_NAME"
   subsystem_name: "SUBSYSTEM_NAME"

pkg/stanza/decoder/decoder.go

@@ -0,0 +1,81 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package decoder // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/decoder"
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/ianaindex"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
+)
+
+type Decoder struct {
+	encoding     encoding.Encoding
+	decoder      *encoding.Decoder
+	decodeBuffer []byte
+}
+
+// New wraps a character set encoding and creates a reusable buffer to reduce allocation.
+// Decoder is not thread-safe and must not be used in multiple goroutines.
+func New(enc encoding.Encoding) *Decoder {
+	return &Decoder{
+		encoding:     enc,
+		decoder:      enc.NewDecoder(),
+		decodeBuffer: make([]byte, 1<<12),
+	}
+}
+
+// Decode converts the bytes in msgBuf to UTF-8 from the configured encoding.
+func (d *Decoder) Decode(msgBuf []byte) ([]byte, error) {
+	for {
+		d.decoder.Reset()
+		nDst, _, err := d.decoder.Transform(d.decodeBuffer, msgBuf, true)
+		if err == nil {
+			return d.decodeBuffer[:nDst], nil
+		}
+		if errors.Is(err, transform.ErrShortDst) {
+			d.decodeBuffer = make([]byte, len(d.decodeBuffer)*2)
+			continue
+		}
+		return nil, fmt.Errorf("transform encoding: %w", err)
+	}
+}
+
+var encodingOverrides = map[string]encoding.Encoding{
+	"":         unicode.UTF8,
+	"nop":      encoding.Nop,
+	"ascii":    unicode.UTF8,
+	"us-ascii": unicode.UTF8,
+	"utf8":     unicode.UTF8,
+	"utf-8":    unicode.UTF8,
+	"utf16":    unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
+	"utf-16":   unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
+}
+
+// LookupEncoding attempts to match the string name provided with a character set encoding.
+func LookupEncoding(enc string) (encoding.Encoding, error) {
+	if e, ok := encodingOverrides[strings.ToLower(enc)]; ok {
+		return e, nil
+	}
+	e, err := ianaindex.IANA.Encoding(enc)
+	if err != nil {
+		return nil, fmt.Errorf("unsupported encoding '%s'", enc)
+	}
+	if e == nil {
+		return nil, fmt.Errorf("no charmap defined for encoding '%s'", enc)
+	}
+	return e, nil
+}
+
+func IsNop(enc string) bool {
+	e, err := LookupEncoding(enc)
+	if err != nil {
+		return false
+	}
+	return e == encoding.Nop
+}

pkg/stanza/docs/operators/README.md

@@ -1,7 +1,7 @@
 ## What is an operator?
 An operator is the most basic unit of log processing. Each operator fulfills a single responsibility, such as reading lines from a file, or parsing JSON from a field. Operators are then chained together in a pipeline to achieve a desired result.
 
-For instance, a user may read lines from a file using the `file_input` operator. From there, the results of this operation may be sent to a `regex_parser` operator that creates fields based on a regex pattern. And then finally, these results may be sent to a `elastic_output` operator that writes each line to Elasticsearch.
+For instance, a user may read lines from a file using the `file_input` operator. From there, the results of this operation may be sent to a `regex_parser` operator that creates fields based on a regex pattern. And then finally, these results may be sent to a `file_output` operator that writes each line to a file on disk.
 
 
 ## What operators are available?

pkg/stanza/fileconsumer/config.go

@@ -12,13 +12,15 @@ import (
 	"go.opentelemetry.io/collector/featuregate"
 	"go.uber.org/zap"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/decoder"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/emit"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/fingerprint"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/header"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/splitter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/matcher"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/tokenize"
 )
 
 const (
@@ -48,7 +50,7 @@ func NewConfig() *Config {
 		IncludeFileNameResolved: false,
 		IncludeFilePathResolved: false,
 		PollInterval:            200 * time.Millisecond,
-		Splitter:                helper.NewSplitterConfig(),
+		Splitter:                tokenize.NewSplitterConfig(),
 		StartAt:                 "end",
 		FingerprintSize:         fingerprint.DefaultSize,
 		MaxLogSize:              defaultMaxLogSize,
@@ -60,19 +62,19 @@ func NewConfig() *Config {
 // Config is the configuration of a file input operator
 type Config struct {
 	matcher.Criteria        `mapstructure:",squash"`
-	IncludeFileName         bool                  `mapstructure:"include_file_name,omitempty"`
-	IncludeFilePath         bool                  `mapstructure:"include_file_path,omitempty"`
-	IncludeFileNameResolved bool                  `mapstructure:"include_file_name_resolved,omitempty"`
-	IncludeFilePathResolved bool                  `mapstructure:"include_file_path_resolved,omitempty"`
-	PollInterval            time.Duration         `mapstructure:"poll_interval,omitempty"`
-	StartAt                 string                `mapstructure:"start_at,omitempty"`
-	FingerprintSize         helper.ByteSize       `mapstructure:"fingerprint_size,omitempty"`
-	MaxLogSize              helper.ByteSize       `mapstructure:"max_log_size,omitempty"`
-	MaxConcurrentFiles      int                   `mapstructure:"max_concurrent_files,omitempty"`
-	MaxBatches              int                   `mapstructure:"max_batches,omitempty"`
-	DeleteAfterRead         bool                  `mapstructure:"delete_after_read,omitempty"`
-	Splitter                helper.SplitterConfig `mapstructure:",squash,omitempty"`
-	Header                  *HeaderConfig         `mapstructure:"header,omitempty"`
+	IncludeFileName         bool                    `mapstructure:"include_file_name,omitempty"`
+	IncludeFilePath         bool                    `mapstructure:"include_file_path,omitempty"`
+	IncludeFileNameResolved bool                    `mapstructure:"include_file_name_resolved,omitempty"`
+	IncludeFilePathResolved bool                    `mapstructure:"include_file_path_resolved,omitempty"`
+	PollInterval            time.Duration           `mapstructure:"poll_interval,omitempty"`
+	StartAt                 string                  `mapstructure:"start_at,omitempty"`
+	FingerprintSize         helper.ByteSize         `mapstructure:"fingerprint_size,omitempty"`
+	MaxLogSize              helper.ByteSize         `mapstructure:"max_log_size,omitempty"`
+	MaxConcurrentFiles      int                     `mapstructure:"max_concurrent_files,omitempty"`
+	MaxBatches              int                     `mapstructure:"max_batches,omitempty"`
+	DeleteAfterRead         bool                    `mapstructure:"delete_after_read,omitempty"`
+	Splitter                tokenize.SplitterConfig `mapstructure:",squash,omitempty"`
+	Header                  *HeaderConfig           `mapstructure:"header,omitempty"`
 }
 
 type HeaderConfig struct {
@@ -130,7 +132,7 @@ func (c Config) buildManager(logger *zap.SugaredLogger, emit emit.Callback, fact
 
 	var hCfg *header.Config
 	if c.Header != nil {
-		enc, err := helper.LookupEncoding(c.Splitter.Encoding)
+		enc, err := decoder.LookupEncoding(c.Splitter.Encoding)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create encoding: %w", err)
 		}
@@ -146,7 +148,7 @@ func (c Config) buildManager(logger *zap.SugaredLogger, emit emit.Callback, fact
 		return nil, err
 	}
 
-	enc, err := helper.LookupEncoding(c.Splitter.Encoding)
+	enc, err := decoder.LookupEncoding(c.Splitter.Encoding)
 	if err != nil {
 		return nil, err
 	}
@@ -218,7 +220,7 @@ func (c Config) validate() error {
 		return errors.New("`max_batches` must not be negative")
 	}
 
-	enc, err := helper.LookupEncoding(c.Splitter.Encoding)
+	enc, err := decoder.LookupEncoding(c.Splitter.Encoding)
 	if err != nil {
 		return err
 	}