Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x](backport #42312) [filebeat] Add support for bulk_max_size and preset #42429

Merged
merged 1 commit into from
Jan 27, 2025
Merged

[8.x](backport #42312) [filebeat] Add support for bulk_max_size and preset #42429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions NOTICE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29374,6 +29374,29 @@ Contents of probable licence file $GOMODCACHE/gotest.tools/[email protected]/LICE
limitations under the License.


--------------------------------------------------------------------------------
Dependency : gotest.tools/v3
Version: v3.5.1
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/gotest.tools/[email protected]/LICENSE:

Copyright 2018 gotest.tools authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.


--------------------------------------------------------------------------------
Dependency : howett.net/plist
Version: v1.0.1
Expand Down Expand Up @@ -72086,29 +72109,6 @@ See the License for the specific language governing permissions and
limitations under the License.


--------------------------------------------------------------------------------
Dependency : gotest.tools/v3
Version: v3.5.1
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/gotest.tools/[email protected]/LICENSE:

Copyright 2018 gotest.tools authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.


--------------------------------------------------------------------------------
Dependency : k8s.io/klog/v2
Version: v2.130.1
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ require (
golang.org/x/term v0.27.0
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gotest.tools/v3 v3.5.1
)

require (
Expand Down
49 changes: 39 additions & 10 deletions libbeat/otelbeat/beatconverter/beatconverter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"go.opentelemetry.io/collector/confmap"

"github.com/elastic/beats/v7/libbeat/cloudid"
elasticsearchtranslate "github.com/elastic/beats/v7/libbeat/otelbeat/oteltranslate/outputs/elasticsearch"
"github.com/elastic/elastic-agent-libs/config"
)

Expand All @@ -47,19 +48,20 @@ func (c converter) Convert(_ context.Context, conf *confmap.Conf) error {
for _, beatreceiver := range supportedReceivers {
var out map[string]any

var beatReceiverConfigKey = "receivers::" + beatreceiver
// check if supported beat receiver is configured. Skip translation logic if not
if v := conf.Get("receivers::" + beatreceiver); v == nil {
if v := conf.Get(beatReceiverConfigKey); v == nil {
continue
}

// handle cloud id if set
if conf.IsSet("receivers::" + beatreceiver + "::cloud") {
if conf.IsSet(beatReceiverConfigKey + "::cloud") {
if err := handleCloudId(beatreceiver, conf); err != nil {
return fmt.Errorf("error handling cloud id %w", err)
}
}

receiverCfg, _ := conf.Sub("receivers::" + beatreceiver)
receiverCfg, _ := conf.Sub(beatReceiverConfigKey)
output, _ := receiverCfg.Sub("output")

if len(output.ToStringMap()) > 1 {
Expand All @@ -70,10 +72,18 @@ func (c converter) Convert(_ context.Context, conf *confmap.Conf) error {
switch key {
case "elasticsearch":
esConfig := config.MustNewConfigFrom(output)
esOTelConfig, err := ToOTelConfig(esConfig)
esOTelConfig, err := elasticsearchtranslate.ToOTelConfig(esConfig)
if err != nil {
return fmt.Errorf("cannot convert elasticsearch config: %w", err)
}

// when output.queue is set by user or it comes from "preset" config, promote it to global level
if ok := esConfig.HasField("queue"); ok {
if err := promoteOutputQueueSettings(beatreceiver, esConfig, conf); err != nil {
return err
}
}

out = map[string]any{
"service::pipelines::logs::exporters": []string{"elasticsearch"},
"exporters": map[string]any{
Expand All @@ -91,14 +101,14 @@ func (c converter) Convert(_ context.Context, conf *confmap.Conf) error {

// Replace output.[configured-output] with output.otelconsumer
out = map[string]any{
"receivers::" + beatreceiver + "::output": nil,
beatReceiverConfigKey + "::output": nil,
}
err := conf.Merge(confmap.NewFromStringMap(out))
if err != nil {
return err
}
out = map[string]any{
"receivers::" + beatreceiver + "::output::otelconsumer": nil,
beatReceiverConfigKey + "::output::otelconsumer": nil,
}

err = conf.Merge(confmap.NewFromStringMap(out))
Expand All @@ -110,9 +120,9 @@ func (c converter) Convert(_ context.Context, conf *confmap.Conf) error {
return nil
}

func handleCloudId(beatreceiver string, conf *confmap.Conf) error {
func handleCloudId(beatReceiverConfigKey string, conf *confmap.Conf) error {

receiverCfg, _ := conf.Sub("receivers::" + beatreceiver)
receiverCfg, _ := conf.Sub("receivers::" + beatReceiverConfigKey)
beatCfg := config.MustNewConfigFrom(receiverCfg.ToStringMap())

// Handle cloud.id the same way Beats does, this will also handle
Expand All @@ -128,7 +138,7 @@ func handleCloudId(beatreceiver string, conf *confmap.Conf) error {
}

out := map[string]any{
"receivers::" + beatreceiver: beatOutput,
"receivers::" + beatReceiverConfigKey: beatOutput,
}
err = conf.Merge(confmap.NewFromStringMap(out))
if err != nil {
Expand All @@ -137,7 +147,26 @@ func handleCloudId(beatreceiver string, conf *confmap.Conf) error {

// we set this to nil to ensure cloudid check does not throw error when output is next set to otelconsumer
out = map[string]any{
"receivers::" + beatreceiver + "::cloud": nil,
"receivers::" + beatReceiverConfigKey + "::cloud": nil,
}
err = conf.Merge(confmap.NewFromStringMap(out))
if err != nil {
return err
}

return nil
}

// promoteOutputQueueSettings promotes output.queue settings to global level
func promoteOutputQueueSettings(beatReceiverConfigKey string, outputConfig *config.C, conf *confmap.Conf) error {

var queueOutput map[string]any
err := outputConfig.Unpack(&queueOutput)
if err != nil {
return err
}
out := map[string]any{
"receivers::" + beatReceiverConfigKey + "::queue": queueOutput["queue"],
}
err = conf.Merge(confmap.NewFromStringMap(out))
if err != nil {
Expand Down
Loading
Loading