Skip to content
This repository has been archived by the owner on Sep 21, 2023. It is now read-only.

Migrate the elasticsearch output to go-elasticsearch #187

Merged
merged 16 commits into from
Dec 1, 2022
Merged

Migrate the elasticsearch output to go-elasticsearch #187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
6166c45
add go-elasticsearch
faec Nov 14, 2022
42e43d5
working on bulk indexing
faec Nov 14, 2022
5b9b64d
cleanup
faec Nov 14, 2022
4e8ed86
better configs
faec Nov 14, 2022
3b47ef0
Merge branch 'main' of github.com:elastic/elastic-agent-shipper into …
faec Nov 14, 2022
009247f
cleanup
faec Nov 15, 2022
b985b24
cleanup
faec Nov 16, 2022
f3dad6a
cleanup
faec Nov 16, 2022
a776544
migrating dependent code
faec Nov 21, 2022
730859b
fix batch acknowledgments?
faec Nov 22, 2022
ac11e31
Merge branch 'main' of github.com:elastic/elastic-agent-shipper into …
faec Nov 22, 2022
d3d8b35
add tls min version
faec Nov 22, 2022
5f47854
mage check
faec Nov 22, 2022
65ad666
mage check again
faec Nov 22, 2022
7b2d0ef
adjust integer signedness
faec Nov 30, 2022
585f5c1
Merge branch 'main' of github.com:elastic/elastic-agent-shipper into …
faec Nov 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
393 changes: 91 additions & 302 deletions NOTICE.txt
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion controller/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ func (r *ServerRunner) Close() (err error) {
}

func outputFromConfig(config output.Config, queue *queue.Queue) (Output, error) {
if config.Elasticsearch != nil {
if config.Elasticsearch != nil && config.Elasticsearch.Enabled {
return elasticsearch.NewElasticSearch(config.Elasticsearch, queue), nil
}
if config.Kafka != nil && config.Kafka.Enabled {
Expand Down
5 changes: 2 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ require (
github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220810153818-dd118efed5a5
github.com/elastic/elastic-agent-client/v7 v7.0.1
github.com/elastic/elastic-agent-shipper-client v0.4.0
github.com/elastic/go-elasticsearch/v8 v8.2.0
github.com/elastic/go-ucfg v0.8.6
github.com/gofrs/uuid v4.2.0+incompatible
github.com/magefile/mage v1.14.0
github.com/stretchr/testify v1.7.1
go.elastic.co/apm v1.15.0
go.elastic.co/go-licence-detector v0.5.0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced
Expand All @@ -35,6 +35,7 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
github.com/elastic/go-licenser v0.4.1 // indirect
github.com/elastic/go-structform v0.0.10 // indirect
github.com/elastic/go-sysinfo v1.8.1 // indirect
Expand Down Expand Up @@ -74,15 +75,13 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/xdg/scram v1.0.3 // indirect
github.com/xdg/stringprep v1.0.3 // indirect
go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 // indirect
go.elastic.co/apm/module/apmhttp/v2 v2.0.0 // indirect
go.elastic.co/apm/v2 v2.1.0 // indirect
go.elastic.co/ecszap v1.0.1 // indirect
go.elastic.co/fastjson v1.1.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.22.0 // indirect
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
Expand Down
9 changes: 3 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,11 +508,13 @@ github.com/elastic/elastic-agent-shipper-client v0.2.0/go.mod h1:OyI2W+Mv3JxlkEF
github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk=
github.com/elastic/elastic-agent-shipper-client v0.4.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q=
github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38=
github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38=
github.com/elastic/go-elasticsearch/v8 v8.2.0 h1:oagGcb1gqxT7yWpQ3E7wMP3NhGRamsKVd7kRdbuI+/Y=
github.com/elastic/go-elasticsearch/v8 v8.2.0/go.mod h1:yY52i2Vj0unLz+N3Nwx1gM5LXwoj3h2dgptNGBYkMLA=
github.com/elastic/go-libaudit/v2 v2.3.2-0.20220729123722-f8f7d5c19e6b/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo=
github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
Expand All @@ -523,7 +525,6 @@ github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE
github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4=
github.com/elastic/go-sysinfo v1.8.1/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM=
Expand Down Expand Up @@ -1539,9 +1540,6 @@ github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q=
go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY=
go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 h1:5UaI4agfuGoGRhpFVb6s63Hj/9xtqYxF7kd6T77tNAw=
go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0/go.mod h1:GmYz+KDp2LDAa2nd/qJ+OkrjWVEoCRIPAWarbUvyt6Y=
go.elastic.co/apm/module/apmhttp/v2 v2.0.0 h1:GNfmK1LD4nE5fYqbLxROCpg1ucyjSFG5iwulxwAJ+3o=
go.elastic.co/apm/module/apmhttp/v2 v2.0.0/go.mod h1:5KmxcNN7hkJh8sVW3Ggl/pYgnwiNenygE46bZoUb9RE=
Expand Down Expand Up @@ -1682,7 +1680,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
Expand Down
9 changes: 4 additions & 5 deletions output/console.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,9 @@ func (out *ConsoleOutput) Start() error {
// time for the output to shut down.
break
}
for i := 0; i < batch.Count(); i++ {
if event, ok := batch.Entry(i).(*messages.Event); ok {
out.send(event)
}
events := batch.Events()
for i := 0; i < len(events); i++ {
out.send(events[i])
}
// This tells the queue that we're done with these events
// and they can be safely discarded. The Beats queue interface
Expand All @@ -48,7 +47,7 @@ func (out *ConsoleOutput) Start() error {
// shipper to track events by their queue IDs so outputs
// can report status back to the server; see
// https://github.com/elastic/elastic-agent-shipper/issues/27.
batch.Done()
batch.Done(uint64(len(events)))
}
}()
return nil
Expand Down
142 changes: 0 additions & 142 deletions output/elasticsearch/bulk.go
View file
Open in desktop

This file was deleted.

98 changes: 0 additions & 98 deletions output/elasticsearch/callbacks.go
View file
Open in desktop

This file was deleted.

Loading