diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 21690cb77058..875c6768a6e8 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -288,6 +288,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Update CEL mito extensions to v1.15.0. {pull}40294[40294]
- Allow cross-region bucket configuration in s3 input. {issue}22161[22161] {pull}40309[40309]
- Improve logging in Okta Entity Analytics provider. {issue}40106[40106] {pull}40347[40347]
+- Document `winlog` input. {issue}40074[40074] {pull}40462[40462]
- Added retry logic to websocket connections in the streaming input. {issue}40271[40271] {pull}40601[40601]
*Auditbeat*
diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc
index 860f98f2356a..3b1d274d59bd 100644
--- a/filebeat/docs/filebeat-options.asciidoc
+++ b/filebeat/docs/filebeat-options.asciidoc
@@ -95,6 +95,8 @@ You can configure {beatname_uc} to use the following inputs:
* <<{beatname_lc}-input-syslog>>
* <<{beatname_lc}-input-tcp>>
* <<{beatname_lc}-input-udp>>
+* <<{beatname_lc}-input-unix>>
+* <<{beatname_lc}-input-winlog>>
include::multiline.asciidoc[]
@@ -157,3 +159,5 @@ include::inputs/input-tcp.asciidoc[]
include::inputs/input-udp.asciidoc[]
include::inputs/input-unix.asciidoc[]
+
+include::inputs/input-winlog.asciidoc[]
diff --git a/filebeat/docs/inputs/input-winlog.asciidoc b/filebeat/docs/inputs/input-winlog.asciidoc
new file mode 100644
index 000000000000..6d58c7d8df8d
--- /dev/null
+++ b/filebeat/docs/inputs/input-winlog.asciidoc
@@ -0,0 +1,459 @@
+:type: winlog
+
+[id="{beatname_lc}-input-{type}"]
+=== winlog input
+
+++++
+winlog
+++++
+
+beta[]
+
+Use the `winlog` input to read Windows event logs. It reads from one
+event log using Windows APIs, filters the events based on user-configured criteria,
+then sends the event data to the configured outputs. It watches the event
+log so that new event data is sent in a timely manner. The read position for
+the event log is persisted to disk to allow the input to resume after
+restarts.
+
+The `winlog` input can capture event data from any event logs running on your system.
+For example, you can capture events such as:
+
+* application events
+* hardware events
+* security events
+* system events
+
+Here is a sample configuration:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Application
+ ignore_older: 72h
+--------------------------------------------------------------------------------
+
+[float]
+=== Configuration options
+
+[float]
+==== `batch_read_size`
+
+The maximum number of event log records to read from the Windows API in a single
+batch. The default batch size is 100. Most Windows versions return an error if
+the value is larger than 1024. *{vista_and_newer}*
+
+{beatname_uc} starts a goroutine (a lightweight thread) to read from each
+individual event log. The goroutine reads a batch of event log records using the
+Windows API, applies any processors to the events, publishes them to the
+configured outputs, and waits for an acknowledgement from the outputs before
+reading additional event log records.
+
+[float]
+==== `name`
+
+The name of the event log to monitor. It must
+have a `name` field, except for those which use a custom XML query.
+A channel is a named stream of events that transports events from an event
+source to an event log. Most channels are tied to specific event publishers.
+You can get a list of available event logs by using the PowerShell
+https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.diagnostics/get-winevent[`Get-WinEvent`] cmdlet
+on Windows Vista or newer. Here is a sample of the output from the command:
+
+[source,sh]
+--------------------------------------------------------------------------------
+PS C:\> Get-WinEvent -ListLog * | Format-List -Property LogName
+LogName : Application
+LogName : HardwareEvents
+LogName : Internet Explorer
+LogName : Key Management Service
+LogName : Security
+LogName : System
+LogName : Windows PowerShell
+LogName : ForwardedEvents
+LogName : Microsoft-Management-UI/Admin
+LogName : Microsoft-Rdms-UI/Admin
+LogName : Microsoft-Rdms-UI/Operational
+LogName : Microsoft-Windows-Windows Firewall With Advanced Security/Firewall
+...
+--------------------------------------------------------------------------------
+
+If `Get-WinEvent` is not available, the https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/get-eventlog[`Get-EventLog`] cmdlet can be used in its
+place.
+
+[source,sh]
+--------------------------------------------------------------------------------
+PS C:\Users\vagrant> Get-EventLog *
+
+ Max(K) Retain OverflowAction Entries Log
+ ------ ------ -------------- ------- ---
+ 20,480 0 OverwriteAsNeeded 75 Application
+ 20,480 0 OverwriteAsNeeded 0 HardwareEvents
+ 512 7 OverwriteOlder 0 Internet Explorer
+ 20,480 0 OverwriteAsNeeded 0 Key Management Service
+ 20,480 0 OverwriteAsNeeded 1,609 Security
+ 20,480 0 OverwriteAsNeeded 1,184 System
+ 15,360 0 OverwriteAsNeeded 464 Windows PowerShell
+--------------------------------------------------------------------------------
+
+You must specify the full name of the channel in the configuration file.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Microsoft-Windows-Windows Firewall With Advanced Security/Firewall
+--------------------------------------------------------------------------------
+
+To read events from an archived `.evtx` file you can specify the `name` as the
+absolute path (it cannot be relative) to the file.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: 'C:\backup\sysmon-2019.08.evtx'
+ no_more_events: stop
+--------------------------------------------------------------------------------
+
+The name key must not be used with custom XML queries.
+
+[float]
+==== `id`
+
+A unique identifier for the event log. This key is required when using a custom
+XML query.
+
+It is used to uniquely identify the event log reader in the registry file. This is
+useful if multiple event logs are being set up to watch the same channel or file. If an
+ID is not given, the `name` value will be used.
+
+This value must be unique.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Application
+ id: application-logs
+ ignore_older: 168h
+--------------------------------------------------------------------------------
+
+[float]
+==== `ignore_older`
+
+If this option is specified, the input filters events that are older than the
+specified amount of time. Valid time units are "ns", "us" (or "µs"), "ms", "s",
+"m", "h". This option is useful when you are beginning to monitor an event log
+that contains older records that you would like to ignore. This field is
+optional.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Application
+ ignore_older: 168h
+--------------------------------------------------------------------------------
+
+[float]
+==== `forwarded`
+
+A boolean flag to indicate that the log contains only events collected from
+remote hosts using the Windows Event Collector. The value defaults to true for
+the ForwardedEvents log and false for any other log. *{vista_and_newer}*
+
+This settings allows {beatname_uc} to optimize reads for forwarded events that are
+already rendered. When the value is true {beatname_uc} does not attempt to render
+the event using message files from the host computer. The Windows Event
+Collector subscription should be configured to use the "RenderedText" format
+(this is the default) to ensure that the events are distributed with messages
+and descriptions.
+
+[float]
+==== `event_id`
+
+An allowlist and blocklist of event IDs. The value is a comma-separated list. The
+accepted values are single event IDs to include (e.g. 4624), a range of event
+IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735).
+*{vista_and_newer}*
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Security
+ event_id: 4624, 4625, 4700-4800, -4735
+--------------------------------------------------------------------------------
+
+[WARNING]
+=======================================
+If you specify more than 22 query conditions (event IDs or event ID ranges), some
+versions of Windows will prevent {beatname_uc} from reading the event log due to
+limits in the query system. If this occurs a similar warning as shown below will
+be logged by {beatname_uc}, and it will continue processing data from other event
+logs.
+
+`WARN EventLog[Application] Open() error. No events will be read from this
+source. The specified query is invalid.`
+
+In some cases, the limit may be lower than 22 conditions. For instance, using a
+mixture of ranges and single event IDs, along with an additional parameter such
+as `ignore older`, results in a limit of 21 conditions.
+
+If you have more than 22 conditions, you can workaround this Windows limitation
+by using a drop_event[drop-event] processor to do the filtering after
+{beatname_uc} has received the events from Windows. The filter shown below is
+equivalent to `event_id: 903, 1024, 4624` but can be expanded beyond 22
+event IDs.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Security
+ processors:
+ - drop_event.when.not.or:
+ - equals.winlog.event_id: 903
+ - equals.winlog.event_id: 1024
+ - equals.winlog.event_id: 4624
+--------------------------------------------------------------------------------
+
+=======================================
+
+[float]
+==== `language`
+
+The language ID the events will be rendered in. The language will be forced regardless
+of the system language. A complete list of language IDs can be found
+https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c[here].
+It defaults to `0`, which indicates to use the system language.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Security
+ event_id: 4624, 4625, 4700-4800, -4735
+ language: 0x0409 # en-US
+--------------------------------------------------------------------------------
+
+[float]
+==== `level`
+
+A list of event levels to include. The value is a comma-separated list of
+levels. *{vista_and_newer}*
+
+[cols="2*", options="header"]
+|===
+|Level
+|Value
+
+|critical, crit
+|1
+
+|error, err
+|2
+
+|warning, warn
+|3
+
+|information, info
+|0 or 4
+
+|verbose
+|5
+|===
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Security
+ level: critical, error, warning
+--------------------------------------------------------------------------------
+
+[float]
+==== `provider`
+
+A list of providers (source names) to include. The value is a YAML list.
+*{vista_and_newer}*
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Application
+ provider:
+ - Application Error
+ - Application Hang
+ - Windows Error Reporting
+ - EMET
+--------------------------------------------------------------------------------
+
+You can obtain a list of providers associated with a log by using PowerShell.
+Here is an example showing the providers associated with the Security log.
+
+[source,sh]
+--------------------------------------------------------------------------------
+PS C:\> (Get-WinEvent -ListLog Security).ProviderNames
+DS
+LSA
+SC Manager
+Security
+Security Account Manager
+ServiceModel 4.0.0.0
+Spooler
+TCP/IP
+VSSAudit
+Microsoft-Windows-Security-Auditing
+Microsoft-Windows-Eventlog
+--------------------------------------------------------------------------------
+
+[float]
+==== `xml_query`
+
+Provide a custom XML query. This option is mutually exclusive with the `name`, `event_id`,
+`ignore_older`, `level`, and `provider` options. These options should be included in
+the XML query directly. Furthermore, an `id` must be provided. Custom XML queries
+provide more flexibility and advanced options than the simpler query options in {beatname_uc}.
+*{vista_and_newer}*
+
+Here is a configuration which will collect DHCP server events from multiple channels:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ id: dhcp-server-logs
+ xml_query: >
+
+
+
+
+
+
+
+--------------------------------------------------------------------------------
+
+XML queries may also be created in Windows Event Viewer using custom views. The query
+can be created using a graphical interface and the corresponding XML can be
+retrieved from the XML tab.
+
+[float]
+==== `include_xml`
+
+Boolean option that controls if the raw XML representation of an event is
+included in the data sent by {beatname_uc}. The default is false.
+*{vista_and_newer}*
+
+The XML representation of the event is useful for troubleshooting purposes. The
+data in the fields reported by {beatname_uc} can be compared to the data in the XML
+to diagnose problems.
+
+Example:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: Microsoft-Windows-Windows Defender/Operational
+ include_xml: true
+--------------------------------------------------------------------------------
+
+[float]
+==== `tags`
+
+A list of tags that the Beat includes in the `tags` field of each published
+event. Tags make it easy to select specific events in Kibana or apply
+conditional filtering in Logstash. These tags will be appended to the list of
+tags specified in the general configuration.
+
+Example:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: CustomLog
+ tags: ["web"]
+--------------------------------------------------------------------------------
+
+[float]
+[[winlog-configuration-fields]]
+==== `fields`
+
+Optional fields that you can specify to add additional information to the
+output. For example, you might add fields that you can use for filtering event
+data. Fields can be scalar values, arrays, dictionaries, or any nested
+combination of these. By default, the fields that you specify here will be
+grouped under a `fields` sub-dictionary in the output document. To store the
+custom fields as top-level fields, set the `fields_under_root` option to true.
+If a duplicate field is declared in the general configuration, then its value
+will be overwritten by the value declared here.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: CustomLog
+ fields:
+ customer_id: 51415432
+--------------------------------------------------------------------------------
+
+[float]
+==== `fields_under_root`
+
+If this option is set to true, the custom <>
+are stored as top-level fields in the output document instead of being grouped
+under a `fields` sub-dictionary. If the custom field names conflict with other
+field names added by {beatname_uc}, then the custom fields overwrite the other
+fields.
+
+[float]
+==== `processors`
+
+A list of processors to apply to the data generated by the event log.
+
+See <> for information about specifying
+processors in your config.
+
+[float]
+==== `index`
+
+If present, this formatted string overrides the index for events from this
+event log (for elasticsearch outputs), or sets the `raw_index` field of the event's
+metadata (for other outputs). This string can only refer to the agent name and
+version and the event timestamp; for access to dynamic fields, use
+`output.elasticsearch.index` or a processor.
+
+Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might
+expand to `"filebeat-myindex-2019.12.13"`.
+
+[float]
+==== `keep_null`
+
+If this option is set to true, fields with `null` values will be published in
+the output document. By default, `keep_null` is set to `false`.
+
+[float]
+==== `no_more_events`
+
+The action that the event log reader should take when it receives a signal from
+Windows that there are no more events to read. It can either `wait` for more
+events to be written (the default behavior) or it can `stop`. The overall
+{beatname_uc} process will stop when all of the individual event log readers have
+stopped. *{vista_and_newer}*
+
+Setting `no_more_events` to `stop` is useful when reading from archived event
+log files where you want to read the whole file then exit.
+
+[float]
+==== `api`
+
+This selects the event log reader implementation that is used to read events
+from the Windows APIs. You should only set this option when testing experimental
+features. When the value is set to `wineventlog-experimental` {beatname_uc} will
+replace the default event log reader with the **experimental** implementation.
+We are evaluating this implementation to see if it can provide increased
+performance and reduce CPU usage. *{vista_and_newer}*
+
+[source,yaml]
+--------------------------------------------------------------------------------
+- type: winlog
+ name: ForwardedEvents
+ api: wineventlog-experimental
+--------------------------------------------------------------------------------
+
+There are a few notable differences in the events:
+
+* Events that contained data under `winlog.user_data` will now have it under
+ `winlog.event_data`.
+* Setting `include_xml: true` has no effect.
diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go
index ddb703dbbb08..d558a477845e 100644
--- a/winlogbeat/eventlog/wineventlog.go
+++ b/winlogbeat/eventlog/wineventlog.go
@@ -270,7 +270,6 @@ func newWinEventLog(options *conf.C) (EventLog, error) {
cache: newMessageFilesCache(id, eventMetadataHandle, freeHandle),
winMetaCache: newWinMetaCache(metaTTL),
logPrefix: fmt.Sprintf("WinEventLog[%s]", id),
- metrics: newInputMetrics(c.Name, id),
}
// Forwarded events should be rendered using RenderEventXML. It is more
@@ -316,6 +315,9 @@ func (l *winEventLog) IsFile() bool {
func (l *winEventLog) Open(state checkpoint.EventLogState) error {
var bookmark win.EvtHandle
var err error
+ // we need to defer metrics initialization since when the event log
+ // is used from winlog input it would register it twice due to CheckConfig calls
+ l.metrics = newInputMetrics(l.channelName, l.id)
if len(state.Bookmark) > 0 {
bookmark, err = win.CreateBookmarkFromXML(state.Bookmark)
} else if state.RecordNumber > 0 && l.channelName != "" {
diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go
index 5e323858f229..3a6a9b2fe92b 100644
--- a/winlogbeat/eventlog/wineventlog_experimental.go
+++ b/winlogbeat/eventlog/wineventlog_experimental.go
@@ -129,7 +129,6 @@ func newWinEventLogExp(options *conf.C) (EventLog, error) {
maxRead: c.BatchReadSize,
renderer: renderer,
log: log,
- metrics: newInputMetrics(c.Name, id),
}
return l, nil
@@ -157,6 +156,9 @@ func (l *winEventLogExp) IsFile() bool {
func (l *winEventLogExp) Open(state checkpoint.EventLogState) error {
l.lastRead = state
+ // we need to defer metrics initialization since when the event log
+ // is used from winlog input it would register it twice due to CheckConfig calls
+ l.metrics = newInputMetrics(l.channelName, l.id)
var err error
l.iterator, err = win.NewEventIterator(