From d3dac6205d3cb3fdd231afd197ba40d787eccf7b Mon Sep 17 00:00:00 2001
From: Marcin Tojek <mtojek@users.noreply.github.com>
Date: Wed, 11 Aug 2021 18:59:21 +0200
Subject: [PATCH] Fix: geo_points can be imported from ECS (#476)

---
 internal/fields/validate.go                   |  5 ++--
 internal/fields/validate_test.go              | 11 ++++++++
 .../fields_tests/_dev/build/build.yml         |  3 +++
 .../fields_tests/_dev/build/docs/README.md    |  5 ++++
 test/packages/fields_tests/changelog.yml      |  6 +++++
 .../first/agent/stream/stream.yml.hbs         |  7 +++++
 .../elasticsearch/ingest_pipeline/default.yml | 10 +++++++
 .../data_stream/first/fields/base-fields.yml  | 12 +++++++++
 .../data_stream/first/fields/geo-fields.yml   |  6 +++++
 .../data_stream/first/manifest.yml            | 13 ++++++++++
 .../data_stream/first/sample_event.json       |  8 ++++++
 test/packages/fields_tests/docs/README.md     | 26 +++++++++++++++++++
 test/packages/fields_tests/manifest.yml       | 22 ++++++++++++++++
 13 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 test/packages/fields_tests/_dev/build/build.yml
 create mode 100644 test/packages/fields_tests/_dev/build/docs/README.md
 create mode 100644 test/packages/fields_tests/changelog.yml
 create mode 100644 test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs
 create mode 100644 test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml
 create mode 100644 test/packages/fields_tests/data_stream/first/fields/base-fields.yml
 create mode 100644 test/packages/fields_tests/data_stream/first/fields/geo-fields.yml
 create mode 100644 test/packages/fields_tests/data_stream/first/manifest.yml
 create mode 100644 test/packages/fields_tests/data_stream/first/sample_event.json
 create mode 100644 test/packages/fields_tests/docs/README.md
 create mode 100644 test/packages/fields_tests/manifest.yml

diff --git a/internal/fields/validate.go b/internal/fields/validate.go
index 6d269ce8e..20bde044f 100644
--- a/internal/fields/validate.go
+++ b/internal/fields/validate.go
@@ -265,8 +265,9 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 	k = strings.ReplaceAll(k, "*", "[^.]+")
 
 	// Workaround for potential geo_point, as "lon" and "lat" fields are not present in field definitions.
-	if def.Type == "geo_point" {
-		k += "\\.(lon|lat)"
+	// Unfortunately we have to assume that imported field could be a geo_point (nasty workaround).
+	if def.Type == "geo_point" || def.External != "" {
+		k += "(\\.lon|\\.lat|)"
 	}
 
 	k = fmt.Sprintf("^%s$", k)
diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go
index 0b7f59fd9..016c644df 100644
--- a/internal/fields/validate_test.go
+++ b/internal/fields/validate_test.go
@@ -223,3 +223,14 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 	require.NoError(t, err)
 	return c
 }
+
+func TestValidate_geo_point(t *testing.T) {
+	validator, err := CreateValidatorForDataStream("../../test/packages/fields_tests/data_stream/first")
+
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	e := readSampleEvent(t, "../../test/packages/fields_tests/data_stream/first/sample_event.json")
+	errs := validator.ValidateDocumentBody(e)
+	require.Empty(t, errs)
+}
diff --git a/test/packages/fields_tests/_dev/build/build.yml b/test/packages/fields_tests/_dev/build/build.yml
new file mode 100644
index 000000000..002aa1565
--- /dev/null
+++ b/test/packages/fields_tests/_dev/build/build.yml
@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@1.10
diff --git a/test/packages/fields_tests/_dev/build/docs/README.md b/test/packages/fields_tests/_dev/build/docs/README.md
new file mode 100644
index 000000000..094b2d224
--- /dev/null
+++ b/test/packages/fields_tests/_dev/build/docs/README.md
@@ -0,0 +1,5 @@
+# Fields Tests
+
+{{event "first"}}
+
+{{fields "first"}}
\ No newline at end of file
diff --git a/test/packages/fields_tests/changelog.yml b/test/packages/fields_tests/changelog.yml
new file mode 100644
index 000000000..7b5f7fd8f
--- /dev/null
+++ b/test/packages/fields_tests/changelog.yml
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link
diff --git a/test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs b/test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs
new file mode 100644
index 000000000..5845510de
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs
@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~
diff --git a/test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml b/test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml
new file mode 100644
index 000000000..81221adf3
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
\ No newline at end of file
diff --git a/test/packages/fields_tests/data_stream/first/fields/base-fields.yml b/test/packages/fields_tests/data_stream/first/fields/base-fields.yml
new file mode 100644
index 000000000..7c798f453
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/fields/base-fields.yml
@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
diff --git a/test/packages/fields_tests/data_stream/first/fields/geo-fields.yml b/test/packages/fields_tests/data_stream/first/fields/geo-fields.yml
new file mode 100644
index 000000000..056a08c6f
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/fields/geo-fields.yml
@@ -0,0 +1,6 @@
+- name: destination.geo.location
+  external: ecs
+- name: geo.location
+  external: ecs
+- name: source.geo.location
+  external: ecs
diff --git a/test/packages/fields_tests/data_stream/first/manifest.yml b/test/packages/fields_tests/data_stream/first/manifest.yml
new file mode 100644
index 000000000..979ef29d6
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/manifest.yml
@@ -0,0 +1,13 @@
+title: "First"
+type: logs
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log
diff --git a/test/packages/fields_tests/data_stream/first/sample_event.json b/test/packages/fields_tests/data_stream/first/sample_event.json
new file mode 100644
index 000000000..97d1717c5
--- /dev/null
+++ b/test/packages/fields_tests/data_stream/first/sample_event.json
@@ -0,0 +1,8 @@
+{
+    "source.geo.location": {
+        "lat": 1.0,
+        "lon": "2.0"
+    },
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
+}
\ No newline at end of file
diff --git a/test/packages/fields_tests/docs/README.md b/test/packages/fields_tests/docs/README.md
new file mode 100644
index 000000000..e1174d949
--- /dev/null
+++ b/test/packages/fields_tests/docs/README.md
@@ -0,0 +1,26 @@
+# Fields Tests
+
+An example event for `first` looks as following:
+
+```json
+{
+    "source.geo.location": {
+        "lat": 1.0,
+        "lon": "2.0"
+    },
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
+}
+```
+
+**Exported fields**
+
+| Field | Description | Type |
+|---|---|---|
+| @timestamp | Event timestamp. | date |
+| data_stream.dataset | Data stream dataset. | constant_keyword |
+| data_stream.namespace | Data stream namespace. | constant_keyword |
+| data_stream.type | Data stream type. | constant_keyword |
+| destination.geo.location | Longitude and latitude. | geo_point |
+| geo.location | Longitude and latitude. | geo_point |
+| source.geo.location | Longitude and latitude. | geo_point |
diff --git a/test/packages/fields_tests/manifest.yml b/test/packages/fields_tests/manifest.yml
new file mode 100644
index 000000000..5025f6ff3
--- /dev/null
+++ b/test/packages/fields_tests/manifest.yml
@@ -0,0 +1,22 @@
+format_version: 1.0.0
+name: fields_tests
+title: "Fields Tests"
+version: 0.0.1
+license: basic
+description: "These are tests of field validation."
+type: integration
+categories:
+  - custom
+release: experimental
+conditions:
+  kibana.version: "^7.15.0"
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations