diff --git a/namespaces/_default_.yml b/namespaces/_default_.yml index 0feb94d..36e3f39 100644 --- a/namespaces/_default_.yml +++ b/namespaces/_default_.yml @@ -1,4 +1,4 @@ -version: 2017.05.23.0 +version: 2017.06.06.0 field_defaults: type: string @@ -14,7 +14,7 @@ _default_: fields: - name: "@timestamp" type: date - format: yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime + format: yyyy-MM-dd HH:mm:ss,SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime example: 2015-01-24T14:06:05.071Z description: > UTC value marking when the log payload was created, or when log payload was first collected if the creation time is not known; diff --git a/namespaces/collectd.yml b/namespaces/collectd.yml index 62437d7..87fe2e8 100644 --- a/namespaces/collectd.yml +++ b/namespaces/collectd.yml @@ -6,36 +6,43 @@ namespace: fields: - name: interval type: float + include_in_all: false description: > Collectd's interval. - name: plugin type: string + include_in_all: false description: > Collectd's plugin. - name: plugin_instance type: string + include_in_all: false description: > Collectd's plugin_instance. - name: type_instance type: string + include_in_all: false description: > Collectd's type_instance. - name: type type: string + include_in_all: false description: > Collectd's type. - name: dstypes type: string + include_in_all: false description: > Collectd's dstypes. - name: processes type: group + include_in_all: false description: > Corresponds to collectd's processes plugin. fields: @@ -154,6 +161,7 @@ namespace: type: group description: > Corresponds to collectd's disk plugin. + include_in_all: false fields: - name: disk_merged type: group @@ -239,6 +247,7 @@ namespace: type: group description: > Corresponds to collectd's interface plugin. + include_in_all: false fields: - name: if_octets type: group @@ -302,6 +311,7 @@ namespace: - name: virt type: group + include_in_all: false description: > Corresponds to collectd's virt plugin. fields: @@ -412,6 +422,7 @@ namespace: - name: CPU type: group + include_in_all: false description: > Corresponds to collectd's CPU plugin. fields: @@ -422,6 +433,7 @@ namespace: - name: df type: group + include_in_all: false description: > Corresponds to collectd's df plugin. fields: @@ -438,6 +450,7 @@ namespace: - name: entropy type: group + include_in_all: false description: > Corresponds to collectd's entropy plugin. fields: @@ -451,6 +464,7 @@ namespace: type: group description: > Corresponds to collectd's nfs plugin. + include_in_all: false fields: - name: nfs_procedure @@ -462,6 +476,7 @@ namespace: type: group description: > Corresponds to collectd's memory plugin. + include_in_all: false fields: - name: memory @@ -478,6 +493,7 @@ namespace: type: group description: > Corresponds to collectd's swap plugin. + include_in_all: false fields: - name: swap @@ -494,6 +510,7 @@ namespace: type: group description: > Corresponds to collectd's load plugin. + include_in_all: false fields: - name: load type: group @@ -519,6 +536,7 @@ namespace: type: group description: > Corresponds to collectd's aggregation plugin. + include_in_all: false fields: - name: percent @@ -530,6 +548,7 @@ namespace: type: group description: > Corresponds to collectd's statsd plugin. + include_in_all: false fields: - name: host_cpu @@ -731,6 +750,7 @@ namespace: type: group description: > Corresponds to collectd's postgresql plugin. + include_in_all: false fields: - name: pg_n_tup_g diff --git a/namespaces/kubernetes.yml b/namespaces/kubernetes.yml index 363bca5..2cc265a 100644 --- a/namespaces/kubernetes.yml +++ b/namespaces/kubernetes.yml @@ -7,6 +7,8 @@ namespace: fields: - name: pod_name type: string + norms: + enabled: true description: > The name of the pod @@ -20,21 +22,29 @@ namespace: - name: pod_id type: string + norms: + enabled: true description: > Kubernetes ID of the pod. - name: namespace_name type: string + norms: + enabled: true description: > The name of the namespace in Kubernetes. - name: namespace_id type: string + norms: + enabled: true description: > ID of the namespace in Kubernetes. - name: host type: string + norms: + enabled: true description: > Kubernetes node name diff --git a/scripts/concat-index-pattern-fields.py b/scripts/concat-index-pattern-fields.py index 97a657c..0ee77de 100755 --- a/scripts/concat-index-pattern-fields.py +++ b/scripts/concat-index-pattern-fields.py @@ -4,14 +4,26 @@ import json doc = {} +names = {} + +def removedupnames(fieldary): + ret = [] + for field in fieldary: + if not field['name'] in names: + names[field['name']] = True + ret.append(field) + return ret + for filename in sys.argv[1:]: with open(filename, "r") as ff: if doc: d2 = json.load(ff) - d2fieldary = json.loads(d2['fields']) + d2fieldary = removedupnames(json.loads(d2['fields'])) docfieldary = json.loads(doc['fields']) docfieldary.extend(d2fieldary) doc['fields'] = json.dumps(docfieldary) else: doc = json.load(ff) -sys.stdout.write(json.dumps(doc, indent=4, sort_keys=True)) + docfieldary = removedupnames(json.loads(doc['fields'])) + doc['fields'] = json.dumps(docfieldary) +sys.stdout.write(json.dumps(doc, indent=4, sort_keys=True, separators=(',', ': '))) diff --git a/scripts/generate_template.py b/scripts/generate_template.py index 1436953..4ded732 100644 --- a/scripts/generate_template.py +++ b/scripts/generate_template.py @@ -77,6 +77,7 @@ def object_types_to_template(template_definition, output, output_index_pattern, json.dump( skeleton, output, indent=2, separators=(',', ': '), sort_keys=True) + output.write('\n') # index pattern stuff time_field_name = "time" @@ -92,6 +93,7 @@ def object_types_to_template(template_definition, output, output_index_pattern, skeleton_index_pattern["fields"] = json.dumps(index_pattern_fields) json.dump( skeleton_index_pattern, output_index_pattern, indent=2, separators=(',', ': '), sort_keys=True) + output_index_pattern.write('\n') def add_mapping_to_skeleton(map_type, skeleton): diff --git a/templates/collectd_metrics/template.yml b/templates/collectd_metrics/template.yml index 12fe602..9657196 100644 --- a/templates/collectd_metrics/template.yml +++ b/templates/collectd_metrics/template.yml @@ -1,10 +1,14 @@ skeleton_path: ../skeleton.json skeleton_index_pattern_path: ../skeleton-index-pattern.json +# exclude all collectd stats fields from _source elasticsearch_template: name: org.ovirt.viaq-collectd index_pattern: "project.ovirt-metrics-*" - order: 10 + order: 20 + _source: + excludes: + - collectd.* namespaces: - collectd.yml diff --git a/templates/openshift/Makefile b/templates/openshift/Makefile index 02e07ee..99687d3 100644 --- a/templates/openshift/Makefile +++ b/templates/openshift/Makefile @@ -1,9 +1,8 @@ -TEMPLATE_NAME = com.redhat.viaq-openshift - .PHONY: all clean all: - python ../../scripts/generate_template.py template.yml ../../namespaces/ + python ../../scripts/generate_template.py template-project.yml ../../namespaces/ + python ../../scripts/generate_template.py template-operations.yml ../../namespaces/ clean: rm *.template.json diff --git a/templates/openshift/template-operations.yml b/templates/openshift/template-operations.yml new file mode 100644 index 0000000..7557a0d --- /dev/null +++ b/templates/openshift/template-operations.yml @@ -0,0 +1,18 @@ +skeleton_path: ../skeleton.json +skeleton_index_pattern_path: ../skeleton-index-pattern.json + +elasticsearch_template: + name: com.redhat.viaq-openshift-operations + index_pattern: ".operations.*" + order: 10 + +namespaces: + - rsyslog.yml + - systemd.yml + - kubernetes.yml + - docker.yml + - pipeline_metadata.yml + - ovirt.yml + - aushape.yml + - tlog.yml +# - openshift/app_log.yml diff --git a/templates/openshift/template.yml b/templates/openshift/template-project.yml similarity index 81% rename from templates/openshift/template.yml rename to templates/openshift/template-project.yml index 20a23f9..3b196d2 100644 --- a/templates/openshift/template.yml +++ b/templates/openshift/template-project.yml @@ -2,8 +2,8 @@ skeleton_path: ../skeleton.json skeleton_index_pattern_path: ../skeleton-index-pattern.json elasticsearch_template: - name: com.redhat.viaq-openshift - index_pattern: "*" + name: com.redhat.viaq-openshift-project + index_pattern: "project.*" order: 10 namespaces: