chore: prometheus monitoring tune (#133)

* increased metric retention up to a week
* added lightmare monitoring by default
* added specs to enable lightmare monitoring

flux/components/monitoring/configs/kustomization.yml

@@ -5,3 +5,4 @@ resources:
   - system
   - kubernetes
   - kubevirt.yml
+  - lightmare.yml

flux/components/monitoring/configs/lightmare.yml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: lightmare-monitoring
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  path: ./flux/components/monitoring/configs/lightmare
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: spectrum
+    namespace: flux-system
+  dependsOn:
+    - name: lightmare
+      namespace: flux-system

flux/components/monitoring/configs/lightmare/kustomization.yml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+resources:
+  - service-monitor.yml
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    grafana_dashboard: "1"
+    app.kubernetes.io/part-of: spectrum-monitoring
+    app.kubernetes.io/component: lightmare
+commonAnnotations:
+  grafana_folder: "lightmare"
+
+# tbd
+# configMapGenerator:
+#   - name: dashboards-k8s-lightmare
+#     files:
+#       - *.json

flux/components/monitoring/configs/lightmare/service-monitor.yml

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: lightmare
+  labels:
+    app.kubernetes.io/part-of: spectrum-monitoring
+    app.kubernetes.io/component: lightmare
+spec:
+  selector:
+    matchLabels:
+      app: lightmare
+  namespaceSelector:
+    matchNames:
+    - lightmare
+  endpoints:
+  - port: metrics
+    scheme: http

flux/components/monitoring/controllers/kube-prometheus-stack/release.yaml

@@ -32,7 +32,7 @@ spec:
       enabled: false
     prometheus:
       prometheusSpec:
-        retention: 24h
+        retention: 168h
         resources:
           requests:
             cpu: 200m
@@ -48,6 +48,7 @@ spec:
               - "kube-state-metrics"
               - "prometheus-node-exporter"
               - "spectrum-monitoring"
+              - "lightmare"
 
         podMonitorNamespaceSelector: {}
         podMonitorSelector: