feat(config): compositeRules severity filters (#245)

.github/workflows/agent.yml

@@ -32,15 +32,14 @@ jobs:
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
           node-version: ${{ matrix.node-version }}
-      - uses: Wandalen/wretry.action@0dd1d5d77d019a6f85beb53d29e2ea2c7294d4f2 # v3.4.0
+      - name: Unit tests
+        uses: Wandalen/wretry.action@0dd1d5d77d019a6f85beb53d29e2ea2c7294d4f2 # v3.4.0
         with:
           command: npm ci && npm run build && npm run test --workspace=src/agent
           attempt_limit: 3
       - name: Install dependencies
         run: npm ci
       - name: Build packages
         run: npm run build
-      - name: Run tests
-        run: npm run test --workspace=src/agent
       - name: Run e2e tests
         run: npm run test:e2e --workspace=src/agent

src/agent/test/FT/compositeRules.spec.ts

@@ -22,6 +22,10 @@ const kUntriggeredCompositeRulesConfigLocation = path.join(
   __dirname,
   "/fixtures/composite-rules-no-mute-untriggered/sigyn.config.json"
 );
+const kSeverityFilterCompositeRulesConfigLocation = path.join(
+  __dirname,
+  "/fixtures/composite-rules-sev-filters/sigyn.config.json"
+);
 const kLogger = new MockLogger();
 const kMockAgent = new MockAgent();
 const kGlobalDispatcher = getGlobalDispatcher();
@@ -33,9 +37,7 @@ describe("Composite Rules", { concurrency: 1 }, () => {
   let rules: any;
 
   before(async() => {
-    if (!fs.existsSync(".temp")) {
-      fs.mkdirSync(".temp");
-    }
+    fs.mkdirSync(".temp", { recursive: true });
 
     initDB(kLogger, { databaseFilename: ".temp/test.sqlite3" });
 
@@ -182,9 +184,7 @@ describe("Composite Rules with muteUntriggered falsy", { concurrency: 1 }, () =>
   let rules: any;
 
   before(async() => {
-    if (!fs.existsSync(".temp")) {
-      fs.mkdirSync(".temp");
-    }
+    fs.mkdirSync(".temp", { recursive: true });
 
     initDB(kLogger, { databaseFilename: ".temp/test.sqlite3" });
 
@@ -241,6 +241,71 @@ describe("Composite Rules with muteUntriggered falsy", { concurrency: 1 }, () =>
   });
 });
 
+describe("Composite Rules with severity filters", { concurrency: 1 }, () => {
+  let config: SigynInitializedConfig;
+  let rules: any;
+
+  before(async() => {
+    fs.mkdirSync(".temp", { recursive: true });
+
+    initDB(kLogger, { databaseFilename: ".temp/test.sqlite3" });
+
+    process.env.GRAFANA_API_TOKEN = "toto";
+    setGlobalDispatcher(kMockAgent);
+
+    const pool = kMockAgent.get("https://discord.com");
+    pool.intercept({
+      method: "POST",
+      path: () => true
+    }).reply(200);
+
+    initDB(kLogger, { databaseFilename: ".temp/test-agent.sqlite3" });
+
+    config = await initConfig(kSeverityFilterCompositeRulesConfigLocation);
+    rules = config.rules.map((ruleConfig) => {
+      const rule = new Rule(ruleConfig, { logger: kLogger });
+      rule.init();
+
+      return rule;
+    });
+  });
+
+  beforeEach(() => {
+    getDB().prepare("DELETE FROM alerts").run();
+  });
+
+  after(() => {
+    setGlobalDispatcher(kGlobalDispatcher);
+  });
+
+  it("should not mute rules that have not triggered alerts when muteUntrigged is false", async() => {
+    resetRuteMuteUntil(rules[0]);
+    resetRuteMuteUntil(rules[1]);
+    resetRuteMuteUntil(rules[2]);
+    resetRuteMuteUntil(rules[3]);
+
+    getDB().prepare("DELETE FROM compositeRuleAlerts").run();
+
+    assert.equal(ruleMuteUntilTimestamp(rules[0]), 0);
+    assert.equal(ruleMuteUntilTimestamp(rules[1]), 0);
+    assert.equal(ruleMuteUntilTimestamp(rules[2]), 0);
+    assert.equal(ruleMuteUntilTimestamp(rules[3]), 0);
+
+    createRuleAlert(rules[0], 5);
+    createRuleAlert(rules[1], 5);
+    createRuleAlert(rules[2], 5);
+    createRuleAlert(rules[3], 5);
+
+    handleCompositeRules(kLogger);
+    await setTimeout(kTimeout);
+
+    assert.ok(ruleMuteUntilTimestamp(rules[0]) > Date.now());
+    assert.equal(ruleMuteUntilTimestamp(rules[1]), 0);
+    assert.ok(ruleMuteUntilTimestamp(rules[2]) > Date.now());
+    assert.ok(ruleMuteUntilTimestamp(rules[3]) > Date.now());
+  });
+});
+
 function createRuleAlert(rule: Rule, times: number) {
   let i = 0;
   while (i++ < times) {

src/agent/test/FT/fixtures/composite-rules-sev-filters/sigyn.config.json

@@ -0,0 +1,107 @@
+{
+  "loki": {
+    "apiUrl": "http://localhost:3100"
+  },
+  "grafana": {
+    "apiUrl": "http://localhost:3000"
+  },
+  "notifiers": {
+    "discord": {
+      "notifier": "discord",
+      "webhookUrl": "https://discord.com/api/webhooks/aaa/bbb"
+    }
+  },
+  "rules": [
+    {
+      "name": "A. Critical severity",
+      "logql": "{env=\"preprod\"} |= `my super logql`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "count": "1",
+          "interval": "1s"
+        },
+        "severity": "critical",
+        "template": {
+          "title": "{ruleName} - Triggered {counter} times!",
+          "content": [
+            "- LogQL: {logql}"
+          ]
+        }
+      }
+    },
+    {
+      "name": "B. Information severity",
+      "logql": "{env=\"preprod\"} |= `my super logql`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "count": "1",
+          "interval": "1s"
+        },
+        "severity": "info",
+        "template": {
+          "title": "{ruleName} - Triggered {counter} times!",
+          "content": [
+            "- LogQL: {logql}"
+          ]
+        }
+      }
+    },
+    {
+      "name": "C. Error severity",
+      "logql": "{env=\"preprod\"} |= `my super logql`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "count": "1",
+          "interval": "1s"
+        },
+        "severity": "error",
+        "template": {
+          "title": "{ruleName} - Triggered {counter} times!",
+          "content": [
+            "- LogQL: {logql}"
+          ]
+        }
+      }
+    },
+    {
+      "name": "D. Critical severity",
+      "logql": "{env=\"preprod\"} |= `my super logql`",
+      "polling": "200ms",
+      "alert": {
+        "on": {
+          "count": "1",
+          "interval": "1s"
+        },
+        "severity": "critical",
+        "template": {
+          "title": "{ruleName} - Triggered {counter} times!",
+          "content": [
+            "- LogQL: {logql}"
+          ]
+        }
+      }
+    }
+  ],
+  "compositeRules": [
+    {
+      "name": "Composite Rule Based on Severity",
+      "template": {
+        "title": "title",
+        "content": ["content"]
+      },
+      "notifCount": 6,
+      "throttle": {
+        "interval": "5m",
+        "count": 3
+      },
+      "ruleCountThreshold": 2,
+      "muteRules": true,
+      "filters": {
+        "severity": ["critical", "error"]
+      }
+    }
+  ]
+}

src/config/docs/composite-rules.md

@@ -90,23 +90,6 @@ Defines the notifiers to send alerts on.
 }
 ```
 
-### `include`
-
-A list of rule to monitor, you can use glob i.e `My Service -*`.  
-By default, the composite rule is based on each rule.
-
-| Type       | Required |
-|------------|----------|
-| `string[]` | ❌       |
-
-### `exclude`
-
-A list of rule to exclude from monitoring, you can use glob i.e `My Service -*`.  
-
-| Type       | Required |
-|------------|----------|
-| `string[]` | ❌       |
-
 ### `notifCount`
 
 The minimum alert to have been sent from each rules to triggers the composite rule.
@@ -166,3 +149,32 @@ Defines the duration for which rules should be muted when `muteRules` is `true`.
 | Type     | Required | Default |
 |----------|----------|---------|
 | `string` | ❌       | `30m`   |
+
+### Filterings rules
+
+The `filters` object allows to filter rules to be included in the composite rule.
+
+### `include`
+
+A list of rule to monitor, you can use glob i.e `My Service -*`.  
+By default, the composite rule is based on each rule.
+
+| Type       | Required |
+|------------|----------|
+| `string[]` | ❌       |
+
+### `exclude`
+
+A list of rule to exclude from monitoring, you can use glob i.e `My Service -*`.  
+
+| Type       | Required |
+|------------|----------|
+| `string[]` | ❌       |
+
+### `severity`
+
+A list of severity to include in the composite rule. Valid values are `information`, `warning`, `error`, `critical`.  
+
+| Type       | Required |
+|------------|----------|
+| `string[]` | ❌       |

src/config/src/schemas/configSchema.json

@@ -191,23 +191,44 @@
             "type": "string",
             "minLength": 1
           },
-          "include": {
-            "title": "Defines the rule to be included",
-            "description": "A list of rule name, glob supported",
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "minItems": 1
-          },
-          "exclude": {
-            "title": "Defines the rule to be excluded",
-            "description": "A list of rule name, glob supported",
-            "type": "array",
-            "items": {
-              "type": "string"
+          "filters": {
+            "type"  : "object",
+            "title": "Defines the filters to apply to the rules",
+            "properties": {
+              "include": {
+                "title": "Defines the rule to be included",
+                "description": "A list of rule name, glob supported",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "minItems": 1
+              },
+              "exclude": {
+                "title": "Defines the rule to be excluded",
+                "description": "A list of rule name, glob supported",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "minItems": 1
+              },
+              "severity": {
+                "type": "array",
+                "title": "Defines the severity to be included",
+                "description": "A list of severity",
+                "items": {
+                  "type": "string",
+                  "enum": [
+                    "critical",
+                    "error",
+                    "warning",
+                    "information"
+                  ]
+                }
+              }
             },
-            "minItems": 1
+            "additionalProperties": false
           },
           "notifCount": {
             "title": "The count of notifications sent by the rules to trigger alert",

src/config/src/types.ts

@@ -110,7 +110,7 @@ export interface SigynAlert {
     minimumLabelCount?: number;
   },
   template: string | SigynAlertTemplate;
-  severity: Extract<AlertSeverity, "critical" | "error" | "warning" | "information">;
+  severity: Extract<AlertSeverity, "critical" | "error" | "warning" | "info">;
   throttle?: {
     count: number;
     interval: string;
@@ -130,7 +130,7 @@ export interface SigynInitializedAlert {
     minimumLabelCount?: number;
   },
   template: SigynInitializedTemplate;
-  severity: Extract<AlertSeverity, "critical" | "error" | "warning" | "information">;
+  severity: Extract<AlertSeverity, "critical" | "error" | "warning" | "info">;
   throttle?: {
     count: number;
     interval: string;
@@ -212,8 +212,11 @@ export interface SigynInitializedSelfMonitoring {
 
 export interface SigynCompositeRule {
   name: string;
-  include?: string[];
-  exclude?: string[];
+  filters?: {
+    include?: string[];
+    exclude?: string[];
+    severity?: Extract<AlertSeverity, "critical" | "error" | "warning" | "info">[];
+  }
   notifCount: number;
   ruleCountThreshold?: number;
   interval?: string;