From 0ecf0f08d9b6b347760a7495f1986b346e4a136c Mon Sep 17 00:00:00 2001 From: maxrichie5 Date: Wed, 6 Dec 2023 16:39:05 -0700 Subject: [PATCH] conditionally show correlation rule output --- panther_analysis_tool/backend/client.py | 28 ++ .../backend/graphql/feature_flags.graphql | 8 + .../backend/public_api_client.py | 26 +- panther_analysis_tool/constants.py | 3 + .../analysis_config_schema.json | 304 ++++++++++++++---- panther_analysis_tool/main.py | 15 +- panther_analysis_tool/schemas.py | 18 -- 7 files changed, 320 insertions(+), 82 deletions(-) create mode 100644 panther_analysis_tool/backend/graphql/feature_flags.graphql diff --git a/panther_analysis_tool/backend/client.py b/panther_analysis_tool/backend/client.py index 3ce4d2b1..08508de6 100644 --- a/panther_analysis_tool/backend/client.py +++ b/panther_analysis_tool/backend/client.py @@ -241,6 +241,7 @@ class BulkUploadResponse: data_models: BulkUploadStatistics lookup_tables: BulkUploadStatistics global_helpers: BulkUploadStatistics + correlation_rules: BulkUploadStatistics @dataclass(frozen=True) @@ -437,6 +438,28 @@ class GenerateEnrichedEventResponse: enriched_event: Dict[str, Any] # json +@dataclass(frozen=True) +class FeatureFlagWithDefault: + flag: str + defaultTreatment: Optional[bool] = None + + +@dataclass(frozen=True) +class FeatureFlagTreatment: + flag: str + treatment: bool + + +@dataclass(frozen=True) +class FeatureFlagsParams: + flags: List[FeatureFlagWithDefault] + + +@dataclass(frozen=True) +class FeatureFlagsResponse: + flags: List[FeatureFlagTreatment] + + class Client(ABC): @abstractmethod def check(self) -> BackendCheckResponse: @@ -516,6 +539,10 @@ def generate_enriched_event_input( ) -> BackendResponse[GenerateEnrichedEventResponse]: pass + @abstractmethod + def feature_flags(self, params: FeatureFlagsParams) -> BackendResponse[FeatureFlagsResponse]: + pass + def backend_response_failed(resp: BackendResponse) -> bool: return resp.status_code >= 400 or resp.data.get("statusCode", 0) >= 400 @@ -532,6 +559,7 @@ def to_bulk_upload_response(data: Any) -> BackendResponse[BulkUploadResponse]: data_models=BulkUploadStatistics(**data.get("dataModels", default_stats)), lookup_tables=BulkUploadStatistics(**data.get("lookupTables", default_stats)), global_helpers=BulkUploadStatistics(**data.get("globalHelpers", default_stats)), + correlation_rules=BulkUploadStatistics(**data.get("correlationRules", default_stats)), ), ) diff --git a/panther_analysis_tool/backend/graphql/feature_flags.graphql b/panther_analysis_tool/backend/graphql/feature_flags.graphql new file mode 100644 index 00000000..dfce20e8 --- /dev/null +++ b/panther_analysis_tool/backend/graphql/feature_flags.graphql @@ -0,0 +1,8 @@ +query GetFeatureFlags($input: GetFeatureFlagsInput!) { + featureFlags(input: $input) { + flags { + flag + treatment + } + } +} \ No newline at end of file diff --git a/panther_analysis_tool/backend/public_api_client.py b/panther_analysis_tool/backend/public_api_client.py index 5dc133ba..6b4bc112 100644 --- a/panther_analysis_tool/backend/public_api_client.py +++ b/panther_analysis_tool/backend/public_api_client.py @@ -20,7 +20,7 @@ import logging import os import time -from dataclasses import dataclass +from dataclasses import dataclass, asdict from pathlib import Path from typing import Any, Dict, List, Optional from urllib.parse import urlparse @@ -31,7 +31,6 @@ from gql.transport.exceptions import TransportQueryError from graphql import DocumentNode, ExecutionResult -from ..constants import VERSION_STRING, ReplayStatus from .client import ( BackendCheckResponse, BackendError, @@ -63,8 +62,12 @@ UpdateSchemaParams, UpdateSchemaResponse, to_bulk_upload_response, + FeatureFlagsParams, + FeatureFlagsResponse, + FeatureFlagTreatment, ) from .errors import is_retryable_error, is_retryable_error_str +from ..constants import VERSION_STRING, ReplayStatus @dataclass(frozen=True) @@ -134,6 +137,9 @@ def stop_replay_mutation(self) -> DocumentNode: def generate_enriched_event_query(self) -> DocumentNode: return self._load("generate_enriched_event") + def feature_flags_query(self) -> DocumentNode: + return self._load("feature_flags") + def _load(self, name: str) -> DocumentNode: if name not in self._cache: self._cache[name] = Path(_get_graphql_content_filepath(name)).read_text() @@ -517,6 +523,22 @@ def generate_enriched_event_input( ), ) + def feature_flags(self, params: FeatureFlagsParams) -> BackendResponse[FeatureFlagsResponse]: + query = self._requests.feature_flags_query() + query_input = {"input": asdict(params)} + res = self._safe_execute(query, variable_values=query_input) + data = res.data.get("featureFlags", {}) # type: ignore + + return BackendResponse( + status_code=200, + data=FeatureFlagsResponse( + flags=[ + FeatureFlagTreatment(flag=flag.get("flag"), treatment=flag.get("treatment")) + for flag in data.get("flags") or [] + ] + ), + ) + def _execute( self, request: DocumentNode, diff --git a/panther_analysis_tool/constants.py b/panther_analysis_tool/constants.py index de181b0b..d379ff10 100644 --- a/panther_analysis_tool/constants.py +++ b/panther_analysis_tool/constants.py @@ -102,3 +102,6 @@ class ReplayStatus: ERROR_COMPUTATION = "ERROR_COMPUTATION" EVALUATION_IN_PROGRESS = "EVALUATION_IN_PROGRESS" COMPUTATION_IN_PROGRESS = "COMPUTATION_IN_PROGRESS" + + +ENABLE_CORRELATION_RULES_FLAG = "EnableCorrelationRules" diff --git a/panther_analysis_tool/detection_schemas/analysis_config_schema.json b/panther_analysis_tool/detection_schemas/analysis_config_schema.json index f0feaad2..2fc5e4cf 100644 --- a/panther_analysis_tool/detection_schemas/analysis_config_schema.json +++ b/panther_analysis_tool/detection_schemas/analysis_config_schema.json @@ -17,7 +17,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["LogType", "Selectors"] + "required": [ + "LogType", + "Selectors" + ] }, "DynamicSeverity": { "properties": { @@ -31,7 +34,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["ChangeTo", "Conditions"] + "required": [ + "ChangeTo", + "Conditions" + ] }, "InlineFilterMatchExpressions": { "items": { @@ -48,7 +54,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition"] + "required": [ + "KeyPath", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -79,7 +88,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition", "Value"] + "required": [ + "KeyPath", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -121,7 +134,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition", "Values"] + "required": [ + "KeyPath", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -140,7 +157,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Table", "Selector", "FieldPath"] + "required": [ + "Table", + "Selector", + "FieldPath" + ] }, "Condition": { "type": "string" @@ -165,7 +186,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Enrichment", "Condition", "Value"] + "required": [ + "Enrichment", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -184,7 +209,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Table", "Selector", "FieldPath"] + "required": [ + "Table", + "Selector", + "FieldPath" + ] }, "Condition": { "type": "string" @@ -220,7 +249,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Enrichment", "Condition", "Values"] + "required": [ + "Enrichment", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -231,7 +264,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["All"] + "required": [ + "All" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -242,7 +277,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["Any"] + "required": [ + "Any" + ] } ], "type": "object" @@ -260,7 +297,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["Key"] + "required": [ + "Key" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -281,7 +320,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["DeepKey"] + "required": [ + "DeepKey" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -292,7 +333,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath"] + "required": [ + "KeyPath" + ] } ], "type": "object" @@ -308,7 +351,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyName", "KeyValue"] + "required": [ + "KeyName", + "KeyValue" + ] }, "LogTypeMap": { "properties": { @@ -324,7 +370,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["PrimaryKey", "AssociatedLogTypes"] + "required": [ + "PrimaryKey", + "AssociatedLogTypes" + ] }, "Mapping": { "properties": { @@ -340,7 +389,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["Name"] + "required": [ + "Name" + ] }, "MatchExpressions": { "items": { @@ -357,7 +408,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition"] + "required": [ + "KeyPath", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -388,7 +442,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition", "Value"] + "required": [ + "KeyPath", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -430,7 +488,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath", "Condition", "Values"] + "required": [ + "KeyPath", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -449,7 +511,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Table", "Selector", "FieldPath"] + "required": [ + "Table", + "Selector", + "FieldPath" + ] }, "Condition": { "type": "string" @@ -474,7 +540,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Enrichment", "Condition", "Value"] + "required": [ + "Enrichment", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -493,7 +563,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Table", "Selector", "FieldPath"] + "required": [ + "Table", + "Selector", + "FieldPath" + ] }, "Condition": { "type": "string" @@ -529,7 +603,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Enrichment", "Condition", "Values"] + "required": [ + "Enrichment", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -540,7 +618,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["All"] + "required": [ + "All" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -551,7 +631,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["Any"] + "required": [ + "Any" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -562,7 +644,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["OnlyOne"] + "required": [ + "OnlyOne" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -573,7 +657,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["None"] + "required": [ + "None" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -587,7 +673,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["Key", "Condition"] + "required": [ + "Key", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -611,7 +700,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["DeepKey", "Condition"] + "required": [ + "DeepKey", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -642,7 +734,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Key", "Condition", "Value"] + "required": [ + "Key", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -684,7 +780,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["Key", "Condition", "Values"] + "required": [ + "Key", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -725,7 +825,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["DeepKey", "Condition", "Value"] + "required": [ + "DeepKey", + "Condition", + "Value" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -777,7 +881,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["DeepKey", "Condition", "Values"] + "required": [ + "DeepKey", + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -790,12 +898,21 @@ }, "Condition": { "type": "string", - "enum": ["AllElements", "AnyElement", "NoElement", "OnlyOneElement"] + "enum": [ + "AllElements", + "AnyElement", + "NoElement", + "OnlyOneElement" + ] } }, "additionalProperties": false, "type": "object", - "required": ["Expressions", "Key", "Condition"] + "required": [ + "Expressions", + "Key", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -811,12 +928,21 @@ }, "Condition": { "type": "string", - "enum": ["AllElements", "AnyElement", "NoElement", "OnlyOneElement"] + "enum": [ + "AllElements", + "AnyElement", + "NoElement", + "OnlyOneElement" + ] } }, "additionalProperties": false, "type": "object", - "required": ["Expressions", "DeepKey", "Condition"] + "required": [ + "Expressions", + "DeepKey", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -829,24 +955,38 @@ }, "Condition": { "type": "string", - "enum": ["AllElements", "AnyElement", "NoElement", "OnlyOneElement"] + "enum": [ + "AllElements", + "AnyElement", + "NoElement", + "OnlyOneElement" + ] } }, "additionalProperties": false, "type": "object", - "required": ["Expressions", "KeyPath", "Condition"] + "required": [ + "Expressions", + "KeyPath", + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { "Condition": { "type": "string", - "enum": ["AlwaysTrue", "AlwaysFalse"] + "enum": [ + "AlwaysTrue", + "AlwaysFalse" + ] } }, "additionalProperties": false, "type": "object", - "required": ["Condition"] + "required": [ + "Condition" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -866,7 +1006,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["Key"] + "required": [ + "Key" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -887,7 +1029,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["DeepKey"] + "required": [ + "DeepKey" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -898,7 +1042,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["KeyPath"] + "required": [ + "KeyPath" + ] } ], "type": "object" @@ -910,14 +1056,20 @@ }, "additionalProperties": false, "type": "object", - "required": ["Condition", "Values"] + "required": [ + "Condition", + "Values" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { "EventEvaluationOrder": { "type": "string", - "enum": ["Chronological", "ReverseChronological"] + "enum": [ + "Chronological", + "ReverseChronological" + ] }, "LookbackWindowMinutes": { "type": "integer" @@ -936,7 +1088,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["TimeoutMinutes"] + "required": [ + "TimeoutMinutes" + ] }, "Sequence": { "items": { @@ -1003,21 +1157,30 @@ }, "additionalProperties": false, "type": "object", - "required": ["From", "To"] + "required": [ + "From", + "To" + ] }, "type": "array" } }, "additionalProperties": false, "type": "object", - "required": ["Schedule", "Sequence"] + "required": [ + "Schedule", + "Sequence" + ] }, { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { "EventEvaluationOrder": { "type": "string", - "enum": ["Chronological", "ReverseChronological"] + "enum": [ + "Chronological", + "ReverseChronological" + ] }, "LookbackWindowMinutes": { "type": "integer" @@ -1036,7 +1199,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["TimeoutMinutes"] + "required": [ + "TimeoutMinutes" + ] }, "Group": { "items": { @@ -1079,7 +1244,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["GroupID", "Match"] + "required": [ + "GroupID", + "Match" + ] }, "type": "array" } @@ -1089,7 +1257,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["Schedule", "Group"] + "required": [ + "Schedule", + "Group" + ] } ], "type": "object" @@ -1137,7 +1308,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["IDs"] + "required": [ + "IDs" + ] }, "QuerySchedule": { "properties": { @@ -1153,7 +1326,9 @@ }, "additionalProperties": false, "type": "object", - "required": ["TimeoutMinutes"] + "required": [ + "TimeoutMinutes" + ] }, "Refresh": { "properties": { @@ -1175,7 +1350,11 @@ }, "additionalProperties": false, "type": "object", - "required": ["RoleARN", "PeriodMinutes", "ObjectPath"] + "required": [ + "RoleARN", + "PeriodMinutes", + "ObjectPath" + ] }, "ReportItem": { "oneOf": [ @@ -1395,7 +1574,9 @@ }, "additionalProperties": true, "type": "object", - "required": ["AnalysisType"] + "required": [ + "AnalysisType" + ] }, "Test": { "properties": { @@ -1419,7 +1600,10 @@ }, "additionalProperties": false, "type": "object", - "required": ["Name", "ExpectedResult"] + "required": [ + "Name", + "ExpectedResult" + ] } } -} +} \ No newline at end of file diff --git a/panther_analysis_tool/main.py b/panther_analysis_tool/main.py index a2db1eed..bd04bbb3 100644 --- a/panther_analysis_tool/main.py +++ b/panther_analysis_tool/main.py @@ -38,7 +38,6 @@ from collections.abc import Mapping from dataclasses import asdict, dataclass from datetime import datetime - # Comment below disabling pylint checks is due to a bug in the CircleCi image with Pylint # It seems to be unable to import the distutils module, however the module is present and importable # in the Python Repl. @@ -95,6 +94,8 @@ BackendError, BulkUploadMultipartError, BulkUploadParams, + FeatureFlagsParams, + FeatureFlagWithDefault, ) from panther_analysis_tool.backend.client import Client as BackendClient from panther_analysis_tool.command import ( @@ -114,6 +115,7 @@ TMP_HELPER_MODULE_LOCATION, VERSION_STRING, AnalysisTypes, + ENABLE_CORRELATION_RULES_FLAG, ) from panther_analysis_tool.destination import FakeDestination from panther_analysis_tool.enriched_event_generator import EnrichedEventGenerator @@ -369,7 +371,14 @@ def upload_zip( else: response = backend.bulk_upload(upload_params) - logging.info("API Response:\n%s", json.dumps(asdict(response.data), indent=4)) + resp_dict = asdict(response.data) + flags_params = FeatureFlagsParams( + flags=[FeatureFlagWithDefault(flag=ENABLE_CORRELATION_RULES_FLAG)] + ) + if not backend.feature_flags(flags_params).data.flags[0].treatment: + del resp_dict["correlation_rules"] + + logging.info("API Response:\n%s", json.dumps(resp_dict, indent=4)) return 0, cli_output.success("Upload succeeded") except BackendError as be_err: @@ -1036,7 +1045,9 @@ def classify_analysis( if not tmp_logtypes: tmp_logtypes = analysis_schema.schema[tmp_logtypes_key] analysis_schema.schema[tmp_logtypes_key] = [str] + analysis_schema.validate(analysis_spec) + # lookup the analysis type id and validate there aren't any conflicts analysis_id = lookup_analysis_id(analysis_spec, analysis_type) if analysis_id in analysis_ids: diff --git a/panther_analysis_tool/schemas.py b/panther_analysis_tool/schemas.py index 58776638..573be031 100644 --- a/panther_analysis_tool/schemas.py +++ b/panther_analysis_tool/schemas.py @@ -230,32 +230,14 @@ def validate( "Detection": object, "Severity": Or("Info", "Low", "Medium", "High", "Critical"), Optional("Description"): str, - Optional("DedupPeriodMinutes"): int, - Optional("InlineFilters"): object, Optional("DisplayName"): And(str, NAME_ID_VALIDATION_REGEX), Optional("OnlyUseBaseRiskScore"): bool, Optional("OutputIds"): [str], Optional("Reference"): str, Optional("Runbook"): str, Optional("SummaryAttributes"): [str], - Optional("Threshold"): int, Optional("Tags"): [str], Optional("Reports"): {str: list}, - Optional("Tests"): [ - { - "Name": str, - Optional( - "LogType" - ): str, # Not needed anymore, optional for backwards compatibility - "ExpectedResult": bool, - "Log": object, - Optional("Mocks"): [MOCK_SCHEMA], - } - ], - Optional("DynamicSeverities"): object, - Optional("AlertTitle"): str, - Optional("AlertContext"): object, - Optional("GroupBy"): object, }, ignore_extra_keys=False, )