Feedly/fetch reports as incidents

Packs/FeedFeedly/Integrations/FeedFeedly/FeedFeedly.py

@@ -1,5 +1,6 @@
 import copy
 from contextlib import suppress
+from typing import Any
 from urllib.parse import parse_qs
 
 from CommonServerPython import *  # noqa: F401
@@ -115,7 +116,9 @@
 
 
 class Client(BaseClient):
-    def fetch_indicators_from_stream(self, stream_id: str, newer_than: float, *, limit: Optional[int] = None) -> list:
+    def fetch_indicators_from_stream(
+        self, stream_id: str, newer_than: float, *, limit: int | None = None, ingest_reports: bool = True
+    ) -> tuple[list, float | None]:
         params = {
             "streamId": stream_id,
             "count": 20,
@@ -137,7 +140,7 @@ def fetch_indicators_from_stream(self, stream_id: str, newer_than: float, *, lim
 
         demisto.debug(f"Fetched {len(objects)} objects from stream {stream_id}")
 
-        indicators = STIX2Parser().parse_stix2_objects(objects)
+        indicators = STIX2Parser(ingest_reports=ingest_reports).parse_stix2_objects(objects)
 
         if limit:
             indicators = indicators[:limit]
@@ -146,7 +149,7 @@ def fetch_indicators_from_stream(self, stream_id: str, newer_than: float, *, lim
             indicator["type"] = indicator.get("indicator_type", "")
             indicator["fields"] = indicator.get("customFields", {})
 
-        return indicators
+        return indicators, extract_next_newer_than(objects)
 
 
 class STIX2Parser:
@@ -180,7 +183,7 @@ class STIX2Parser:
         "vulnerability",
     ]
 
-    def __init__(self):
+    def __init__(self, ingest_reports: bool):
         self.indicator_regexes = [
             re.compile(INDICATOR_EQUALS_VAL_PATTERN),
             re.compile(INDICATOR_IN_VAL_PATTERN),
@@ -194,6 +197,8 @@ def __init__(self):
         self.id_to_object: dict[str, Any] = {}
         self.parsed_object_id_to_object: dict[str, Any] = {}
 
+        self.ingest_reports = ingest_reports
+
     @staticmethod
     def get_indicator_publication(indicator: dict[str, Any]):
         """
@@ -300,13 +305,14 @@ def parse_attack_pattern(attack_pattern_obj: dict[str, Any]) -> list[dict[str, A
 
         return [attack_pattern]
 
-    @staticmethod
-    def parse_report(report_obj: dict[str, Any]):
+    def parse_report(self, report_obj: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
         """
         Parses a single report object
         :param report_obj: report object
         :return: report extracted from the report object in cortex format
         """
+        if not self.ingest_reports:
+            return [], []
         object_refs = report_obj.get("object_refs", [])
         new_relationships = []
         for obj_id in object_refs:
@@ -957,7 +963,7 @@ def test_module(client: Client, params: dict) -> str:  # pragma: no cover
 
 
 def get_indicators_command(
-    client: Client, params: dict[str, str], args: dict[str, str]
+    client: Client, params: dict[str, Any], args: dict[str, Any]
 ) -> CommandResults:  # pragma: no cover
     """Wrapper for retrieving indicators from the feed to the war-room.
     Args:
@@ -967,14 +973,19 @@ def get_indicators_command(
     Returns:
         Outputs.
     """
-    indicators = client.fetch_indicators_from_stream(
-        params["feedly_stream_id"], newer_than=time.time() - 24 * 3600, limit=int(args.get("limit", "10"))
+    indicators, _ = client.fetch_indicators_from_stream(
+        params["feedly_stream_id"],
+        newer_than=time.time() - 24 * 3600,
+        limit=int(args.get("limit", "10")),
+        ingest_reports=args.get("ingest_articles_as_indicators", True),
     )
     demisto.createIndicators(indicators)  # type: ignore
     return CommandResults(readable_output=f"Created {len(indicators)} indicators.")
 
 
-def fetch_indicators_command(client: Client, params: dict[str, str], context: dict[str, str]) -> list[dict]:
+def fetch_indicators_command(
+    client: Client, params: dict[str, str], context: dict[str, str]
+) -> tuple[list, float | None]:  # pragma: no cover
     """Wrapper for fetching indicators from the feed to the Indicators tab.
     Args:
         client: Client object with request
@@ -984,10 +995,34 @@ def fetch_indicators_command(client: Client, params: dict[str, str], context: di
         Indicators.
     """
     return client.fetch_indicators_from_stream(
-        params["feedly_stream_id"], newer_than=float(context.get("last_successful_run", time.time() - 7 * 24 * 3600))
+        params["feedly_stream_id"],
+        newer_than=get_newer_than_timestamp(params, context),
+        ingest_reports=params.get("ingest_articles_as_indicators", True),  # type: ignore
     )
 
 
+def get_newer_than_timestamp(params: dict[str, str], context: dict[str, str]) -> float:  # pragma: no cover
+    stream_id = params["feedly_stream_id"]
+    if "/tag/" in stream_id:
+        saved_timestamp = context.get("last_run")
+    else:
+        saved_timestamp = context.get("last_fetched_article_crawled_time")
+
+    return float(saved_timestamp or (time.time() - int(params.get("days_to_backfill", 7)) * 24 * 3600))
+
+
+def extract_next_newer_than(stix_objects: list[dict[str, str]]) -> float | None:
+    if not stix_objects:
+        return None
+    return datetime.fromisoformat(
+        max(stix_object["created"] for stix_object in stix_objects if stix_object.get("type") == "report")
+    ).timestamp()
+
+
+def set_next_newer_than(last_article_time: float, now: float) -> None:  # pragma: no cover
+    demisto.setLastRun({"last_fetched_article_crawled_time": last_article_time, "last_run": now})
+
+
 def main():  # pragma: no cover
     params = demisto.params()
 
@@ -1012,11 +1047,11 @@ def main():  # pragma: no cover
 
         elif command == "fetch-indicators":
             now = time.time()
-            indicators = fetch_indicators_command(client, params, demisto.getLastRun())
+            indicators, last_fetched_article_time = fetch_indicators_command(client, params, demisto.getLastRun())
             for indicators_batch in batch(indicators, batch_size=2000):
                 demisto.createIndicators(indicators_batch)  # type: ignore
-            demisto.setLastRun({"last_successful_run": str(now)})
-
+            if indicators:
+                set_next_newer_than(last_fetched_article_time, now)  # type: ignore
         else:
             raise NotImplementedError(f"Command {command} is not implemented.")
 

Packs/FeedFeedly/Integrations/FeedFeedly/FeedFeedly.yml

@@ -81,6 +81,12 @@ configuration:
   type: 0
   defaultvalue: '7'
   additionalinfo: Number of days to fetch articles from when running the integration for the first time
+- display: Ingest Articles as Indicators
+  name: ingest_articles_as_indicators
+  required: false
+  type: 8
+  defaultvalue: 'true'
+  additionalinfo: When selected, the integration will ingest articles as indicators. If not selected, you may want to use the IncidentsFeedly integration to ingest articles as incidents. The current FeedFeedly integration will still be needed to ingest entities as indicators, and to create relationships between indicators.
 - additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified.
   defaultvalue: 'true'
   display: Incremental feed
@@ -116,7 +122,7 @@ script:
     description: Gets indicators from the feed.
     execution: false
     name: feedly-get-indicators
-  dockerimage: demisto/python3:3.10.14.94490
+  dockerimage: demisto/python3:3.11.9.105369
   feed: true
   isfetch: false
   longRunning: false

Packs/FeedFeedly/Integrations/FeedFeedly/FeedFeedly_description.md

@@ -1,6 +1,8 @@
 ## Feedly
 
-Use the Feedly integration to import articles with entities, indicators, and relationships from your Feedly boards and folders.
+Use the FeedFeedly integration to import articles with entities, indicators, and relationships as indicators from your Feedly boards and folders.
+
+**Note** There is a second integration `IncidentsFeedly` that can be used to ingest articles as incidents instead of indicators. This `FeedFeedly` integration is still needed, to ingest entities (intrusion sets, malware, TTPs) as indicators, and relationships between them. 
 
 **Disclaimer** You will need the Feedly for Threat Intelligence package to enable this integration. You can learn more about our product here: https://feedly.com/i/landing/threatIntelligence
 

Packs/FeedFeedly/Integrations/FeedFeedly/FeedFeedly_test.py

@@ -30,8 +30,12 @@ def test_build_iterator(requests_mock):
         response = file.read()
     requests_mock.get(URL, text=response)
     expected_ips = {"31.31.194.65", "95.213.205.83", "77.223.124.212"}
-    client = Client(base_url=URL, verify=False, proxy=False,)
-    indicators = client.fetch_indicators_from_stream("tag/enterpriseName/category/uuid", 0)
+    client = Client(
+        base_url=URL,
+        verify=False,
+        proxy=False,
+    )
+    indicators = client.fetch_indicators_from_stream("tag/enterpriseName/category/uuid", 0)[0]
     ip_indicators = {indicator["value"] for indicator in indicators if indicator["type"] == "IP"}
     assert expected_ips == ip_indicators
 

Packs/FeedFeedly/Integrations/FeedFeedly/README.md

@@ -21,6 +21,7 @@ Ingest articles with indicators, entities and relationships from Feedly into XSO
     |  |  | False |
     | Stream ID | The stream id you want to fetch articles from. You can find it in Feedly by going to the stream, clicking on \`...\` > \`Sharing\`, then \`Copy ID\` in the \`Feedly API Stream ID\` section. | True |
     | Days to fetch for first run | Number of days to fetch articles from when running the integration for the first time | True |
+    | Ingest Articles as Indicators | When selected, the integration will ingest articles as indicators. If not selected, you may want to use the IncidentsFeedly integration to ingest articles as incidents. The current FeedFeedly integration will still be needed to ingest entities as indicators, and to create relationships between indicators. | False |
     | Incremental feed | Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified. | False |
 
 4. Click **Test** to validate the URLs, token, and connection.

Packs/FeedFeedly/README.md

@@ -1,3 +1,5 @@
 The Feedly feed provides access to articles with entities, IoCs, and relationships from your Feedly account.
 
+You can ingest the articles as indicators using this pack. If you prefer to ingest them as incidents, you need to also install the FeedlyArticles pack.
+
 In order to access the Feedly feed, you'll need to generate an access token. You can do it from [here](https://feedly.com/i/team/api).

Packs/FeedFeedly/ReleaseNotes/1_1_0.md

@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### Feedly Feed
+
+- Added reference to the new incident feed integration.
+- Fixed an issue with newerThan param

Packs/FeedFeedly/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Feedly",
     "description": "Import Articles from Feedly with enriched IOCs",
     "support": "partner",
-    "currentVersion": "1.0.4",
+    "currentVersion": "1.1.0",
     "author": "Feedly",
     "url": "https://feedly.com/i/landing/threatIntelligence",
     "email": "[email protected]",

Packs/FeedlyArticles/.secrets-ignore

@@ -0,0 +1,2 @@
+https://api.feedly.com
+https://feedly.com

Packs/FeedlyArticles/Classifiers/classifier-Feedly_-_Report_Mapper.json

@@ -0,0 +1,99 @@
+{
+	"brands": null,
+	"cacheVersn": 0,
+	"defaultIncidentType": "",
+	"definitionId": "",
+	"description": "",
+	"feed": false,
+	"fromServerVersion": "",
+	"id": "Feedly - Report Mapper",
+	"incidentSamples": null,
+	"indicatorSamples": null,
+	"instanceIds": null,
+	"itemVersion": "",
+	"keyTypeMap": {},
+	"locked": false,
+	"logicalVersion": 10,
+	"mapping": {
+		"dbot_classification_incident_type_all": {
+			"dontMapEventToLabels": true,
+			"internalMapping": {
+				"Additional Email Addresses": {
+					"simple": "indicators.Email"
+				},
+				"CVE List": {
+					"simple": "indicators.CVE"
+				},
+				"Detected IPs": {
+					"simple": "indicators.IP"
+				},
+				"Domain Name": {
+					"simple": "indicators.Domain"
+				},
+				"Event ID": {
+					"simple": "event_id"
+				},
+				"Feedly Malware Names": {
+					"simple": "indicators.Malware"
+				},
+				"Feedly Threat Actor Names": {
+					"simple": "indicators.Threat Actor"
+				},
+				"Feedly crawled date": {
+					"simple": "create_time"
+				},
+				"Feedly url": {
+					"simple": "feedly_url"
+				},
+				"File MD5": {
+					"simple": "indicators.File"
+				},
+				"MITRE Technique ID": {
+					"simple": "indicators.TTP"
+				},
+				"MITRE Technique Name": {
+					"simple": "indicators.Attack Pattern"
+				},
+				"Source name": {
+					"simple": "source.name"
+				},
+				"Source url": {
+					"simple": "source.url"
+				},
+				"Tags": {
+					"complex": {
+						"filters": [],
+						"root": "tags",
+						"transformers": []
+					}
+				},
+				"Threat Name": {
+					"simple": "indicators.Malware"
+				},
+				"URLs": {
+					"simple": "indicators.URL"
+				},
+				"Use Case Description": {
+					"simple": "content"
+				},
+				"name": {
+					"simple": "name"
+				}
+			}
+		}
+	},
+	"name": "Feedly - Report Mapper",
+	"nameRaw": "Feedly - Report Mapper",
+	"packID": "",
+	"packName": "",
+	"propagationLabels": [
+		"all"
+	],
+	"sourceClassifierId": "",
+	"system": false,
+	"toServerVersion": "",
+	"transformer": {},
+	"type": "mapping-incoming",
+	"unclassifiedCases": null,
+	"version": -1
+}

Packs/FeedlyArticles/IncidentFields/incident_feedlycrawleddate.json

@@ -0,0 +1,28 @@
+{
+    "id": "incident_feedlycrawleddate",
+    "version": -1,
+    "modified": "2024-10-10T12:19:12.752862283Z",
+    "name": "Feedly crawled date",
+    "ownerOnly": false,
+    "cliName": "feedlycrawleddate",
+    "type": "shortText",
+    "closeForm": false,
+    "editForm": true,
+    "required": false,
+    "neverSetAsRequired": false,
+    "isReadOnly": false,
+    "useAsKpi": false,
+    "locked": false,
+    "system": false,
+    "content": true,
+    "group": 0,
+    "hidden": false,
+    "openEnded": false,
+    "associatedToAll": true,
+    "unmapped": false,
+    "unsearchable": false,
+    "caseInsensitive": true,
+    "sla": 0,
+    "threshold": 72,
+    "fromVersion": "6.10.0"
+}

Packs/FeedlyArticles/IncidentFields/incident_feedlymalwarenames.json

@@ -0,0 +1,28 @@
+{
+    "id": "incident_feedlymalwarenames",
+    "version": -1,
+    "modified": "2024-11-13T14:27:30.49621132Z",
+    "name": "Feedly Malware Names",
+    "ownerOnly": false,
+    "cliName": "feedlymalwarenames",
+    "type": "multiSelect",
+    "closeForm": false,
+    "editForm": true,
+    "required": false,
+    "neverSetAsRequired": false,
+    "isReadOnly": false,
+    "useAsKpi": false,
+    "locked": false,
+    "system": false,
+    "content": true,
+    "group": 0,
+    "hidden": false,
+    "openEnded": true,
+    "associatedToAll": true,
+    "unmapped": false,
+    "unsearchable": false,
+    "caseInsensitive": true,
+    "sla": 0,
+    "threshold": 72,
+    "fromVersion": "6.10.0"
+}