Model extensions. Closes #405 . (#407)

* making pendng migration

* add cowrie session model and extend IOC model

* apply psf black formatting

* change ioc model default for more simple handling in _add_ioc function

* fix error in _add_ioc: new IOC instances not able to access ManyToMany relation with GeneralHoneypot

* minor model modifications

* remove unnecessary exception class

* add method to extract more information about attackers from TPot

* rewrite data extraction process for general honeypot class to extract more data

* rewrite data extraction process for cowrie class to extract more data and add cowrie session data extraction

* revert already made migration

* rename times_seen to attack_count

* minor model tweaks

* add model migration

* add data migration

* fill attack and interaction count correctly

* Rename header in frontend code

* base first_seen and last_seen on TPot timestamps instead of extraction time

* add model tests

* change default value of login_attempts to 0

* minor improvements

* increment IOCs login attempt counter on detection in cowrie session extraction

* bump alpine from 3.18 to 3.21 in frontend build

api/serializers.py

@@ -72,7 +72,7 @@ class FeedsResponseSerializer(serializers.Serializer):
     payload_request = serializers.BooleanField()
     first_seen = serializers.DateField(format="%Y-%m-%d")
     last_seen = serializers.DateField(format="%Y-%m-%d")
-    times_seen = serializers.IntegerField()
+    attack_count = serializers.IntegerField()
 
     def validate_feed_type(self, feed_type):
         logger.debug(f"FeedsResponseSerializer - validation feed_type: '{feed_type}'")

api/views.py

@@ -159,10 +159,10 @@ def get_queryset(request, feed_type, valid_feed_types, attack_type, age, format_
         # ... and at least detected 10 different days
         number_of_days_seen = 10
         query_dict["number_of_days_seen__gte"] = number_of_days_seen
-        # if ordering == "feed_type" or None replace it with the default value "-times_seen"
+        # if ordering == "feed_type" or None replace it with the default value "-attack_count"
         # ordering by "feed_type" is done in feed_response function
         if ordering is None or ordering == "feed_type" or ordering == "-feed_type":
-            ordering = "-times_seen"
+            ordering = "-attack_count"
         iocs = IOC.objects.exclude(general_honeypot__active=False).filter(**query_dict).order_by(ordering).prefetch_related("general_honeypot")[:5000]
 
     # save request source for statistics
@@ -233,7 +233,7 @@ def feeds_response(request, iocs, feed_type, valid_feed_types, format_, dict_onl
                 PAYLOAD_REQUEST: ioc.payload_request,
                 "first_seen": ioc.first_seen.strftime("%Y-%m-%d"),
                 "last_seen": ioc.last_seen.strftime("%Y-%m-%d"),
-                "times_seen": ioc.times_seen,
+                "attack_count": ioc.attack_count,
                 "feed_type": ioc_feed_type,
             }
 

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Frontend 
-FROM node:lts-alpine3.18 as frontend-build
+FROM node:lts-alpine3.21 as frontend-build
 
 WORKDIR /
 # copy react source code

frontend/src/components/feeds/tableColumns.jsx

@@ -35,8 +35,8 @@ const feedsTableColumns = [
     maxWidth: 70,
   },
   {
-    Header: "Times Seen",
-    accessor: "times_seen",
+    Header: "Attack Count",
+    accessor: "attack_count",
     maxWidth: 60,
   },
 ];

frontend/tests/components/feeds/Feeds.test.jsx

@@ -20,7 +20,7 @@ jest.mock("@certego/certego-ui", () => {
           PAYLOAD_REQUEST: true,
           first_seen: "2023-03-15",
           last_seen: "2023-03-15",
-          times_seen: 1,
+          attack_count: 1,
           feed_type: "log4j",
         },
       ],

greedybear/admin.py

@@ -24,7 +24,7 @@ class IOCModelAdmin(admin.ModelAdmin):
         "last_seen",
         "days_seen",
         "number_of_days_seen",
-        "times_seen",
+        "attack_count",
         "related_urls",
         "scanner",
         "payload_request",

greedybear/consts.py

@@ -14,3 +14,5 @@
 
 DOMAIN = "domain"
 IP = "ip"
+
+ATTACK_DATA_FIELDS = ["@timestamp", "src_ip", "dest_port", "ip_rep", "geoip"]

greedybear/cronjobs/attacks.py

@@ -1,6 +1,7 @@
 # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear
 # See the file 'LICENSE' for copying permission.
 from abc import ABCMeta
+from collections import defaultdict
 from datetime import datetime
 from ipaddress import IPv4Address
 
@@ -12,13 +13,11 @@
 
 
 class ExtractAttacks(Cronjob, metaclass=ABCMeta):
-    class IOCWhitelist(Exception):
-        pass
-
     def __init__(self, minutes_back=None):
         super().__init__()
         self.first_time_run = False
         self.minutes_back = minutes_back
+        self.whitelist = set(Sensors.objects.all())
 
     @property
     def minutes_back_to_lookup(self):
@@ -31,61 +30,66 @@ def minutes_back_to_lookup(self):
             minutes = 11 if LEGACY_EXTRACTION else EXTRACTION_INTERVAL
         return minutes
 
-    def _add_ioc(self, ioc, attack_type, related_urls=None, log4j=False, cowrie=False, general=""):  # FEEDS
-        self.log.info(f"saving ioc {ioc} for attack_type {attack_type} and related_urls {related_urls}")
-        try:
-            today = datetime.today().date()
-            ioc_type = self._get_ioc_type(ioc)
-            try:
-                ioc_instance = IOC.objects.get(name=ioc)
-            except IOC.DoesNotExist:
-                self._check_if_allowed(ioc)
-                ioc_instance = IOC(
-                    name=ioc,
-                    type=ioc_type,
-                    days_seen=[today],
-                )
-                if related_urls:
-                    ioc_instance.related_urls = related_urls
-            else:
-                ioc_instance.last_seen = datetime.utcnow()
-                ioc_instance.times_seen += 1
-                if today not in ioc_instance.days_seen:
-                    ioc_instance.days_seen.append(today)
-                    ioc_instance.number_of_days_seen += 1
-                if related_urls:
-                    for related_url in related_urls:
-                        if related_url and related_url not in ioc_instance.related_urls:
-                            ioc_instance.related_urls.append(related_url)
-
-            if attack_type == SCANNER:
-                ioc_instance.scanner = True
-            if attack_type == PAYLOAD_REQUEST:
-                ioc_instance.payload_request = True
-
-            if log4j:
-                ioc_instance.log4j = True
-
-            if cowrie:
-                ioc_instance.cowrie = True
-
-            if ioc_instance:
-                ioc_instance.save()
-
-            # FEEDS - add general honeypot to list, if it is no already in it
-            if general and general not in ioc_instance.general_honeypot.all():
-                ioc_instance.general_honeypot.add(GeneralHoneypot.objects.get(name=general))
-
-        except self.IOCWhitelist:
+    def _add_ioc(self, ioc, attack_type: str, general=None) -> bool:
+        self.log.info(f"saving ioc {ioc} for attack_type {attack_type}")
+        if ioc.name in self.whitelist:
             self.log.info(f"not saved {ioc} because is whitelisted")
+            return False
 
-    def _check_if_allowed(self, ioc):
         try:
-            Sensors.objects.get(address=ioc)
-        except Sensors.DoesNotExist:
-            pass
+            ioc_record = IOC.objects.get(name=ioc.name)
+        except IOC.DoesNotExist:
+            # Create
+            ioc_record = ioc
+            ioc_record.save()
         else:
-            raise self.IOCWhitelist()
+            # Update
+            ioc_record.last_seen = ioc.last_seen
+            ioc_record.attack_count += 1
+            ioc_record.interaction_count += ioc.interaction_count
+            ioc_record.related_urls = sorted(set(ioc_record.related_urls + ioc.related_urls))
+            ioc_record.destination_ports = sorted(set(ioc_record.destination_ports + ioc.destination_ports))
+            ioc_record.ip_reputation = ioc.ip_reputation
+            ioc_record.asn = ioc.asn
+            ioc_record.login_attempts += ioc.login_attempts
+
+        if general is not None:
+            if general not in ioc_record.general_honeypot.all():
+                ioc_record.general_honeypot.add(GeneralHoneypot.objects.get(name=general))
+
+        if len(ioc_record.days_seen) == 0 or ioc_record.days_seen[-1] != ioc_record.last_seen.date():
+            ioc_record.days_seen.append(ioc_record.last_seen.date())
+            ioc_record.number_of_days_seen = len(ioc_record.days_seen)
+        ioc_record.scanner = attack_type == SCANNER
+        ioc_record.payload_request = attack_type == PAYLOAD_REQUEST
+        ioc_record.save()
+
+    def _get_attacker_data(self, honeypot, fields: list) -> list:
+        hits_by_ip = defaultdict(list)
+        search = self._base_search(honeypot)
+        search.source(fields)
+        for hit in search.iterate():
+            if "src_ip" not in hit:
+                continue
+            hits_by_ip[hit.src_ip].append(hit.to_dict())
+        iocs = []
+        for ip, hits in hits_by_ip.items():
+            dest_ports = [hit["dest_port"] for hit in hits if "dest_port" in hit]
+            ioc = IOC(
+                name=ip,
+                type=self._get_ioc_type(ip),
+                interaction_count=len(hits),
+                ip_reputation=hits[0].get("ip_rep", ""),
+                asn=hits[0].get("geoip", {}).get("asn"),
+                destination_ports=sorted(set(dest_ports)),
+                login_attempts=len(hits) if honeypot.name == "Heralding" else 0,
+            )
+            timestamps = [hit["@timestamp"] for hit in hits if "@timestamp" in hit]
+            if timestamps:
+                ioc.first_seen = datetime.fromisoformat(min(timestamps))
+                ioc.last_seen = datetime.fromisoformat(max(timestamps))
+            iocs.append(ioc)
+        return iocs
 
     def _get_ioc_type(self, ioc):
         try:

greedybear/cronjobs/cowrie.py

@@ -1,12 +1,13 @@
 # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear
 # See the file 'LICENSE' for copying permission.
 import re
+from collections import defaultdict
 from urllib.parse import urlparse
 
-from greedybear.consts import PAYLOAD_REQUEST, SCANNER
+from greedybear.consts import ATTACK_DATA_FIELDS, PAYLOAD_REQUEST, SCANNER
 from greedybear.cronjobs.attacks import ExtractAttacks
 from greedybear.cronjobs.honeypots import Honeypot
-from greedybear.models import IOC
+from greedybear.models import IOC, CowrieSession
 from greedybear.regex import REGEX_URL_PROTOCOL
 
 
@@ -30,25 +31,13 @@ def _cowrie_lookup(self):
         )
 
     def _get_scanners(self):
-        search = self._base_search(self.cowrie)
-        search = search.filter("terms", eventid=["cowrie.login.failed", "cowrie.session.file_upload"])
-        # get no more than X IPs a day
-        search.aggs.bucket(
-            "attacker_ips",
-            "terms",
-            field="src_ip.keyword",
-            size=1000,
-        )
-        agg_response = search[0:0].execute()
-        for tag in agg_response.aggregations.attacker_ips.buckets:
-            if not tag.key:
-                self.log.warning(f"why tag.key is empty? tag: {tag}")
-                continue
-            self.log.info(f"found IP {tag.key} by honeypot cowrie")
-            scanner_ip = str(tag.key)
-            self._add_ioc(scanner_ip, SCANNER, cowrie=True)
+        for ioc in self._get_attacker_data(self.cowrie, ATTACK_DATA_FIELDS):
+            ioc.cowrie = True
+            self.log.info(f"found IP {ioc.name} by honeypot cowrie")
+            self._add_ioc(ioc, attack_type=SCANNER)
             self.added_scanners += 1
-            self._extract_possible_payload_in_messages(scanner_ip)
+            self._extract_possible_payload_in_messages(ioc.name)
+            self._get_sessions(ioc.name)
 
     def _extract_possible_payload_in_messages(self, scanner_ip):
         # looking for URLs inside attacks payloads
@@ -64,12 +53,13 @@ def _extract_possible_payload_in_messages(self, scanner_ip):
                 self.log.info(f"found hidden URL {payload_url}" f" in payload from attacker {scanner_ip}")
                 payload_hostname = urlparse(payload_url).hostname
                 self.log.info(f"extracted hostname {payload_hostname} from {payload_url}")
-                self._add_ioc(
-                    payload_hostname,
-                    PAYLOAD_REQUEST,
-                    related_urls=[payload_url],
+                ioc = IOC(
+                    name=payload_hostname,
+                    type=self._get_ioc_type(payload_hostname),
                     cowrie=True,
+                    related_urls=[payload_url],
                 )
+                self._add_ioc(ioc, attack_type=PAYLOAD_REQUEST)
                 self._add_fks(scanner_ip, payload_hostname)
 
     def _get_url_downloads(self):
@@ -81,15 +71,58 @@ def _get_url_downloads(self):
         for hit in hits:
             self.log.info(f"found IP {hit.src_ip} trying to execute download from {hit.url}")
             scanner_ip = str(hit.src_ip)
-            self._add_ioc(scanner_ip, SCANNER, cowrie=True)
+            ioc = IOC(name=scanner_ip, type=self._get_ioc_type(scanner_ip), cowrie=True)
+            self._add_ioc(ioc, attack_type=SCANNER)
             self.added_ip_downloads += 1
             download_url = str(hit.url)
             if download_url:
                 hostname = urlparse(download_url).hostname
-                self._add_ioc(hostname, PAYLOAD_REQUEST, related_urls=[download_url], cowrie=True)
+                ioc = IOC(
+                    name=hostname,
+                    type=self._get_ioc_type(hostname),
+                    cowrie=True,
+                    related_urls=[download_url],
+                )
+                self._add_ioc(ioc, attack_type=PAYLOAD_REQUEST)
                 self.added_url_downloads += 1
                 self._add_fks(scanner_ip, hostname)
 
+    def _get_sessions(self, scanner_ip: str):
+        self.log.info(f"adding cowrie sessions from {scanner_ip}")
+        search = self._base_search(self.cowrie)
+        search = search.filter("term", src_ip=scanner_ip)
+        search = search.source(["session", "eventid", "timestamp", "duration", "message", "username", "password"])
+        hits_per_session = defaultdict(list)
+
+        for hit in search.iterate():
+            hits_per_session[int(hit.session, 16)].append(hit)
+
+        for sid, hits in hits_per_session.items():
+            try:
+                session_record = CowrieSession.objects.get(session_id=sid)
+            except CowrieSession.DoesNotExist:
+                session_record = CowrieSession(session_id=sid)
+
+            session_record.source = IOC.objects.filter(name=scanner_ip).first()
+            for hit in hits:
+                match hit.eventid:
+                    case "cowrie.session.connect":
+                        session_record.start_time = hit.timestamp
+                    case "cowrie.login.failed" | "cowrie.login.success":
+                        session_record.login_attempt = True
+                        session_record.credentials.append(f"{hit.username} | {hit.password}")
+                        session_record.source.login_attempts += 1
+                    case "cowrie.command.input":
+                        session_record.command_execution = True
+                    case "cowrie.session.closed":
+                        session_record.duration = hit.duration
+                session_record.interaction_count += 1
+
+            session_record.source.save()
+            session_record.save()
+
+        self.log.info(f"{len(hits_per_session)} sessions added")
+
     def _add_fks(self, scanner_ip, hostname):
         self.log.info(f"adding foreign keys for the following iocs: {scanner_ip}, {hostname}")
         scanner_ip_instance = IOC.objects.filter(name=scanner_ip).first()

greedybear/cronjobs/general.py

@@ -1,10 +1,10 @@
 # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear
 # See the file 'LICENSE' for copying permission.
 
-from greedybear.consts import SCANNER
+from greedybear.consts import ATTACK_DATA_FIELDS, SCANNER
 from greedybear.cronjobs.attacks import ExtractAttacks
 from greedybear.cronjobs.honeypots import Honeypot
-from greedybear.models import GeneralHoneypot
+from greedybear.models import IOC, GeneralHoneypot
 
 
 class ExtractGeneral(ExtractAttacks):
@@ -18,23 +18,10 @@ def _general_lookup(self):
         self.log.info(f"added {self.added_scanners} scanners for {self.hp.name}")
 
     def _get_scanners(self):
-        search = self._base_search(self.hp)
-        name = self.hp.name
-        # get no more than X IPs a day
-        search.aggs.bucket(
-            "attacker_ips",
-            "terms",
-            field="src_ip.keyword",
-            size=1000,
-        )
-        agg_response = search[0:0].execute()
-        for tag in agg_response.aggregations.attacker_ips.buckets:
-            if not tag.key:
-                self.log.warning(f"why tag.key is empty? tag: {tag}")
-                continue
-            self.log.info(f"found IP {tag.key} by honeypot {name}")
-            scanner_ip = str(tag.key)
-            self._add_ioc(scanner_ip, SCANNER, general=name)
+        honeypot_name = self.hp.name
+        for ioc in self._get_attacker_data(self.hp, ATTACK_DATA_FIELDS):
+            self.log.info(f"found IP {ioc.name} by honeypot {honeypot_name}")
+            self._add_ioc(ioc, attack_type=SCANNER, general=honeypot_name)
             self.added_scanners += 1
 
     def run(self):