-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from JiscCTI/splunk-9-2
Splunk UF 9.0.9
- Loading branch information
Showing
17 changed files
with
287 additions
and
184 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| SPDX-FileCopyrightText: 2023-2024 Jisc Services Limited | ||
| SPDX-FileContributor: Joe Pitt | ||
|
|
||
| SPDX-License-Identifier: GPL-3.0-only |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| # SPDX-FileCopyrightText: 2024 Jisc Services Limited | ||
| # SPDX-FileContributor: Joe Pitt | ||
| # | ||
| # SPDX-License-Identifier: GPL-3.0-only | ||
|
|
||
| name: Update DockerHub | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| workflow_dispatch: | ||
| jobs: | ||
| run: | ||
| name: Update DockerHub Pages | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| # Environment Setup | ||
| - id: checkout | ||
| name: Checkout Project | ||
| uses: actions/checkout@v4 | ||
| - id: splunk-forwarder | ||
| name: MISP Docker Forwarder App for Splunk | ||
| uses: peter-evans/dockerhub-description@v4 | ||
| with: | ||
| username: ${{ vars.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| repository: ${{ vars.DOCKERHUB_ORGANISATION }}/misp-splunk-forwarder | ||
| short-description: Splunk Universal Forwarder configured for use with ${{ vars.DOCKERHUB_ORGANISATION }}/misp-web. | ||
| readme-filepath: docs/misp-splunk-forwarder.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,17 @@ | ||
| # SPDX-FileCopyrightText: 2023 Jisc Services Limited | ||
| # SPDX-FileCopyrightText: 2023-2024 Jisc Services Limited | ||
| # SPDX-FileContributor: Joe Pitt | ||
| # | ||
| # SPDX-License-Identifier: GPL-3.0-only | ||
|
|
||
| FROM splunk/universalforwarder:9.0.7 | ||
| LABEL org.opencontainers.image.title="misp-splunk-forwarder" org.opencontainers.image.version=v1.0.0\ | ||
| FROM splunk/universalforwarder:9.0.9 | ||
| LABEL org.opencontainers.image.title="misp-splunk-forwarder" org.opencontainers.image.version=v1.0.1\ | ||
| org.opencontainers.image.ref.name="misp-splunk-forwarder"\ | ||
| org.opencontainers.image.description="Self configuring Splunk Universal Forwarder for MISP."\ | ||
| org.opencontainers.image.authors="Jisc <[email protected]"\ | ||
| org.opencontainers.image.base.name="hub.docker.com/splunk/universalforwarder" | ||
| ENV HEC_URI=https://splunk.example.com:8088 HEC_KEY=00000000-1111-2222-3333-444444444444 HEC_SSL=true HEC_VERIFY=false\ | ||
| INDEX=default FQDN=misp.example.com HTTPS_PORT=443 | ||
| ENV FQDN=misp.local HTTPS_PORT=443 SPLUNK_HEC_KEY=00000000-1111-2222-3333-444444444444\ | ||
| SPLUNK_HEC_URI=https://splunk.example.com:8088 SPLUNK_HEC_VERIFY=false SPLUNK_INDEX=default\ | ||
| SPLUNK_PASSWORD=ChangeMeChangeMeChangeMe | ||
| VOLUME "/opt/splunkforwarder/etc/" "/opt/splunkforwarder/var/" "/opt/misp_docker/" | ||
| COPY --chown=ansible:ansible configure.py /sbin/configure.py | ||
| COPY --chown=ansible:ansible entrypoint.sh /sbin/entrypoint.sh | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,80 +2,87 @@ | |
|
|
||
| """Test if remote feeds are reachable and output Splunk CIM-compliant Web events""" | ||
|
|
||
| # SPDX-FileCopyrightText: 2023 Jisc Services Limited | ||
| # SPDX-FileCopyrightText: 2023-2024 Jisc Services Limited | ||
| # SPDX-FileContributor: Joe Pitt | ||
| # | ||
| # SPDX-License-Identifier: GPL-3.0-only | ||
|
|
||
| from configparser import ConfigParser | ||
| from json import dumps, loads | ||
| from os.path import dirname, join | ||
| from sys import path | ||
| from sys import exit as sys_exit, path | ||
| from time import time | ||
| from urllib.parse import urlparse | ||
|
|
||
| from urllib3 import disable_warnings | ||
| from urllib3.exceptions import InsecureRequestWarning | ||
|
|
||
| try: | ||
| path.insert(0, join(dirname(__file__), "..", "lib")) | ||
| from requests import get | ||
| except ImportError: | ||
| raise ImportError("Failed to load requests") | ||
| path.insert(0, join(dirname(__file__), "..", "lib")) | ||
| # Import must happen after PATH is altered | ||
| from requests import get # pylint: disable=wrong-import-position | ||
|
|
||
| # pylint: disable-next=wrong-import-position | ||
| from requests.exceptions import ( | ||
| JSONDecodeError, | ||
| RequestException, | ||
| ) | ||
|
|
||
| __author__ = "Joe Pitt" | ||
| __copyright__ = "Copyright 2023, Jisc Services Limited" | ||
| __copyright__ = "Copyright 2023-2024, Jisc Services Limited" | ||
| __email__ = "[email protected]" | ||
| __license__ = "GPL-3.0-only" | ||
| __maintainer__ = "Joe Pitt" | ||
| __status__ = "Production" | ||
| __version__ = "1.0.0" | ||
| __version__ = "1.0.1" | ||
|
|
||
| disable_warnings(InsecureRequestWarning) | ||
|
|
||
| JobsConfigFile = "/opt/misp_docker/misp_maintenance_jobs.ini" | ||
| JobsConfig = ConfigParser() | ||
| JobsConfig.read(JobsConfigFile) | ||
| JOBS_CONF = "/opt/misp_docker/misp_maintenance_jobs.ini" | ||
| jobsConfig = ConfigParser() | ||
| jobsConfig.read(JOBS_CONF) | ||
|
|
||
| Headers = { | ||
| "Authorization": JobsConfig.get("DEFAULT", "AuthKey"), | ||
| headers = { | ||
| "Authorization": jobsConfig.get("DEFAULT", "AuthKey"), | ||
| "Accept": "application/json", | ||
| "Content-type": "application/json", | ||
| "User-Agent": "misp_test_feeds/{}".format(__version__), | ||
| "User-Agent": f"misp_test_feeds/{__version__}", | ||
| } | ||
|
|
||
| try: | ||
| feeds = get( | ||
| "{}/feeds/index".format(JobsConfig.get("DEFAULT", "BaseUrl")), | ||
| headers=Headers, | ||
| f"{jobsConfig.get('DEFAULT', 'BaseUrl')}/feeds/index", | ||
| headers=headers, | ||
| timeout=5, | ||
| verify=JobsConfig.getboolean("DEFAULT", "VerifyTls"), | ||
| verify=jobsConfig.getboolean("DEFAULT", "VerifyTls"), | ||
| ) | ||
| except Exception as e: | ||
| except RequestException as e: | ||
| # Shorten exception type to just final class | ||
| exceptionType = str(type(e)) | ||
| if "<class '" in exceptionType: | ||
| exceptionType = exceptionType[8:-2] | ||
| exceptionType = exceptionType.split(".")[-1] | ||
| EXCEPTION_TYPE = str(type(e)) | ||
| if "<class '" in EXCEPTION_TYPE: | ||
| EXCEPTION_TYPE = EXCEPTION_TYPE[8:-2] | ||
| # Benefit of use-maxsplit-arg unclear | ||
| # pylint: disable-next=use-maxsplit-arg | ||
| EXCEPTION_TYPE = EXCEPTION_TYPE.split(".")[-1] | ||
| result = {} | ||
| result["_time"] = time() | ||
| result["action"] = "error" | ||
| result["app"] = "MISP" | ||
| result["authentication_method"] = "api" | ||
| result["reason"] = "{} getting feed list".format(exceptionType) | ||
| result["src_host"] = JobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
| result["reason"] = f"{EXCEPTION_TYPE} getting feed list" | ||
| result["src_host"] = jobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
|
|
||
| print(dumps(result, sort_keys=True)) | ||
| exit() | ||
| sys_exit() | ||
|
|
||
| if feeds.status_code == 200: | ||
| for feed in feeds.json(): | ||
| if type(feed) != dict: | ||
| if not isinstance(feed, dict): | ||
| result = {} | ||
| result["_time"] = time() | ||
| result["error"] = "Expected dict got {}".format(type(feed)) | ||
| result["error"] = f"Expected dict got {type(feed)}" | ||
| try: | ||
| result["value"] = str(feed) | ||
| except Exception: | ||
| except ValueError: | ||
| result["value"] = "Non-serialisable" | ||
| print(dumps(result, sort_keys=True)) | ||
| continue | ||
|
|
@@ -86,25 +93,25 @@ | |
|
|
||
| url = feed["url"] | ||
| if feed["source_format"] == "misp": | ||
| url = "{}/manifest.json".format(url) | ||
| url = f"{url}/manifest.json" | ||
|
|
||
| Headers = None | ||
| FEED_HEADERS = None | ||
| if feed["headers"] is not None and len(feed["headers"]) > 0: | ||
| Headers = dict( | ||
| FEED_HEADERS = dict( | ||
| header.split(": ", 1) for header in feed["headers"].split("\r\n") | ||
| ) | ||
| try: | ||
| start = time() | ||
| response = get(url, headers=Headers, timeout=5) | ||
| response = get(url, headers=FEED_HEADERS, timeout=5) | ||
| # Web CIM duration is in milliseconds | ||
| duration = round((time() - start) * 1000, 3) | ||
| except Exception as e: | ||
| except RequestException as e: | ||
| # Shorten exception type to just final class | ||
| exceptionType = str(type(e)) | ||
| if "<class '" in exceptionType: | ||
| exceptionType = exceptionType[8:-2] | ||
| exceptionType = exceptionType.split(".")[-1] | ||
| error = exceptionType | ||
| EXCEPTION_TYPE = str(type(e)) | ||
| if "<class '" in EXCEPTION_TYPE: | ||
| EXCEPTION_TYPE = EXCEPTION_TYPE[8:-2] | ||
| EXCEPTION_TYPE = EXCEPTION_TYPE.split(".")[-1] | ||
| error = EXCEPTION_TYPE | ||
|
|
||
| urlParts = urlparse(url) | ||
| result = {} | ||
|
|
@@ -119,19 +126,20 @@ | |
| if "duration" in locals(): | ||
| result["duration"] = duration | ||
| if "error" in locals(): | ||
| result["error_code"] = error | ||
| # this line can only be called if error is assigned | ||
| result["error_code"] = error # pylint: disable=used-before-assignment | ||
| if "response" in locals() and response.headers.get("Content-Type") is not None: | ||
| result["http_content_type"] = response.headers.get("Content-Type").split( | ||
| ";" | ||
| )[0] | ||
| result["http_method"] = "GET" | ||
| result["src_host"] = JobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
| result["src_host"] = jobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
| if "response" in locals(): | ||
| result["status"] = response.status_code | ||
| if urlParts.path != "": | ||
| result["uri_path"] = urlParts.path | ||
| if urlParts.query != "": | ||
| result["uri_query"] = "?{}".format(urlParts.query) | ||
| result["uri_query"] = f"?{urlParts.query}" | ||
| result["url_domain"] = urlParts.hostname | ||
| result["url_length"] = len(url) | ||
| result["url"] = url | ||
|
|
@@ -147,13 +155,11 @@ | |
| result["action"] = "error" | ||
| result["app"] = "MISP" | ||
| result["authentication_method"] = "api" | ||
| result["reason"] = "{} - {} getting feed list".format( | ||
| feeds.status_code, feeds.reason | ||
| ) | ||
| result["reason"] = f"{feeds.status_code} - {feeds.reason} getting feed list" | ||
| try: | ||
| result["response"] = feeds.json() | ||
| except Exception: | ||
| except JSONDecodeError: | ||
| result["response"] = feeds.content.decode(errors="replace") | ||
| result["src_host"] = JobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
| result["src_host"] = jobsConfig.get("DEFAULT", "BaseUrl").split(":")[1][2:] | ||
|
|
||
| print(dumps(result, sort_keys=True)) | ||
Oops, something went wrong.