Decrypt passwords from environment provider sub suite information (#42)

.gitignore

@@ -10,13 +10,15 @@
 __pycache__/*
 .cache/*
 .*.swp
+.tags
 
 # Project files
 .ropeproject
 .project
 .pydevproject
 .settings
 .idea
+.python-version
 
 # Package files
 *.egg

requirements.txt

@@ -16,5 +16,6 @@
 # scipy==1.0
 #
 packageurl-python==0.9.1
-etos_lib==2.1.0
+cryptography~=41.0
+etos_lib==3.2.2
 jsontas==1.3.0

setup.cfg

@@ -24,7 +24,8 @@ package_dir =
 # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD!
 setup_requires = pyscaffold>=3.2a0,<3.3a0
 install_requires =
-    etos_lib==2.1.0
+    etos_lib==3.2.2
+    cryptography~=41.0
     packageurl-python==0.9.1
     jsontas==1.3.0
 

src/etos_test_runner/__init__.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 """ETOS test runner module."""
 import os
+import logging
 from importlib.metadata import version, PackageNotFoundError
 from etos_lib.logging.logger import setup_logging
 
@@ -26,3 +27,10 @@
 DEV = os.getenv("DEV", "false").lower() == "true"
 ENVIRONMENT = "development" if DEV else "production"
 setup_logging("ETOS Test Runner", VERSION, ENVIRONMENT)
+# JSONTas would print all passwords as they are decrypted,
+# which is not safe, so we disable propagation on the loggers.
+# Propagation needs to be set to 0 instead of disabling the
+# logger or setting the loglevel higher because of how the
+# etos library sets up logging.
+logging.getLogger("JSONTas").propagate = 0
+logging.getLogger("Dataset").propagate = 0

src/etos_test_runner/etr.py

@@ -24,13 +24,17 @@
 import importlib
 import pkgutil
 from pprint import pprint
+from collections import OrderedDict
 
 from etos_lib import ETOS
 from etos_lib.logging.logger import FORMAT_CONFIG
+from jsontas.jsontas import JsonTas
 
 from etos_test_runner import VERSION
 from etos_test_runner.lib.testrunner import TestRunner
 from etos_test_runner.lib.iut import Iut
+from etos_test_runner.lib.custom_dataset import CustomDataset
+from etos_test_runner.lib.decrypt import Decrypt, decrypt
 
 
 # Remove spam from pika.
@@ -63,6 +67,10 @@ class ETR:
     def __init__(self):
         """Initialize ETOS library and start eiffel publisher."""
         self.etos = ETOS("ETOS Test Runner", os.getenv("HOSTNAME"), "ETOS Test Runner")
+        if os.getenv("ETOS_ENCRYPTION_KEY"):
+            os.environ["RABBITMQ_PASSWORD"] = decrypt(
+                os.environ["RABBITMQ_PASSWORD"], os.getenv("ETOS_ENCRYPTION_KEY")
+            )
 
         self.etos.config.rabbitmq_publisher_from_environment()
         # ETR will print the entire environment just before executing.
@@ -81,15 +89,24 @@ def graceful_shutdown(*args):
 
     def download_and_load(self):
         """Download and load test json."""
-        generator = self.etos.http.wait_for_request(self.tests_url)
+        generator = self.etos.http.wait_for_request(self.tests_url, as_json=False)
         for response in generator:
-            json_config = response
+            json_config = response.json(object_pairs_hook=OrderedDict)
             break
-        self.etos.config.set("test_config", json_config)
-        self.etos.config.set("context", json_config.get("context"))
-        self.etos.config.set("artifact", json_config.get("artifact"))
-        self.etos.config.set("main_suite_id", json_config.get("test_suite_started_id"))
-        self.etos.config.set("suite_id", json_config.get("suite_id"))
+        dataset = CustomDataset()
+        dataset.add("decrypt", Decrypt)
+        config = JsonTas(dataset).run(json_config)
+
+        # ETR will print the entire environment just before executing.
+        # Hide the encryption key.
+        if os.getenv("ETOS_ENCRYPTION_KEY"):
+            os.environ["ETOS_ENCRYPTION_KEY"] = "*********"
+
+        self.etos.config.set("test_config", config)
+        self.etos.config.set("context", config.get("context"))
+        self.etos.config.set("artifact", config.get("artifact"))
+        self.etos.config.set("main_suite_id", config.get("test_suite_started_id"))
+        self.etos.config.set("suite_id", config.get("suite_id"))
 
     def _run_tests(self):
         """Run tests in ETOS test runner.
@@ -161,6 +178,8 @@ def main(args):
 
 def run():
     """Entry point to ETR."""
+    # Disable sending logs for now.
+    os.environ["ETOS_ENABLE_SENDING_LOGS"] = "false"
     logging.basicConfig(level=logging.INFO, stream=sys.stdout)
     main(sys.argv[1:])
 

src/etos_test_runner/lib/custom_dataset.py

@@ -0,0 +1,37 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Custom dataset module."""
+from jsontas.dataset import Dataset
+
+
+class CustomDataset(Dataset):
+    """Custom dataset for ETR to decrypt secrets.
+
+    This custom dataset removes all default JsonTas datastructures
+    as we are going to run JsonTas on the sub suite information
+    retrieved from the environment provider.
+    This sub suite information is quite large and if we keep the
+    default datastructures the ETR would be susceptible to remote
+    code execution. This custom dataset shall only be used when
+    decrypting secrets.
+    """
+
+    def __init__(self):
+        """Initialize an empty dataset."""
+        super().__init__()
+        # pylint:disable=unused-private-member
+        # It is used by the parent class.
+        self.__dataset = {}

src/etos_test_runner/lib/decrypt.py

@@ -0,0 +1,47 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""JSONTas decrypt string data structure module."""
+import os
+from cryptography.fernet import Fernet
+from jsontas.data_structures.datastructure import DataStructure
+
+# pylint:disable=too-few-public-methods
+
+
+def decrypt(value, key):
+    """Decrypt a string.
+
+    :param value: Data to decrypt.
+    :type value: str
+    :param key: Encryption key to decrypt data with.
+    :type key: str
+    :return: Decrypted data.
+    :rtype: str
+    """
+    return Fernet(key).decrypt(value).decode()
+
+
+class Decrypt(DataStructure):
+    """Decrypt an encrypted string."""
+
+    def execute(self):
+        """Execute datastructure.
+
+        :return: Name of key. None, to tel JSONTas to not override key name, and decrypted value.
+        """
+        key = os.getenv("ETOS_ENCRYPTION_KEY")
+        assert key is not None, "ETOS_ENCRYPTION_KEY environment variable must be set"
+        return None, decrypt(self.data.get("value"), key)

tests/scenarios/test_full_execution.py

@@ -15,13 +15,15 @@
 # limitations under the License.
 """Tests full executions."""
 import os
+import json
 import logging
+from collections import OrderedDict
 from copy import deepcopy
 from shutil import rmtree
 from contextlib import contextmanager
 from pathlib import Path
 from unittest import TestCase
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 from etos_lib.lib.debug import Debug
 from etos_test_runner.etr import ETR
 
@@ -115,7 +117,11 @@ def _patch_wait_for_request(self):
         self.patchers.append(patcher)
         self.wait_for_request = patcher.start()
         self.suite = deepcopy(SUITE)
-        self.wait_for_request.return_value = [self.suite]
+        response = Mock()
+        response.json.return_value = json.loads(
+            json.dumps(self.suite), object_pairs_hook=OrderedDict
+        )
+        self.wait_for_request.return_value = [response]
 
     def _patch_http_request(self):
         """Patch the ETOS library http request method."""