Bugfix/api error handling

cobalt_purestorage/configuration.py

@@ -51,6 +51,12 @@ class Settings(BaseSettings):
         description="Minimum age of Access Keys in seconds",
     )
 
+    failure_mode_key_age: int = Field(
+        43200,
+        env="FAILURE_MODE_KEY_AGE",
+        description="In the case of API Failure, the amount of time to ",
+    )
+
     access_key_age_variance: int = Field(
         900,
         env="ACCESS_KEY_AGE_VARIANCE",

cobalt_purestorage/k8s.py

@@ -4,6 +4,8 @@
 
 import kubernetes
 
+import base64
+
 from cobalt_purestorage.configuration import config
 from cobalt_purestorage.logging_utils import format_stacktrace
 
@@ -62,3 +64,22 @@ def update_secret(self, namespace, secret_name, secret_key, secret_body):
         else:
             logger.error("specified secret does not exist")
             raise ValueError("secret does not exist")
+
+    def get_secret(self, namespace, secret_name, secret_key):
+        """get pre-existing secret"""
+
+        secret_exists = self._secret_exist(namespace, secret_name)
+
+        if secret_exists:
+            try:
+                data = self.v1.read_namespaced_secret(secret_name, namespace).data
+                decoded = base64.b64decode(data.get(secret_key)).decode("utf-8")
+                return decoded
+
+            except kubernetes.client.exceptions.ApiException:
+                logger.error(format_stacktrace())
+                raise RuntimeError("error getting k8s secret")
+
+        else:
+            logger.error("specified secret does not exist")
+            raise ValueError("secret does not exist")

cobalt_purestorage/pure_storage.py

@@ -15,6 +15,9 @@
 logger = logging.getLogger(__name__)
 
 
+class FBAPIError(Exception):
+    pass
+
 class PureStorageFlashBlade:
     """Service class for the PureStorage FlashBlade API"""
 
@@ -36,15 +39,15 @@ def _create_client(self, url, token, timeout):
 
             try:
                 client = Client(url, api_token=token, timeout=timeout)
-                return client
+                # return client
 
-            except requests.exceptions.ConnectionError:
+            except (requests.exceptions.ConnectionError, PureError):
                 logger.error(format_stacktrace())
-                raise RuntimeError("Could not instantiate FlashBlade client")
+                raise FBAPIError("Could not instantiate FlashBlade client")
+                # raise RuntimeError("Could not instantiate FlashBlade client")
 
-            except PureError:
-                logger.error(format_stacktrace())
-                raise RuntimeError("Could not instantiate FlashBlade Client")
+            else:
+                return client
 
     def object_store_user_exists(self, name):
         """Given an Object Store User name,

cobalt_purestorage/rotator.py

@@ -8,7 +8,7 @@
 
 from cobalt_purestorage.configuration import config
 from cobalt_purestorage.k8s import K8S
-from cobalt_purestorage.pure_storage import PureStorageFlashBlade
+from cobalt_purestorage.pure_storage import PureStorageFlashBlade, FBAPIError
 
 logging.basicConfig(level=config.log_level)
 logger = logging.getLogger(__name__)
@@ -41,6 +41,26 @@ def key_too_recent(keys):
     return False
 
 
+def bump_aws_credentials(old_credentials):
+    """Given old AWS credentials, bump the expiration and return
+    credentials in the format expected by the AWS SDK.
+    """
+
+    expiry_ts = (
+        datetime.utcnow().replace(microsecond=0) + timedelta(seconds=config.failure_mode_key_age)
+    ).isoformat()
+
+    refreshed_credentials = old_credentials
+    refreshed_credentials.update(
+        {
+            "Expiration": f"{expiry_ts}Z",
+        }
+    )
+
+    logger.debug(f"Credentials updated to expire at {expiry_ts}")
+
+    return refreshed_credentials
+
 def generate_aws_credentials(credentials):
     """Given FlashBlade credentials, return the credentials
     in the format expected by the AWS SDK.
@@ -96,6 +116,18 @@ def update_k8s(refreshed_credentials, user_name):
     )
     logger.info(f"Updated k8s. User: {user_name}")
 
+def get_k8s_secret_decoded(user_name):
+    """Get the k8s secret."""
+
+    k8s = K8S()
+    secret = k8s.get_secret(
+        config.k8s_namespace,
+        config.k8s_secret_name,
+        config.k8s_secret_key
+    )
+    # logger.info(f"Fetched k8s secret")
+
+    return secret
 
 def update_local(refreshed_credentials, user_name):
     """Given a credentials dict, write it out to the local filesystem."""
@@ -112,64 +144,80 @@ def main():
         logger.error("No Interesting Users are configured, exiting...")
         return
 
-    fb = PureStorageFlashBlade()
+    try:
+        fb = PureStorageFlashBlade()
 
-    for user_name in config.interesting_users:
-        logger.debug(f"Begin operations for user: {user_name}")
-        # check username is valid
-        if not fb.object_store_user_exists(user_name):
-            logger.error(f"User {user_name} does not appear to be a valid user...")
-            continue
+    except FBAPIError:
+        logger.warning(
+            f"FB API Error: Will bump expiration field of AWS credential to be FAILURE_MODE_KEY_AGE seconds in the future."
+        )
 
-        if keys := fb.get_access_keys_for_user(user_name):
-            logger.debug(f"Keys for user {user_name}: {keys}")
-            # sort keys to identify the oldest key for deletion
-            sorted_keys = sorted(keys, key=lambda d: d["created"])
+        # get kubernetes secret
+        credentials = get_k8s_secret_decoded()
 
-            # if existing key not too young create a new key
-            if len(keys) == 1:
-                logger.info(f"One key found. User: {user_name}")
+        # bump the expiration field.
+        update_credentials(
+            bump_aws_credentials(credentials),
+            "N/A"
+        )
 
-                if not key_too_recent(keys):
-                    if credentials := fb.post_object_store_access_keys(user_name):
+    else:
+        for user_name in config.interesting_users:
+            logger.debug(f"Begin operations for user: {user_name}")
+            # check username is valid
+            if not fb.object_store_user_exists(user_name):
+                logger.error(f"User {user_name} does not appear to be a valid user...")
+                continue
+
+            if keys := fb.get_access_keys_for_user(user_name):
+                logger.debug(f"Keys for user {user_name}: {keys}")
+                # sort keys to identify the oldest key for deletion
+                sorted_keys = sorted(keys, key=lambda d: d["created"])
+
+                # if existing key not too young create a new key
+                if len(keys) == 1:
+                    logger.info(f"One key found. User: {user_name}")
+
+                    if not key_too_recent(keys):
+                        if credentials := fb.post_object_store_access_keys(user_name):
+                            logger.info(
+                                f"New key created. User: {user_name}, Key: {credentials['name']}"
+                            )
+                            update_credentials(
+                                generate_aws_credentials(credentials), user_name
+                            )
+                    else:
+                        logger.warning(f"Keys are too young, ignoring. User: {user_name}")
+
+                # if existing keys not too young, delete oldest then create new
+                if len(keys) == 2:
+                    logger.info(f"Two keys found. User: {user_name}")
+
+                    if not key_too_recent(keys):
+                        fb.delete_object_store_access_keys([sorted_keys[0]["name"]])
                         logger.info(
-                            f"New key created. User: {user_name}, Key: {credentials['name']}"
+                            f"Oldest key deleted. User: {user_name}, Key: {sorted_keys[0]['name']}"
                         )
-                        update_credentials(
-                            generate_aws_credentials(credentials), user_name
-                        )
-                else:
-                    logger.warning(f"Keys are too young, ignoring. User: {user_name}")
-
-            # if existing keys not too young, delete oldest then create new
-            if len(keys) == 2:
-                logger.info(f"Two keys found. User: {user_name}")
-
-                if not key_too_recent(keys):
-                    fb.delete_object_store_access_keys([sorted_keys[0]["name"]])
+                        if credentials := fb.post_object_store_access_keys(user_name):
+                            logger.info(
+                                f"New key created. User: {user_name}, Key: {credentials['name']}"
+                            )
+                            update_credentials(
+                                generate_aws_credentials(credentials), user_name
+                            )
+
+                    else:
+                        logger.warning(f"Keys are too young, ignoring. User: {user_name}")
+
+                # hmmm, the FlashBlade only allows a max of two keys per user
+                if len(keys) > 2:
+                    logger.warning(f"More than two keys found. User: {user_name}")
+
+            else:
+                # no keys, create a new one
+                logger.info(f"No keys found. User: {user_name}")
+                if credentials := fb.post_object_store_access_keys(user_name):
                     logger.info(
-                        f"Oldest key deleted. User: {user_name}, Key: {sorted_keys[0]['name']}"
+                        f"New key created. User: {user_name}, Key: {credentials['name']}"
                     )
-                    if credentials := fb.post_object_store_access_keys(user_name):
-                        logger.info(
-                            f"New key created. User: {user_name}, Key: {credentials['name']}"
-                        )
-                        update_credentials(
-                            generate_aws_credentials(credentials), user_name
-                        )
-
-                else:
-                    logger.warning(f"Keys are too young, ignoring. User: {user_name}")
-
-            # hmmm, the FlashBlade only allows a max of two keys per user
-            if len(keys) > 2:
-                logger.warning(f"More than two keys found. User: {user_name}")
-
-        else:
-            # no keys, create a new one
-            logger.info(f"No keys found. User: {user_name}")
-            if credentials := fb.post_object_store_access_keys(user_name):
-                logger.info(
-                    f"New key created. User: {user_name}, Key: {credentials['name']}"
-                )
-                update_credentials(generate_aws_credentials(credentials), user_name)
+                    update_credentials(generate_aws_credentials(credentials), user_name)

tests/test_k8s.py

@@ -4,12 +4,13 @@
 
 import kubernetes
 import pytest
+import base64
 
 from cobalt_purestorage.k8s import K8S
 
 
 def mock_api_response(items=[]):
-    """Mock the kubernetes api response"""
+    """Mock the kubernetes api response for list request"""
 
     resp = {"kind": "SecretList", "apiVersion": "v1", "items": items}
 
@@ -95,3 +96,39 @@ def test_update_secret(mock_exists, mock_v1, mock_config, expected):
         mock_exists.assert_called_with(namespace, secret_name)
         mock_config.load_incluster_config.assert_called_once()
         mock_v1.return_value.patch_namespaced_secret.assert_not_called()
+
+
+@patch("cobalt_purestorage.configuration.config.k8s_mode", True)
+@patch("cobalt_purestorage.configuration.config.kubeconfig", None)
+@patch("cobalt_purestorage.k8s.kubernetes.config")
+@patch("cobalt_purestorage.k8s.kubernetes.client.CoreV1Api")
+@patch("cobalt_purestorage.k8s.K8S._secret_exist")
+@pytest.mark.parametrize("exists", [True, False])
+def test_get_secret(mock_exists, mock_v1, mock_config, exists):
+
+    # Arrange
+    namespace = "pytest"
+    secret_name = "pytest"
+    secret_key = "pytest"
+    secret_body = "pytest"
+    secret_body_encoded = base64.b64encode(secret_body.encode("utf-8")) # is it good practice to be dependent on base64 module like this?
+
+    data = {secret_key: secret_body_encoded}
+
+    k8s = K8S()
+
+    k8s.v1.read_namespaced_secret.return_value.data = data # is this a good way to structure it? it isn't calling the function, we are mocking the entire response. 
+    mock_exists.return_value = exists
+
+    # Assert
+    if exists:
+        response = k8s.get_secret(namespace, secret_name, secret_key)
+        assert isinstance(response, str)
+        assert response == secret_body
+    else:
+        with pytest.raises(ValueError):
+            response = k8s.get_secret(namespace, secret_name, secret_key)
+
+    mock_exists.assert_called_once()
+    mock_exists.assert_called_with(namespace,secret_name)
+    mock_config.load_incluster_config.assert_called_once()

tests/test_pure_storage.py

@@ -6,7 +6,7 @@
 import requests
 from pypureclient.flashblade import Client
 
-from cobalt_purestorage.pure_storage import PureStorageFlashBlade
+from cobalt_purestorage.pure_storage import PureStorageFlashBlade, FBAPIError
 
 MOCK_FB_URL = "169.254.99.99"
 
@@ -84,7 +84,7 @@ def test_init_conn_failure(requests_mock):
         exc=requests.exceptions.ConnectTimeout,
     )
 
-    with pytest.raises(RuntimeError):
+    with pytest.raises(FBAPIError):
         fb = PureStorageFlashBlade()
 
 
@@ -103,7 +103,7 @@ def test_init_pure_failure(requests_mock):
     # mock that the api token is incorrect
     requests_mock.post(f"https://{MOCK_FB_URL}/api/login", status_code=401)
 
-    with pytest.raises(RuntimeError):
+    with pytest.raises(FBAPIError):
         fb = PureStorageFlashBlade()