From 32ba37d68874edab2f12824f176b1e177e9e30c9 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 14:43:01 +0000 Subject: [PATCH 01/73] Updating with logic for expire token --- gestalt/vault.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index f906109..5401d39 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -13,6 +13,7 @@ from gestalt.provider import Provider +EXPIRATION_THRESHOLD_DAYS = 5 class Vault(Provider): def __init__( @@ -39,7 +40,7 @@ def __init__( self._scheme: str = scheme self._run_worker = True self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.kubes_token_queue: Queue[Tuple[str, str, str]] = Queue() + self.kubes_token: Tuple[str, str, str] = ("", "", "") self._vault_client: Optional[hvac.Client] = None self._secret_expiry_times: Dict[str, datetime] = dict() @@ -80,6 +81,7 @@ def connect(self) -> None: if self._role and self._jwt: try: + hvac.api.auth_methods.Kubernetes( self.vault_client.adapter).login(role=self._role, jwt=self._jwt) @@ -96,7 +98,7 @@ def connect(self) -> None: token["data"]["id"], token["data"]["ttl"], ) - self.kubes_token_queue.put(kubes_token) + self.kubes_token = kubes_token except hvac.exceptions.InvalidPath: raise RuntimeError( "Gestalt Error: Kubernetes auth couldn't be performed") @@ -113,7 +115,7 @@ def connect(self) -> None: name="kubes-token-renew", target=self.worker, daemon=True, - args=(self.kubes_token_queue, ), + args=(self.kubes_token, ), ) kubernetes_ttl_renew.start() self._is_connected = True @@ -150,6 +152,19 @@ def get( if key in self._secret_expiry_times and not self._is_secret_expired( key): return self._secret_values[key] + + # verify if the token still valid, in case not connect() + token_details = self.vault_client.auth.token.lookup_self() + if token_details is not None and token_details['data'] is not None: + expire_time = datetime.fromisoformat(token_details['data']['expire_time']) + threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) + delta_time = expire_time - datetime.now() + if delta_time < threshold: + self.connect() + else: + print("Token still valid for: {delta_time} days") + else: + print("Token information not retreived") try: response = retry_call( @@ -213,23 +228,20 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt - def worker(self, token_queue: Queue) -> None: # type: ignore + def worker(self, kube_token: Tuple) -> None: # type: ignore """ Worker function to renew lease on expiry """ try: while self._run_worker: - if not token_queue.empty(): - token_type, token_id, token_duration = token = token_queue.get( - ) + if not kube_token: + token_type, token_id, token_duration = token = kube_token if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") elif token_type == "dynamic": self.vault_client.sys.renew_lease(token_id) print("dynamic token for the app has been renewed") - token_queue.task_done() - token_queue.put_nowait(token) sleep((token_duration / 3) * 2) except hvac.exceptions.InvalidPath: raise RuntimeError( From 02a9f7da13dcf4c9faf0cf041c9247fa37f92aa8 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 15:08:23 +0000 Subject: [PATCH 02/73] Updating version with added fix --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e60b1..f056180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) +## [3.4.2] - 2024-08-05 + +### Fixed +- Adding logic to check when the token is about to expire to re-connect. This fix cases for services that are running longer that token's ttl without restarting. Causing requests to get a Permission denied error. + + ## [3.4.1] - 2024-07-12 ### Fixed diff --git a/setup.py b/setup.py index b7a74d3..a998403 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.1", + version="3.4.2", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From b40e417a926974565de750718873a63f45807211 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 15:12:22 +0000 Subject: [PATCH 03/73] adding pre release with suffix beta --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a998403..7c236ba 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2", + version="3.4.2-beta", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 3555becbf9fb64419f6689cece85df869bbfa18e Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 15:35:11 +0000 Subject: [PATCH 04/73] Updating tests --- tests/test_gestalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index b398e58..0cb4f46 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -496,7 +496,7 @@ def test_set_default_bad_type_set_config(): def test_vault_setup(): vault = Vault(role=None, jwt=None) - assert vault.vault_client.is_authenticated() is True + assert vault.vault_client.is_authenticated() is False def test_vault_interpolation(secret_setup): From 38b9721ffa2ba270c7bd3cc2e951a01e24a5d7a2 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 15:51:18 +0000 Subject: [PATCH 05/73] Refactoring code and fixing bugs --- gestalt/vault.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 5401d39..4908645 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -40,7 +40,7 @@ def __init__( self._scheme: str = scheme self._run_worker = True self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.kubes_token: Tuple[str, str, str] = ("", "", "") + self.kubes_token: Optional[Tuple[str, str, str]] = None self._vault_client: Optional[hvac.Client] = None self._secret_expiry_times: Dict[str, datetime] = dict() @@ -154,17 +154,7 @@ def get( return self._secret_values[key] # verify if the token still valid, in case not connect() - token_details = self.vault_client.auth.token.lookup_self() - if token_details is not None and token_details['data'] is not None: - expire_time = datetime.fromisoformat(token_details['data']['expire_time']) - threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) - delta_time = expire_time - datetime.now() - if delta_time < threshold: - self.connect() - else: - print("Token still valid for: {delta_time} days") - else: - print("Token information not retreived") + self._validate_token_expiration() try: response = retry_call( @@ -234,7 +224,7 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore """ try: while self._run_worker: - if not kube_token: + if kube_token: token_type, token_id, token_duration = token = kube_token if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) @@ -255,3 +245,16 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore @property def scheme(self) -> str: return self._scheme + + def _validate_token_expiration(self): + token_details = self.vault_client.auth.token.lookup_self() + if token_details is not None and token_details['data'] is not None: + expire_time = datetime.fromisoformat(token_details['data']['expire_time']) + threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) + delta_time = expire_time - datetime.now() + if delta_time < threshold: + self.connect() + else: + print(f"Token still valid for: {delta_time} days") + else: + print("Token information not retreived") From f328e831a610c4bceca5674069b81d9fa5ecc14a Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 16:14:57 +0000 Subject: [PATCH 06/73] Updating --- gestalt/vault.py | 2 +- tests/test_gestalt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 4908645..0bed77a 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -153,7 +153,7 @@ def get( key): return self._secret_values[key] - # verify if the token still valid, in case not connect() + # verify if the token still valid, in case not, call connect() self._validate_token_expiration() try: diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 0cb4f46..b398e58 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -496,7 +496,7 @@ def test_set_default_bad_type_set_config(): def test_vault_setup(): vault = Vault(role=None, jwt=None) - assert vault.vault_client.is_authenticated() is False + assert vault.vault_client.is_authenticated() is True def test_vault_interpolation(secret_setup): From 8420b72f21aeae7ff75fc54f27c942aa2e255112 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 5 Aug 2024 16:52:20 +0000 Subject: [PATCH 07/73] Fixing str convertion issue --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0bed77a..96d4c25 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -249,7 +249,7 @@ def scheme(self) -> str: def _validate_token_expiration(self): token_details = self.vault_client.auth.token.lookup_self() if token_details is not None and token_details['data'] is not None: - expire_time = datetime.fromisoformat(token_details['data']['expire_time']) + expire_time = datetime.fromisoformat(str(token_details['data']['expire_time'])) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() if delta_time < threshold: From 265f44f0774b3ff22c80a3f3806c8335a272c455 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 17:42:28 +0000 Subject: [PATCH 08/73] Handling none cases and adding fail message --- gestalt/vault.py | 7 ++++++- tests/test_gestalt.py | 11 +++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 96d4c25..5315a75 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -248,7 +248,12 @@ def scheme(self) -> str: def _validate_token_expiration(self): token_details = self.vault_client.auth.token.lookup_self() - if token_details is not None and token_details['data'] is not None: + if token_details['data'] is not None: + + # Validate expire_time is present + if token_details['data']['expire_time'] is not None: + raise ValueError(f"Cannot parse to expire_time, value is None: {token_details['data']}") + expire_time = datetime.fromisoformat(str(token_details['data']['expire_time'])) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index b398e58..6cc6a57 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -643,11 +643,11 @@ def test_vault_start_dynamic_lease(mock_vault_workers): return_value=mock_response) with mock_vault_client_patch as mock_vault_client_read: mock_dynamic_token_queue = Mock() - mock_kube_token_queue = Mock() + mock_kube_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") with patch( "gestalt.vault.Queue", - side_effect=[mock_dynamic_token_queue, mock_kube_token_queue], - ) as mock_queues: + side_effect=[mock_dynamic_token_queue], + ) as mock_queue: v = Vault(role=None, jwt=None) g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testmount.json") @@ -657,9 +657,8 @@ def test_vault_start_dynamic_lease(mock_vault_workers): mock_vault_client_read.assert_called() mock_dynamic_token_queue.put_nowait.assert_called() + assert mock_kube_token == ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") mock_vault_client_read.stop() mock_dynamic_token_queue.stop() - mock_kube_token_queue.stop() - mock_queues.stop() - mock_vault_client_read.stop() + mock_queue.stop() From 9b75d7203e871a1fed80e295cb759c9c30309399 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 17:47:31 +0000 Subject: [PATCH 09/73] Updating --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 5315a75..2f065bc 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -251,8 +251,8 @@ def _validate_token_expiration(self): if token_details['data'] is not None: # Validate expire_time is present - if token_details['data']['expire_time'] is not None: - raise ValueError(f"Cannot parse to expire_time, value is None: {token_details['data']}") + if token_details['data']['expire_time'] is None: + raise ValueError(f"Cannot parse expire_time, value is None: {token_details['data']}") expire_time = datetime.fromisoformat(str(token_details['data']['expire_time'])) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) From 27eed33b4fc0f2ad49ff468d15eacdfb5a803db5 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 17:53:12 +0000 Subject: [PATCH 10/73] Updating --- gestalt/vault.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 2f065bc..c7b9f0c 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -249,15 +249,16 @@ def scheme(self) -> str: def _validate_token_expiration(self): token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: - # Validate expire_time is present if token_details['data']['expire_time'] is None: - raise ValueError(f"Cannot parse expire_time, value is None: {token_details['data']}") + print(f"Cannot parse expire_time, value is None: {token_details['data']}") + return None expire_time = datetime.fromisoformat(str(token_details['data']['expire_time'])) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() if delta_time < threshold: + print(f"Re-auth with vault.") self.connect() else: print(f"Token still valid for: {delta_time} days") From 960425f651e9bf00faf809824516b9c5c768e15f Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 17:55:59 +0000 Subject: [PATCH 11/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index c7b9f0c..6f42aed 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -250,7 +250,7 @@ def _validate_token_expiration(self): token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: # Validate expire_time is present - if token_details['data']['expire_time'] is None: + if token_details['data'] is None or token_details['data']['expire_time'] is None: print(f"Cannot parse expire_time, value is None: {token_details['data']}") return None From 74ad6c0b473cf2686c94429e41bfdfe7a9d1bdb9 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 18:02:13 +0000 Subject: [PATCH 12/73] Updating --- gestalt/vault.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 6f42aed..0f5b11a 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -249,12 +249,18 @@ def scheme(self) -> str: def _validate_token_expiration(self): token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: + + expire_time = None + if 'expire_time' not in token_details['data']['expire_time']: + print("Key 'expire_time' does not exist in token_details['data']") + return None + # Validate expire_time is present - if token_details['data'] is None or token_details['data']['expire_time'] is None: - print(f"Cannot parse expire_time, value is None: {token_details['data']}") + if expire_time is None: + print("Cannot parse expire_time, value is None") return None - expire_time = datetime.fromisoformat(str(token_details['data']['expire_time'])) + expire_time = str(expire_time) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() if delta_time < threshold: From 4bff3b968679007cd2a0bdce0515efb18e349c45 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 18:03:50 +0000 Subject: [PATCH 13/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0f5b11a..55cdee1 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -251,7 +251,7 @@ def _validate_token_expiration(self): if token_details['data'] is not None: expire_time = None - if 'expire_time' not in token_details['data']['expire_time']: + if 'expire_time' not in token_details['data']: print("Key 'expire_time' does not exist in token_details['data']") return None From 8b270cadfd983d7ae06a6fc1f6c5cc549f447fa3 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 18:33:53 +0000 Subject: [PATCH 14/73] Updating --- tests/test_gestalt.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 6cc6a57..b1742ef 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -579,16 +579,15 @@ def except_once(self, **kwargs): with patch("gestalt.vault.sleep", side_effect=except_once, autospec=True) as mock_sleep: with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - v.connect() + vault = Vault(role="test-role", jwt="test-jwt") + vault.connect() mock_k8s_renew.start.assert_called() - test_token_queue = Queue(maxsize=0) - test_token_queue.put(("dynamic", 1, 100)) + test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") with pytest.raises(RuntimeError): - v.worker(test_token_queue) + vault.worker(test_token) mock_sleep.assert_called() mock_client().sys.renew_lease.assert_called() @@ -611,16 +610,15 @@ def except_once(self, **kwargs): with patch("gestalt.vault.sleep", side_effect=except_once, autospec=True) as mock_sleep: with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - v.connect() + vault = Vault(role="test-role", jwt="test-jwt") + vault.connect() mock_k8s_renew.start.assert_called() - test_token_queue = Queue(maxsize=0) - test_token_queue.put(("kubernetes", 1, 100)) + test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") with pytest.raises(RuntimeError): - v.worker(test_token_queue) + vault.worker(test_token) mock_sleep.assert_called() mock_client().auth.token.renew.assert_called() From cb75b8da641b49dc5b9ffe824926aaeeae5ba574 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 18:38:07 +0000 Subject: [PATCH 15/73] Updating --- tests/test_gestalt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index b1742ef..3ad95ad 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -579,15 +579,15 @@ def except_once(self, **kwargs): with patch("gestalt.vault.sleep", side_effect=except_once, autospec=True) as mock_sleep: with patch("gestalt.vault.hvac.Client") as mock_client: - vault = Vault(role="test-role", jwt="test-jwt") - vault.connect() + v = Vault(role="test-role", jwt="test-jwt") + v.connect() mock_k8s_renew.start.assert_called() test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") with pytest.raises(RuntimeError): - vault.worker(test_token) + v.worker(test_token) mock_sleep.assert_called() mock_client().sys.renew_lease.assert_called() @@ -610,15 +610,15 @@ def except_once(self, **kwargs): with patch("gestalt.vault.sleep", side_effect=except_once, autospec=True) as mock_sleep: with patch("gestalt.vault.hvac.Client") as mock_client: - vault = Vault(role="test-role", jwt="test-jwt") - vault.connect() + v = Vault(role="test-role", jwt="test-jwt") + v.connect() mock_k8s_renew.start.assert_called() test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") with pytest.raises(RuntimeError): - vault.worker(test_token) + v.worker(test_token) mock_sleep.assert_called() mock_client().auth.token.renew.assert_called() From db0b201aa0eda832cf3cc331d2ab622363035a73 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 19:32:21 +0000 Subject: [PATCH 16/73] Updating --- gestalt/vault.py | 1 + tests/test_gestalt.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 55cdee1..4b7540b 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -226,6 +226,7 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore while self._run_worker: if kube_token: token_type, token_id, token_duration = token = kube_token + print(f"Kube Token Values: {token_type}, {token_id}, {token_duration}") if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 3ad95ad..cbf5b47 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -584,8 +584,8 @@ def except_once(self, **kwargs): mock_k8s_renew.start.assert_called() - test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") - + test_token = ("kubernetes", 1, 100) + with pytest.raises(RuntimeError): v.worker(test_token) @@ -612,10 +612,11 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() + print(f"Run worker active: {v._run_worker}") mock_k8s_renew.start.assert_called() - test_token = ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") + test_token = ("kubernetes", 1, 100) with pytest.raises(RuntimeError): v.worker(test_token) From 3394686f7ada2d9301f4f93b97b8eef17f417f76 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 19:42:17 +0000 Subject: [PATCH 17/73] Updating --- gestalt/vault.py | 1 - tests/test_gestalt.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 4b7540b..55cdee1 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -226,7 +226,6 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore while self._run_worker: if kube_token: token_type, token_id, token_duration = token = kube_token - print(f"Kube Token Values: {token_type}, {token_id}, {token_duration}") if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index cbf5b47..b7a2c0f 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -584,7 +584,7 @@ def except_once(self, **kwargs): mock_k8s_renew.start.assert_called() - test_token = ("kubernetes", 1, 100) + test_token = ("dynamic", 1, 100) with pytest.raises(RuntimeError): v.worker(test_token) @@ -612,7 +612,6 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() - print(f"Run worker active: {v._run_worker}") mock_k8s_renew.start.assert_called() From ce0aa2a629d5192a70d70180c26fe55fd186ef27 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 19:44:02 +0000 Subject: [PATCH 18/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 55cdee1..6c325ce 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -246,7 +246,7 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore def scheme(self) -> str: return self._scheme - def _validate_token_expiration(self): + def _validate_token_expiration(self) -> None: token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: From 94819bcd5a9a0136cb839cef25ea136309a3162d Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 19:50:03 +0000 Subject: [PATCH 19/73] Updating --- gestalt/vault.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 6c325ce..2850487 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -15,6 +15,7 @@ EXPIRATION_THRESHOLD_DAYS = 5 + class Vault(Provider): def __init__( self, @@ -81,7 +82,6 @@ def connect(self) -> None: if self._role and self._jwt: try: - hvac.api.auth_methods.Kubernetes( self.vault_client.adapter).login(role=self._role, jwt=self._jwt) @@ -253,8 +253,7 @@ def _validate_token_expiration(self) -> None: expire_time = None if 'expire_time' not in token_details['data']: print("Key 'expire_time' does not exist in token_details['data']") - return None - + return None # Validate expire_time is present if expire_time is None: print("Cannot parse expire_time, value is None") From 287e50230772f317613a914cc91dfdaead909e72 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 20:07:13 +0000 Subject: [PATCH 20/73] Formatting with yapf --- gestalt/vault.py | 28 +++++++++++++++------------- tests/test_gestalt.py | 5 +++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 2850487..d9d1d3c 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -17,6 +17,7 @@ class Vault(Provider): + def __init__( self, cert: Optional[Tuple[str, str]] = None, @@ -126,13 +127,12 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - def get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def get(self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from @@ -152,7 +152,7 @@ def get( if key in self._secret_expiry_times and not self._is_secret_expired( key): return self._secret_values[key] - + # verify if the token still valid, in case not, call connect() self._validate_token_expiration() @@ -245,20 +245,22 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore @property def scheme(self) -> str: return self._scheme - + def _validate_token_expiration(self) -> None: token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: - + expire_time = None if 'expire_time' not in token_details['data']: - print("Key 'expire_time' does not exist in token_details['data']") - return None + print( + "Key 'expire_time' does not exist in token_details['data']" + ) + return None # Validate expire_time is present if expire_time is None: print("Cannot parse expire_time, value is None") return None - + expire_time = str(expire_time) threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index b7a2c0f..34a179c 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -585,7 +585,7 @@ def except_once(self, **kwargs): mock_k8s_renew.start.assert_called() test_token = ("dynamic", 1, 100) - + with pytest.raises(RuntimeError): v.worker(test_token) @@ -655,7 +655,8 @@ def test_vault_start_dynamic_lease(mock_vault_workers): mock_vault_client_read.assert_called() mock_dynamic_token_queue.put_nowait.assert_called() - assert mock_kube_token == ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", "2764799") + assert mock_kube_token == ("kubernetes", "hvs.CAESIEkz-UO8yvfC8v", + "2764799") mock_vault_client_read.stop() mock_dynamic_token_queue.stop() From 59a8dee7e9ff911bee69c064807608d2f8374dee Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 20:16:23 +0000 Subject: [PATCH 21/73] Formatting with yapf --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index d9d1d3c..fa255f8 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -270,4 +270,4 @@ def _validate_token_expiration(self) -> None: else: print(f"Token still valid for: {delta_time} days") else: - print("Token information not retreived") + print("Token information not retreived") \ No newline at end of file From c48c5508533ed0a209c6ffa42becc1795cb67ca8 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 20:31:48 +0000 Subject: [PATCH 22/73] Formatting with yapf --- gestalt/vault.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index fa255f8..52157a7 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -17,7 +17,6 @@ class Vault(Provider): - def __init__( self, cert: Optional[Tuple[str, str]] = None, @@ -127,12 +126,13 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - def get(self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from @@ -270,4 +270,4 @@ def _validate_token_expiration(self) -> None: else: print(f"Token still valid for: {delta_time} days") else: - print("Token information not retreived") \ No newline at end of file + print("Token information not retreived") From 0c6cb51af6ec2eae39604d78bae742d485f42e76 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 21:16:46 +0000 Subject: [PATCH 23/73] Threshold to expire in 10 min to test --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 52157a7..e438cc8 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -262,7 +262,7 @@ def _validate_token_expiration(self) -> None: return None expire_time = str(expire_time) - threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) + threshold = timedelta(minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() if delta_time < threshold: print(f"Re-auth with vault.") From f5994d74830e6a14ac8af4c1d0a83679905bc805 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 21:18:52 +0000 Subject: [PATCH 24/73] Threshold to expire in 10 min to test --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index e438cc8..f8799d8 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -264,7 +264,7 @@ def _validate_token_expiration(self) -> None: expire_time = str(expire_time) threshold = timedelta(minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() - if delta_time < threshold: + if delta_time <= threshold: print(f"Re-auth with vault.") self.connect() else: From 1d3e19e1d78ea60bc624957871ecd8d0bbe57e0c Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 6 Aug 2024 21:29:29 +0000 Subject: [PATCH 25/73] Threshold to expire in 10 min to test --- gestalt/vault.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index f8799d8..47aa48e 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -262,7 +262,8 @@ def _validate_token_expiration(self) -> None: return None expire_time = str(expire_time) - threshold = timedelta(minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) + threshold = timedelta( + minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) delta_time = expire_time - datetime.now() if delta_time <= threshold: print(f"Re-auth with vault.") From 8824d546f96e4d75d1f551266d769d2c62c9a54b Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Wed, 7 Aug 2024 19:32:40 +0000 Subject: [PATCH 26/73] Updating with gestalt beta version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7c236ba..0a7b328 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2-beta", + version="3.4.2-beta1", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From a626abc5cc1df6f5fd1c6b2b10426d010ec66268 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 13:36:24 +0000 Subject: [PATCH 27/73] Updating to test --- gestalt/vault.py | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 47aa48e..492e3aa 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -256,6 +256,8 @@ def _validate_token_expiration(self) -> None: "Key 'expire_time' does not exist in token_details['data']" ) return None + + expire_time = token_details['data']['expire_time'] # Validate expire_time is present if expire_time is None: print("Cannot parse expire_time, value is None") diff --git a/setup.py b/setup.py index 0a7b328..9ee29f5 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2-beta1", + version="3.4.2-beta2", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 8c54bfa0fc202ddadc38fae683aa8481feda46f5 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 13:42:14 +0000 Subject: [PATCH 28/73] Updating to test --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 492e3aa..07e1a60 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -256,7 +256,7 @@ def _validate_token_expiration(self) -> None: "Key 'expire_time' does not exist in token_details['data']" ) return None - + expire_time = token_details['data']['expire_time'] # Validate expire_time is present if expire_time is None: From ea0b4ca9b18e16df1bfda85ad3c0a92b2f8ed578 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 17:12:15 +0000 Subject: [PATCH 29/73] Updating to test --- gestalt/vault.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 07e1a60..d536492 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -1,5 +1,5 @@ import os -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from queue import Queue from threading import Thread from time import sleep @@ -263,10 +263,12 @@ def _validate_token_expiration(self) -> None: print("Cannot parse expire_time, value is None") return None - expire_time = str(expire_time) + expire_time = datetime.strptime(str(expire_time), '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) threshold = timedelta( minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) - delta_time = expire_time - datetime.now() + current_time = datetime.now(timezone.utc) + delta_time = expire_time - current_time + if delta_time <= threshold: print(f"Re-auth with vault.") self.connect() From fbcfb2d70e457c0a424463cb99ad92c0da3cb86e Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 17:13:31 +0000 Subject: [PATCH 30/73] Updating to test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ee29f5..43257fb 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2-beta2", + version="3.4.2-beta3", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From f7a69b5fe2091c3ea27cff21ce6bec4ce4e2ca2b Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 19:04:11 +0000 Subject: [PATCH 31/73] Updating to test gestalt beta version --- gestalt/vault.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index d536492..0fcb4e1 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -262,7 +262,10 @@ def _validate_token_expiration(self) -> None: if expire_time is None: print("Cannot parse expire_time, value is None") return None - + + # Truncate the fractional seconds to 6 digits before parsing + expire_time = expire_time[:26] + 'Z' + expire_time = datetime.strptime(str(expire_time), '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) threshold = timedelta( minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) From 2ebd14e70ce05785913f125a4e286679e5ed3a83 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:00:37 +0000 Subject: [PATCH 32/73] Updating to test gestalt beta version --- gestalt/vault.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0fcb4e1..3feb21d 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -268,7 +268,7 @@ def _validate_token_expiration(self) -> None: expire_time = datetime.strptime(str(expire_time), '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) threshold = timedelta( - minutes=10) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) + hours=24) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) current_time = datetime.now(timezone.utc) delta_time = expire_time - current_time diff --git a/setup.py b/setup.py index 43257fb..760329b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2-beta3", + version="3.4.2-beta5", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 47d7966f87538865aa584cd0a670bdc8844c4588 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:08:18 +0000 Subject: [PATCH 33/73] Updating to test gestalt beta version --- gestalt/vault.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 3feb21d..e56d0a4 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -12,6 +12,7 @@ from retry.api import retry_call from gestalt.provider import Provider +from dateutil.parser import isoparse EXPIRATION_THRESHOLD_DAYS = 5 @@ -263,10 +264,15 @@ def _validate_token_expiration(self) -> None: print("Cannot parse expire_time, value is None") return None - # Truncate the fractional seconds to 6 digits before parsing - expire_time = expire_time[:26] + 'Z' + # Use isoparse to correctly parse the datetime string + expire_time = isoparse(expire_time) + + # Ensure the parsed time is in UTC + if expire_time.tzinfo is None: + expire_time = expire_time.replace(tzinfo=timezone.utc) + else: + expire_time = expire_time.astimezone(timezone.utc) - expire_time = datetime.strptime(str(expire_time), '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) threshold = timedelta( hours=24) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) current_time = datetime.now(timezone.utc) From d4515d8563656a55aee94e842f67a698430814b0 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:08:51 +0000 Subject: [PATCH 34/73] Updating to test gestalt beta version --- gestalt/vault.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index e56d0a4..0c3a1b4 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -263,21 +263,21 @@ def _validate_token_expiration(self) -> None: if expire_time is None: print("Cannot parse expire_time, value is None") return None - + # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) - + # Ensure the parsed time is in UTC if expire_time.tzinfo is None: expire_time = expire_time.replace(tzinfo=timezone.utc) else: expire_time = expire_time.astimezone(timezone.utc) - + threshold = timedelta( hours=24) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) current_time = datetime.now(timezone.utc) delta_time = expire_time - current_time - + if delta_time <= threshold: print(f"Re-auth with vault.") self.connect() From aee3fdd7f9767438761bb26bb714417d747bb0ad Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:29:53 +0000 Subject: [PATCH 35/73] Adding dateutil dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index be8829c..9cfdffc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ hvac>=1.0.2,<1.1.0 jsonpath-ng==1.5.3 retry==0.9.2 types-retry==0.9.9 +python-dateutil>=2.8.0 From c17080c7224034ad7be89405aa025f73944d8afd Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:30:58 +0000 Subject: [PATCH 36/73] Adding dateutil dependency --- requirements.test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.test.txt b/requirements.test.txt index 8f6a296..3f71f88 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -12,3 +12,4 @@ retry==0.9.2 types-retry==0.9.9 jsonpath-ng==1.5.3 pytest-asyncio==0.19.0 +python-dateutil>=2.8.0 From 81acd2115a77f6a6e87545011a8f1b46b02ea4d9 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Mon, 12 Aug 2024 20:38:09 +0000 Subject: [PATCH 37/73] Adding dateutil dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9cfdffc..9c7f928 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ jsonpath-ng==1.5.3 retry==0.9.2 types-retry==0.9.9 python-dateutil>=2.8.0 +types-python-dateutil>=0.1.0 From d1dd4656ac8f65bb75d63f73e3e5059c05384a65 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 15:26:42 +0000 Subject: [PATCH 38/73] Updating with beta test --- gestalt/vault.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0c3a1b4..ee0c981 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -14,7 +14,7 @@ from gestalt.provider import Provider from dateutil.parser import isoparse -EXPIRATION_THRESHOLD_DAYS = 5 +EXPIRATION_THRESHOLD_DAYS = 7 class Vault(Provider): @@ -273,8 +273,7 @@ def _validate_token_expiration(self) -> None: else: expire_time = expire_time.astimezone(timezone.utc) - threshold = timedelta( - hours=24) # timedelta(days=EXPIRATION_THRESHOLD_DAYS) + threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) current_time = datetime.now(timezone.utc) delta_time = expire_time - current_time @@ -282,6 +281,6 @@ def _validate_token_expiration(self) -> None: print(f"Re-auth with vault.") self.connect() else: - print(f"Token still valid for: {delta_time} days") + print(f"Token still valid for: {delta_time.days} days") else: print("Token information not retreived") From fed95bef56ddaaf4d6b89a96caa8c186be6079ce Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 15:33:28 +0000 Subject: [PATCH 39/73] Upgrading version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 760329b..a998403 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2-beta5", + version="3.4.2", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 780a95c097ebd8635d1945af65b03da89399f3f6 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 19:12:35 +0000 Subject: [PATCH 40/73] Upgrading version --- gestalt/vault.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index ee0c981..0578df5 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -1,7 +1,7 @@ import os from datetime import datetime, timedelta, timezone from queue import Queue -from threading import Thread +from threading import Thread, Event from time import sleep from typing import Any, Dict, List, Optional, Tuple, Union @@ -58,6 +58,8 @@ def __init__( self.delay = delay self.tries = tries + + self.stop_event = Event() @property def vault_client(self) -> hvac.Client: @@ -112,13 +114,16 @@ def connect(self) -> None: daemon=True, args=(self.dynamic_token_queue, ), ) # noqa: F841 - kubernetes_ttl_renew = Thread( + + # + self._stop_all_threads(self.kubernetes_ttl_renew) + self.kubernetes_ttl_renew = Thread( name="kubes-token-renew", target=self.worker, daemon=True, args=(self.kubes_token, ), ) - kubernetes_ttl_renew.start() + self.kubernetes_ttl_renew.start() self._is_connected = True def stop(self) -> None: @@ -226,7 +231,7 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore try: while self._run_worker: if kube_token: - token_type, token_id, token_duration = token = kube_token + token_type, token_id, token_duration = kube_token if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") @@ -250,7 +255,6 @@ def scheme(self) -> str: def _validate_token_expiration(self) -> None: token_details = self.vault_client.auth.token.lookup_self() if token_details['data'] is not None: - expire_time = None if 'expire_time' not in token_details['data']: print( @@ -265,7 +269,7 @@ def _validate_token_expiration(self) -> None: return None # Use isoparse to correctly parse the datetime string - expire_time = isoparse(expire_time) + expire_time = self._parse_expire_time(expire_time) # Ensure the parsed time is in UTC if expire_time.tzinfo is None: @@ -284,3 +288,17 @@ def _validate_token_expiration(self) -> None: print(f"Token still valid for: {delta_time.days} days") else: print("Token information not retreived") + + def _parse_expire_time(expire_time): + try: + expire_time = isoparse(expire_time) + return expire_time + except ValueError as e: + raise RuntimeError(f" Error: Failed to parse expire_time: {expire_time}. Error: {e}") + + def _stop_all_threads(self, thread: Thread): + if thread and thread.is_alive(): + self.stop_event.set() # Signal the thread to stop + thread.join() # Wait for the thread to finish + print("Thread stopped.") + self.stop_event.clear() \ No newline at end of file From cc48ad240350fa27f5ac31366e83063d77588715 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 19:22:16 +0000 Subject: [PATCH 41/73] Updating --- gestalt/vault.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0578df5..4483d3e 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -58,7 +58,7 @@ def __init__( self.delay = delay self.tries = tries - + self.stop_event = Event() @property @@ -114,8 +114,8 @@ def connect(self) -> None: daemon=True, args=(self.dynamic_token_queue, ), ) # noqa: F841 - - # + + # self._stop_all_threads(self.kubernetes_ttl_renew) self.kubernetes_ttl_renew = Thread( name="kubes-token-renew", @@ -294,11 +294,13 @@ def _parse_expire_time(expire_time): expire_time = isoparse(expire_time) return expire_time except ValueError as e: - raise RuntimeError(f" Error: Failed to parse expire_time: {expire_time}. Error: {e}") + raise RuntimeError( + f" Error: Failed to parse expire_time: {expire_time}. Error: {e}" + ) def _stop_all_threads(self, thread: Thread): if thread and thread.is_alive(): self.stop_event.set() # Signal the thread to stop thread.join() # Wait for the thread to finish print("Thread stopped.") - self.stop_event.clear() \ No newline at end of file + self.stop_event.clear() From 63d963f495705c9ea57fdf051e9a9ceeaebfb44c Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 19:25:46 +0000 Subject: [PATCH 42/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 4483d3e..1a63314 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -229,7 +229,7 @@ def worker(self, kube_token: Tuple) -> None: # type: ignore Worker function to renew lease on expiry """ try: - while self._run_worker: + while not self.stop_event.is_set() and self._run_worker: if kube_token: token_type, token_id, token_duration = kube_token if token_type == "kubernetes": From 076b0caa8809b0e566a27d7cea52a3f2360833b1 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 19:28:49 +0000 Subject: [PATCH 43/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 1a63314..d03e874 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -58,7 +58,7 @@ def __init__( self.delay = delay self.tries = tries - + self.kubernetes_ttl_renew = None self.stop_event = Event() @property From 3452d8aabc32b312ba1ad8e79bfb008f016bffc8 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 20:03:18 +0000 Subject: [PATCH 44/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index d03e874..ba91f2c 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -299,7 +299,7 @@ def _parse_expire_time(expire_time): ) def _stop_all_threads(self, thread: Thread): - if thread and thread.is_alive(): + if thread is not None and thread.is_alive(): self.stop_event.set() # Signal the thread to stop thread.join() # Wait for the thread to finish print("Thread stopped.") From 3698abc13ddf544f58e943af7cb8d3e5192bf7ed Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 20:47:25 +0000 Subject: [PATCH 45/73] Updating --- gestalt/vault.py | 60 +++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index ba91f2c..0fb8315 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -59,7 +59,6 @@ def __init__( self.delay = delay self.tries = tries self.kubernetes_ttl_renew = None - self.stop_event = Event() @property def vault_client(self) -> hvac.Client: @@ -115,15 +114,14 @@ def connect(self) -> None: args=(self.dynamic_token_queue, ), ) # noqa: F841 - # - self._stop_all_threads(self.kubernetes_ttl_renew) - self.kubernetes_ttl_renew = Thread( - name="kubes-token-renew", - target=self.worker, - daemon=True, - args=(self.kubes_token, ), - ) - self.kubernetes_ttl_renew.start() + if self.kubernetes_ttl_renew is None: + self.kubernetes_ttl_renew = Thread( + name="kubes-token-renew", + target=self.worker, + daemon=True, + ) + self.kubernetes_ttl_renew.start() + self._is_connected = True def stop(self) -> None: @@ -224,14 +222,14 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt - def worker(self, kube_token: Tuple) -> None: # type: ignore + def worker(self) -> None: # type: ignore """ Worker function to renew lease on expiry """ try: - while not self.stop_event.is_set() and self._run_worker: - if kube_token: - token_type, token_id, token_duration = kube_token + while self._run_worker: + if self.kube_token: + token_type, token_id, token_duration = self.kube_token if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") @@ -269,8 +267,8 @@ def _validate_token_expiration(self) -> None: return None # Use isoparse to correctly parse the datetime string - expire_time = self._parse_expire_time(expire_time) - + expire_time = isoparse(expire_time) + # Ensure the parsed time is in UTC if expire_time.tzinfo is None: expire_time = expire_time.replace(tzinfo=timezone.utc) @@ -289,18 +287,18 @@ def _validate_token_expiration(self) -> None: else: print("Token information not retreived") - def _parse_expire_time(expire_time): - try: - expire_time = isoparse(expire_time) - return expire_time - except ValueError as e: - raise RuntimeError( - f" Error: Failed to parse expire_time: {expire_time}. Error: {e}" - ) - - def _stop_all_threads(self, thread: Thread): - if thread is not None and thread.is_alive(): - self.stop_event.set() # Signal the thread to stop - thread.join() # Wait for the thread to finish - print("Thread stopped.") - self.stop_event.clear() + # def _parse_expire_time(expire_time: str): + # try: + # expire_time = isoparse(expire_time) + # return expire_time + # except ValueError as e: + # raise RuntimeError( + # f" Error: Failed to parse expire_time: {expire_time}. Error: {e}" + # ) + + # def _stop_all_threads(self, thread: Optional[Thread]): + # if thread is not None and thread.is_alive(): + # self.stop_event.set() # Signal the thread to stop + # thread.join() # Wait for the thread to finish + # print("Thread stopped.") + # self.stop_event.clear() From b5fbd302c6a7fd1d04b52dadddb90de4d9cfb594 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 20:47:56 +0000 Subject: [PATCH 46/73] Updating --- gestalt/vault.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0fb8315..cde2546 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -286,19 +286,3 @@ def _validate_token_expiration(self) -> None: print(f"Token still valid for: {delta_time.days} days") else: print("Token information not retreived") - - # def _parse_expire_time(expire_time: str): - # try: - # expire_time = isoparse(expire_time) - # return expire_time - # except ValueError as e: - # raise RuntimeError( - # f" Error: Failed to parse expire_time: {expire_time}. Error: {e}" - # ) - - # def _stop_all_threads(self, thread: Optional[Thread]): - # if thread is not None and thread.is_alive(): - # self.stop_event.set() # Signal the thread to stop - # thread.join() # Wait for the thread to finish - # print("Thread stopped.") - # self.stop_event.clear() From 318502e46777d247368a7815c1250d52faac7f62 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 20:54:10 +0000 Subject: [PATCH 47/73] Updating tests --- tests/test_gestalt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 34a179c..7e33173 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -581,13 +581,12 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() + v.kube_token = ("dynamic", 1, 100) mock_k8s_renew.start.assert_called() - test_token = ("dynamic", 1, 100) - with pytest.raises(RuntimeError): - v.worker(test_token) + v.worker() mock_sleep.assert_called() mock_client().sys.renew_lease.assert_called() @@ -612,13 +611,12 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() + v.kube_token = ("dynamic", 1, 100) mock_k8s_renew.start.assert_called() - test_token = ("kubernetes", 1, 100) - with pytest.raises(RuntimeError): - v.worker(test_token) + v.worker() mock_sleep.assert_called() mock_client().auth.token.renew.assert_called() From 2023ad1b1cd0efeb2193addb25bc8f7f1bcb79a4 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 20:57:36 +0000 Subject: [PATCH 48/73] Updating tests --- tests/test_gestalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 7e33173..0ad718b 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -611,7 +611,7 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() - v.kube_token = ("dynamic", 1, 100) + v.kube_token = ("kubernetes", 1, 100) mock_k8s_renew.start.assert_called() From ac156557963b4e96c42e7dc46f6d9675e7887fb2 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:03:18 +0000 Subject: [PATCH 49/73] Updating --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index cde2546..6a5e900 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -228,8 +228,8 @@ def worker(self) -> None: # type: ignore """ try: while self._run_worker: - if self.kube_token: - token_type, token_id, token_duration = self.kube_token + if self.kubes_token: + token_type, token_id, token_duration = self.kubes_token if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") From 4ea6986fe58f59e03a1b7236d4a8d35d75199f38 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:05:26 +0000 Subject: [PATCH 50/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 6a5e900..02e1027 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -58,7 +58,7 @@ def __init__( self.delay = delay self.tries = tries - self.kubernetes_ttl_renew = None + self.kubernetes_ttl_renew: Optional[Thread] = None @property def vault_client(self) -> hvac.Client: From c3a839bf58b1279986d4bf61ebf546ea2279305b Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:08:12 +0000 Subject: [PATCH 51/73] Updating --- tests/test_gestalt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 0ad718b..d3029d4 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -581,7 +581,7 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() - v.kube_token = ("dynamic", 1, 100) + v.kubes_token = ("dynamic", 1, 100) mock_k8s_renew.start.assert_called() @@ -611,7 +611,7 @@ def except_once(self, **kwargs): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") v.connect() - v.kube_token = ("kubernetes", 1, 100) + v.kubes_token = ("kubernetes", 1, 100) mock_k8s_renew.start.assert_called() From a6a9481a302ba2f9522bdb4202b10ad2250b71bb Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:27:47 +0000 Subject: [PATCH 52/73] Updating --- gestalt/vault.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 02e1027..107c0da 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -121,7 +121,7 @@ def connect(self) -> None: daemon=True, ) self.kubernetes_ttl_renew.start() - + self._is_connected = True def stop(self) -> None: @@ -222,7 +222,7 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt - def worker(self) -> None: # type: ignore + def worker(self) -> None: """ Worker function to renew lease on expiry """ @@ -236,7 +236,7 @@ def worker(self) -> None: # type: ignore elif token_type == "dynamic": self.vault_client.sys.renew_lease(token_id) print("dynamic token for the app has been renewed") - sleep((token_duration / 3) * 2) + sleep((token_duration / 3) * 2) # type: ignore except hvac.exceptions.InvalidPath: raise RuntimeError( "Gestalt Error: The lease path or mount is set incorrectly") @@ -268,7 +268,7 @@ def _validate_token_expiration(self) -> None: # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) - + # Ensure the parsed time is in UTC if expire_time.tzinfo is None: expire_time = expire_time.replace(tzinfo=timezone.utc) From 2a86d9d4459c5ccf90f4026606f1b03f226440de Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:30:37 +0000 Subject: [PATCH 53/73] Updating --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9c7f928..9cfdffc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,3 @@ jsonpath-ng==1.5.3 retry==0.9.2 types-retry==0.9.9 python-dateutil>=2.8.0 -types-python-dateutil>=0.1.0 From f202635a73ad47a03d160b310227f82af667026e Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Tue, 13 Aug 2024 21:35:38 +0000 Subject: [PATCH 54/73] Updating --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9cfdffc..9c7f928 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ jsonpath-ng==1.5.3 retry==0.9.2 types-retry==0.9.9 python-dateutil>=2.8.0 +types-python-dateutil>=0.1.0 From 69afcf30ea1dc9f35e82278fe8def6c957a44585 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Wed, 14 Aug 2024 15:35:05 +0000 Subject: [PATCH 55/73] Updating version with beta --- gestalt/vault.py | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 107c0da..7b926ef 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -252,6 +252,7 @@ def scheme(self) -> str: def _validate_token_expiration(self) -> None: token_details = self.vault_client.auth.token.lookup_self() + print(token_details) if token_details['data'] is not None: expire_time = None if 'expire_time' not in token_details['data']: diff --git a/setup.py b/setup.py index a998403..cca8acb 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2", + version="3.4.2beta7", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 28b1537ac5e073e5ba806d05e5b3dfd4a1c5f3a6 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 15:10:39 +0000 Subject: [PATCH 56/73] Removing thread and worker. --- gestalt/vault.py | 101 +++++++++++++++++++++-------------------------- setup.py | 2 +- 2 files changed, 47 insertions(+), 56 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 7b926ef..72ba779 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -58,7 +58,7 @@ def __init__( self.delay = delay self.tries = tries - self.kubernetes_ttl_renew: Optional[Thread] = None + # self.kubernetes_ttl_renew: Optional[Thread] = None @property def vault_client(self) -> hvac.Client: @@ -99,6 +99,7 @@ def connect(self) -> None: "kubernetes", token["data"]["id"], token["data"]["ttl"], + token["data"]['expire_time'], ) self.kubes_token = kubes_token except hvac.exceptions.InvalidPath: @@ -107,20 +108,20 @@ def connect(self) -> None: except requests.exceptions.ConnectionError: raise RuntimeError("Gestalt Error: Couldn't connect to Vault") - dynamic_ttl_renew = Thread( - name="dynamic-token-renew", - target=self.worker, - daemon=True, - args=(self.dynamic_token_queue, ), - ) # noqa: F841 - - if self.kubernetes_ttl_renew is None: - self.kubernetes_ttl_renew = Thread( - name="kubes-token-renew", - target=self.worker, - daemon=True, - ) - self.kubernetes_ttl_renew.start() + # dynamic_ttl_renew = Thread( + # name="dynamic-token-renew", + # target=self.worker, + # daemon=True, + # args=(self.dynamic_token_queue, ), + # ) # noqa: F841 + + # if self.kubernetes_ttl_renew is None: + # self.kubernetes_ttl_renew = Thread( + # name="kubes-token-renew", + # target=self.worker, + # daemon=True, + # ) + # self.kubernetes_ttl_renew.start() self._is_connected = True @@ -222,51 +223,39 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt - def worker(self) -> None: - """ - Worker function to renew lease on expiry - """ - try: - while self._run_worker: - if self.kubes_token: - token_type, token_id, token_duration = self.kubes_token - if token_type == "kubernetes": - self.vault_client.auth.token.renew(token_id) - print("kubernetes token for the app has been renewed") - elif token_type == "dynamic": - self.vault_client.sys.renew_lease(token_id) - print("dynamic token for the app has been renewed") - sleep((token_duration / 3) * 2) # type: ignore - except hvac.exceptions.InvalidPath: - raise RuntimeError( - "Gestalt Error: The lease path or mount is set incorrectly") - except requests.exceptions.ConnectionError: - raise RuntimeError( - "Gestalt Error: Gestalt couldn't connect to Vault") - except Exception as err: - raise RuntimeError(f"Gestalt Error: {err}") + # def worker(self) -> None: + # """ + # Worker function to renew lease on expiry + # """ + # try: + # while self._run_worker: + # if self.kubes_token: + # token_type, token_id, token_duration = self.kubes_token + # if token_type == "kubernetes": + # self.vault_client.auth.token.renew(token_id) + # print("kubernetes token for the app has been renewed") + # elif token_type == "dynamic": + # self.vault_client.sys.renew_lease(token_id) + # print("dynamic token for the app has been renewed") + # sleep((token_duration / 3) * 2) # type: ignore + # except hvac.exceptions.InvalidPath: + # raise RuntimeError( + # "Gestalt Error: The lease path or mount is set incorrectly") + # except requests.exceptions.ConnectionError: + # raise RuntimeError( + # "Gestalt Error: Gestalt couldn't connect to Vault") + # except Exception as err: + # raise RuntimeError(f"Gestalt Error: {err}") @property def scheme(self) -> str: return self._scheme def _validate_token_expiration(self) -> None: - token_details = self.vault_client.auth.token.lookup_self() - print(token_details) - if token_details['data'] is not None: - expire_time = None - if 'expire_time' not in token_details['data']: - print( - "Key 'expire_time' does not exist in token_details['data']" - ) - return None - - expire_time = token_details['data']['expire_time'] - # Validate expire_time is present - if expire_time is None: - print("Cannot parse expire_time, value is None") - return None - + # token_details = self.vault_client.auth.token.lookup_self() + expire_time = self.kubes_token[3] + print(self.kubes_token) + if expire_time is not None: # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) @@ -286,4 +275,6 @@ def _validate_token_expiration(self) -> None: else: print(f"Token still valid for: {delta_time.days} days") else: - print("Token information not retreived") + print( + f"Can't reconnect cause token: {self.kubes_token}, expire_time is None" + ) diff --git a/setup.py b/setup.py index cca8acb..acf7542 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2beta7", + version="3.4.2beta8", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From f131f0683f04553c221d2f67b0c6ff04e08875e4 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 15:39:07 +0000 Subject: [PATCH 57/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 72ba779..fe7dd30 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -251,7 +251,7 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], def scheme(self) -> str: return self._scheme - def _validate_token_expiration(self) -> None: + def _validate_token_expiration(self) -> None: # type: ignore # token_details = self.vault_client.auth.token.lookup_self() expire_time = self.kubes_token[3] print(self.kubes_token) From ff09477313831c9e4c30ca64b29c83208cbce8d6 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 15:52:14 +0000 Subject: [PATCH 58/73] Updating --- gestalt/vault.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index fe7dd30..a8a2012 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -253,9 +253,9 @@ def scheme(self) -> str: def _validate_token_expiration(self) -> None: # type: ignore # token_details = self.vault_client.auth.token.lookup_self() - expire_time = self.kubes_token[3] print(self.kubes_token) - if expire_time is not None: + if self.kubes_token is not None: + expire_time = self.kubes_token[3] # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) diff --git a/setup.py b/setup.py index acf7542..793b038 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2beta8", + version="3.4.2beta10", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From 68a1164becf8561722d597b4bc86446af5bf71d0 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 18:15:50 +0000 Subject: [PATCH 59/73] Updating --- gestalt/vault.py | 7 ++++--- setup.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index a8a2012..785894c 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -14,7 +14,7 @@ from gestalt.provider import Provider from dateutil.parser import isoparse -EXPIRATION_THRESHOLD_DAYS = 7 +EXPIRATION_THRESHOLD_HOURS = 2 class Vault(Provider): @@ -102,6 +102,7 @@ def connect(self) -> None: token["data"]['expire_time'], ) self.kubes_token = kubes_token + print(f"Token LookUp after K8s Login: {self.kubes_token}") except hvac.exceptions.InvalidPath: raise RuntimeError( "Gestalt Error: Kubernetes auth couldn't be performed") @@ -253,7 +254,7 @@ def scheme(self) -> str: def _validate_token_expiration(self) -> None: # type: ignore # token_details = self.vault_client.auth.token.lookup_self() - print(self.kubes_token) + print(f"Token Stored after K8s Login: {self.kubes_token}") if self.kubes_token is not None: expire_time = self.kubes_token[3] # Use isoparse to correctly parse the datetime string @@ -265,7 +266,7 @@ def _validate_token_expiration(self) -> None: # type: ignore else: expire_time = expire_time.astimezone(timezone.utc) - threshold = timedelta(days=EXPIRATION_THRESHOLD_DAYS) + threshold = timedelta(days=EXPIRATION_THRESHOLD_HOURS) current_time = datetime.now(timezone.utc) delta_time = expire_time - current_time diff --git a/setup.py b/setup.py index 793b038..0193268 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2beta10", + version="3.4.2beta11", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From b49276c99c162a85c094db567745bf4c3aa12729 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 20:07:22 +0000 Subject: [PATCH 60/73] Updating --- gestalt/vault.py | 12 +++---- tests/test_gestalt.py | 84 +++++++++++++++++++++---------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 785894c..1647eb1 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -14,7 +14,7 @@ from gestalt.provider import Provider from dateutil.parser import isoparse -EXPIRATION_THRESHOLD_HOURS = 2 +EXPIRATION_THRESHOLD_HOURS = 1 class Vault(Provider): @@ -266,15 +266,15 @@ def _validate_token_expiration(self) -> None: # type: ignore else: expire_time = expire_time.astimezone(timezone.utc) - threshold = timedelta(days=EXPIRATION_THRESHOLD_HOURS) current_time = datetime.now(timezone.utc) - delta_time = expire_time - current_time + # in hours + delta_time = (expire_time - current_time).total_seconds() / 3600 - if delta_time <= threshold: - print(f"Re-auth with vault.") + if delta_time < EXPIRATION_THRESHOLD_HOURS: + print(f"Re-auth with vault") self.connect() else: - print(f"Token still valid for: {delta_time.days} days") + print(f"Token still valid for: {delta_time} days") else: print( f"Can't reconnect cause token: {self.kubes_token}, expire_time is None" diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index d3029d4..14cd8af 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -566,64 +566,64 @@ def test_vault_lazy_connect(mock_vault_workers, mock_vault_k8s_auth): mock_client().auth.token.lookup_self.assert_called() -def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): - mock_dynamic_renew, mock_k8s_renew = mock_vault_workers +# def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): +# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - mock_sleep = None +# mock_sleep = None - def except_once(self, **kwargs): - # side effect used to exit the worker loop after one call - if mock_sleep.call_count == 1: - raise hvac.exceptions.VaultError("some error") +# def except_once(self, **kwargs): +# # side effect used to exit the worker loop after one call +# if mock_sleep.call_count == 1: +# raise hvac.exceptions.VaultError("some error") - with patch("gestalt.vault.sleep", side_effect=except_once, - autospec=True) as mock_sleep: - with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - v.connect() - v.kubes_token = ("dynamic", 1, 100) +# with patch("gestalt.vault.sleep", side_effect=except_once, +# autospec=True) as mock_sleep: +# with patch("gestalt.vault.hvac.Client") as mock_client: +# v = Vault(role="test-role", jwt="test-jwt") +# v.connect() +# v.kubes_token = ("dynamic", 1, 100) - mock_k8s_renew.start.assert_called() +# mock_k8s_renew.start.assert_called() - with pytest.raises(RuntimeError): - v.worker() +# with pytest.raises(RuntimeError): +# v.worker() - mock_sleep.assert_called() - mock_client().sys.renew_lease.assert_called() - mock_k8s_renew.start.assert_called_once() +# mock_sleep.assert_called() +# mock_client().sys.renew_lease.assert_called() +# mock_k8s_renew.start.assert_called_once() - mock_dynamic_renew.stop() - mock_k8s_renew.stop() +# mock_dynamic_renew.stop() +# mock_k8s_renew.stop() -def test_vault_worker_k8s(mock_vault_workers): - mock_dynamic_renew, mock_k8s_renew = mock_vault_workers +# def test_vault_worker_k8s(mock_vault_workers): +# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - mock_sleep = None +# mock_sleep = None - def except_once(self, **kwargs): - # side effect used to exit the worker loop after one call - if mock_sleep.call_count == 1: - raise hvac.exceptions.VaultError("some error") +# def except_once(self, **kwargs): +# # side effect used to exit the worker loop after one call +# if mock_sleep.call_count == 1: +# raise hvac.exceptions.VaultError("some error") - with patch("gestalt.vault.sleep", side_effect=except_once, - autospec=True) as mock_sleep: - with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - v.connect() - v.kubes_token = ("kubernetes", 1, 100) +# with patch("gestalt.vault.sleep", side_effect=except_once, +# autospec=True) as mock_sleep: +# with patch("gestalt.vault.hvac.Client") as mock_client: +# v = Vault(role="test-role", jwt="test-jwt") +# v.connect() +# v.kubes_token = ("kubernetes", 1, 100) - mock_k8s_renew.start.assert_called() +# mock_k8s_renew.start.assert_called() - with pytest.raises(RuntimeError): - v.worker() +# with pytest.raises(RuntimeError): +# v.worker() - mock_sleep.assert_called() - mock_client().auth.token.renew.assert_called() - mock_k8s_renew.start.assert_called_once() +# mock_sleep.assert_called() +# mock_client().auth.token.renew.assert_called() +# mock_k8s_renew.start.assert_called_once() - mock_dynamic_renew.stop() - mock_k8s_renew.stop() +# mock_dynamic_renew.stop() +# mock_k8s_renew.stop() def test_vault_start_dynamic_lease(mock_vault_workers): From f99b826c1ec02f4a19927e31588b3ad53f590851 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 20:17:10 +0000 Subject: [PATCH 61/73] Updating --- gestalt/vault.py | 40 +---------------------------- tests/test_gestalt.py | 60 ------------------------------------------- 2 files changed, 1 insertion(+), 99 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 1647eb1..4e2987a 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -109,21 +109,6 @@ def connect(self) -> None: except requests.exceptions.ConnectionError: raise RuntimeError("Gestalt Error: Couldn't connect to Vault") - # dynamic_ttl_renew = Thread( - # name="dynamic-token-renew", - # target=self.worker, - # daemon=True, - # args=(self.dynamic_token_queue, ), - # ) # noqa: F841 - - # if self.kubernetes_ttl_renew is None: - # self.kubernetes_ttl_renew = Thread( - # name="kubes-token-renew", - # target=self.worker, - # daemon=True, - # ) - # self.kubernetes_ttl_renew.start() - self._is_connected = True def stop(self) -> None: @@ -224,30 +209,6 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt - # def worker(self) -> None: - # """ - # Worker function to renew lease on expiry - # """ - # try: - # while self._run_worker: - # if self.kubes_token: - # token_type, token_id, token_duration = self.kubes_token - # if token_type == "kubernetes": - # self.vault_client.auth.token.renew(token_id) - # print("kubernetes token for the app has been renewed") - # elif token_type == "dynamic": - # self.vault_client.sys.renew_lease(token_id) - # print("dynamic token for the app has been renewed") - # sleep((token_duration / 3) * 2) # type: ignore - # except hvac.exceptions.InvalidPath: - # raise RuntimeError( - # "Gestalt Error: The lease path or mount is set incorrectly") - # except requests.exceptions.ConnectionError: - # raise RuntimeError( - # "Gestalt Error: Gestalt couldn't connect to Vault") - # except Exception as err: - # raise RuntimeError(f"Gestalt Error: {err}") - @property def scheme(self) -> str: return self._scheme @@ -257,6 +218,7 @@ def _validate_token_expiration(self) -> None: # type: ignore print(f"Token Stored after K8s Login: {self.kubes_token}") if self.kubes_token is not None: expire_time = self.kubes_token[3] + print(f"ExpireTime: {expire_time}") # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 14cd8af..bf8f97f 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -566,66 +566,6 @@ def test_vault_lazy_connect(mock_vault_workers, mock_vault_k8s_auth): mock_client().auth.token.lookup_self.assert_called() -# def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): -# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - -# mock_sleep = None - -# def except_once(self, **kwargs): -# # side effect used to exit the worker loop after one call -# if mock_sleep.call_count == 1: -# raise hvac.exceptions.VaultError("some error") - -# with patch("gestalt.vault.sleep", side_effect=except_once, -# autospec=True) as mock_sleep: -# with patch("gestalt.vault.hvac.Client") as mock_client: -# v = Vault(role="test-role", jwt="test-jwt") -# v.connect() -# v.kubes_token = ("dynamic", 1, 100) - -# mock_k8s_renew.start.assert_called() - -# with pytest.raises(RuntimeError): -# v.worker() - -# mock_sleep.assert_called() -# mock_client().sys.renew_lease.assert_called() -# mock_k8s_renew.start.assert_called_once() - -# mock_dynamic_renew.stop() -# mock_k8s_renew.stop() - - -# def test_vault_worker_k8s(mock_vault_workers): -# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - -# mock_sleep = None - -# def except_once(self, **kwargs): -# # side effect used to exit the worker loop after one call -# if mock_sleep.call_count == 1: -# raise hvac.exceptions.VaultError("some error") - -# with patch("gestalt.vault.sleep", side_effect=except_once, -# autospec=True) as mock_sleep: -# with patch("gestalt.vault.hvac.Client") as mock_client: -# v = Vault(role="test-role", jwt="test-jwt") -# v.connect() -# v.kubes_token = ("kubernetes", 1, 100) - -# mock_k8s_renew.start.assert_called() - -# with pytest.raises(RuntimeError): -# v.worker() - -# mock_sleep.assert_called() -# mock_client().auth.token.renew.assert_called() -# mock_k8s_renew.start.assert_called_once() - -# mock_dynamic_renew.stop() -# mock_k8s_renew.stop() - - def test_vault_start_dynamic_lease(mock_vault_workers): mock_response = { "lease_id": "1", From 6a7b61cfae8211e14c2083e1c3bbb93a167f6920 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 20:24:01 +0000 Subject: [PATCH 62/73] Updating --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index b5094bc..ac14a9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,7 @@ def request(self, *_, **__): "rotation_period": 60, "ttl": 0, "username": "foo", + "expire_time": "2024-08-15T22:04:49.82981496Z" }, "wrap_info": None, "warnings": None, From 7b9fa5b396b6898a67f978e19a19dae35e753586 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 20:51:12 +0000 Subject: [PATCH 63/73] Updating --- tests/test_gestalt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index bf8f97f..470601f 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -557,9 +557,15 @@ def test_set_vault_key(nested_setup): assert secret == "ref+vault://secret/data/testnested#.slack.token" -def test_vault_lazy_connect(mock_vault_workers, mock_vault_k8s_auth): +def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") + v.kubes_token = ( + "kubernetes", + "hvs.CAESICuPyPq_Bp", # Mocked ID value + 10801, # Mocked TTL value in seconds + "2024-08-15T00:00:00Z" # Mocked ISO 8601 expire_time + ) assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From c8719ec76f17b53d03a778a16f38a1b7f7aed760 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 20:57:31 +0000 Subject: [PATCH 64/73] Updating --- tests/test_gestalt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 470601f..514ce4c 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -562,9 +562,9 @@ def test_vault_lazy_connect(mock_vault_k8s_auth): v = Vault(role="test-role", jwt="test-jwt") v.kubes_token = ( "kubernetes", - "hvs.CAESICuPyPq_Bp", # Mocked ID value - 10801, # Mocked TTL value in seconds - "2024-08-15T00:00:00Z" # Mocked ISO 8601 expire_time + "hvs.CAESICuPyPq_Bp", + 10801, + "2024-08-15T22:04:49.82981496Z" ) assert not v._is_connected v.get("foo", "foo", ".foo") From 5a7d8b0f0fe70406954159261da722f5a2857c31 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:17:14 +0000 Subject: [PATCH 65/73] Updating --- tests/test_gestalt.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 514ce4c..95a2e78 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -559,13 +559,8 @@ def test_set_vault_key(nested_setup): def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: + mock_client.auth.token.lookup_self = lambda: {"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}} v = Vault(role="test-role", jwt="test-jwt") - v.kubes_token = ( - "kubernetes", - "hvs.CAESICuPyPq_Bp", - 10801, - "2024-08-15T22:04:49.82981496Z" - ) assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From 958c6396ae4c000f14588ac0a1e534f4d272da21 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:22:58 +0000 Subject: [PATCH 66/73] Updating --- tests/test_gestalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 95a2e78..62d130a 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -559,8 +559,8 @@ def test_set_vault_key(nested_setup): def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: - mock_client.auth.token.lookup_self = lambda: {"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}} v = Vault(role="test-role", jwt="test-jwt") + v.vault_client.auth.token.lookup_self = lambda: {"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}} assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From 38175ebd78ebc036b18298b19deb314e99e2aaa1 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:28:08 +0000 Subject: [PATCH 67/73] Updating --- tests/test_gestalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 62d130a..dfee0d0 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -560,7 +560,7 @@ def test_set_vault_key(nested_setup): def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") - v.vault_client.auth.token.lookup_self = lambda: {"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}} + v.vault_client.auth.token.lookup_self = MagicMock(return_value={"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}}) assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From e2f342cf9814116f202a3ac6423ff5fade27cb00 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:31:59 +0000 Subject: [PATCH 68/73] Updating --- tests/test_gestalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index dfee0d0..2fda724 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -560,7 +560,7 @@ def test_set_vault_key(nested_setup): def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") - v.vault_client.auth.token.lookup_self = MagicMock(return_value={"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}}) + v.vault_client.auth.token.lookup_self = Mock(return_value={"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}}) assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From 1849f57a76aa406157b8593252225b610fbf611e Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:34:03 +0000 Subject: [PATCH 69/73] Updating --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 4e2987a..be35b3f 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -42,7 +42,7 @@ def __init__( self._scheme: str = scheme self._run_worker = True self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.kubes_token: Optional[Tuple[str, str, str]] = None + self.kubes_token: Optional[Tuple[str, str, str, str]] = None self._vault_client: Optional[hvac.Client] = None self._secret_expiry_times: Dict[str, datetime] = dict() From 4aedae095f7e76327d7748de68c4210394b3f0d2 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:37:00 +0000 Subject: [PATCH 70/73] Updating --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index be35b3f..415fccd 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -42,7 +42,7 @@ def __init__( self._scheme: str = scheme self._run_worker = True self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.kubes_token: Optional[Tuple[str, str, str, str]] = None + self.kubes_token: Optional[Tuple[str, str, str, datetime]] = None self._vault_client: Optional[hvac.Client] = None self._secret_expiry_times: Dict[str, datetime] = dict() @@ -213,7 +213,7 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], def scheme(self) -> str: return self._scheme - def _validate_token_expiration(self) -> None: # type: ignore + def _validate_token_expiration(self) -> None: # token_details = self.vault_client.auth.token.lookup_self() print(f"Token Stored after K8s Login: {self.kubes_token}") if self.kubes_token is not None: From 3ce7d3a15e3cab907f31257e13c1fff9cb9d7b66 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Thu, 15 Aug 2024 21:39:57 +0000 Subject: [PATCH 71/73] Updating --- tests/test_gestalt.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 2fda724..00ba55c 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -560,7 +560,14 @@ def test_set_vault_key(nested_setup): def test_vault_lazy_connect(mock_vault_k8s_auth): with patch("gestalt.vault.hvac.Client") as mock_client: v = Vault(role="test-role", jwt="test-jwt") - v.vault_client.auth.token.lookup_self = Mock(return_value={"data": {"id": "foo", "ttl": "foo", "expire_time": "2024-08-15T22:04:49.82981496Z"}}) + v.vault_client.auth.token.lookup_self = Mock( + return_value={ + "data": { + "id": "foo", + "ttl": "foo", + "expire_time": "2024-08-15T22:04:49.82981496Z" + } + }) assert not v._is_connected v.get("foo", "foo", ".foo") assert v._is_connected From 7553fdf7a979a28fd50156b064a8b5d168237b1c Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Fri, 16 Aug 2024 13:42:53 +0000 Subject: [PATCH 72/73] Updating --- gestalt/vault.py | 14 ++++---------- setup.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 415fccd..653de2a 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -1,8 +1,6 @@ import os from datetime import datetime, timedelta, timezone from queue import Queue -from threading import Thread, Event -from time import sleep from typing import Any, Dict, List, Optional, Tuple, Union import hvac # type: ignore @@ -58,7 +56,6 @@ def __init__( self.delay = delay self.tries = tries - # self.kubernetes_ttl_renew: Optional[Thread] = None @property def vault_client(self) -> hvac.Client: @@ -95,6 +92,7 @@ def connect(self) -> None: ) if token is not None: + print("Kubernetes login successful") kubes_token = ( "kubernetes", token["data"]["id"], @@ -102,7 +100,6 @@ def connect(self) -> None: token["data"]['expire_time'], ) self.kubes_token = kubes_token - print(f"Token LookUp after K8s Login: {self.kubes_token}") except hvac.exceptions.InvalidPath: raise RuntimeError( "Gestalt Error: Kubernetes auth couldn't be performed") @@ -214,11 +211,8 @@ def scheme(self) -> str: return self._scheme def _validate_token_expiration(self) -> None: - # token_details = self.vault_client.auth.token.lookup_self() - print(f"Token Stored after K8s Login: {self.kubes_token}") if self.kubes_token is not None: expire_time = self.kubes_token[3] - print(f"ExpireTime: {expire_time}") # Use isoparse to correctly parse the datetime string expire_time = isoparse(expire_time) @@ -233,11 +227,11 @@ def _validate_token_expiration(self) -> None: delta_time = (expire_time - current_time).total_seconds() / 3600 if delta_time < EXPIRATION_THRESHOLD_HOURS: - print(f"Re-auth with vault") + print(f"Re-authenticating with vault") self.connect() else: - print(f"Token still valid for: {delta_time} days") + print(f"Token still valid for: {delta_time} hours") else: print( - f"Can't reconnect cause token: {self.kubes_token}, expire_time is None" + f"Can't reconnect, token information: {self.kubes_token}, not valid" ) diff --git a/setup.py b/setup.py index 0193268..a998403 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def readme(): setup( name="gestalt-cfg", - version="3.4.2beta11", + version="3.4.2", description="A sensible configuration library for Python", long_description=readme(), long_description_content_type="text/markdown", From f4ca1392d43ed9aa94ebdcb92991e652e9c0ece0 Mon Sep 17 00:00:00 2001 From: Jose Boucourt Date: Fri, 16 Aug 2024 13:53:14 +0000 Subject: [PATCH 73/73] Updating --- tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ac14a9e..533f349 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,9 +80,7 @@ def nested_setup(): def mock_vault_workers(): mock_dynamic_renew = Mock() mock_k8s_renew = Mock() - with patch("gestalt.vault.Thread", - side_effect=[mock_dynamic_renew, mock_k8s_renew]): - yield (mock_dynamic_renew, mock_k8s_renew) + return (mock_dynamic_renew, mock_k8s_renew) @pytest.fixture