actions/sync_gws_accounts.py: handle more exceptions (#120)

* actions/sync_gws_accounts.py: handle more exceptions

* <bot> update dependencies*.log files(s)

* fix tests

* fix

---------

Co-authored-by: github-actions <[email protected]>

actions/sync_gws_accounts.py

@@ -19,15 +19,14 @@
 import string
 import time
 
-from google.auth.exceptions import RefreshError
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
-from googleapiclient.errors import HttpError
 
 from krs.ldap import LDAP
 from krs.token import get_rest_client
 from krs.users import list_users
 
+from actions.util import retry_execute
 
 logger = logging.getLogger('sync_gws_accounts')
 SHADOWEXPIRE_DAYS_REMAINING_CUTOFF_FOR_ELIGIBILITY = -365 * 2
@@ -45,7 +44,7 @@ def get_gws_accounts(gws_users_client):
     user_list = []
     request = gws_users_client.list(customer='my_customer', maxResults=500)
     while request is not None:
-        response = request.execute()
+        response = retry_execute(request)
         user_list.extend(response.get('users', []))
         request = gws_users_client.list_next(request, response)
     return dict((u['primaryEmail'].split('@')[0], u) for u in user_list)
@@ -63,23 +62,9 @@ def add_canonical_alias(gws_users_client, kc_attrs):
                      f'doesn\'t have attribute canonical_email')
         return
     logger.info(f'adding to {kc_attrs["username"]} alias {kc_attrs["attributes"]["canonical_email"]}')
-    attempt = saved_exception = None  # keep code linter happy
-    for attempt in range(1, 8):
-        time.sleep(2 ** attempt)
-        try:
-            gws_users_client.aliases().insert(
-                userKey=f'{kc_attrs["username"]}@icecube.wisc.edu',
-                body={'alias': kc_attrs["attributes"]["canonical_email"]}).execute()
-            break
-        except HttpError as e:
-            if e.status_code == 412:  # precondition failed (user creation not complete?)
-                saved_exception = e
-                continue
-            else:
-                raise
-    else:
-        logger.error(f'giving up on alias creation after {attempt} attempts')
-        logger.error(f'saved exception: {saved_exception}')
+    retry_execute(gws_users_client.aliases().insert(
+        userKey=f'{kc_attrs["username"]}@icecube.wisc.edu',
+        body={'alias': kc_attrs["attributes"]["canonical_email"]}))
 
 
 def set_canonical_sendas(gws_creds, kc_attrs):
@@ -108,24 +93,7 @@ def set_canonical_sendas(gws_creds, kc_attrs):
         "isDefault": True,
         "treatAsAlias": True,
     }
-    attempt = saved_exception = None  # keep code linter happy
-    for attempt in range(1, 9):
-        time.sleep(2 ** attempt)
-        try:
-            sendas.create(userId='me', body=body).execute()
-            break
-        except RefreshError as e:  # insert into group allowing gmail probably still being processed
-            saved_exception = e
-            continue
-        except HttpError as e:
-            if e.status_code == 400:  # bad request: the alias is probably still being created
-                saved_exception = e
-                continue
-            else:
-                raise
-    else:
-        logger.error(f'giving up on sendAs setting after {attempt} attempts')
-        logger.error(f'saved exception: {saved_exception}')
+    retry_execute(sendas.create(userId='me', body=body))
 
 
 def is_eligible(account_attrs, shadow_expire):
@@ -192,21 +160,24 @@ def create_missing_eligible_accounts(gws_users_client, gws_accounts, ldap_accoun
                              'givenName': attrs['firstName'],
                              'familyName': attrs['lastName']},
                          'password': ''.join(random.choices(string.ascii_letters, k=16))}
-            gws_users_client.insert(body=user_body).execute()
+            retry_execute(gws_users_client.insert(body=user_body))
             created_usernames.append(username)
             if attrs.get('attributes', {}).get('canonical_email'):
                 add_canonical_alias(gws_users_client, attrs)
                 set_canonical_sendas(gws_creds, attrs)
         else:
-            logger.debug(f'ignoring ineligible user {username}')
+            logger.debug(f'ignoring existing or ineligible user {username}')
     return created_usernames
 
 
 async def sync_gws_accounts(gws_users_client, ldap_client, keycloak_client,
                             gws_creds, dryrun=False):
+    """This function looks like this to enable unit testing without needing to simulate
+    clients for Google API, LDAP, and KeyCloak.
+    """
     kc_accounts = await list_users(rest_client=keycloak_client)
     gws_accounts = get_gws_accounts(gws_users_client)
-    ldap_accounts = ldap_client.list_users(attrs=['shadowExpire'])
+    ldap_accounts = ldap_client.list_users(attrs=['shadowExpire'])  # noqa pycharm bug?
 
     create_missing_eligible_accounts(gws_users_client, gws_accounts, ldap_accounts,
                                      kc_accounts, gws_creds, dryrun)

actions/sync_gws_mailing_lists.py

@@ -44,6 +44,8 @@
 from krs.users import user_info
 from krs.email import send_email
 
+from actions.util import retry_execute
+
 logger = logging.getLogger('sync_gws_mailing_lists')
 
 MESSAGE_FOOTER = "\n\nThis message was generated by the sync_gws_mailing_lists robot."
@@ -87,7 +89,7 @@ def get_gws_group_members(group_email, gws_members_client) -> list:
     ret = []
     req = gws_members_client.list(groupKey=group_email)
     while req is not None:
-        res = req.execute()
+        res = retry_execute(req)
         if 'members' in res:
             ret.extend(res['members'])
         req = gws_members_client.list_next(req, res)
@@ -108,7 +110,7 @@ async def get_gws_members_from_keycloak_group(group_path, role, keycloak_client)
             # address. This doesn't make sense and is not allowed.
             preferred = None
         if preferred:
-            # Preferred addresses are controlled by users so we need to do some sanitizing
+            # Preferred addresses are controlled by users, so we need to do some sanitizing
             preferred = preferred.lower()
             # If a user has a preferred mailing list email, also add their canonical
             # (IceCube) email as a no-mail member, since they may not be able to log
@@ -146,7 +148,7 @@ async def sync_kc_group_to_gws(kc_group, group_email, keycloak_client, gws_membe
         if email not in actual_membership:
             logger.info(f"Inserting into {group_email} {body} (dryrun={dryrun})")
             if not dryrun:
-                gws_members_client.insert(groupKey=group_email, body=body).execute()
+                retry_execute(gws_members_client.insert(groupKey=group_email, body=body))
                 if send_notifications:
                     send_email(email, f"You have been subscribed to {group_email}",
                                SUBSCRIPTION_MESSAGE.format(
@@ -160,8 +162,8 @@ async def sync_kc_group_to_gws(kc_group, group_email, keycloak_client, gws_membe
         elif body['role'] != actual_membership[email]['role']:
             logger.info(f"Patching in {group_email} role of {email} to {body['role']} (dryrun={dryrun})")
             if not dryrun:
-                gws_members_client.patch(groupKey=group_email, memberKey=email,
-                                         body={'email': email, 'role': body['role']}).execute()
+                retry_execute(gws_members_client.patch(groupKey=group_email, memberKey=email,
+                                                       body={'email': email, 'role': body['role']}))
                 if send_notifications:
                     send_email(email, f"Your member role in {group_email} has changed",
                                ROLE_CHANGE_MESSAGE.format(
@@ -175,7 +177,7 @@ async def sync_kc_group_to_gws(kc_group, group_email, keycloak_client, gws_membe
         if actual_membership[email]['role'] != 'OWNER':
             logger.info(f"Removing from {group_email} {email} (dryrun={dryrun})")
             if not dryrun:
-                gws_members_client.delete(groupKey=group_email, memberKey=email).execute()
+                retry_execute(gws_members_client.delete(groupKey=group_email, memberKey=email))
                 if send_notifications:
                     send_email(email, f"You have been unsubscribed from {group_email}",
                                UNSUBSCRIPTION_MESSAGE.format(
@@ -207,7 +209,7 @@ async def sync_gws_mailing_lists(gws_members_client, gws_groups_client, keycloak
         send_notifications (bool): whether to send email notifications
         dryrun (bool): Perform a mock run with no changes made
     """
-    res = gws_groups_client.list(customer='my_customer').execute()
+    res = retry_execute(gws_groups_client.list(customer='my_customer'))
     gws_group_emails = [g['email'] for g in res.get('groups', [])]
 
     kc_ml_root_group = await group_info('/mail', rest_client=keycloak_client)

actions/util.py

@@ -1,7 +1,10 @@
 import pathlib
 import subprocess
 import tempfile
+import time
 
+from google.auth.exceptions import RefreshError
+from googleapiclient.errors import HttpError
 
 QUOTAS = {
     # production dirs
@@ -36,6 +39,56 @@
 ]
 
 
+class RetryError(Exception):
+    def __init__(self, sleep_time_history, exception_history):
+        self.sleep_time_history = sleep_time_history
+        self.exception_history = exception_history
+        super().__init__()
+
+    def __repr__(self):
+        return (f"RetryError(sleep_time_history={self.sleep_time_history}, "
+                f"exception_history={self.exception_history})")
+
+
+def retry_execute(request, max_attempts=8):
+    """Retry calling request.execute() with exponential backoff.
+
+    Args:
+        request: object with .execute() method
+        max_attempts: maximum number of re-attempts
+
+    Returns:
+        Return value of request.execute()
+
+    Raises:
+        RetryError if reached max_attempts
+    """
+    if not hasattr(request, 'execute'):
+        raise AttributeError(f"{type(request)} object has no attribute 'execute'")
+    sleep_time_history = []
+    exception_history = []
+    for attempt in range(max_attempts):
+        sleep_time = 2 ** attempt - 1
+        time.sleep(sleep_time)
+        sleep_time_history.append(sleep_time)
+        try:
+            return request.execute()
+        except RefreshError as e:
+            # Refreshing the credentials' access token failed. This can be transient, so retry.
+            exception_history.append(e)
+            continue
+        except HttpError as e:
+            exception_history.append(e)
+            if e.status_code in (400, 412):
+                # Both 400 (bad request) and 412 (precondition failed) could be caused
+                # by a prerequisite resource not being ready, so retrying makes sense.
+                continue
+            else:
+                raise RetryError(sleep_time_history, exception_history)
+    else:
+        raise RetryError(sleep_time_history, exception_history)
+
+
 def ssh(host, *args):
     """Run command on remote machine via ssh."""
     cmd = ['ssh'] + ssh_opts + [f'{host}'] + list(args)

dependencies-from-Dockerfile.log

@@ -1,15 +1,15 @@
-aio-pika==9.3.1
-aiormq==6.7.7
+aio-pika==9.4.0
+aiormq==6.8.0
 asyncache==0.3.1
 cachetools==5.3.2
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
 cryptography==41.0.7
-dnspython==2.4.2
+dnspython==2.5.0
 google-api-core==2.15.0
-google-api-python-client==2.112.0
-google-auth==2.26.1
+google-api-python-client==2.114.0
+google-auth==2.26.2
 google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.2.0
 googleapis-common-protos==1.62.0
@@ -19,8 +19,8 @@ ldap3==2.9.1
 motor==3.3.2
 multidict==6.0.4
 oauthlib==3.2.2
-pamqp==3.2.1
-protobuf==4.25.1
+pamqp==3.3.0
+protobuf==4.25.2
 pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
@@ -35,7 +35,7 @@ requests-oauthlib==1.3.1
 rsa==4.9
 tornado==6.4
 typing_extensions==4.9.0
-Unidecode==1.3.7
+Unidecode==1.3.8
 uritemplate==4.1.1
 urllib3==2.1.0
 wipac-dev-tools==1.8.2