Sync 2025-01-07

README.md

@@ -88,7 +88,7 @@ _The current installation process does not work on Ubuntu 22.04_
 Run the following commands:
 
 ```
-sudo apt update; sudo apt install python3-pip sshpass git python3-virtualenv python3.9
+sudo apt update; sudo apt install python3-pip sshpass git python3.9-venv python3.9-dev python3.9 -y
 ```
 
 #### Pulling optscale-deploy scripts
@@ -125,13 +125,11 @@ ansible-playbook -e "ansible_ssh_user=<user>" -k -K -i "<ip address>," ansible/k
 ```
 
 where `<user>` - actual username; `<ip address>` - host ip address,
-ip address should be private address of the machine, you can check it with
+ip address should be private address of the machine, you can check it with the command `ip a`.
 
-```
-ip a
-```
+**Note:** do not use `127.0.0.1` or `localhost` as the hostname. Instead, prefer providing the server's hostname (check with the command `hostname`) and make sure it is resolveable from host that the Ansible Playbooks ran from (if needed, add to the ``/etc/hosts`` files).
 
-If your deployment server is the service-host server, add `"ansible_connection=local"` to the ansible command.
+If your deployment server is the service-host server, add `-e "ansible_connection=local"` to the ansible command.
 
 #### Creating user overlay
 

auth/auth_server/handlers/v2/signin.py

@@ -66,7 +66,6 @@ async def post(self, **url_params):
         data = self._request_body()
         data.update(url_params)
         data.update({'ip': self.get_ip_addr()})
-        data.update({'redirect_uri': self.request.headers.get('Origin')})
         await self._validate_params(**data)
         res = await run_task(self.controller.signin, **data)
         self.set_status(201)

auth/auth_server/handlers/v2/users.py

@@ -101,7 +101,7 @@ async def patch(self, user_id, **kwargs):
             Required permission: EDIT_USER_INFO or ACTIVATE_USER or
                 RESET_USER_PASSWORD
         parameters:
-        -   name: id
+        -   name: user_id
             in: path
             description: ID of user to modify
             required: true

bumiworker/bumiworker/modules/archive/inactive_users.py

@@ -1,7 +1,8 @@
 import logging
 
 from bumiworker.bumiworker.consts import ArchiveReason
-from bumiworker.bumiworker.modules.inactive_users_base import ArchiveInactiveUsersBase
+from bumiworker.bumiworker.modules.inactive_users_base import (
+    ArchiveInactiveUsersBase)
 from bumiworker.bumiworker.modules.recommendations.inactive_users import (
     InactiveUsers as InactiveUsersRecommendation)
 
@@ -12,6 +13,7 @@
 class InactiveUsers(ArchiveInactiveUsersBase, InactiveUsersRecommendation):
     SUPPORTED_CLOUD_TYPES = [
         'aws_cnr',
+        'gcp_cnr',
         'nebius'
     ]
 

bumiworker/bumiworker/modules/recommendations/abandoned_images.py

@@ -9,7 +9,7 @@
 DEFAULT_DAYS_THRESHOLD = 7
 BULK_SIZE = 1000
 SUPPORTED_CLOUD_TYPES = [
-    'nebius'
+    'gcp_cnr', 'nebius'
 ]
 
 
@@ -90,7 +90,7 @@ def _get(self):
                             'cloud_account_id': image['cloud_account_id'],
                             'cloud_account_name': account['name'],
                             'cloud_type': account['type'],
-                            'folder_id': image['meta']['folder_id'],
+                            'folder_id': image['meta'].get('folder_id'),
                             'last_used': last_used_map.get(
                                 image['cloud_resource_id'], 0),
                             'first_seen': image['first_seen'],

bumiworker/bumiworker/modules/recommendations/inactive_users.py

@@ -5,13 +5,16 @@
 
 
 DEFAULT_DAYS_THRESHOLD = 90
+INTERVAL = 300
+GCP_METRIC_NAME = 'iam.googleapis.com/service_account/authn_events_count'
 MSEC_IN_SEC = 1000
 LOG = logging.getLogger(__name__)
 
 
 class InactiveUsers(InactiveUsersBase):
     SUPPORTED_CLOUD_TYPES = [
         'aws_cnr',
+        'gcp_cnr',
         'nebius'
     ]
 
@@ -28,6 +31,8 @@ def list_users(self, cloud_adapter):
             result = []
             for folder_id in cloud_adapter.folders:
                 result.extend(cloud_adapter.service_accounts_list(folder_id))
+        elif cloud_type == 'gcp_cnr':
+            result = cloud_adapter.service_accounts_list()
         else:
             result = cloud_adapter.list_users()
         return result
@@ -56,6 +61,33 @@ def is_outdated(last_used_):
                 'last_used': int(last_used.timestamp())
             }
 
+    def handle_gcp_user(self, user, now, cloud_adapter, days_threshold):
+        last_used = 0
+        service_account_id = user.unique_id
+        inactive_threshold = self._get_inactive_threshold(days_threshold)
+        end_date = now
+        # there is no created_at for service account, so extend dates range to
+        # try to get last_used
+        start_date = now - inactive_threshold - inactive_threshold
+        service_account_usage = cloud_adapter.get_metric(
+            GCP_METRIC_NAME, [service_account_id], INTERVAL, start_date,
+            end_date, id_field='unique_id'
+        )
+        used_dates = [
+            point.interval.end_time for data in service_account_usage
+            for point in data.points if point.value.double_value != 0
+        ]
+        if used_dates:
+            last_used_dt = max(used_dates)
+            last_used = int(last_used_dt.timestamp())
+            if not self._is_outdated(now, last_used_dt, inactive_threshold):
+                return
+        return {
+            'user_name': user.display_name,
+            'user_id': service_account_id,
+            'last_used': last_used
+        }
+
     def handle_nebius_user(self, user, now, cloud_adapter, days_threshold):
         service_account_id = user['id']
         folder_id = user['folderId']
@@ -99,12 +131,13 @@ def handle_nebius_user(self, user, now, cloud_adapter, days_threshold):
 
     def handle_user(self, user, now, cloud_adapter, days_threshold):
         cloud_type = cloud_adapter.config['type']
-        if cloud_type == 'aws_cnr':
-            return self.handle_aws_user(user, now, cloud_adapter,
-                                        days_threshold)
-        else:
-            return self.handle_nebius_user(user, now, cloud_adapter,
-                                           days_threshold)
+        cloud_func_map = {
+            "aws_cnr": self.handle_aws_user,
+            "gcp_cnr": self.handle_gcp_user,
+            "nebius": self.handle_nebius_user,
+        }
+        func = cloud_func_map[cloud_type]
+        return func(user, now, cloud_adapter, days_threshold)
 
 
 def main(organization_id, config_client, created_at, **kwargs):

bumiworker/bumiworker/modules/recommendations/insecure_security_groups.py

@@ -159,8 +159,7 @@ def _get_aws_insecure(self, config, resources, excluded_pools,
             if s_groups is None:
                 continue
             security_groups_map = region_sg_map[region]
-            for group in s_groups:
-                group_id = group['GroupId']
+            for group_id in s_groups:
                 instances = security_groups_map.get(group_id, [])
                 instances.append(instance)
                 security_groups_map[group_id] = instances

bumiworker/bumiworker/modules/recommendations/s3_public_buckets.py

@@ -5,6 +5,7 @@
 
 SUPPORTED_CLOUD_TYPES = [
     'aws_cnr',
+    'gcp_cnr',
     'nebius'
 ]
 

bumiworker/bumiworker/modules/service/saving_spike_notification.py

@@ -64,6 +64,7 @@ def _get(self):
             modules_data.sort(key=lambda x: x['saving'], reverse=True)
             task = {
                 "object_id": self.organization_id,
+                "organization_id": self.organization_id,
                 "object_type": "organization",
                 "action": "saving_spike",
                 "meta": {"previous_total": previous_total,

diworker/diworker/importers/gcp.py

@@ -27,7 +27,7 @@ def detect_period_start(self):
             if last_exp_date:
                 self.period_start = last_exp_date.replace(
                     hour=0, minute=0, second=0, microsecond=0) - timedelta(
-                    days=1)
+                    days=3)
         if not self.period_start:
             super().detect_period_start()
 

docker_images/cleanmongodb/clean-mongo-db.py

@@ -29,6 +29,11 @@ def __init__(self):
             # linked to cloud_account_id
             self.mongo_client.restapi.raw_expenses: ROWS_LIMIT,
             self.mongo_client.restapi.resources: ROWS_LIMIT,
+            # linked to organization_id
+            self.mongo_client.restapi.archived_recommendations: ROWS_LIMIT,
+            self.mongo_client.restapi.checklists: ROWS_LIMIT,
+            self.mongo_client.restapi.webhook_observer: ROWS_LIMIT,
+            self.mongo_client.restapi.webhook_logs: ROWS_LIMIT,
             # linked to run_id
             self.mongo_client.arcee.console: ROWS_LIMIT,
             self.mongo_client.arcee.log: ROWS_LIMIT,
@@ -318,14 +323,17 @@ def split_chunk_by_files(self, chunk, available_rows_count, filename,
         return result
 
     def _delete_by_organization(self, org_id, token, infra_token):
-        if not token:
-            self.update_cleaned_at(organization_id=org_id)
-            return
+        restapi_collections = [
+            self.mongo_client.restapi.archived_recommendations,
+            self.mongo_client.restapi.checklists,
+            # delete clusters resources
+            self.mongo_client.restapi.resources,
+            self.mongo_client.restapi.webhook_observer,
+            self.mongo_client.restapi.webhook_logs
+        ]
         keeper_collections = [
             self.mongo_client.keeper.event
         ]
-        # delete clusters resources
-        restapi_collections = [self.mongo_client.restapi.resources]
         # delete ml objects
         arcee_collections = [self.mongo_client.arcee.dataset,
                              self.mongo_client.arcee.metric,
@@ -342,6 +350,10 @@ def _delete_by_organization(self, org_id, token, infra_token):
         for collection in restapi_collections:
             self.limits[collection] = self.delete_in_chunks(
                 collection, 'organization_id', org_id)
+
+        if not token:
+            self.update_cleaned_at(organization_id=org_id)
+            return
         for collection in arcee_collections:
             self.limits[collection] = self.delete_in_chunks(
                 collection, 'token', token)