Merge pull request #27 from softwareone-platform/feat/MPT-4914-add-li…

…st-org-datasources-endpoint

feat: Add endpoints to list an organization's cloud accounts (MPT-4914, MPT-4915)

app/api_clients/api_modifier.py

@@ -1,8 +1,11 @@
 import httpx
 
 from app import settings
-from app.api_clients.base import APIClientError, BaseAPIClient, BearerAuth
-from app.utils import get_api_modifier_jwt_token
+from app.api_clients.base import (
+    APIClientError,
+    APIModifierJWTTokenAuth,
+    BaseAPIClient,
+)
 
 
 class APIModifierClientError(APIClientError):
@@ -11,9 +14,7 @@ class APIModifierClientError(APIClientError):
 
 class APIModifierClient(BaseAPIClient):
     base_url = settings.api_modifier_base_url
-
-    def get_auth(self) -> BearerAuth:
-        return BearerAuth(get_api_modifier_jwt_token())
+    default_auth = APIModifierJWTTokenAuth()
 
     async def create_user(self, email: str, display_name: str, password: str) -> httpx.Response:
         response = await self.httpx_client.post(

app/api_clients/base.py

@@ -8,24 +8,10 @@
 
 import httpx
 
-logger = logging.getLogger(__name__)
-
-
-class HeaderAuth(httpx.Auth):
-    def __init__(self, header_name: str, header_value: str):
-        self.header_name = header_name
-        self.header_value = header_value
-
-    def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
-        if self.header_name not in request.headers:  # pragma: no cover
-            request.headers[self.header_name] = self.header_value
-
-        yield request
+from app import settings
+from app.utils import get_api_modifier_jwt_token
 
-
-class BearerAuth(HeaderAuth):
-    def __init__(self, token: str):
-        super().__init__("Authorization", f"Bearer {token}")
+logger = logging.getLogger(__name__)
 
 
 class APIClientError(Exception):
@@ -41,9 +27,29 @@ def __init__(self, message: str):
         super().__init__(f"{self.client_name} API client error: {message}")
 
 
+class APIModifierJWTTokenAuth(httpx.Auth):
+    def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
+        # NOTE: Needs to be re-generated for each request as it exipres after a certain time
+        jwt_token = get_api_modifier_jwt_token()
+
+        request.headers["Authorization"] = f"Bearer {jwt_token}"
+
+        yield request
+
+
+class OptscaleClusterSecretAuth(httpx.Auth):
+    def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
+        request.headers["Secret"] = settings.opt_cluster_secret
+
+        yield request
+
+
+HEADERS_TO_REDACT_IN_LOGS = {"authorization", "secret"}
+
+
 class BaseAPIClient(abc.ABC):
     base_url: ClassVar[str]
-    auth: ClassVar[httpx.Auth | None] = None
+    default_auth: ClassVar[httpx.Auth | None] = None
 
     _clients_by_name: ClassVar[dict[str, type[Self]]] = {}
 
@@ -62,8 +68,8 @@ def get_clients_by_name(cls) -> dict[str, type[Self]]:
 
     def __init__(self):
         self.httpx_client = httpx.AsyncClient(
-            base_url=self.get_base_url(),
-            auth=self.get_auth(),
+            base_url=self.base_url,
+            auth=self.default_auth,
             event_hooks={"request": [self._log_request], "response": [self._log_response]},
         )
 
@@ -85,11 +91,8 @@ async def __aexit__(
 
     def _get_headers_to_log(self, headers: httpx.Headers) -> dict[str, str]:
         return {
-            key: value
+            key: (value if key.lower() not in HEADERS_TO_REDACT_IN_LOGS else "REDACTED")
             for key, value in headers.items()
-            if not (
-                isinstance(self.auth, HeaderAuth) and key.lower() == self.auth.header_name.lower()
-            )
         }
 
     async def _log_request(self, request: httpx.Request) -> None:
@@ -140,13 +143,3 @@ async def _log_response(self, response: httpx.Response) -> None:
             response.status_code,
             extra=structured_log_data,
         )
-
-    # ===========================================
-    # Methods for dynamic class fields evaliation
-    # ===========================================
-
-    def get_auth(self) -> httpx.Auth | None:
-        return self.auth
-
-    def get_base_url(self) -> str:
-        return self.base_url

app/api_clients/optscale.py

@@ -1,7 +1,9 @@
+from uuid import UUID
+
 import httpx
 
 from app import settings
-from app.api_clients.base import APIClientError, BaseAPIClient
+from app.api_clients.base import APIClientError, BaseAPIClient, OptscaleClusterSecretAuth
 
 
 class OptscaleClientError(APIClientError):
@@ -10,12 +12,36 @@ class OptscaleClientError(APIClientError):
 
 class OptscaleClient(BaseAPIClient):
     base_url = settings.opt_api_base_url
+    default_auth = OptscaleClusterSecretAuth()
 
     async def reset_password(self, email: str) -> httpx.Response:
         response = await self.httpx_client.post(
             "/restore_password",
             json={"email": email},
+            auth=None,
         )
 
         response.raise_for_status()
         return response
+
+    async def fetch_cloud_accounts_for_organization(
+        self, organization_id: UUID | str
+    ) -> httpx.Response:
+        response = await self.httpx_client.get(
+            f"/organizations/{organization_id}/cloud_accounts",
+            params={
+                "details": "true",
+            },
+        )
+        response.raise_for_status()
+        return response
+
+    async def fetch_cloud_account_by_id(self, cloud_account_id: UUID | str) -> httpx.Response:
+        response = await self.httpx_client.get(
+            f"/cloud_accounts/{cloud_account_id}",
+            params={
+                "details": "true",
+            },
+        )
+        response.raise_for_status()
+        return response

app/api_clients/optscale_auth.py

@@ -1,7 +1,7 @@
 import httpx
 
 from app import settings
-from app.api_clients.base import APIClientError, BaseAPIClient, HeaderAuth
+from app.api_clients.base import APIClientError, BaseAPIClient, OptscaleClusterSecretAuth
 
 
 class OptscaleAuthClientError(APIClientError):
@@ -16,7 +16,7 @@ def __init__(self, email: str):
 
 class OptscaleAuthClient(BaseAPIClient):
     base_url = settings.opt_auth_base_url
-    auth = HeaderAuth("Secret", settings.opt_cluster_secret)
+    default_auth = OptscaleClusterSecretAuth()
 
     async def get_existing_user_info(self, email: str) -> httpx.Response:
         response = await self.httpx_client.get(

app/enums.py

@@ -12,3 +12,16 @@ class EntitlementStatus(str, enum.Enum):
     NEW = "new"
     ACTIVE = "active"
     TERMINATED = "terminated"
+
+
+@enum.unique
+class CloudAccountType(str, enum.Enum):
+    AWS_CNR = "aws_cnr"
+    AZURE_CNR = "azure_cnr"
+    AZURE_TENANT = "azure_tenant"
+    GCP_CNR = "gcp_cnr"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def _missing_(cls, value):
+        return cls.UNKNOWN

app/routers/organizations.py

@@ -1,16 +1,19 @@
+from typing import Annotated
 from uuid import UUID
 
 import svcs
-from fastapi import APIRouter, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi_pagination.limit_offset import LimitOffsetPage
 
 from app.api_clients import APIModifierClient
+from app.api_clients.optscale import OptscaleClient
 from app.auth import CurrentSystem
 from app.db.handlers import NotFoundError
 from app.db.models import Organization
+from app.enums import CloudAccountType
 from app.pagination import paginate
 from app.repositories import OrganizationRepository
-from app.schemas import OrganizationCreate, OrganizationRead, from_orm
+from app.schemas import CloudAccountRead, OrganizationCreate, OrganizationRead, from_orm
 from app.utils import wrap_http_error_in_502
 
 router = APIRouter()
@@ -70,13 +73,90 @@ async def create_organization(
         return from_orm(OrganizationRead, db_organization)
 
 
-@router.get("/{id}", response_model=OrganizationRead)
-async def get_organization_by_id(id: UUID, organization_repo: OrganizationRepository):
+async def fetch_organization_or_404(
+    organization_id: UUID, organization_repo: OrganizationRepository
+) -> Organization:
     try:
-        db_organization = await organization_repo.get(id=id)
-        return from_orm(OrganizationRead, db_organization)
+        return await organization_repo.get(id=organization_id)
     except NotFoundError as e:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=str(e),
         ) from e
+
+
+@router.get("/{organization_id}", response_model=OrganizationRead)
+async def get_organization_by_id(
+    organization: Annotated[Organization, Depends(fetch_organization_or_404)],
+):
+    return from_orm(OrganizationRead, organization)
+
+
+@router.get("/{organization_id}/cloud-accounts", response_model=list[CloudAccountRead])
+async def get_cloud_accounts_by_organization_id(
+    organization: Annotated[Organization, Depends(fetch_organization_or_404)],
+    services: svcs.fastapi.DepContainer,
+):
+    if organization.organization_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=(
+                f"Organization {organization.name} has no associated "
+                "FinOps for Cloud organization"
+            ),
+        )
+
+    optscale_client = await services.aget(OptscaleClient)
+
+    async with wrap_http_error_in_502(
+        f"Error fetching cloud accounts for organization {organization.name}"
+    ):
+        response = await optscale_client.fetch_cloud_accounts_for_organization(
+            organization_id=organization.organization_id
+        )
+
+    cloud_accounts = response.json()["cloud_accounts"]
+
+    return [
+        CloudAccountRead(
+            id=acc["id"],
+            organization_id=organization.id,
+            type=CloudAccountType(acc["type"]),
+            resources_changed_this_month=acc["details"]["tracked"],
+            expenses_so_far_this_month=acc["details"]["cost"],
+            expenses_forecast_this_month=acc["details"]["forecast"],
+        )
+        for acc in cloud_accounts
+    ]
+
+
+@router.get("/{organization_id}/cloud-accounts/{cloud_account_id}", response_model=CloudAccountRead)
+async def get_cloud_account_by_id(
+    organization: Annotated[Organization, Depends(fetch_organization_or_404)],
+    cloud_account_id: UUID,
+    services: svcs.fastapi.DepContainer,
+):
+    if organization.organization_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=(
+                f"Organization {organization.name} has no associated "
+                "FinOps for Cloud organization"
+            ),
+        )
+
+    optscale_client = await services.aget(OptscaleClient)
+
+    async with wrap_http_error_in_502(f"Error fetching cloud account with ID {cloud_account_id}"):
+        response = await optscale_client.fetch_cloud_account_by_id(cloud_account_id)
+
+    cloud_account = response.json()
+
+    return CloudAccountRead(
+        id=cloud_account["id"],
+        organization_id=organization.id,
+        type=CloudAccountType(cloud_account["type"]),
+        resources_changed_this_month=cloud_account["details"]["tracked"],
+        expenses_so_far_this_month=cloud_account["details"]["cost"],
+        expenses_forecast_this_month=cloud_account["details"]["forecast"],
+    )

app/schemas.py

@@ -7,7 +7,7 @@
 from pydantic import BaseModel, ConfigDict, Field
 
 from app.db.models import Base
-from app.enums import ActorType, EntitlementStatus
+from app.enums import ActorType, CloudAccountType, EntitlementStatus
 
 
 def from_orm[M: Base, S: BaseModel](cls: type[S], db_model: M) -> S:
@@ -120,3 +120,12 @@ class UserCreate(UserBase):
 
 class UserRead(UserBase):
     id: uuid.UUID
+
+
+class CloudAccountRead(BaseSchema):
+    id: uuid.UUID
+    organization_id: uuid.UUID
+    type: CloudAccountType
+    resources_changed_this_month: int
+    expenses_so_far_this_month: float
+    expenses_forecast_this_month: float

tests/conftest.py

@@ -71,7 +71,7 @@ async def db_session() -> AsyncGenerator[AsyncSession, None]:
     await db_engine.dispose()
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 async def api_client(fastapi_app: FastAPI):
     async with AsyncClient(
         transport=ASGITransport(app=fastapi_app), base_url="http://v1/"