Merge branch 'main' into acouch/issue-3306-save-opps-buttons

.github/workflows/ci-cron-vulnerability-scans.yml

@@ -6,7 +6,7 @@ name: CI Cron Vulnerability Scans
 on:
   workflow_dispatch:
   schedule:
-    # Run every day at (8am ET, 11am PT) right before the start of the workday
+    # Run every day at (8am ET, 5am PT) before the start of the workday
     - cron: "0 12 * * *"
 
 jobs:

.template-infra/app-analytics.yml

@@ -1,3 +1,4 @@
-_commit: 929a959
+_commit: platform-cli-migration/v0.4.0
 _src_path: https://github.com/navapbc/template-infra
 app_name: analytics
+template: app

.template-infra/app-api.yml

@@ -1,3 +1,4 @@
-_commit: 929a959
+_commit: platform-cli-migration/v0.4.0
 _src_path: https://github.com/navapbc/template-infra
 app_name: api
+template: app

.template-infra/app-frontend.yml

@@ -1,3 +1,4 @@
-_commit: 929a959
+_commit: platform-cli-migration/v0.4.0
 _src_path: https://github.com/navapbc/template-infra
 app_name: frontend
+template: app

.template-infra/base.yml

@@ -1,3 +1,3 @@
-_commit: 929a959
+_commit: platform-cli-migration/v0.4.0
 _src_path: https://github.com/navapbc/template-infra
-app_name: template-only
+template: base

analytics/README.md

@@ -2,7 +2,7 @@
 
 ## Introduction
 
-This package encapsulates a data pipeline service. The service is responsible for extracting project data from GitHub and transforming the extracted data into rows in a data warehouse. 
+This package encapsulates a data pipeline service. The service is responsible for extracting project data from GitHub and transforming the extracted data into rows in a data warehouse. We're using Metabase to provide data visualization and Business Intelligence for the data warehouse. As an example, our [dashboard that demonstrates the flow of Simpler Grants.gov Opportunity Data from the Operational DB to the Data Warehouse](http://metabase-prod-899895252.us-east-1.elb.amazonaws.com/dashboard/100-operational-data).
 
 ## Project Directory Structure
 

analytics/src/analytics/integrations/github/validation.py

@@ -1,10 +1,30 @@
 """Pydantic schemas for validating GitHub API responses."""
 
 # pylint: disable=no-self-argument
+# mypy: disable-error-code="literal-required"
+# This mypy disable is needed so we can use constants for field aliases
+
 from datetime import datetime, timedelta
 
 from pydantic import BaseModel, Field, computed_field, model_validator
 
+# Declare constants for the fields that need to be aliased from the GitHub data
+# so that we only have to change these values in one place.
+#
+# We need to declare them at the module-level instead of class-level
+# because pydantic throws an error when using class level constants.
+
+# Issue content aliases
+ISSUE_TYPE = "issueType"
+CLOSED_AT = "closedAt"
+CREATED_AT = "createdAt"
+PARENT = "parent"
+# Iteration aliases
+ITERATION_ID = "iterationId"
+START_DATE = "startDate"
+# Single select aliases
+OPTION_ID = "optionId"
+
 
 def safe_default_factory(data: dict, keys_to_replace: list[str]) -> dict:
     """
@@ -40,19 +60,20 @@ class IssueType(BaseModel):
 class IssueContent(BaseModel):
     """Schema for core issue metadata."""
 
+    # The fields that we're parsing from the raw GitHub output
     title: str
     url: str
     closed: bool
-    created_at: str = Field(alias="createdAt")
-    closed_at: str | None = Field(alias="closedAt", default=None)
-    issue_type: IssueType = Field(alias="type", default_factory=IssueType)
+    created_at: str = Field(alias=CREATED_AT)
+    closed_at: str | None = Field(alias=CLOSED_AT, default=None)
+    issue_type: IssueType = Field(alias=ISSUE_TYPE, default_factory=IssueType)
     parent: IssueParent = Field(default_factory=IssueParent)
 
     @model_validator(mode="before")
     def replace_none_with_defaults(cls, values) -> dict:  # noqa: ANN001, N805
-        """Replace None with default_factory instances."""
+        """Replace keys that are set to None with default_factory instances."""
         # Replace None with default_factory instances
-        return safe_default_factory(values, ["type", "parent"])
+        return safe_default_factory(values, [ISSUE_TYPE, PARENT])
 
 
 # #############################################
@@ -63,9 +84,9 @@ def replace_none_with_defaults(cls, values) -> dict:  # noqa: ANN001, N805
 class IterationValue(BaseModel):
     """Schema for iteration field values like Sprint or Quad."""
 
-    iteration_id: str | None = Field(alias="iterationId", default=None)
+    iteration_id: str | None = Field(alias=ITERATION_ID, default=None)
     title: str | None = None
-    start_date: str | None = Field(alias="startDate", default=None)
+    start_date: str | None = Field(alias=START_DATE, default=None)
     duration: int | None = None
 
     @computed_field
@@ -82,7 +103,7 @@ def end_date(self) -> str | None:
 class SingleSelectValue(BaseModel):
     """Schema for single select field values like Status or Pillar."""
 
-    option_id: str | None = Field(alias="optionId", default=None)
+    option_id: str | None = Field(alias=OPTION_ID, default=None)
     name: str | None = None
 
 
@@ -112,7 +133,7 @@ class ProjectItem(BaseModel):
 
     @model_validator(mode="before")
     def replace_none_with_defaults(cls, values) -> dict:  # noqa: ANN001, N805
-        """Replace None with default_factory instances."""
+        """Replace keys that are set to None with default_factory instances."""
         return safe_default_factory(
             values,
             ["sprint", "points", "quad", "pillar", "status"],

analytics/tests/conftest.py

@@ -287,6 +287,12 @@ def reset_aws_env_vars(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
 
 
+@pytest.fixture(autouse=True)
+def use_cdn(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Set up CDN URL environment variable for tests."""
+    monkeypatch.setenv("CDN_URL", "http://localhost:4566")
+
+
 @pytest.fixture
 def mock_s3() -> boto3.resource:
     """Instantiate an S3 bucket resource."""

analytics/tests/integrations/github/test_validation.py

@@ -24,7 +24,7 @@
         "title": "Test Parent",
         "url": "https://github.com/test/repo/issues/2",
     },
-    "type": {
+    "issueType": {
         "name": "Bug",
     },
 }

api/local.env

@@ -150,3 +150,5 @@ DEPLOY_GITHUB_REF=main
 DEPLOY_GITHUB_SHA=ffaca647223e0b6e54344122eefa73401f5ec131
 DEPLOY_TIMESTAMP=2024-12-02T21:25:18Z
 DEPLOY_WHOAMI=local-developer
+
+CDN_URL=http://localhost:4566/local-mock-public-bucket

api/openapi.generated.yml

@@ -628,6 +628,53 @@ paths:
       security:
       - ApiKeyAuth: []
   /v1/users/{user_id}/saved-searches/{saved_search_id}:
+    put:
+      parameters:
+      - in: path
+        name: user_id
+        schema:
+          type: string
+        required: true
+      - in: path
+        name: saved_search_id
+        schema:
+          type: string
+        required: true
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/UserUpdateSavedSearchResponse'
+          description: Successful response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
+          description: Validation error
+        '401':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
+          description: Authentication error
+        '404':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
+          description: Not found
+      tags:
+      - User v1
+      summary: User Update Saved Search
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UserUpdateSavedSearchRequest'
+      security:
+      - ApiJwtAuth: []
     delete:
       parameters:
       - in: path
@@ -837,6 +884,7 @@ components:
         page_size:
           type: integer
           minimum: 1
+          maximum: 5000
           description: The size of the page to fetch
           example: 25
         page_offset:
@@ -978,6 +1026,7 @@ components:
         page_size:
           type: integer
           minimum: 1
+          maximum: 5000
           description: The size of the page to fetch
           example: 25
         page_offset:
@@ -1005,40 +1054,6 @@ components:
           - $ref: '#/components/schemas/AgencyPaginationV1'
       required:
       - pagination
-    AgencyContactInfo:
-      type: object
-      properties:
-        contact_name:
-          type: string
-          description: Full name of the agency contact person
-        address_line_1:
-          type: string
-          description: Primary street address of the agency
-        address_line_2:
-          type:
-          - string
-          - 'null'
-          description: Additional address information (suite, unit, etc.)
-        city:
-          type: string
-          description: City where the agency is located
-        state:
-          type: string
-          description: State where the agency is located
-        zip_code:
-          type: string
-          description: Postal code for the agency address
-        phone_number:
-          type: string
-          description: Contact phone number for the agency
-        primary_email:
-          type: string
-          description: Main email address for agency communications
-        secondary_email:
-          type:
-          - string
-          - 'null'
-          description: Alternative email address for agency communications
     AgencyResponse:
       type: object
       properties:
@@ -1048,35 +1063,11 @@ components:
           type: string
         agency_code:
           type: string
-        sub_agency_code:
-          type:
-          - string
-          - 'null'
-        assistance_listing_number:
-          type: string
-        agency_submission_notification_setting:
-          type: string
         top_level_agency:
           type:
           - object
           allOf:
           - $ref: '#/components/schemas/AgencyResponse'
-        agency_contact_info:
-          type:
-          - object
-          - 'null'
-          anyOf:
-          - $ref: '#/components/schemas/AgencyContactInfo'
-          - type: 'null'
-        agency_download_file_types:
-          type: array
-          description: List of download file types supported by the agency
-          items:
-            enum:
-            - xml
-            - pdf
-            type:
-            - string
         created_at:
           type: string
           format: date-time
@@ -1469,6 +1460,7 @@ components:
         page_size:
           type: integer
           minimum: 1
+          maximum: 5000
           description: The size of the page to fetch
           example: 25
         page_offset:
@@ -1490,6 +1482,14 @@ components:
           maxLength: 100
           description: Query string which searches against several text fields
           example: research
+        query_operator:
+          description: Query operator for combining search conditions
+          example: OR
+          enum:
+          - AND
+          - OR
+          type:
+          - string
         filters:
           type:
           - object
@@ -2294,6 +2294,28 @@ components:
           type: integer
           description: The HTTP status code
           example: 200
+    UserUpdateSavedSearchRequest:
+      type: object
+      properties:
+        name:
+          type: string
+          description: Name of the saved search
+          example: Example search
+      required:
+      - name
+    UserUpdateSavedSearchResponse:
+      type: object
+      properties:
+        message:
+          type: string
+          description: The message to return
+          example: Success
+        data:
+          example: null
+        status_code:
+          type: integer
+          description: The HTTP status code
+          example: 200
     UserDeleteSavedSearchResponse:
       type: object
       properties:

api/src/adapters/aws/aws_session.py

@@ -1,11 +1,33 @@
-import os
-
 import boto3
 
+from src.util.env_config import PydanticBaseEnvConfig
+
+
+class BaseAwsConfig(PydanticBaseEnvConfig):
+    is_local_aws: bool = False
+
+
+_base_aws_config: BaseAwsConfig | None = None
+
+
+def get_base_aws_config() -> BaseAwsConfig:
+    global _base_aws_config
+    if _base_aws_config is None:
+        _base_aws_config = BaseAwsConfig()
+
+    return _base_aws_config
+
+
+def is_local_aws() -> bool:
+    """Whether we are running against local AWS which affects the credentials we use (forces them to be not real)"""
+    return get_base_aws_config().is_local_aws
+
 
 def get_boto_session() -> boto3.Session:
-    is_local = bool(os.getenv("IS_LOCAL_AWS", False))
-    if is_local:
-        return boto3.Session(aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS")
+    if is_local_aws():
+        # Locally, set fake creds in a region we don't actually use so we can't hit actual AWS resources
+        return boto3.Session(
+            aws_access_key_id="NO_CREDS", aws_secret_access_key="NO_CREDS", region_name="us-west-2"
+        )
 
     return boto3.Session()