Merge pull request #1191 from ChildMindInstitute/release/1.3.20

Release 1.3.20

.env.default

@@ -86,4 +86,5 @@ MONGO__AES_KEY=
 # RABBITMQ__USE_SSL=False
 RABBITMQ__URL=rabbitmq
 
-
+# Secret key for data encryption. Use this key only for local development
+SECRETS__SECRET_KEY=0eb7f5d4c1367199c21e9a2ec793b5a481b60fe2af24464bcb18ac7fa48a645f

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    allow:
+      - dependency-type: "all"
+    ignore:
+      - dependency-name: "pipenv"
+    commit-message:
+      prefix: "[pip] Dependabot:"
+      prefix-development: "[pip-dev] Dependabot:"
+    pull-request-branch-name:
+      separator: "-"

src/apps/answers/api.py

@@ -54,11 +54,13 @@
 from apps.workspaces.service.check_access import CheckAccessService
 from infrastructure.database import atomic, session_manager
 from infrastructure.database.deps import get_session
+from infrastructure.http import get_tz_utc_offset
 
 
 async def create_answer(
     user: User = Depends(get_current_user),
     schema: AppletAnswerCreate = Body(...),
+    tz_offset: int | None = Depends(get_tz_utc_offset()),
     session=Depends(get_session),
     answer_session=Depends(get_answer_session),
 ) -> None:
@@ -69,13 +71,16 @@ async def create_answer(
         except NotValidAppletHistory:
             raise InvalidVersionError()
         service = AnswerService(session, user.id, answer_session)
+        if tz_offset is not None and schema.answer.tz_offset is None:
+            schema.answer.tz_offset = tz_offset // 60  # value in minutes
         async with atomic(answer_session):
             answer = await service.create_answer(schema)
         await service.create_report_from_answer(answer)
 
 
 async def create_anonymous_answer(
     schema: AppletAnswerCreate = Body(...),
+    tz_offset: int | None = Depends(get_tz_utc_offset()),
     session=Depends(get_session),
     answer_session=Depends(get_answer_session),
 ) -> None:
@@ -84,6 +89,8 @@ async def create_anonymous_answer(
         assert anonymous_respondent
 
         service = AnswerService(session, anonymous_respondent.id, answer_session)
+        if tz_offset is not None and schema.answer.tz_offset is None:
+            schema.answer.tz_offset = tz_offset // 60  # value in minutes
         async with atomic(answer_session):
             answer = await service.create_answer(schema)
         await service.create_report_from_answer(answer)
@@ -374,7 +381,7 @@ async def applet_answers_export(
     total_answers = data.total_answers
     for answer in data.answers:
         if answer.is_manager:
-            answer.respondent_secret_id = f"[admin account] ({answer.respondent_email})"
+            answer.respondent_secret_id = f"[admin account] ({answer.respondent_secret_id})"
 
     if activities_last_version:
         applet = await AppletService(session, user.id).get(applet_id)

src/apps/answers/crud/answers.py

@@ -172,6 +172,8 @@ async def get_applet_answers(
                 reviewed_answer_id.label("reviewed_answer_id"),
                 reviewed_answer_id.label("reviewed_answer_id"),
                 AnswerSchema.client,
+                AnswerItemSchema.tz_offset,
+                AnswerItemSchema.scheduled_event_id,
             )
             .select_from(AnswerSchema)
             .join(AnswerItemSchema, AnswerItemSchema.answer_id == AnswerSchema.id)

src/apps/answers/db/schemas.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Boolean, Column, Date, DateTime, ForeignKey, Text, Time, Unicode
+from sqlalchemy import Boolean, Column, Date, DateTime, ForeignKey, Integer, Text, Time, Unicode
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy_utils import StringEncryptedType
 
@@ -51,3 +51,4 @@ class AnswerItemSchema(Base):
     is_assessment = Column(Boolean())
     migrated_data = Column(JSONB())
     assessment_activity_id = Column(Text(), nullable=True, index=True)
+    tz_offset = Column(Integer, nullable=True, comment="Local timezone offset in minutes")

src/apps/answers/domain/answers.py

@@ -61,6 +61,7 @@ class ItemAnswerCreate(InternalModel):
     scheduled_event_id: str | None = None
     local_end_date: datetime.date | None = None
     local_end_time: datetime.time | None = None
+    tz_offset: int | None = None
 
     @validator("item_ids")
     def convert_item_ids(cls, value: list[uuid.UUID]):
@@ -111,6 +112,7 @@ class AssessmentAnswerCreate(InternalModel):
 class AnswerDate(InternalModel):
     created_at: datetime.datetime
     answer_id: uuid.UUID
+    end_datetime: datetime.datetime
 
 
 class ReviewActivity(InternalModel):
@@ -129,6 +131,7 @@ class SummaryActivity(InternalModel):
 class PublicAnswerDate(PublicModel):
     created_at: datetime.datetime
     answer_id: uuid.UUID
+    end_datetime: datetime.datetime
 
 
 class PublicReviewActivity(PublicModel):
@@ -278,6 +281,8 @@ class UserAnswerDataBase(BaseModel):
     start_datetime: datetime.datetime | None = None
     end_datetime: datetime.datetime | None = None
     migrated_date: datetime.datetime | None = None
+    tz_offset: int | None = None
+    scheduled_event_id: uuid.UUID | str | None = None
     applet_history_id: str
     activity_history_id: str | None
     flow_history_id: str | None

src/apps/answers/service.py

@@ -188,6 +188,7 @@ async def _create_answer(self, applet_answer: AppletAnswerCreate):
             scheduled_event_id=item_answer.scheduled_event_id,
             local_end_date=item_answer.local_end_date,
             local_end_time=item_answer.local_end_time,
+            tz_offset=item_answer.tz_offset,
         )
 
         await AnswerItemsCRUD(self.answer_session).create(item_answer)
@@ -263,7 +264,9 @@ async def get_review_activities(
                     continue
                 answer_item_duplicate.add(key)
                 activity_map[activity_id].answer_dates.append(
-                    AnswerDate(created_at=answer_item.created_at, answer_id=answer.id)
+                    AnswerDate(
+                        created_at=answer_item.created_at, answer_id=answer.id, end_datetime=answer_item.end_datetime
+                    )
                 )
         return list(activity_map.values())
 

src/apps/answers/tests/test_answers.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import re
 import uuid
 
 import pytest
@@ -978,8 +979,14 @@ async def test_answers_export(self, mock_kiq_report, client, tom, applet):
                 height=1080,
             ),
         )
+        tz_str = "US/Pacific"
+        tz_offset = -420
 
-        response = await client.post(self.answer_url, data=create_data)
+        response = await client.post(
+            self.answer_url,
+            data=create_data,
+            headers={"x-timezone": tz_str},
+        )
 
         assert response.status_code == 201
 
@@ -1034,8 +1041,14 @@ async def test_answers_export(self, mock_kiq_report, client, tom, applet):
             "respondentSecretId", "reviewedAnswerId", "userPublicKey",
             "version", "submitId", "scheduledDatetime", "startDatetime",
             "endDatetime", "legacyProfileId", "migratedDate", "client",
+            "tzOffset", "scheduledEventId",
         }
-        assert int(answer['startDatetime'] * 1000) == 1690188679657
+        assert int(answer["startDatetime"] * 1000) == 1690188679657
+        assert answer["tzOffset"] == tz_offset
+        assert re.match(
+            r"\[admin account\] \([0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12}\)",
+            answer["respondentSecretId"]
+        )
         # fmt: on
 
         assert set(assessment.keys()) == expected_keys

src/apps/answers/tests/test_answers_arbitrary.py

@@ -846,6 +846,7 @@ async def test_answers_export(self, mock_kiq_report, arbitrary_client, tom, appl
             "respondentSecretId", "reviewedAnswerId", "userPublicKey",
             "version", "submitId", "scheduledDatetime", "startDatetime",
             "endDatetime", "legacyProfileId", "migratedDate", "client",
+            "tzOffset", "scheduledEventId",
         }
         assert int(answer['startDatetime'] * 1000) == 1690188679657
         # fmt: on
@@ -1169,6 +1170,7 @@ async def test_answers_arbitrary_export(self, mock_kiq_report, arbitrary_session
             "respondentSecretId", "reviewedAnswerId", "userPublicKey",
             "version", "submitId", "scheduledDatetime", "startDatetime",
             "endDatetime", "legacyProfileId", "migratedDate", "client",
+            "tzOffset", "scheduledEventId",
         }
         assert int(answer['startDatetime'] * 1000) == 1690188679657
         # fmt: on

src/apps/applets/service/applet.py

@@ -158,6 +158,9 @@ async def _create(
                 encryption=create_data.encryption.dict() if create_data.encryption else None,
                 extra_fields=create_data.extra_fields,
                 creator_id=creator_id,
+                stream_enabled=create_data.stream_enabled,
+                stream_ip_address=create_data.stream_ip_address,
+                stream_port=create_data.stream_port,
             )
         )
         return AppletFull.from_orm(schema)

src/apps/applets/tests/fixtures/applets.py

@@ -130,9 +130,9 @@ def applet_base_data(encryption: Encryption) -> AppletBase:
         pinned_at=None,
         retention_period=None,
         retention_type=None,
-        stream_enabled=False,
-        stream_ip_address=None,
-        stream_port=None,
+        stream_enabled=True,
+        stream_ip_address="127.0.0.1",
+        stream_port=2323,
         encryption=encryption,
     )
 

src/apps/applets/tests/test_applet.py

@@ -68,6 +68,19 @@ async def test_create_applet_with_minimal_data(
 
         response = await client.get(self.applet_detail_url.format(pk=response.json()["result"]["id"]))
         assert response.status_code == http.HTTPStatus.OK
+        result = response.json()["result"]
+        assert result["streamIpAddress"] == str(applet_minimal_data.stream_ip_address)
+        assert result["streamPort"] == applet_minimal_data.stream_port
+        assert result["displayName"] == applet_minimal_data.display_name
+        assert result["image"] == applet_minimal_data.image
+        assert result["watermark"] == applet_minimal_data.watermark
+        assert result["link"] == applet_minimal_data.link
+        assert result["pinnedAt"] == applet_minimal_data.pinned_at
+        assert result["retentionPeriod"] == applet_minimal_data.retention_period
+        assert result["retentionType"] == applet_minimal_data.retention_type
+        assert result["reportServerIp"] == applet_minimal_data.report_server_ip
+        assert result["reportPublicKey"] == applet_minimal_data.report_public_key
+        assert result["reportRecipients"] == applet_minimal_data.report_recipients
 
     async def test_creating_applet_failed_by_duplicate_activity_name(
         self, client: TestClient, tom: User, applet_minimal_data: AppletCreate

src/apps/file/api/file.py

@@ -27,6 +27,7 @@
     FilePresignRequest,
     LogFileExistenceResponse,
     PresignedUrl,
+    WebmTargetExtenstion,
 )
 from apps.file.enums import FileScopeEnum
 from apps.file.errors import FileNotFoundError
@@ -151,9 +152,12 @@ async def convert_not_supported_image(file: UploadFile):
     return None
 
 
-def _get_keys_and_bucket_for_media(orig_key: str) -> tuple[str, str, str]:
+def _get_keys_and_bucket_for_media(
+    orig_key: str, target_extension: WebmTargetExtenstion | None = None
+) -> tuple[str, str, str]:
     if orig_key.lower().endswith(".webm"):
-        target_key = orig_key + ".mp3"
+        extension = target_extension if target_extension else WebmTargetExtenstion.MP3
+        target_key = orig_key + extension
         upload_key = f"{settings.cdn.bucket}/{orig_key}"
         bucket = settings.cdn.bucket_operations
     else:
@@ -396,7 +400,7 @@ async def generate_presigned_media_url(
     cdn_client: CDNClient = Depends(get_media_bucket),
 ) -> Response[PresignedUrl]:
     orig_key = cdn_client.generate_key(FileScopeEnum.CONTENT, user.id, f"{uuid.uuid4()}/{body.file_name}")
-    target_key, upload_key, bucket = _get_keys_and_bucket_for_media(orig_key)
+    target_key, upload_key, bucket = _get_keys_and_bucket_for_media(orig_key, body.target_extension)
     data = cdn_client.generate_presigned_post(bucket, upload_key)
     return Response(
         result=PresignedUrl(
@@ -441,7 +445,7 @@ async def generate_presigned_logs_url(
 ) -> Response[PresignedUrl]:
     service = LogFileService(user.id, cdn_client)
     key = f"{service.device_key_prefix(device_id=device_id)}/{body.file_id}"
-    data = cdn_client.generate_presigned_post(settings.cdn.bucket, key)
+    data = cdn_client.generate_presigned_post(cdn_client.config.bucket, key)
     return Response(
         result=PresignedUrl(upload_url=data["url"], fields=data["fields"], url=cdn_client.generate_private_url(key))
     )

src/apps/file/domain.py

@@ -1,8 +1,15 @@
+import enum
+
 from pydantic import HttpUrl
 
 from apps.shared.domain import PublicModel
 
 
+class WebmTargetExtenstion(str, enum.Enum):
+    MP3 = ".mp3"
+    MP4 = ".mp4"
+
+
 class ContentUploadedFile(PublicModel):
     key: str
     url: str | None
@@ -37,6 +44,7 @@ class LogFileExistenceResponse(FileExistenceResponse):
 
 class FileNameRequest(PublicModel):
     file_name: str
+    target_extension: WebmTargetExtenstion | None = None
 
 
 class FileIdRequest(PublicModel):

src/apps/file/test_file.py

@@ -10,6 +10,7 @@
 from sqlalchemy.orm import Query
 
 from apps.applets.domain.applet_full import AppletFull
+from apps.file.domain import WebmTargetExtenstion
 from apps.file.enums import FileScopeEnum
 from apps.file.services import LogFileService
 from apps.shared.test import BaseTest
@@ -223,19 +224,29 @@ async def test_generate_presigned_url_for_answers(
         assert resp.json()["result"]["url"] == url
 
     @pytest.mark.usefixtures("mock_presigned_post")
-    async def test_generate_presigned_log_url(self, client: TestClient, device_tom: str, tom: User):
+    async def test_generate_presigned_log_url__logs_are_uploaded_to_the_answer_bucket(
+        self, client: TestClient, device_tom: str, tom: User, mocker: MockerFixture
+    ):
+        bucket_answer_name = "bucket_answer_test"
+        settings.cdn.bucket_answer = bucket_answer_name
+        config = CdnConfig(
+            endpoint_url=settings.cdn.endpoint_url,
+            access_key=settings.cdn.access_key,
+            secret_key=settings.cdn.secret_key,
+            region=settings.cdn.region,
+            bucket=settings.cdn.bucket_answer,
+            ttl_signed_urls=settings.cdn.ttl_signed_urls,
+        )
+        cdn_client = CDNClient(config, env="env")
+        mocker.patch("infrastructure.dependency.cdn.get_log_bucket", return_value=cdn_client)
         await client.login(self.login_url, "[email protected]", "Test1234!")
         file_name = "test.txt"
         resp = await client.post(self.log_upload_url.format(device_id=device_tom), data={"file_id": file_name})
         assert resp.status_code == http.HTTPStatus.OK
         key = resp.json()["result"]["fields"]["key"]
-        expected_key = LogFileService(
-            tom.id,
-            CDNClient(
-                CdnConfig(region="region", bucket="bucket", secret_key="secret_key", access_key="access_key"), "env"
-            ),
-        ).key(device_tom, file_name)
+        expected_key = LogFileService(tom.id, cdn_client).key(device_tom, file_name)
         assert key == expected_key
+        assert bucket_answer_name in resp.json()["result"]["uploadUrl"]
 
     @pytest.mark.usefixtures("mock_presigned_post")
     @pytest.mark.parametrize("file_name", ("test.webm", "test.WEBM"))
@@ -349,3 +360,34 @@ async def test_answer_existance_for_heic_format_for_arbitrary(
         mock_check_existance.assert_awaited_once_with(settings.cdn.bucket_operations, exp_check_key)
         assert result[0]["url"].endswith(exp_converted_file_name)
         assert ARBITRARY_BUCKET_NAME in result[0]["url"]
+
+    @pytest.mark.usefixtures("mock_presigned_post")
+    @pytest.mark.parametrize(
+        "file_name,target_extension,exp_file_name",
+        (
+            ("test.webm", WebmTargetExtenstion.MP3, f"test.webm{WebmTargetExtenstion.MP3}"),
+            ("test.webm", WebmTargetExtenstion.MP4, f"test.webm{WebmTargetExtenstion.MP4}"),
+            ("test.webm", None, f"test.webm{WebmTargetExtenstion.MP3}"),
+            ("test.webm", "without extension", f"test.webm{WebmTargetExtenstion.MP3}"),
+        ),
+    )
+    async def test_generate_upload_url__webm_to_mp3_mp4(
+        self, client: TestClient, file_name: str, target_extension: WebmTargetExtenstion, exp_file_name: str
+    ):
+        await client.login(self.login_url, "[email protected]", "Test1234!")
+        data = {"file_name": file_name, "target_extension": target_extension}
+        if target_extension == "without extension":
+            del data["target_extension"]
+        resp = await client.post(self.upload_media_url, data=data)
+        assert resp.status_code == http.HTTPStatus.OK
+        result = resp.json()["result"]
+        assert result["url"].endswith(exp_file_name)
+
+    @pytest.mark.parametrize("not_valid_extesion", ("mp3", "mp2"))
+    async def test_generate_upload_url__webm_to_mp3_mp4_not_valid_extension(
+        self, client: TestClient, not_valid_extesion: str
+    ):
+        await client.login(self.login_url, "[email protected]", "Test1234!")
+        data = {"file_name": "test.webm", "target_extension": not_valid_extesion}
+        resp = await client.post(self.upload_media_url, data=data)
+        assert resp.status_code == http.HTTPStatus.UNPROCESSABLE_ENTITY