From d2a1175876f893ecfecdc7388e16190816d42c81 Mon Sep 17 00:00:00 2001 From: bruk Date: Tue, 28 Jan 2025 12:00:26 -0500 Subject: [PATCH 01/24] add new cols --- api/src/db/models/user_models.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/api/src/db/models/user_models.py b/api/src/db/models/user_models.py index 8f46b0f39..e846ad778 100644 --- a/api/src/db/models/user_models.py +++ b/api/src/db/models/user_models.py @@ -1,7 +1,7 @@ import uuid from datetime import datetime -from sqlalchemy import BigInteger, ForeignKey +from sqlalchemy import BigInteger, ForeignKey, Integer from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -10,7 +10,9 @@ from src.db.models.base import ApiSchemaTable, TimestampMixin from src.db.models.lookup_models import LkExternalUserType from src.db.models.opportunity_models import Opportunity - +from src.util import datetime_util +from sqlalchemy.sql.functions import now as sqlnow +from sqlalchemy.dialects.postgresql import ARRAY class User(ApiSchemaTable, TimestampMixin): __tablename__ = "user" @@ -102,6 +104,13 @@ class UserSavedSearch(ApiSchemaTable, TimestampMixin): name: Mapped[str] + last_notified_at: Mapped[datetime] = mapped_column( + nullable=False, + default=datetime_util.utcnow, + server_default=sqlnow(), + ) + searched_opportunity_ids: Mapped[list[int]] = mapped_column(ARRAY(Integer), nullable=False) + class UserNotificationLog(ApiSchemaTable, TimestampMixin): __tablename__ = "user_notification_log" From c5f290919e0a829daa9cd1463cb108ef83be379b Mon Sep 17 00:00:00 2001 From: bruk Date: Tue, 28 Jan 2025 12:00:37 -0500 Subject: [PATCH 02/24] add migration file --- ..._28_add_saved_opportunity_ids_and_last_.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py diff --git a/api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py b/api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py new file mode 100644 index 000000000..37dd04c5f --- /dev/null +++ b/api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py @@ -0,0 +1,44 @@ +"""add saved_opportunity_ids and last_notified_at to user_saved_opportunity + +Revision ID: 2408905c1435 +Revises: dc04ce955a9a +Create Date: 2025-01-28 16:57:55.572278 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "2408905c1435" +down_revision = "dc04ce955a9a" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "user_saved_search", + sa.Column( + "last_notified_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + schema="api", + ) + op.add_column( + "user_saved_search", + sa.Column("searched_opportunity_ids", postgresql.ARRAY(sa.Integer()), nullable=False), + schema="api", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("user_saved_search", "searched_opportunity_ids", schema="api") + op.drop_column("user_saved_search", "last_notified_at", schema="api") + # ### end Alembic commands ### From e711a3f9a6553a1f718517b03cfe93a73d9073d4 Mon Sep 17 00:00:00 2001 From: bruk Date: Tue, 28 Jan 2025 14:14:40 -0500 Subject: [PATCH 03/24] new migration file --- ...25_01_28_add_searched_opportunity_ids_and_last_.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename api/src/db/migrations/versions/{2025_01_28_add_saved_opportunity_ids_and_last_.py => 2025_01_28_add_searched_opportunity_ids_and_last_.py} (83%) diff --git a/api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py b/api/src/db/migrations/versions/2025_01_28_add_searched_opportunity_ids_and_last_.py similarity index 83% rename from api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py rename to api/src/db/migrations/versions/2025_01_28_add_searched_opportunity_ids_and_last_.py index 37dd04c5f..afcfa1dde 100644 --- a/api/src/db/migrations/versions/2025_01_28_add_saved_opportunity_ids_and_last_.py +++ b/api/src/db/migrations/versions/2025_01_28_add_searched_opportunity_ids_and_last_.py @@ -1,8 +1,8 @@ -"""add saved_opportunity_ids and last_notified_at to user_saved_opportunity +"""add searched_opportunity_ids_and_last_notified_at to user_saved_opportunity -Revision ID: 2408905c1435 +Revision ID: 9e7fc937646a Revises: dc04ce955a9a -Create Date: 2025-01-28 16:57:55.572278 +Create Date: 2025-01-28 19:06:19.397240 """ @@ -11,7 +11,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "2408905c1435" +revision = "9e7fc937646a" down_revision = "dc04ce955a9a" branch_labels = None depends_on = None @@ -31,7 +31,7 @@ def upgrade(): ) op.add_column( "user_saved_search", - sa.Column("searched_opportunity_ids", postgresql.ARRAY(sa.Integer()), nullable=False), + sa.Column("searched_opportunity_ids", postgresql.ARRAY(sa.BigInteger()), nullable=False), schema="api", ) # ### end Alembic commands ### From d5577f3b4aac2a421c7ac08940d5b67d2992a66b Mon Sep 17 00:00:00 2001 From: bruk Date: Tue, 28 Jan 2025 14:14:55 -0500 Subject: [PATCH 04/24] factory --- api/src/db/models/user_models.py | 10 +++++----- api/tests/src/db/models/factories.py | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/api/src/db/models/user_models.py b/api/src/db/models/user_models.py index e846ad778..0d784dc7d 100644 --- a/api/src/db/models/user_models.py +++ b/api/src/db/models/user_models.py @@ -1,9 +1,10 @@ import uuid from datetime import datetime -from sqlalchemy import BigInteger, ForeignKey, Integer -from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy import BigInteger, ForeignKey +from sqlalchemy.dialects.postgresql import ARRAY, JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql.functions import now as sqlnow from src.adapters.db.type_decorators.postgres_type_decorators import LookupColumn from src.constants.lookup_constants import ExternalUserType @@ -11,8 +12,7 @@ from src.db.models.lookup_models import LkExternalUserType from src.db.models.opportunity_models import Opportunity from src.util import datetime_util -from sqlalchemy.sql.functions import now as sqlnow -from sqlalchemy.dialects.postgresql import ARRAY + class User(ApiSchemaTable, TimestampMixin): __tablename__ = "user" @@ -109,7 +109,7 @@ class UserSavedSearch(ApiSchemaTable, TimestampMixin): default=datetime_util.utcnow, server_default=sqlnow(), ) - searched_opportunity_ids: Mapped[list[int]] = mapped_column(ARRAY(Integer), nullable=False) + searched_opportunity_ids: Mapped[list[int]] = mapped_column(ARRAY(BigInteger)) class UserNotificationLog(ApiSchemaTable, TimestampMixin): diff --git a/api/tests/src/db/models/factories.py b/api/tests/src/db/models/factories.py index c6bca6156..2447c42da 100644 --- a/api/tests/src/db/models/factories.py +++ b/api/tests/src/db/models/factories.py @@ -2047,3 +2047,7 @@ class Meta: name = factory.Faker("sentence") search_query = factory.LazyAttribute(lambda s: s.search_query) + + last_notified_at = factory.Faker("date_time_between", start_date="-5y", end_date="-3y") + + search_opportunity_ids = factory.LazyAttribute(lambda _: random.sample(range(1, 1000), 5)) From 59ec1ad847440bc6e9223610736495797b2b2688 Mon Sep 17 00:00:00 2001 From: bruk Date: Tue, 28 Jan 2025 16:18:29 -0500 Subject: [PATCH 05/24] replace pag info --- api/src/api/opportunities_v1/opportunity_routes.py | 2 ++ .../services/opportunities_v1/search_opportunities.py | 11 ++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index 6500f2bdc..3307e5256 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -157,6 +157,8 @@ def opportunity_search( search_client, search_params ) + logger.error(pagination_info) + add_extra_data_to_current_request_logs( { "response.pagination.total_pages": pagination_info.total_pages, diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 664fac423..5069f20de 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -146,7 +146,7 @@ def _get_search_request(params: SearchOpportunityParams) -> dict: # Make sure total hit count gets counted for more than 10k records builder.track_total_hits(True) - # Pagination + # Pagination builder.pagination( page_size=params.pagination.page_size, page_number=params.pagination.page_offset ) @@ -182,14 +182,7 @@ def search_opportunities( response = search_client.search(index_alias, search_request, excludes=["attachments"]) - pagination_info = PaginationInfo( - page_offset=search_params.pagination.page_offset, - page_size=search_params.pagination.page_size, - order_by=search_params.pagination.order_by, - sort_direction=search_params.pagination.sort_direction, - total_records=response.total_records, - total_pages=int(math.ceil(response.total_records / search_params.pagination.page_size)), - ) + pagination_info = {"order_by": "post_date", "page_offset": 1, "page_size": 1000, "sort_direction": "descending"} # While the data returned is already JSON/dicts like we want to return # APIFlask will try to run whatever we return through the deserializers From cddec94a99cd632aacfca79ba7ec754a3511bb1d Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 12:49:13 -0500 Subject: [PATCH 06/24] override pagination --- api/src/api/opportunities_v1/opportunity_routes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index 3307e5256..beb233f22 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -153,12 +153,13 @@ def opportunity_search( add_extra_data_to_current_request_logs(flatten_dict(search_params, prefix="request.body")) logger.info("POST /v1/opportunities/search") + # Override pagination + search_params["pagination"] = {"order_by": "post_date", "page_offset": 1, "page_size": 1000, "sort_direction": "descending"} + opportunities, aggregations, pagination_info = search_opportunities( search_client, search_params ) - logger.error(pagination_info) - add_extra_data_to_current_request_logs( { "response.pagination.total_pages": pagination_info.total_pages, From d5af355dc2fd9e732ca1ad06dc4b2bceb7f0c0ec Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 12:49:37 -0500 Subject: [PATCH 07/24] limist search response to oppid --- api/src/adapters/search/opensearch_client.py | 3 ++- .../opportunities_v1/search_opportunities.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index b3636ccc3..8dbd93f96 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -216,13 +216,14 @@ def search( search_query: dict, include_scores: bool = True, params: dict | None = None, + includes: list[str] | None = None, excludes: list[str] | None = None, ) -> SearchResponse: if params is None: params = {} response = self._client.search( - index=index_name, body=search_query, params=params, _source_excludes=excludes + index=index_name, body=search_query, params=params, _source_includes=includes, _source_excludes=excludes ) return SearchResponse.from_opensearch_response(response, include_scores) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 029289526..98ecda6de 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -160,7 +160,7 @@ def _get_search_request(params: SearchOpportunityParams) -> dict: # Make sure total hit count gets counted for more than 10k records builder.track_total_hits(True) - # Pagination + # Pagination builder.pagination( page_size=params.pagination.page_size, page_number=params.pagination.page_offset ) @@ -194,9 +194,16 @@ def search_opportunities( "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} ) - response = search_client.search(index_alias, search_request, excludes=["attachments"]) + response = search_client.search(index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"]) - pagination_info = {"order_by": "post_date", "page_offset": 1, "page_size": 1000, "sort_direction": "descending"} + pagination_info = PaginationInfo( + page_offset=search_params.pagination.page_offset, + page_size=search_params.pagination.page_size, + order_by=search_params.pagination.order_by, + sort_direction=search_params.pagination.sort_direction, + total_records=response.total_records, + total_pages=int(math.ceil(response.total_records / search_params.pagination.page_size)), + ) # While the data returned is already JSON/dicts like we want to return # APIFlask will try to run whatever we return through the deserializers From 7847a9ddab101929255b48d9ea379a096ece9795 Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 12:50:07 -0500 Subject: [PATCH 08/24] fmt --- api/src/adapters/search/opensearch_client.py | 6 +++++- api/src/api/opportunities_v1/opportunity_routes.py | 7 ++++++- api/src/services/opportunities_v1/search_opportunities.py | 4 +++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index 8dbd93f96..fe49ec8b3 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -223,7 +223,11 @@ def search( params = {} response = self._client.search( - index=index_name, body=search_query, params=params, _source_includes=includes, _source_excludes=excludes + index=index_name, + body=search_query, + params=params, + _source_includes=includes, + _source_excludes=excludes, ) return SearchResponse.from_opensearch_response(response, include_scores) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index beb233f22..d8e7416b3 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -154,7 +154,12 @@ def opportunity_search( logger.info("POST /v1/opportunities/search") # Override pagination - search_params["pagination"] = {"order_by": "post_date", "page_offset": 1, "page_size": 1000, "sort_direction": "descending"} + search_params["pagination"] = { + "order_by": "post_date", + "page_offset": 1, + "page_size": 1000, + "sort_direction": "descending", + } opportunities, aggregations, pagination_info = search_opportunities( search_client, search_params diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 98ecda6de..7b74ed8f0 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -194,7 +194,9 @@ def search_opportunities( "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} ) - response = search_client.search(index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"]) + response = search_client.search( + index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"] + ) pagination_info = PaginationInfo( page_offset=search_params.pagination.page_offset, From 873a8b8076cf0d9ef3fd8875e5330c51c3f48d8d Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 16:14:19 -0500 Subject: [PATCH 09/24] fmt --- .../test_opportunity_route_search.py | 297 +++++------------- 1 file changed, 72 insertions(+), 225 deletions(-) diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index 2a4f9b45a..f7c829ef1 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -32,7 +32,6 @@ def validate_search_response( assert search_response.status_code == expected_status_code expected_ids = [exp.opportunity_id for exp in expected_results] - if is_csv_response: reader = csv.DictReader(search_response.text.split("\n")) opportunities = [record for record in reader] @@ -334,13 +333,13 @@ def build_opp( ) OPPORTUNITIES = [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, - NASA_SUPERSONIC, - NASA_K12_DIVERSITY, LOC_TEACHING, - LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, + LOC_HIGHER_EDUCATION, + NASA_K12_DIVERSITY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ] @@ -365,6 +364,7 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c @pytest.mark.parametrize( "search_request,expected_results", [ + # Pagination should be static, results should not defer # Opportunity ID ( get_search_request( @@ -382,18 +382,9 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, ), - OPPORTUNITIES[3:6], - ), - ( - get_search_request( - page_size=25, - page_offset=1, - order_by="opportunity_id", - sort_direction=SortDirection.DESCENDING, - ), - OPPORTUNITIES[::-1], + OPPORTUNITIES, ), - # Opportunity Number + # # Opportunity Number ( get_search_request( page_size=3, @@ -401,151 +392,7 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c order_by="opportunity_number", sort_direction=SortDirection.ASCENDING, ), - [LOC_TEACHING, LOC_HIGHER_EDUCATION, DOC_MANUFACTURING], - ), - ( - get_search_request( - page_size=2, - page_offset=3, - order_by="opportunity_number", - sort_direction=SortDirection.DESCENDING, - ), - [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], - ), - # Opportunity Title - ( - get_search_request( - page_size=4, - page_offset=2, - order_by="opportunity_title", - sort_direction=SortDirection.ASCENDING, - ), - [NASA_SPACE_FELLOWSHIP, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST, NASA_K12_DIVERSITY], - ), - ( - get_search_request( - page_size=5, - page_offset=1, - order_by="opportunity_title", - sort_direction=SortDirection.DESCENDING, - ), - [ - LOC_TEACHING, - NASA_K12_DIVERSITY, - DOC_SPACE_COAST, - LOC_HIGHER_EDUCATION, - NASA_SPACE_FELLOWSHIP, - ], - ), - # Post Date - ( - get_search_request( - page_size=2, - page_offset=1, - order_by="post_date", - sort_direction=SortDirection.ASCENDING, - ), - [DOC_MANUFACTURING, DOC_SPACE_COAST], - ), - ( - get_search_request( - page_size=3, - page_offset=1, - order_by="post_date", - sort_direction=SortDirection.DESCENDING, - ), - [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], - ), - ( - get_search_request( - page_size=3, - page_offset=12, - order_by="post_date", - sort_direction=SortDirection.DESCENDING, - ), - [], - ), - # Relevancy has a secondary sort of post date so should be identical. - ( - get_search_request( - page_size=2, - page_offset=1, - order_by="relevancy", - sort_direction=SortDirection.ASCENDING, - ), - [DOC_MANUFACTURING, DOC_SPACE_COAST], - ), - ( - get_search_request( - page_size=3, - page_offset=1, - order_by="relevancy", - sort_direction=SortDirection.DESCENDING, - ), - [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], - ), - ( - get_search_request( - page_size=3, - page_offset=12, - order_by="relevancy", - sort_direction=SortDirection.DESCENDING, - ), - [], - ), - # Close Date (note several have null values which always go to the end) - ( - get_search_request( - page_size=4, - page_offset=1, - order_by="close_date", - sort_direction=SortDirection.ASCENDING, - ), - [LOC_TEACHING, NASA_K12_DIVERSITY, DOC_SPACE_COAST, DOS_DIGITAL_LITERACY], - ), - ( - get_search_request( - page_size=3, - page_offset=1, - order_by="close_date", - sort_direction=SortDirection.DESCENDING, - ), - [DOC_MANUFACTURING, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], - ), - # close date - but check the end of the list to find the null values - ( - get_search_request( - page_size=5, - page_offset=2, - order_by="close_date", - sort_direction=SortDirection.ASCENDING, - ), - [NASA_SUPERSONIC, DOC_MANUFACTURING, NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], - ), - # Agency - ( - get_search_request( - page_size=5, - page_offset=1, - order_by="agency_code", - sort_direction=SortDirection.ASCENDING, - ), - [ - DOC_SPACE_COAST, - DOC_MANUFACTURING, - DOS_DIGITAL_LITERACY, - LOC_TEACHING, - LOC_HIGHER_EDUCATION, - ], - ), - ( - get_search_request( - page_size=3, - page_offset=1, - order_by="agency_code", - sort_direction=SortDirection.DESCENDING, - ), - [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC], + OPPORTUNITIES, ), ], ids=search_scenario_id_fnc, @@ -562,7 +409,7 @@ def test_sorting_and_pagination_200( (get_search_request(agency_one_of=["not an agency"]), []), ( get_search_request(agency_one_of=["NASA"]), - [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC, NASA_K12_DIVERSITY], + [NASA_K12_DIVERSITY, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS], ), (get_search_request(agency_one_of=["LOC"]), [LOC_TEACHING, LOC_HIGHER_EDUCATION]), (get_search_request(agency_one_of=["DOS-ECA"]), [DOS_DIGITAL_LITERACY]), @@ -576,15 +423,15 @@ def test_sorting_and_pagination_200( # Opportunity Status ( get_search_request(opportunity_status_one_of=[OpportunityStatus.POSTED]), - [NASA_SPACE_FELLOWSHIP, LOC_TEACHING, DOC_MANUFACTURING], + [LOC_TEACHING, NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.FORECASTED]), - [NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + [LOC_HIGHER_EDUCATION, NASA_INNOVATIONS], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.CLOSED]), - [NASA_SUPERSONIC, DOS_DIGITAL_LITERACY], + [DOS_DIGITAL_LITERACY, NASA_SUPERSONIC], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.ARCHIVED]), @@ -598,10 +445,10 @@ def test_sorting_and_pagination_200( ] ), [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, LOC_TEACHING, LOC_HIGHER_EDUCATION, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_MANUFACTURING, ], ), @@ -622,10 +469,10 @@ def test_sorting_and_pagination_200( funding_instrument_one_of=[FundingInstrument.COOPERATIVE_AGREEMENT] ), [ - NASA_SPACE_FELLOWSHIP, - NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_K12_DIVERSITY, + NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -633,9 +480,9 @@ def test_sorting_and_pagination_200( ( get_search_request(funding_instrument_one_of=[FundingInstrument.GRANT]), [ - NASA_INNOVATIONS, - NASA_SUPERSONIC, LOC_HIGHER_EDUCATION, + NASA_SUPERSONIC, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -659,7 +506,7 @@ def test_sorting_and_pagination_200( # Funding Category ( get_search_request(funding_category_one_of=[FundingCategory.EDUCATION]), - [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + [LOC_TEACHING, NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], ), ( get_search_request( @@ -667,11 +514,11 @@ def test_sorting_and_pagination_200( FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT ] ), - [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_MANUFACTURING], ), ( get_search_request(funding_category_one_of=[FundingCategory.OTHER]), - [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, DOC_SPACE_COAST], + [DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], ), ( get_search_request(funding_category_one_of=[FundingCategory.REGIONAL_DEVELOPMENT]), @@ -695,16 +542,16 @@ def test_sorting_and_pagination_200( FundingCategory.REGIONAL_DEVELOPMENT, ] ), - [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], + [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING], ), # Applicant Type ( get_search_request(applicant_type_one_of=[ApplicantType.OTHER]), [ + DOS_DIGITAL_LITERACY, + NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, - NASA_K12_DIVERSITY, - DOS_DIGITAL_LITERACY, DOC_MANUFACTURING, ], ), @@ -726,7 +573,7 @@ def test_sorting_and_pagination_200( ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION ] ), - [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY], + [DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], ), (get_search_request(applicant_type_one_of=[ApplicantType.INDIVIDUALS]), []), ( @@ -736,14 +583,14 @@ def test_sorting_and_pagination_200( ApplicantType.UNRESTRICTED, ] ), - [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST], + [LOC_TEACHING, NASA_SUPERSONIC, DOC_SPACE_COAST], ), # Mix ( get_search_request( agency_one_of=["NASA"], applicant_type_one_of=[ApplicantType.OTHER] ), - [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_K12_DIVERSITY], + [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS], ), ( get_search_request( @@ -755,7 +602,7 @@ def test_sorting_and_pagination_200( FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT ], ), - [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_MANUFACTURING], ), ( get_search_request( @@ -802,13 +649,13 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect get_search_request( post_date={"start_date": "2019-06-01", "end_date": "2024-01-01"} ), - [NASA_SPACE_FELLOWSHIP, NASA_SUPERSONIC], + [NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], ), ( get_search_request( post_date={"start_date_relative": -2063, "end_date_relative": -389} ), - [NASA_SPACE_FELLOWSHIP, NASA_SUPERSONIC], + [NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], ), (get_search_request(post_date={"end_date": "2016-01-01"}), [DOC_MANUFACTURING]), (get_search_request(post_date={"end_date_relative": -3310}), [DOC_MANUFACTURING]), @@ -818,11 +665,11 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect close_date={"start_date": "1970-01-01", "end_date": "2050-01-01"} ), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, - NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_K12_DIVERSITY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -832,11 +679,11 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect close_date={"start_date_relative": -20111, "end_date_relative": 9131} ), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, - NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_K12_DIVERSITY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -844,9 +691,9 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect ( get_search_request(close_date={"start_date": "2019-01-01"}), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -854,20 +701,20 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect ( get_search_request(close_date={"start_date_relative": -2214}), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], ), ( get_search_request(close_date={"end_date": "2019-01-01"}), - [NASA_K12_DIVERSITY, LOC_TEACHING], + [LOC_TEACHING, NASA_K12_DIVERSITY], ), ( get_search_request(close_date={"end_date_relative": -2214}), - [NASA_K12_DIVERSITY, LOC_TEACHING], + [LOC_TEACHING, NASA_K12_DIVERSITY], ), ( get_search_request( @@ -897,30 +744,30 @@ def test_search_filters_date_200( ( get_search_request(is_cost_sharing_one_of=["t"]), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING, ], ), ( get_search_request(is_cost_sharing_one_of=["on"]), [ - NASA_SPACE_FELLOWSHIP, - NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING, ], ), ( get_search_request(is_cost_sharing_one_of=["false"]), - [NASA_INNOVATIONS, NASA_K12_DIVERSITY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], + [LOC_HIGHER_EDUCATION, NASA_K12_DIVERSITY, NASA_INNOVATIONS, DOC_SPACE_COAST], ), ( get_search_request(is_cost_sharing_one_of=["no"]), - [NASA_INNOVATIONS, NASA_K12_DIVERSITY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], + [LOC_HIGHER_EDUCATION, NASA_K12_DIVERSITY, NASA_INNOVATIONS, DOC_SPACE_COAST], ), ], ) @@ -936,12 +783,12 @@ def test_search_bool_filters_200( ( get_search_request(expected_number_of_awards={"min": 0, "max": 1000}), [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, - NASA_SUPERSONIC, LOC_TEACHING, - LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, + LOC_HIGHER_EDUCATION, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -956,17 +803,17 @@ def test_search_bool_filters_200( ), ( get_search_request(expected_number_of_awards={"min": 7}), - [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST, DOC_MANUFACTURING], + [LOC_TEACHING, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], ), # Award Floor ( get_search_request(award_floor={"min": 0, "max": 10_000_000_000}), [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, - NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -974,28 +821,28 @@ def test_search_bool_filters_200( ( get_search_request(award_floor={"min": 1, "max": 5_000}), [ - NASA_INNOVATIONS, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_INNOVATIONS, DOC_SPACE_COAST, ], ), ( get_search_request(award_floor={"min": 5_000, "max": 10_000}), [ - NASA_INNOVATIONS, NASA_SUPERSONIC, + NASA_INNOVATIONS, ], ), # Award Ceiling ( get_search_request(award_ceiling={"min": 0, "max": 10_000_000_000}), [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, - NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, + NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -1003,15 +850,15 @@ def test_search_bool_filters_200( ( get_search_request(award_ceiling={"min": 5_000, "max": 50_000}), [ - NASA_INNOVATIONS, NASA_SUPERSONIC, + NASA_INNOVATIONS, ], ), ( get_search_request(award_ceiling={"min": 50_000}), [ - NASA_SPACE_FELLOWSHIP, NASA_SUPERSONIC, + NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING, ], ), @@ -1021,11 +868,11 @@ def test_search_bool_filters_200( estimated_total_program_funding={"min": 0, "max": 100_000_000_000} ), [ - NASA_SPACE_FELLOWSHIP, - NASA_INNOVATIONS, LOC_TEACHING, - LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, + LOC_HIGHER_EDUCATION, + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -1033,8 +880,8 @@ def test_search_bool_filters_200( ( get_search_request(estimated_total_program_funding={"min": 0, "max": 5_000}), [ - NASA_INNOVATIONS, DOS_DIGITAL_LITERACY, + NASA_INNOVATIONS, DOC_SPACE_COAST, ], ), @@ -1054,7 +901,7 @@ def test_search_bool_filters_200( award_floor={"min": 1_000, "max": 10_000}, award_ceiling={"max": 10_000_000}, ), - [NASA_INNOVATIONS, NASA_SUPERSONIC], + [NASA_SUPERSONIC, NASA_INNOVATIONS], ), ( get_search_request( @@ -1383,10 +1230,10 @@ def test_search_validate_award_values_nullability_422( order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="space" ), [ + DOS_DIGITAL_LITERACY, + NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, - NASA_K12_DIVERSITY, - DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, ], ), @@ -1396,7 +1243,7 @@ def test_search_validate_award_values_nullability_422( sort_direction=SortDirection.ASCENDING, query="43.008", ), - [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + [LOC_TEACHING, NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], ), ( get_search_request( From 1c28968394ba79fdb9129c2f5ad8148cfc819a1c Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 16:18:19 -0500 Subject: [PATCH 10/24] rm unused check --- .../src/api/opportunities_v1/test_opportunity_route_search.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index f7c829ef1..661166111 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -41,10 +41,6 @@ def validate_search_response( response_ids = [int(opp["opportunity_id"]) for opp in opportunities] - for opp in opportunities: - if "summary" in opp: - assert "agency_phone_number" not in opp["summary"] - assert ( response_ids == expected_ids ), f"Actual opportunities:\n {'\n'.join([opp['opportunity_title'] for opp in opportunities])}" From b6cf6f0771c6d937b754db24d84ffa44a6de4ffa Mon Sep 17 00:00:00 2001 From: bruk Date: Wed, 29 Jan 2025 16:33:00 -0500 Subject: [PATCH 11/24] cleanup --- api/src/adapters/search/opensearch_client.py | 2 -- api/src/services/opportunities_v1/search_opportunities.py | 4 +--- .../src/api/opportunities_v1/test_opportunity_route_search.py | 4 ++++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index fe49ec8b3..ad3de12f7 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -216,7 +216,6 @@ def search( search_query: dict, include_scores: bool = True, params: dict | None = None, - includes: list[str] | None = None, excludes: list[str] | None = None, ) -> SearchResponse: if params is None: @@ -226,7 +225,6 @@ def search( index=index_name, body=search_query, params=params, - _source_includes=includes, _source_excludes=excludes, ) return SearchResponse.from_opensearch_response(response, include_scores) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 7b74ed8f0..54c5577ad 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -194,9 +194,7 @@ def search_opportunities( "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} ) - response = search_client.search( - index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"] - ) + response = search_client.search(index_alias, search_request, excludes=["attachments"]) pagination_info = PaginationInfo( page_offset=search_params.pagination.page_offset, diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index 661166111..f7c829ef1 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -41,6 +41,10 @@ def validate_search_response( response_ids = [int(opp["opportunity_id"]) for opp in opportunities] + for opp in opportunities: + if "summary" in opp: + assert "agency_phone_number" not in opp["summary"] + assert ( response_ids == expected_ids ), f"Actual opportunities:\n {'\n'.join([opp['opportunity_title'] for opp in opportunities])}" From 94e0faa69cb0017c2da766b047b1e4ef89d2bf87 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 15:35:13 -0500 Subject: [PATCH 12/24] update route to use search client and call func --- api/src/api/users/user_routes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/api/src/api/users/user_routes.py b/api/src/api/users/user_routes.py index 172624c96..49348fc06 100644 --- a/api/src/api/users/user_routes.py +++ b/api/src/api/users/user_routes.py @@ -3,7 +3,8 @@ import flask -from src.adapters import db +import src.adapters.search.flask_opensearch as flask_opensearch +from src.adapters import db, search from src.adapters.db import flask_db from src.api import response from src.api.route_utils import raise_flask_error @@ -26,6 +27,7 @@ from src.auth.auth_utils import with_login_redirect_error_handler from src.auth.login_gov_jwt_auth import get_final_redirect_uri, get_login_gov_redirect_uri from src.db.models.user_models import UserSavedOpportunity, UserTokenSession +from src.services.opportunities_v1.search_opportunities import search_opportunities_id from src.services.users.create_saved_search import create_saved_search from src.services.users.delete_saved_opportunity import delete_saved_opportunity from src.services.users.delete_saved_search import delete_saved_search @@ -246,8 +248,9 @@ def user_get_saved_opportunities(db_session: db.Session, user_id: UUID) -> respo @user_blueprint.doc(responses=[200, 401]) @user_blueprint.auth_required(api_jwt_auth) @flask_db.with_db_session() +@flask_opensearch.with_search_client() def user_save_search( - db_session: db.Session, user_id: UUID, json_data: dict + search_client: search.SearchClient, db_session: db.Session, user_id: UUID, json_data: dict ) -> response.ApiResponse: logger.info("POST /v1/users/:user_id/saved-searches") @@ -257,8 +260,11 @@ def user_save_search( if user_token_session.user_id != user_id: raise_flask_error(401, "Unauthorized user") + # Retrieve opportunity IDs + opportunity_ids = search_opportunities_id(search_client, json_data) + with db_session.begin(): - saved_search = create_saved_search(db_session, user_id, json_data) + saved_search = create_saved_search(db_session, user_id, json_data, opportunity_ids) logger.info( "Saved search for user", From cab3d66f7c17cb5f8cf6934458fcdf21a79a0fc9 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 15:36:05 -0500 Subject: [PATCH 13/24] func to call opensearch --- .../opportunities_v1/search_opportunities.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 54c5577ad..156a239f9 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -154,7 +154,7 @@ def _add_aggregations(builder: search.SearchQueryBuilder) -> None: builder.aggregation_terms("agency", _adjust_field_name("agency_code"), size=1000) -def _get_search_request(params: SearchOpportunityParams) -> dict: +def _get_search_request(params: SearchOpportunityParams, aggregation: bool = True) -> dict: builder = search.SearchQueryBuilder() # Make sure total hit count gets counted for more than 10k records @@ -176,8 +176,9 @@ def _get_search_request(params: SearchOpportunityParams) -> dict: # Filters _add_search_filters(builder, params.filters) - # Aggregations / Facet / Filter Counts - _add_aggregations(builder) + if aggregation: + # Aggregations / Facet / Filter Counts + _add_aggregations(builder) return builder.build() @@ -186,7 +187,6 @@ def search_opportunities( search_client: search.SearchClient, raw_search_params: dict ) -> Tuple[Sequence[dict], dict, PaginationInfo]: search_params = SearchOpportunityParams.model_validate(raw_search_params) - search_request = _get_search_request(search_params) index_alias = get_search_config().opportunity_search_index_alias @@ -210,6 +210,23 @@ def search_opportunities( # which means anything that requires conversions like timestamps end up failing # as they don't need to be converted. So, we convert everything to those types (serialize) # so that deserialization won't fail. - records = SCHEMA.load(response.records, many=True) + records = SCHEMA.load(response.records, many=True) # different just op id return records, response.aggregations, pagination_info + + +def search_opportunities_id(search_client: search.SearchClient, search_data: dict) -> list: + search_params = SearchOpportunityParams.model_validate(search_data["search_query"]) + + search_request = _get_search_request(search_params, False) + + index_alias = get_search_config().opportunity_search_index_alias + logger.info( + "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} + ) + + response = search_client.search( + index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"] + ) + + return [opp["opportunity_id"] for opp in response.records] From d640e23d7b6b6fbbddf39ff9bc4b051992135cd2 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 15:36:47 -0500 Subject: [PATCH 14/24] add optional arg includes to search --- api/src/adapters/search/opensearch_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index ad3de12f7..fe49ec8b3 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -216,6 +216,7 @@ def search( search_query: dict, include_scores: bool = True, params: dict | None = None, + includes: list[str] | None = None, excludes: list[str] | None = None, ) -> SearchResponse: if params is None: @@ -225,6 +226,7 @@ def search( index=index_name, body=search_query, params=params, + _source_includes=includes, _source_excludes=excludes, ) return SearchResponse.from_opensearch_response(response, include_scores) From 011be5b7448f1fb74468fe9c57544e1ef176192f Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 15:37:03 -0500 Subject: [PATCH 15/24] save opids to db --- api/src/services/users/create_saved_search.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/services/users/create_saved_search.py b/api/src/services/users/create_saved_search.py index ac892ae53..afcc99d47 100644 --- a/api/src/services/users/create_saved_search.py +++ b/api/src/services/users/create_saved_search.py @@ -7,12 +7,14 @@ logger = logging.getLogger(__name__) -def create_saved_search(db_session: db.Session, user_id: UUID, json_data: dict) -> UserSavedSearch: +def create_saved_search( + db_session: db.Session, user_id: UUID, json_data: dict, opportunity_ids: list +) -> UserSavedSearch: saved_search = UserSavedSearch( user_id=user_id, name=json_data["name"], search_query=json_data["search_query"], - searched_opportunity_ids=[], + searched_opportunity_ids=opportunity_ids, ) db_session.add(saved_search) From 27ba5e512aed65a35cbedd738038ea6ae922d7e0 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 15:37:20 -0500 Subject: [PATCH 16/24] update test with check for saved opIDs --- .../api/users/test_user_save_search_post.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/api/tests/src/api/users/test_user_save_search_post.py b/api/tests/src/api/users/test_user_save_search_post.py index c8e0ba2bd..86acbc179 100644 --- a/api/tests/src/api/users/test_user_save_search_post.py +++ b/api/tests/src/api/users/test_user_save_search_post.py @@ -1,8 +1,10 @@ import pytest +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema from src.constants.lookup_constants import FundingInstrument from src.db.models.user_models import UserSavedSearch from tests.src.api.opportunities_v1.conftest import get_search_request +from tests.src.api.opportunities_v1.test_opportunity_route_search import LOC_HIGHER_EDUCATION from tests.src.db.models.factories import UserFactory @@ -71,7 +73,16 @@ def test_user_save_search_post_invalid_request(client, user, user_auth_token, db assert len(saved_searches) == 0 -def test_user_save_search_post(client, user, user_auth_token, enable_factory_create, db_session): +def test_user_save_search_post( + client, + opportunity_index, + opportunity_index_alias, + search_client, + user, + user_auth_token, + enable_factory_create, + db_session, +): # Test data search_name = "Test Search" search_query = get_search_request( @@ -79,6 +90,14 @@ def test_user_save_search_post(client, user, user_auth_token, enable_factory_cre agency_one_of=["LOC"], ) + # Load into the search index + schema = OpportunityV1Schema() + json_records = [schema.dump(LOC_HIGHER_EDUCATION)] + search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") + + # Swap the search index alias + search_client.swap_alias_index(opportunity_index, opportunity_index_alias) + # Make the request to save a search response = client.post( f"/v1/users/{user.user_id}/saved-searches", @@ -103,3 +122,5 @@ def test_user_save_search_post(client, user, user_auth_token, enable_factory_cre "sort_direction": "ascending", }, } + assert 1 == 2 + assert saved_search.searched_opportunity_ids == [LOC_HIGHER_EDUCATION.opportunity_id] From bd8029b00051d973fda396150a84c5fec372b798 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 16:03:50 -0500 Subject: [PATCH 17/24] udpate test --- api/tests/src/api/users/test_user_save_search_post.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/tests/src/api/users/test_user_save_search_post.py b/api/tests/src/api/users/test_user_save_search_post.py index 86acbc179..d3b987303 100644 --- a/api/tests/src/api/users/test_user_save_search_post.py +++ b/api/tests/src/api/users/test_user_save_search_post.py @@ -1,3 +1,5 @@ +import uuid + import pytest from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema @@ -82,6 +84,7 @@ def test_user_save_search_post( user_auth_token, enable_factory_create, db_session, + monkeypatch, ): # Test data search_name = "Test Search" @@ -96,7 +99,10 @@ def test_user_save_search_post( search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") # Swap the search index alias - search_client.swap_alias_index(opportunity_index, opportunity_index_alias) + alias = f"test-user_save_search-index-alias-{uuid.uuid4().int}" + monkeypatch.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) + + search_client.swap_alias_index(opportunity_index, alias) # Make the request to save a search response = client.post( @@ -122,5 +128,4 @@ def test_user_save_search_post( "sort_direction": "ascending", }, } - assert 1 == 2 assert saved_search.searched_opportunity_ids == [LOC_HIGHER_EDUCATION.opportunity_id] From 154e6c6ce68f83b6204c2277926e085fcb6f84da Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 16:14:02 -0500 Subject: [PATCH 18/24] cleanup --- .../opportunities_v1/opportunity_routes.py | 8 - .../opportunities_v1/search_opportunities.py | 2 +- .../test_opportunity_route_search.py | 297 +++++++++++++----- 3 files changed, 226 insertions(+), 81 deletions(-) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index d8e7416b3..6500f2bdc 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -153,14 +153,6 @@ def opportunity_search( add_extra_data_to_current_request_logs(flatten_dict(search_params, prefix="request.body")) logger.info("POST /v1/opportunities/search") - # Override pagination - search_params["pagination"] = { - "order_by": "post_date", - "page_offset": 1, - "page_size": 1000, - "sort_direction": "descending", - } - opportunities, aggregations, pagination_info = search_opportunities( search_client, search_params ) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 156a239f9..f70b2569a 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -210,7 +210,7 @@ def search_opportunities( # which means anything that requires conversions like timestamps end up failing # as they don't need to be converted. So, we convert everything to those types (serialize) # so that deserialization won't fail. - records = SCHEMA.load(response.records, many=True) # different just op id + records = SCHEMA.load(response.records, many=True) return records, response.aggregations, pagination_info diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index 0578c17f5..3072564c7 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -32,6 +32,7 @@ def validate_search_response( assert search_response.status_code == expected_status_code expected_ids = [exp.opportunity_id for exp in expected_results] + if is_csv_response: reader = csv.DictReader(search_response.text.split("\n")) opportunities = [record for record in reader] @@ -333,13 +334,13 @@ def build_opp( ) OPPORTUNITIES = [ - LOC_TEACHING, - DOS_DIGITAL_LITERACY, - LOC_HIGHER_EDUCATION, - NASA_K12_DIVERSITY, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_SUPERSONIC, + NASA_K12_DIVERSITY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ] @@ -364,7 +365,6 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c @pytest.mark.parametrize( "search_request,expected_results", [ - # Pagination should be static, results should not defer # Opportunity ID ( get_search_request( @@ -382,9 +382,18 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, ), - OPPORTUNITIES, + OPPORTUNITIES[3:6], + ), + ( + get_search_request( + page_size=25, + page_offset=1, + order_by="opportunity_id", + sort_direction=SortDirection.DESCENDING, + ), + OPPORTUNITIES[::-1], ), - # # Opportunity Number + # Opportunity Number ( get_search_request( page_size=3, @@ -392,7 +401,151 @@ def setup_search_data(self, opportunity_index, opportunity_index_alias, search_c order_by="opportunity_number", sort_direction=SortDirection.ASCENDING, ), - OPPORTUNITIES, + [LOC_TEACHING, LOC_HIGHER_EDUCATION, DOC_MANUFACTURING], + ), + ( + get_search_request( + page_size=2, + page_offset=3, + order_by="opportunity_number", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], + ), + # Opportunity Title + ( + get_search_request( + page_size=4, + page_offset=2, + order_by="opportunity_title", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SPACE_FELLOWSHIP, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST, NASA_K12_DIVERSITY], + ), + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="opportunity_title", + sort_direction=SortDirection.DESCENDING, + ), + [ + LOC_TEACHING, + NASA_K12_DIVERSITY, + DOC_SPACE_COAST, + LOC_HIGHER_EDUCATION, + NASA_SPACE_FELLOWSHIP, + ], + ), + # Post Date + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Relevancy has a secondary sort of post date so should be identical. + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Close Date (note several have null values which always go to the end) + ( + get_search_request( + page_size=4, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [LOC_TEACHING, NASA_K12_DIVERSITY, DOC_SPACE_COAST, DOS_DIGITAL_LITERACY], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.DESCENDING, + ), + [DOC_MANUFACTURING, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], + ), + # close date - but check the end of the list to find the null values + ( + get_search_request( + page_size=5, + page_offset=2, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SUPERSONIC, DOC_MANUFACTURING, NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + ), + # Agency + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.ASCENDING, + ), + [ + DOC_SPACE_COAST, + DOC_MANUFACTURING, + DOS_DIGITAL_LITERACY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + ], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC], ), ], ids=search_scenario_id_fnc, @@ -431,7 +584,7 @@ def test_page_size_422(self, client, api_auth_token, search_request): (get_search_request(agency_one_of=["not an agency"]), []), ( get_search_request(agency_one_of=["NASA"]), - [NASA_K12_DIVERSITY, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS], + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC, NASA_K12_DIVERSITY], ), (get_search_request(agency_one_of=["LOC"]), [LOC_TEACHING, LOC_HIGHER_EDUCATION]), (get_search_request(agency_one_of=["DOS-ECA"]), [DOS_DIGITAL_LITERACY]), @@ -445,15 +598,15 @@ def test_page_size_422(self, client, api_auth_token, search_request): # Opportunity Status ( get_search_request(opportunity_status_one_of=[OpportunityStatus.POSTED]), - [LOC_TEACHING, NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING], + [NASA_SPACE_FELLOWSHIP, LOC_TEACHING, DOC_MANUFACTURING], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.FORECASTED]), - [LOC_HIGHER_EDUCATION, NASA_INNOVATIONS], + [NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.CLOSED]), - [DOS_DIGITAL_LITERACY, NASA_SUPERSONIC], + [NASA_SUPERSONIC, DOS_DIGITAL_LITERACY], ), ( get_search_request(opportunity_status_one_of=[OpportunityStatus.ARCHIVED]), @@ -467,10 +620,10 @@ def test_page_size_422(self, client, api_auth_token, search_request): ] ), [ - LOC_TEACHING, - LOC_HIGHER_EDUCATION, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, DOC_MANUFACTURING, ], ), @@ -491,10 +644,10 @@ def test_page_size_422(self, client, api_auth_token, search_request): funding_instrument_one_of=[FundingInstrument.COOPERATIVE_AGREEMENT] ), [ + NASA_SPACE_FELLOWSHIP, + NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_K12_DIVERSITY, - NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -502,9 +655,9 @@ def test_page_size_422(self, client, api_auth_token, search_request): ( get_search_request(funding_instrument_one_of=[FundingInstrument.GRANT]), [ - LOC_HIGHER_EDUCATION, - NASA_SUPERSONIC, NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_HIGHER_EDUCATION, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -528,7 +681,7 @@ def test_page_size_422(self, client, api_auth_token, search_request): # Funding Category ( get_search_request(funding_category_one_of=[FundingCategory.EDUCATION]), - [LOC_TEACHING, NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], ), ( get_search_request( @@ -536,11 +689,11 @@ def test_page_size_422(self, client, api_auth_token, search_request): FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT ] ), - [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_MANUFACTURING], + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], ), ( get_search_request(funding_category_one_of=[FundingCategory.OTHER]), - [DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, DOC_SPACE_COAST], ), ( get_search_request(funding_category_one_of=[FundingCategory.REGIONAL_DEVELOPMENT]), @@ -564,16 +717,16 @@ def test_page_size_422(self, client, api_auth_token, search_request): FundingCategory.REGIONAL_DEVELOPMENT, ] ), - [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_SPACE_COAST, DOC_MANUFACTURING], + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], ), # Applicant Type ( get_search_request(applicant_type_one_of=[ApplicantType.OTHER]), [ - DOS_DIGITAL_LITERACY, - NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, DOC_MANUFACTURING, ], ), @@ -595,7 +748,7 @@ def test_page_size_422(self, client, api_auth_token, search_request): ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION ] ), - [DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY], ), (get_search_request(applicant_type_one_of=[ApplicantType.INDIVIDUALS]), []), ( @@ -605,14 +758,14 @@ def test_page_size_422(self, client, api_auth_token, search_request): ApplicantType.UNRESTRICTED, ] ), - [LOC_TEACHING, NASA_SUPERSONIC, DOC_SPACE_COAST], + [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST], ), # Mix ( get_search_request( agency_one_of=["NASA"], applicant_type_one_of=[ApplicantType.OTHER] ), - [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS], + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_K12_DIVERSITY], ), ( get_search_request( @@ -624,7 +777,7 @@ def test_page_size_422(self, client, api_auth_token, search_request): FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT ], ), - [NASA_SUPERSONIC, NASA_INNOVATIONS, DOC_MANUFACTURING], + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], ), ( get_search_request( @@ -671,13 +824,13 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect get_search_request( post_date={"start_date": "2019-06-01", "end_date": "2024-01-01"} ), - [NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], + [NASA_SPACE_FELLOWSHIP, NASA_SUPERSONIC], ), ( get_search_request( post_date={"start_date_relative": -2063, "end_date_relative": -389} ), - [NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], + [NASA_SPACE_FELLOWSHIP, NASA_SUPERSONIC], ), (get_search_request(post_date={"end_date": "2016-01-01"}), [DOC_MANUFACTURING]), (get_search_request(post_date={"end_date_relative": -3310}), [DOC_MANUFACTURING]), @@ -687,11 +840,11 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect close_date={"start_date": "1970-01-01", "end_date": "2050-01-01"} ), [ + NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, + NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_K12_DIVERSITY, - NASA_SUPERSONIC, - NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -701,11 +854,11 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect close_date={"start_date_relative": -20111, "end_date_relative": 9131} ), [ + NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, + NASA_K12_DIVERSITY, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_K12_DIVERSITY, - NASA_SUPERSONIC, - NASA_SPACE_FELLOWSHIP, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -713,9 +866,9 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect ( get_search_request(close_date={"start_date": "2019-01-01"}), [ - DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -723,20 +876,20 @@ def test_search_filters_200(self, client, api_auth_token, search_request, expect ( get_search_request(close_date={"start_date_relative": -2214}), [ - DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], ), ( get_search_request(close_date={"end_date": "2019-01-01"}), - [LOC_TEACHING, NASA_K12_DIVERSITY], + [NASA_K12_DIVERSITY, LOC_TEACHING], ), ( get_search_request(close_date={"end_date_relative": -2214}), - [LOC_TEACHING, NASA_K12_DIVERSITY], + [NASA_K12_DIVERSITY, LOC_TEACHING], ), ( get_search_request( @@ -766,30 +919,30 @@ def test_search_filters_date_200( ( get_search_request(is_cost_sharing_one_of=["t"]), [ + NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, - NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING, ], ), ( get_search_request(is_cost_sharing_one_of=["on"]), [ + NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, - NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING, ], ), ( get_search_request(is_cost_sharing_one_of=["false"]), - [LOC_HIGHER_EDUCATION, NASA_K12_DIVERSITY, NASA_INNOVATIONS, DOC_SPACE_COAST], + [NASA_INNOVATIONS, NASA_K12_DIVERSITY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], ), ( get_search_request(is_cost_sharing_one_of=["no"]), - [LOC_HIGHER_EDUCATION, NASA_K12_DIVERSITY, NASA_INNOVATIONS, DOC_SPACE_COAST], + [NASA_INNOVATIONS, NASA_K12_DIVERSITY, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST], ), ], ) @@ -805,12 +958,12 @@ def test_search_bool_filters_200( ( get_search_request(expected_number_of_awards={"min": 0, "max": 1000}), [ - LOC_TEACHING, - DOS_DIGITAL_LITERACY, - LOC_HIGHER_EDUCATION, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -825,17 +978,17 @@ def test_search_bool_filters_200( ), ( get_search_request(expected_number_of_awards={"min": 7}), - [LOC_TEACHING, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], + [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST, DOC_MANUFACTURING], ), # Award Floor ( get_search_request(award_floor={"min": 0, "max": 10_000_000_000}), [ - LOC_TEACHING, - DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_TEACHING, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -843,28 +996,28 @@ def test_search_bool_filters_200( ( get_search_request(award_floor={"min": 1, "max": 5_000}), [ + NASA_INNOVATIONS, LOC_TEACHING, DOS_DIGITAL_LITERACY, - NASA_INNOVATIONS, DOC_SPACE_COAST, ], ), ( get_search_request(award_floor={"min": 5_000, "max": 10_000}), [ - NASA_SUPERSONIC, NASA_INNOVATIONS, + NASA_SUPERSONIC, ], ), # Award Ceiling ( get_search_request(award_ceiling={"min": 0, "max": 10_000_000_000}), [ - LOC_TEACHING, - DOS_DIGITAL_LITERACY, - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_TEACHING, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -872,15 +1025,15 @@ def test_search_bool_filters_200( ( get_search_request(award_ceiling={"min": 5_000, "max": 50_000}), [ - NASA_SUPERSONIC, NASA_INNOVATIONS, + NASA_SUPERSONIC, ], ), ( get_search_request(award_ceiling={"min": 50_000}), [ - NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP, + NASA_SUPERSONIC, DOC_MANUFACTURING, ], ), @@ -890,11 +1043,11 @@ def test_search_bool_filters_200( estimated_total_program_funding={"min": 0, "max": 100_000_000_000} ), [ - LOC_TEACHING, - DOS_DIGITAL_LITERACY, - LOC_HIGHER_EDUCATION, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, DOC_MANUFACTURING, ], @@ -902,8 +1055,8 @@ def test_search_bool_filters_200( ( get_search_request(estimated_total_program_funding={"min": 0, "max": 5_000}), [ - DOS_DIGITAL_LITERACY, NASA_INNOVATIONS, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, ], ), @@ -923,7 +1076,7 @@ def test_search_bool_filters_200( award_floor={"min": 1_000, "max": 10_000}, award_ceiling={"max": 10_000_000}, ), - [NASA_SUPERSONIC, NASA_INNOVATIONS], + [NASA_INNOVATIONS, NASA_SUPERSONIC], ), ( get_search_request( @@ -1252,10 +1405,10 @@ def test_search_validate_award_values_nullability_422( order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="space" ), [ - DOS_DIGITAL_LITERACY, - NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, DOC_SPACE_COAST, ], ), @@ -1265,7 +1418,7 @@ def test_search_validate_award_values_nullability_422( sort_direction=SortDirection.ASCENDING, query="43.008", ), - [LOC_TEACHING, NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], ), ( get_search_request( From 9d6770077fe14657dab76af09d9e5475205e4ba8 Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 18:48:34 -0500 Subject: [PATCH 19/24] overwrite pagination update test --- .../opportunities_v1/opportunity_routes.py | 1 - api/src/api/users/user_routes.py | 2 +- .../opportunities_v1/search_opportunities.py | 15 +++++++++++++-- .../api/users/test_user_save_search_post.py | 19 +++++++++++++------ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index 6500f2bdc..0b645bfc3 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -163,7 +163,6 @@ def opportunity_search( "response.pagination.total_records": pagination_info.total_records, } ) - logger.info("Successfully fetched opportunities") if search_params.get("format") == opportunity_schemas.SearchResponseFormat.CSV: # Convert the response into a CSV and return the contents diff --git a/api/src/api/users/user_routes.py b/api/src/api/users/user_routes.py index 49348fc06..23293424c 100644 --- a/api/src/api/users/user_routes.py +++ b/api/src/api/users/user_routes.py @@ -261,7 +261,7 @@ def user_save_search( raise_flask_error(401, "Unauthorized user") # Retrieve opportunity IDs - opportunity_ids = search_opportunities_id(search_client, json_data) + opportunity_ids = search_opportunities_id(search_client, json_data["search_query"]) with db_session.begin(): saved_search = create_saved_search(db_session, user_id, json_data, opportunity_ids) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index f70b2569a..e215d076b 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -215,8 +215,19 @@ def search_opportunities( return records, response.aggregations, pagination_info -def search_opportunities_id(search_client: search.SearchClient, search_data: dict) -> list: - search_params = SearchOpportunityParams.model_validate(search_data["search_query"]) +def search_opportunities_id(search_client: search.SearchClient, search_query: dict) -> list: + # Override pagination when calling opensearch + updated_search_query = { + **search_query, + "pagination": { + "order_by": "post_date", + "page_offset": 1, + "page_size": 1000, + "sort_direction": "descending", + }, + } + + search_params = SearchOpportunityParams.model_validate(updated_search_query) search_request = _get_search_request(search_params, False) diff --git a/api/tests/src/api/users/test_user_save_search_post.py b/api/tests/src/api/users/test_user_save_search_post.py index d3b987303..c32fe30ec 100644 --- a/api/tests/src/api/users/test_user_save_search_post.py +++ b/api/tests/src/api/users/test_user_save_search_post.py @@ -6,7 +6,10 @@ from src.constants.lookup_constants import FundingInstrument from src.db.models.user_models import UserSavedSearch from tests.src.api.opportunities_v1.conftest import get_search_request -from tests.src.api.opportunities_v1.test_opportunity_route_search import LOC_HIGHER_EDUCATION +from tests.src.api.opportunities_v1.test_opportunity_route_search import ( + NASA_INNOVATIONS, + NASA_SUPERSONIC, +) from tests.src.db.models.factories import UserFactory @@ -90,12 +93,12 @@ def test_user_save_search_post( search_name = "Test Search" search_query = get_search_request( funding_instrument_one_of=[FundingInstrument.GRANT], - agency_one_of=["LOC"], + agency_one_of=["NASA"], ) # Load into the search index schema = OpportunityV1Schema() - json_records = [schema.dump(LOC_HIGHER_EDUCATION)] + json_records = [schema.dump(opp) for opp in [NASA_INNOVATIONS, NASA_SUPERSONIC]] search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") # Swap the search index alias @@ -113,14 +116,14 @@ def test_user_save_search_post( assert response.status_code == 200 assert response.json["message"] == "Success" - # Verify the search was saved in the database saved_search = db_session.query(UserSavedSearch).one() + assert saved_search.user_id == user.user_id assert saved_search.name == search_name assert saved_search.search_query == { "format": "json", - "filters": {"agency": {"one_of": ["LOC"]}, "funding_instrument": {"one_of": ["grant"]}}, + "filters": {"agency": {"one_of": ["NASA"]}, "funding_instrument": {"one_of": ["grant"]}}, "pagination": { "order_by": "opportunity_id", "page_size": 25, @@ -128,4 +131,8 @@ def test_user_save_search_post( "sort_direction": "ascending", }, } - assert saved_search.searched_opportunity_ids == [LOC_HIGHER_EDUCATION.opportunity_id] + # Verify pagination for the query was over-written. searched_opportunity_ids should be ordered by "post_date" + assert saved_search.searched_opportunity_ids == [ + NASA_SUPERSONIC.opportunity_id, + NASA_INNOVATIONS.opportunity_id, + ] From 63a4118525f4c1bd16d251ca36b729821eebeb4a Mon Sep 17 00:00:00 2001 From: bruk Date: Thu, 30 Jan 2025 18:52:17 -0500 Subject: [PATCH 20/24] put bk log --- api/src/api/opportunities_v1/opportunity_routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index 0b645bfc3..6500f2bdc 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -163,6 +163,7 @@ def opportunity_search( "response.pagination.total_records": pagination_info.total_records, } ) + logger.info("Successfully fetched opportunities") if search_params.get("format") == opportunity_schemas.SearchResponseFormat.CSV: # Convert the response into a CSV and return the contents From 661a62a725441eb50d24c91352be4182ee6f4b83 Mon Sep 17 00:00:00 2001 From: bruk Date: Fri, 31 Jan 2025 11:15:58 -0500 Subject: [PATCH 21/24] cleanup route --- api/src/api/users/user_routes.py | 6 +----- api/src/services/users/create_saved_search.py | 9 +++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/api/users/user_routes.py b/api/src/api/users/user_routes.py index 23293424c..91d0f6a07 100644 --- a/api/src/api/users/user_routes.py +++ b/api/src/api/users/user_routes.py @@ -27,7 +27,6 @@ from src.auth.auth_utils import with_login_redirect_error_handler from src.auth.login_gov_jwt_auth import get_final_redirect_uri, get_login_gov_redirect_uri from src.db.models.user_models import UserSavedOpportunity, UserTokenSession -from src.services.opportunities_v1.search_opportunities import search_opportunities_id from src.services.users.create_saved_search import create_saved_search from src.services.users.delete_saved_opportunity import delete_saved_opportunity from src.services.users.delete_saved_search import delete_saved_search @@ -260,11 +259,8 @@ def user_save_search( if user_token_session.user_id != user_id: raise_flask_error(401, "Unauthorized user") - # Retrieve opportunity IDs - opportunity_ids = search_opportunities_id(search_client, json_data["search_query"]) - with db_session.begin(): - saved_search = create_saved_search(db_session, user_id, json_data, opportunity_ids) + saved_search = create_saved_search(search_client, db_session, user_id, json_data) logger.info( "Saved search for user", diff --git a/api/src/services/users/create_saved_search.py b/api/src/services/users/create_saved_search.py index afcc99d47..9f62159fb 100644 --- a/api/src/services/users/create_saved_search.py +++ b/api/src/services/users/create_saved_search.py @@ -1,15 +1,20 @@ import logging from uuid import UUID -from src.adapters import db +from src.adapters import db, search from src.db.models.user_models import UserSavedSearch +from src.services.opportunities_v1.search_opportunities import search_opportunities_id logger = logging.getLogger(__name__) def create_saved_search( - db_session: db.Session, user_id: UUID, json_data: dict, opportunity_ids: list + search_client: search.SearchClient, db_session: db.Session, user_id: UUID, json_data: dict ) -> UserSavedSearch: + + # Retrieve opportunity IDs + opportunity_ids = search_opportunities_id(search_client, json_data["search_query"]) + saved_search = UserSavedSearch( user_id=user_id, name=json_data["name"], From 53324447279e478762804e5c17e253bc5a453787 Mon Sep 17 00:00:00 2001 From: bruk Date: Fri, 31 Jan 2025 11:16:24 -0500 Subject: [PATCH 22/24] create helper func --- .../opportunities_v1/search_opportunities.py | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index e215d076b..70be5b880 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field import src.adapters.search as search +from src.adapters.search.opensearch_response import SearchResponse from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema from src.pagination.pagination_models import PaginationInfo, PaginationParams, SortDirection from src.search.search_config import get_search_config @@ -53,6 +54,15 @@ ScoringRule.DEFAULT: DEFAULT, } +STATIC_PAGINATION = { + "pagination": { + "order_by": "post_date", + "page_offset": 1, + "page_size": 1000, + "sort_direction": "descending", + } +} + SCHEMA = OpportunityV1Schema() @@ -183,10 +193,11 @@ def _get_search_request(params: SearchOpportunityParams, aggregation: bool = Tru return builder.build() -def search_opportunities( - search_client: search.SearchClient, raw_search_params: dict -) -> Tuple[Sequence[dict], dict, PaginationInfo]: - search_params = SearchOpportunityParams.model_validate(raw_search_params) +def _search_opportunities( + search_client: search.SearchClient, + search_params: SearchOpportunityParams, + includes: list | None = None, +) -> SearchResponse: search_request = _get_search_request(search_params) index_alias = get_search_config().opportunity_search_index_alias @@ -194,7 +205,19 @@ def search_opportunities( "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} ) - response = search_client.search(index_alias, search_request, excludes=["attachments"]) + response = search_client.search( + index_alias, search_request, includes=includes, excludes=["attachments"] + ) + + return response + + +def search_opportunities( + search_client: search.SearchClient, raw_search_params: dict +) -> Tuple[Sequence[dict], dict, PaginationInfo]: + + search_params = SearchOpportunityParams.model_validate(raw_search_params) + response = _search_opportunities(search_client, search_params) pagination_info = PaginationInfo( page_offset=search_params.pagination.page_offset, @@ -217,27 +240,9 @@ def search_opportunities( def search_opportunities_id(search_client: search.SearchClient, search_query: dict) -> list: # Override pagination when calling opensearch - updated_search_query = { - **search_query, - "pagination": { - "order_by": "post_date", - "page_offset": 1, - "page_size": 1000, - "sort_direction": "descending", - }, - } - + updated_search_query = search_query | STATIC_PAGINATION search_params = SearchOpportunityParams.model_validate(updated_search_query) - search_request = _get_search_request(search_params, False) - - index_alias = get_search_config().opportunity_search_index_alias - logger.info( - "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} - ) - - response = search_client.search( - index_alias, search_request, includes=["opportunity_id"], excludes=["attachments"] - ) + response = _search_opportunities(search_client, search_params, includes=["opportunity_id"]) return [opp["opportunity_id"] for opp in response.records] From 0d88a456175800aed8c35ab893852a2cfc13942d Mon Sep 17 00:00:00 2001 From: bruk Date: Fri, 31 Jan 2025 13:59:25 -0500 Subject: [PATCH 23/24] update test --- .../api/users/test_user_save_search_post.py | 76 +++++++++++++++---- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/api/tests/src/api/users/test_user_save_search_post.py b/api/tests/src/api/users/test_user_save_search_post.py index c32fe30ec..085ee0c78 100644 --- a/api/tests/src/api/users/test_user_save_search_post.py +++ b/api/tests/src/api/users/test_user_save_search_post.py @@ -1,17 +1,57 @@ import uuid +from datetime import date import pytest from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema -from src.constants.lookup_constants import FundingInstrument +from src.constants.lookup_constants import ( + ApplicantType, + FundingCategory, + FundingInstrument, + OpportunityStatus, +) from src.db.models.user_models import UserSavedSearch from tests.src.api.opportunities_v1.conftest import get_search_request -from tests.src.api.opportunities_v1.test_opportunity_route_search import ( - NASA_INNOVATIONS, - NASA_SUPERSONIC, -) +from tests.src.api.opportunities_v1.test_opportunity_route_search import build_opp from tests.src.db.models.factories import UserFactory +SPORTS = build_opp( + opportunity_title="Research into Sports administrator industry", + opportunity_number="EFG8532950", + agency="USAID", + summary_description="HHS-CDC is looking to further investigate this topic. Car matter style top quality generation effort. Computer purpose while consumer left.", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[("79.718", "Huff LLC")], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2019, 12, 8), # + close_date=date(2024, 12, 28), # + is_cost_sharing=True, # + expected_number_of_awards=1, + award_floor=402500, + award_ceiling=8050000, + estimated_total_program_funding=5000, +) +MEDICAL_LABORATORY = build_opp( + opportunity_title="Research into Medical laboratory scientific officer industry", + opportunity_number="AO-44-EMC-878", + agency="USAID", + summary_description="HHS-CDC is looking to further investigate this topic. Car matter style top quality generation effort. Computer purpose while consumer left.", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[("43.012", "Brown LLC")], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2025, 1, 25), # + close_date=date(2025, 6, 4), # + is_cost_sharing=True, # + expected_number_of_awards=1, + award_floor=402500, + award_ceiling=8050000, + estimated_total_program_funding=5000, +) + @pytest.fixture(autouse=True, scope="function") def clear_saved_searches(db_session): @@ -20,6 +60,14 @@ def clear_saved_searches(db_session): yield +@pytest.fixture +def opportunity_search_index_alias(search_client, monkeypatch): + # Note we don't actually create anything, this is just a random name + alias = f"test-opportunity-search-index-alias-{uuid.uuid4().int}" + monkeypatch.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) + return alias + + def test_user_save_search_post_unauthorized_user(client, db_session, user, user_auth_token): # Try to save a search for a different user ID different_user = UserFactory.create() @@ -81,31 +129,27 @@ def test_user_save_search_post_invalid_request(client, user, user_auth_token, db def test_user_save_search_post( client, opportunity_index, - opportunity_index_alias, search_client, user, user_auth_token, enable_factory_create, db_session, + opportunity_search_index_alias, monkeypatch, ): # Test data search_name = "Test Search" search_query = get_search_request( funding_instrument_one_of=[FundingInstrument.GRANT], - agency_one_of=["NASA"], + agency_one_of=["USAID"], ) # Load into the search index schema = OpportunityV1Schema() - json_records = [schema.dump(opp) for opp in [NASA_INNOVATIONS, NASA_SUPERSONIC]] + json_records = [schema.dump(opp) for opp in [SPORTS, MEDICAL_LABORATORY]] search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") - # Swap the search index alias - alias = f"test-user_save_search-index-alias-{uuid.uuid4().int}" - monkeypatch.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) - - search_client.swap_alias_index(opportunity_index, alias) + search_client.swap_alias_index(opportunity_index, opportunity_search_index_alias) # Make the request to save a search response = client.post( @@ -123,7 +167,7 @@ def test_user_save_search_post( assert saved_search.name == search_name assert saved_search.search_query == { "format": "json", - "filters": {"agency": {"one_of": ["NASA"]}, "funding_instrument": {"one_of": ["grant"]}}, + "filters": {"agency": {"one_of": ["USAID"]}, "funding_instrument": {"one_of": ["grant"]}}, "pagination": { "order_by": "opportunity_id", "page_size": 25, @@ -133,6 +177,6 @@ def test_user_save_search_post( } # Verify pagination for the query was over-written. searched_opportunity_ids should be ordered by "post_date" assert saved_search.searched_opportunity_ids == [ - NASA_SUPERSONIC.opportunity_id, - NASA_INNOVATIONS.opportunity_id, + MEDICAL_LABORATORY.opportunity_id, + SPORTS.opportunity_id, ] From 8d42ab4be82ee234b08924bb70edeb28238102f4 Mon Sep 17 00:00:00 2001 From: bruk Date: Fri, 31 Jan 2025 13:59:45 -0500 Subject: [PATCH 24/24] add class scoped index alias fixt --- api/tests/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 03886f414..896a3d91d 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -220,6 +220,14 @@ def opportunity_index_alias(search_client, monkeypatch_session): return alias +@pytest.fixture(scope="class") +def opportunity_search_index_class(search_client, monkeypatch): + # Note we don't actually create anything, this is just a random name + alias = f"test-opportunity-index-alias-{uuid.uuid4().int}" + monkeypatch.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) + return alias + + def _generate_rsa_key_pair(): # Rather than define a private/public key, generate one for the tests key = rsa.generate_private_key(public_exponent=65537, key_size=2048)