diff --git a/components/nmdc_runtime/workflow_execution_activity/core.py b/components/nmdc_runtime/workflow_execution_activity/core.py index 8a550e80..ad7d020d 100644 --- a/components/nmdc_runtime/workflow_execution_activity/core.py +++ b/components/nmdc_runtime/workflow_execution_activity/core.py @@ -94,7 +94,9 @@ def insert_into_keys( workflow: Workflow, data_objects: list[DataObject] ) -> dict[str, Any]: """Insert data object url into correct workflow input field.""" - workflow_dict = workflow.dict() + workflow_dict = workflow.model_dump( + mode="json", + ) for key in workflow_dict["inputs"]: for do in data_objects: if workflow_dict["inputs"][key] == str(do.data_object_type): diff --git a/nmdc_runtime/api/core/util.py b/nmdc_runtime/api/core/util.py index 5aad716d..48d15ed1 100644 --- a/nmdc_runtime/api/core/util.py +++ b/nmdc_runtime/api/core/util.py @@ -98,6 +98,6 @@ def generate_secret(length=12): def json_clean(data, model, exclude_unset=False) -> dict: """Run data through a JSON serializer for a pydantic model.""" if not isinstance(data, (dict, BaseModel)): - raise TypeError("`data` must be a pydantic model or its .dict()") + raise TypeError("`data` must be a pydantic model or its .model_dump()") m = model(**data) if isinstance(data, dict) else data return json.loads(m.json(exclude_unset=exclude_unset)) diff --git a/nmdc_runtime/api/endpoints/objects.py b/nmdc_runtime/api/endpoints/objects.py index 13cfa3aa..0dd3b443 100644 --- a/nmdc_runtime/api/endpoints/objects.py +++ b/nmdc_runtime/api/endpoints/objects.py @@ -78,7 +78,7 @@ def create_object( """ id_supplied = supplied_object_id( - mdb, client_site, object_in.dict(exclude_unset=True) + mdb, client_site, object_in.model_dump(mode="json", exclude_unset=True) ) drs_id = local_part( id_supplied if id_supplied is not None else generate_one_id(mdb, S3_ID_NS) @@ -255,7 +255,9 @@ def update_object( status_code=status.HTTP_403_FORBIDDEN, detail=f"client authorized for different site_id than {object_mgr_site}", ) - doc_object_patched = merge(doc, object_patch.dict(exclude_unset=True)) + doc_object_patched = merge( + doc, object_patch.model_dump(mode="json", exclude_unset=True) + ) mdb.operations.replace_one({"id": object_id}, doc_object_patched) return doc_object_patched diff --git a/nmdc_runtime/api/endpoints/operations.py b/nmdc_runtime/api/endpoints/operations.py index 1f1f67b3..ecb4d33e 100644 --- a/nmdc_runtime/api/endpoints/operations.py +++ b/nmdc_runtime/api/endpoints/operations.py @@ -61,12 +61,16 @@ def update_operation( detail=f"client authorized for different site_id than {site_id_op}", ) op_patch_metadata = merge( - op_patch.dict(exclude_unset=True).get("metadata", {}), + op_patch.model_dump(mode="json", exclude_unset=True).get("metadata", {}), pick(["site_id", "job", "model"], doc_op.get("metadata", {})), ) doc_op_patched = merge( doc_op, - assoc(op_patch.dict(exclude_unset=True), "metadata", op_patch_metadata), + assoc( + op_patch.model_dump(mode="json", exclude_unset=True), + "metadata", + op_patch_metadata, + ), ) mdb.operations.replace_one({"id": op_id}, doc_op_patched) return doc_op_patched diff --git a/nmdc_runtime/api/endpoints/queries.py b/nmdc_runtime/api/endpoints/queries.py index 4e1c49c9..11417698 100644 --- a/nmdc_runtime/api/endpoints/queries.py +++ b/nmdc_runtime/api/endpoints/queries.py @@ -75,9 +75,9 @@ def run_query( id=qid, saved_at=saved_at, ) - mdb.queries.insert_one(query.dict(exclude_unset=True)) + mdb.queries.insert_one(query.model_dump(mode="json", exclude_unset=True)) cmd_response = _run_query(query, mdb) - return unmongo(cmd_response.dict(exclude_unset=True)) + return unmongo(cmd_response.model_dump(mode="json", exclude_unset=True)) @router.get("/queries/{query_id}", response_model=Query) @@ -107,7 +107,7 @@ def rerun_query( check_can_delete(user) cmd_response = _run_query(query, mdb) - return unmongo(cmd_response.dict(exclude_unset=True)) + return unmongo(cmd_response.model_dump(mode="json", exclude_unset=True)) def _run_query(query, mdb) -> CommandResponse: @@ -131,12 +131,12 @@ def _run_query(query, mdb) -> CommandResponse: detail="Failed to back up to-be-deleted documents. operation aborted.", ) - q_response = mdb.command(query.cmd.dict(exclude_unset=True)) + q_response = mdb.command(query.cmd.model_dump(mode="json", exclude_unset=True)) cmd_response: CommandResponse = command_response_for(q_type)(**q_response) query_run = ( QueryRun(qid=query.id, ran_at=ran_at, result=cmd_response) if cmd_response.ok else QueryRun(qid=query.id, ran_at=ran_at, error=cmd_response) ) - mdb.query_runs.insert_one(query_run.dict(exclude_unset=True)) + mdb.query_runs.insert_one(query_run.model_dump(mode="json", exclude_unset=True)) return cmd_response diff --git a/nmdc_runtime/api/endpoints/runs.py b/nmdc_runtime/api/endpoints/runs.py index c49b7800..8bd9f22d 100644 --- a/nmdc_runtime/api/endpoints/runs.py +++ b/nmdc_runtime/api/endpoints/runs.py @@ -94,5 +94,9 @@ def post_run_event( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Supplied run_event.run.id does not match run_id given in request URL.", ) - mdb.run_events.insert_one(run_event.dict()) + mdb.run_events.insert_one( + run_event.model_dump( + mode="json", + ) + ) return _get_run_summary(run_event.run.id, mdb) diff --git a/nmdc_runtime/api/endpoints/search.py b/nmdc_runtime/api/endpoints/search.py index 5fe80d2c..4813f7d2 100644 --- a/nmdc_runtime/api/endpoints/search.py +++ b/nmdc_runtime/api/endpoints/search.py @@ -25,7 +25,9 @@ def data_objects( req: DataObjectListRequest = Depends(), mdb: MongoDatabase = Depends(get_mongo_db), ): - filter_ = list_request_filter_to_mongo_filter(req.dict(exclude_unset=True)) + filter_ = list_request_filter_to_mongo_filter( + req.model_dump(mode="json", exclude_unset=True) + ) max_page_size = filter_.pop("max_page_size", None) page_token = filter_.pop("page_token", None) req = ListRequest( diff --git a/nmdc_runtime/api/endpoints/sites.py b/nmdc_runtime/api/endpoints/sites.py index f63fd993..76adfdc1 100644 --- a/nmdc_runtime/api/endpoints/sites.py +++ b/nmdc_runtime/api/endpoints/sites.py @@ -56,7 +56,11 @@ def create_site( status_code=status.HTTP_409_CONFLICT, detail=f"site with supplied id {site.id} already exists", ) - mdb.sites.insert_one(site.dict()) + mdb.sites.insert_one( + site.model_dump( + mode="json", + ) + ) refresh_minter_requesters_from_sites() rv = mdb.users.update_one( {"username": user.username}, @@ -165,7 +169,11 @@ def put_object_in_site( }, } ) - mdb.operations.insert_one(op.dict()) + mdb.operations.insert_one( + op.model_dump( + mode="json", + ) + ) return op diff --git a/nmdc_runtime/api/endpoints/users.py b/nmdc_runtime/api/endpoints/users.py index 82ea7d75..4ba7a1b3 100644 --- a/nmdc_runtime/api/endpoints/users.py +++ b/nmdc_runtime/api/endpoints/users.py @@ -36,7 +36,11 @@ async def login_for_access_token( detail="Incorrect username or password", headers={"WWW-Authenticate": "Bearer"}, ) - access_token_expires = timedelta(**ACCESS_TOKEN_EXPIRES.dict()) + access_token_expires = timedelta( + **ACCESS_TOKEN_EXPIRES.model_dump( + mode="json", + ) + ) access_token = create_access_token( data={"sub": f"user:{user.username}"}, expires_delta=access_token_expires ) @@ -51,7 +55,11 @@ async def login_for_access_token( headers={"WWW-Authenticate": "Bearer"}, ) # TODO make below an absolute time - access_token_expires = timedelta(**ACCESS_TOKEN_EXPIRES.dict()) + access_token_expires = timedelta( + **ACCESS_TOKEN_EXPIRES.model_dump( + mode="json", + ) + ) access_token = create_access_token( data={"sub": f"client:{form_data.client_id}"}, expires_delta=access_token_expires, @@ -69,7 +77,9 @@ async def login_for_access_token( return { "access_token": access_token, "token_type": "bearer", - "expires": ACCESS_TOKEN_EXPIRES.dict(), + "expires": ACCESS_TOKEN_EXPIRES.model_dump( + mode="json", + ), } @@ -95,8 +105,10 @@ def create_user( check_can_create_user(requester) mdb.users.insert_one( UserInDB( - **user_in.dict(), + **user_in.model_dump( + mode="json", + ), hashed_password=get_password_hash(user_in.password), - ).dict(exclude_unset=True) + ).model_dump(mode="json", exclude_unset=True) ) return mdb.users.find_one({"username": user_in.username}) diff --git a/nmdc_runtime/api/endpoints/util.py b/nmdc_runtime/api/endpoints/util.py index 1846083f..9ab2daa6 100644 --- a/nmdc_runtime/api/endpoints/util.py +++ b/nmdc_runtime/api/endpoints/util.py @@ -463,9 +463,11 @@ def _create_object( mdb: MongoDatabase, object_in: DrsObjectIn, mgr_site, drs_id, self_uri ): drs_obj = DrsObject( - **object_in.dict(exclude_unset=True), id=drs_id, self_uri=self_uri + **object_in.model_dump(exclude_unset=True, mode="json"), + id=drs_id, + self_uri=self_uri, ) - doc = drs_obj.dict(exclude_unset=True) + doc = drs_obj.model_dump(exclude_unset=True, mode="json") doc["_mgr_site"] = mgr_site # manager site try: mdb.objects.insert_one(doc) @@ -526,16 +528,22 @@ def _claim_job(job_id: str, mdb: MongoDatabase, site: Site): "workflow": job.workflow, "config": job.config, } - ).dict(exclude_unset=True), + ).model_dump(mode="json", exclude_unset=True), "site_id": site.id, "model": dotted_path_for(JobOperationMetadata), }, } ) - mdb.operations.insert_one(op.dict()) - mdb.jobs.replace_one({"id": job.id}, job.dict(exclude_unset=True)) + mdb.operations.insert_one( + op.model_dump( + mode="json", + ) + ) + mdb.jobs.replace_one( + {"id": job.id}, job.model_dump(mode="json", exclude_unset=True) + ) - return op.dict(exclude_unset=True) + return op.model_dump(mode="json", exclude_unset=True) @lru_cache diff --git a/nmdc_runtime/api/main.py b/nmdc_runtime/api/main.py index c80ffb4a..476d57e0 100644 --- a/nmdc_runtime/api/main.py +++ b/nmdc_runtime/api/main.py @@ -235,7 +235,9 @@ def ensure_initial_resources_on_boot(): collection_boot = import_module(f"nmdc_runtime.api.boot.{collection_name}") for model in collection_boot.construct(): - doc = model.dict() + doc = model.model_dump( + mode="json", + ) mdb[collection_name].replace_one({"id": doc["id"]}, doc, upsert=True) username = os.getenv("API_ADMIN_USER") @@ -247,7 +249,7 @@ def ensure_initial_resources_on_boot(): username=username, hashed_password=get_password_hash(os.getenv("API_ADMIN_PASS")), site_admin=[os.getenv("API_SITE_ID")], - ).dict(exclude_unset=True), + ).model_dump(mode="json", exclude_unset=True), upsert=True, ) mdb.users.create_index("username") @@ -268,7 +270,9 @@ def ensure_initial_resources_on_boot(): ), ) ], - ).dict(), + ).model_dump( + mode="json", + ), upsert=True, ) diff --git a/nmdc_runtime/api/models/run.py b/nmdc_runtime/api/models/run.py index 49fa37ba..43bf734e 100644 --- a/nmdc_runtime/api/models/run.py +++ b/nmdc_runtime/api/models/run.py @@ -93,7 +93,11 @@ def _add_run_requested_event(run_spec: RunUserSpec, mdb: MongoDatabase, user: Us time=now(as_str=True), inputs=run_spec.inputs, ) - mdb.run_events.insert_one(event.dict()) + mdb.run_events.insert_one( + event.model_dump( + mode="json", + ) + ) return run_id @@ -113,7 +117,9 @@ def _add_run_started_event(run_id: str, mdb: MongoDatabase): job=requested.job, type=RunEventType.STARTED, time=now(as_str=True), - ).dict() + ).model_dump( + mode="json", + ) ) return run_id @@ -134,7 +140,9 @@ def _add_run_fail_event(run_id: str, mdb: MongoDatabase): job=requested.job, type=RunEventType.FAIL, time=now(as_str=True), - ).dict() + ).model_dump( + mode="json", + ) ) return run_id @@ -156,6 +164,8 @@ def _add_run_complete_event(run_id: str, mdb: MongoDatabase, outputs: List[str]) type=RunEventType.COMPLETE, time=now(as_str=True), outputs=outputs, - ).dict() + ).model_dump( + mode="json", + ) ) return run_id diff --git a/nmdc_runtime/core/exceptions/token.py b/nmdc_runtime/core/exceptions/token.py index c5e9b1c3..afe00871 100644 --- a/nmdc_runtime/core/exceptions/token.py +++ b/nmdc_runtime/core/exceptions/token.py @@ -1,4 +1,4 @@ -from core.exceptions import CustomException +from nmdc_runtime.core.exceptions import CustomException class DecodeTokenException(CustomException): diff --git a/nmdc_runtime/minter/adapters/repository.py b/nmdc_runtime/minter/adapters/repository.py index 25382731..879bdcc8 100644 --- a/nmdc_runtime/minter/adapters/repository.py +++ b/nmdc_runtime/minter/adapters/repository.py @@ -97,7 +97,9 @@ def mint(self, req_mint: MintingRequest) -> list[Identifier]: ) ) for id_ in ids: - self.db[id_.id] = id_.dict() + self.db[id_.id] = id_.model_dump( + mode="json", + ) return ids def bind(self, req_bind: BindingRequest) -> Identifier: @@ -184,7 +186,14 @@ def mint(self, req_mint: MintingRequest) -> list[Identifier]: ) for id_name in not_taken ] - self.db["minter.id_records"].insert_many([i.dict() for i in ids]) + self.db["minter.id_records"].insert_many( + [ + i.model_dump( + mode="json", + ) + for i in ids + ] + ) collected.extend(ids) if len(collected) == req_mint.how_many: break diff --git a/nmdc_runtime/minter/entrypoints/fastapi_app.py b/nmdc_runtime/minter/entrypoints/fastapi_app.py index d0ac097f..3d4b7efc 100644 --- a/nmdc_runtime/minter/entrypoints/fastapi_app.py +++ b/nmdc_runtime/minter/entrypoints/fastapi_app.py @@ -37,7 +37,13 @@ def mint_ids( requester = Entity(id=site.id) try: minted = s.mint( - MintingRequest(service=service, requester=requester, **req_mint.dict()) + MintingRequest( + service=service, + requester=requester, + **req_mint.model_dump( + mode="json", + ), + ) ) return [d.id for d in minted] except MinterError as e: diff --git a/nmdc_runtime/site/drsobjects/ingest.py b/nmdc_runtime/site/drsobjects/ingest.py index b4b7dc38..26a26f43 100644 --- a/nmdc_runtime/site/drsobjects/ingest.py +++ b/nmdc_runtime/site/drsobjects/ingest.py @@ -44,7 +44,11 @@ def claim_metadata_ingest_jobs( ) jobs = [] while True: - rv = client.list_jobs(lr.dict()).json() + rv = client.list_jobs( + lr.model_dump( + mode="json", + ) + ).json() jobs.extend(rv["resources"]) if "next_page_token" not in rv: break diff --git a/nmdc_runtime/site/ops.py b/nmdc_runtime/site/ops.py index 828fe4e8..265498ab 100644 --- a/nmdc_runtime/site/ops.py +++ b/nmdc_runtime/site/ops.py @@ -267,7 +267,11 @@ def get_operation(context): def produce_curated_db(context, op: Operation): client: RuntimeApiSiteClient = context.resources.runtime_api_site_client mdb: MongoDatabase = context.resources.mongo.db - op = Operation[ResultT, JobOperationMetadata](**op.dict()) + op = Operation[ResultT, JobOperationMetadata]( + **op.model_dump( + mode="json", + ) + ) op_meta: JobOperationMetadata = op.metadata job_id = op_meta.job.id job = mdb.jobs.find_one({"id": job_id}) @@ -350,7 +354,12 @@ def filter_ops_undone_expired() -> str: @op(required_resource_keys={"runtime_api_site_client"}) def list_operations(context, filter_: str) -> list: client = context.resources.runtime_api_site_client - ops = [op.dict() for op in client.list_operations({"filter": filter_})] + ops = [ + op.model_dump( + mode="json", + ) + for op in client.list_operations({"filter": filter_}) + ] context.log.info(str(len(ops))) return ops @@ -466,7 +475,7 @@ def perform_changesheet_updates(context, sheet_in: ChangesheetIn): op = Operation(**mdb.operations.find_one({"id": op_id})) op.done = True op.result = {"update_cmd": json.dumps(update_cmd)} - op_doc = op.dict(exclude_unset=True) + op_doc = op.model_dump(mode="json", exclude_unset=True) mdb.operations.replace_one({"id": op_id}, op_doc) return ["/operations/" + op_doc["id"]] diff --git a/nmdc_runtime/site/repository.py b/nmdc_runtime/site/repository.py index 44270249..22b9a9c0 100644 --- a/nmdc_runtime/site/repository.py +++ b/nmdc_runtime/site/repository.py @@ -405,7 +405,9 @@ def claim_and_run_apply_changesheet_jobs(_context): def done_object_put_ops(_context): client = get_runtime_api_site_client(run_config_frozen__normal_env) ops = [ - op.dict() + op.model_dump( + mode="json", + ) for op in client.list_operations( { "filter": json.dumps( diff --git a/nmdc_runtime/site/resources.py b/nmdc_runtime/site/resources.py index 79cae368..983ab206 100644 --- a/nmdc_runtime/site/resources.py +++ b/nmdc_runtime/site/resources.py @@ -60,7 +60,9 @@ def request(self, method, url_path, params_or_json_data=None): self.ensure_token() kwargs = {"url": self.base_url + url_path, "headers": self.headers} if isinstance(params_or_json_data, BaseModel): - params_or_json_data = params_or_json_data.dict(exclude_unset=True) + params_or_json_data = params_or_json_data.model_dump( + mode="json", exclude_unset=True + ) if method.upper() == "GET": kwargs["params"] = params_or_json_data else: diff --git a/tests/files/nmdc_bsm-12-7mysck21.json b/tests/files/nmdc_bsm-12-7mysck21.json new file mode 100644 index 00000000..d0571f47 --- /dev/null +++ b/tests/files/nmdc_bsm-12-7mysck21.json @@ -0,0 +1,48 @@ +{ + "analysis_type": [ + "metagenomics" + ], + "biosample_categories": [ + "NEON" + ], + "collection_date": { + "has_raw_value": "2014-07-15T18:00Z" + }, + "depth": { + "has_maximum_numeric_value": 1, + "has_minimum_numeric_value": 0, + "has_unit": "meters" + }, + "elev": 1179.5, + "env_broad_scale": { + "term": { + "id": "ENVO:01000253", + "name": "freshwater river biome" + } + }, + "env_local_scale": { + "term": { + "id": "ENVO:03600095", + "name": "stream run" + } + }, + "env_medium": { + "term": { + "id": "ENVO:01001057", + "name": "environment associated with a plant part or small plant" + } + }, + "geo_loc_name": { + "has_raw_value": "USA: Colorado, Arikaree River" + }, + "id": "nmdc:bsm-12-7mysck21", + "lat_lon": { + "latitude": 39.758206, + "longitude": -102.447148 + }, + "name": "ARIK.20140715.AMC.EPIPHYTON.5", + "part_of": [ + "nmdc:sty-11-34xj1150" + ], + "type": "nmdc:Biosample" +} diff --git a/tests/files/nmdc_sty-11-pzmd0x14.json b/tests/files/nmdc_sty-11-pzmd0x14.json new file mode 100644 index 00000000..114437c0 --- /dev/null +++ b/tests/files/nmdc_sty-11-pzmd0x14.json @@ -0,0 +1,65 @@ +{ + "id": "nmdc:sty-11-pzmd0x14", + "name": "National Ecological Observatory Network: benthic metagenomes (DP1.20279.001)", + "type": "nmdc:Study", + "title": "National Ecological Observatory Network: benthic metagenomes (DP1.20279.001)", + "description": "The National Science Foundation's National Ecological Observatory Network (NEON) is a continental-scale observation facility operated by Battelle and designed to collect long-term open access ecological data to better understand how U.S. ecosystems are changing.", + "websites": [ + "https://www.neonscience.org/", + "https://data.neonscience.org/data-products/DP1.20279.001", + "https://data.neonscience.org/api/v0/documents/NEON_metagenomes_userGuide_vE.pdf" + ], + "funding_sources": [ + "NSF#1724433 National Ecological Observatory Network: Operations Activities" + ], + "principal_investigator": { + "name": "Kate Thibault", + "email": "kthibault@battelleecology.org", + "orcid": "orcid:0000-0003-3477-6424", + "has_raw_value": "Kate Thibault" + }, + "has_credit_associations": [ + { + "applies_to_person": { + "name": "Hugh Cross", + "email": "crossh@battelleecology.org", + "orcid": "orcid:0000-0002-6745-9479" + }, + "applied_roles": [ + "Methodology", + "Data curation" + ] + }, + { + "applies_to_person": { + "name": "Kate Thibault", + "email": "kthibault@battelleecology.org", + "orcid": "orcid:0000-0003-3477-6424" + }, + "applied_roles": [ + "Principal Investigator" + ] + }, + { + "applies_to_person": { + "name": "Stephanie Parker", + "email": "sparker@battelleecology.org", + "orcid": "0000-0002-7180-7245" + }, + "applied_roles": [ + "Methodology", + "Data curation" + ] + } + ], + "study_image": [ + { + "url": "https://portal.nersc.gov/project/m3408/profile_images/nmdc_sty-11-34xj1150.jpg" + } + ], + "gold_study_identifiers": [], + "part_of": [ + "nmdc:sty-11-nxrz9m96" + ], + "study_category": "consortium" +} diff --git a/tests/integration/test_minter_repository.py b/tests/integration/test_minter_repository.py index 96199670..45ad0b56 100644 --- a/tests/integration/test_minter_repository.py +++ b/tests/integration/test_minter_repository.py @@ -29,7 +29,12 @@ def test_mint_and_resolve(): s: InMemoryIDStore = get_test_inmemoryidstore() req_mint = minting_request() id_: Identifier = next(i for i in s.mint(req_mint)) - req_res = ResolutionRequest(id_name=id_.name, **req_mint.dict()) + req_res = ResolutionRequest( + id_name=id_.name, + **req_mint.model_dump( + mode="json", + ), + ) assert s.resolve(req_res) is not None @@ -37,9 +42,23 @@ def test_mint_and_delete(): s: InMemoryIDStore = get_test_inmemoryidstore() req_mint = minting_request() id_: Identifier = next(i for i in s.mint(req_mint)) - req_del = DeleteRequest(id_name=id_.name, **req_mint.dict()) + req_del = DeleteRequest( + id_name=id_.name, + **req_mint.model_dump( + mode="json", + ), + ) s.delete(req_del) - assert s.resolve(ResolutionRequest(**req_del.dict())) is None + assert ( + s.resolve( + ResolutionRequest( + **req_del.model_dump( + mode="json", + ) + ) + ) + is None + ) def test_mongo_mint_one(): @@ -70,7 +89,12 @@ def test_mongo_mint_and_resolve(): req_mint = minting_request() id_: Identifier = next(i for i in s.mint(req_mint)) - req_res = ResolutionRequest(id_name=id_.name, **req_mint.dict()) + req_res = ResolutionRequest( + id_name=id_.name, + **req_mint.model_dump( + mode="json", + ), + ) assert s.resolve(req_res) is not None @@ -80,7 +104,21 @@ def test_mongo_mint_and_delete(): req_mint = minting_request() id_: Identifier = next(i for i in s.mint(req_mint)) - req_del = DeleteRequest(id_name=id_.name, **req_mint.dict()) + req_del = DeleteRequest( + id_name=id_.name, + **req_mint.model_dump( + mode="json", + ), + ) s.delete(req_del) - assert s.resolve(ResolutionRequest(**req_del.dict())) is None + assert ( + s.resolve( + ResolutionRequest( + **req_del.model_dump( + mode="json", + ) + ) + ) + is None + ) assert s.db["minter.id_records"].count_documents({}) == 0 diff --git a/tests/test_api/test_endpoints.py b/tests/test_api/test_endpoints.py index 1dd677cc..387154b0 100644 --- a/tests/test_api/test_endpoints.py +++ b/tests/test_api/test_endpoints.py @@ -1,4 +1,6 @@ +import json import os +import re import pytest import requests @@ -7,13 +9,17 @@ from toolz import get_in from nmdc_runtime.api.core.auth import get_password_hash +from nmdc_runtime.api.core.metadata import df_from_sheet_in, _validate_changesheet from nmdc_runtime.api.core.util import generate_secret, dotted_path_for from nmdc_runtime.api.db.mongo import get_mongo_db +from nmdc_runtime.api.endpoints.util import persist_content_and_get_drs_object from nmdc_runtime.api.models.job import Job, JobOperationMetadata +from nmdc_runtime.api.models.metadata import ChangesheetIn from nmdc_runtime.api.models.site import SiteInDB, SiteClientInDB from nmdc_runtime.api.models.user import UserInDB, UserIn, User from nmdc_runtime.site.repository import run_config_frozen__normal_env from nmdc_runtime.site.resources import get_mongo, RuntimeApiSiteClient +from nmdc_runtime.util import REPO_ROOT_DIR def ensure_test_resources(mdb): @@ -26,7 +32,7 @@ def ensure_test_resources(mdb): username=username, hashed_password=get_password_hash(password), site_admin=[site_id], - ).dict(exclude_unset=True), + ).model_dump(mode="json", exclude_unset=True), upsert=True, ) @@ -42,7 +48,9 @@ def ensure_test_resources(mdb): hashed_secret=get_password_hash(client_secret), ) ], - ).dict(), + ).model_dump( + mode="json", + ), upsert=True, ) wf_id = "test" @@ -50,7 +58,9 @@ def ensure_test_resources(mdb): prev_ops = {"metadata.job.id": job_id, "metadata.site_id": site_id} mdb.operations.delete_many(prev_ops) job = Job(**{"id": job_id, "workflow": {"id": wf_id}, "config": {}, "claims": []}) - mdb.jobs.replace_one({"id": job_id}, job.dict(exclude_unset=True), upsert=True) + mdb.jobs.replace_one( + {"id": job_id}, job.model_dump(mode="json", exclude_unset=True), upsert=True + ) return { "site_client": { "site_id": site_id, @@ -58,7 +68,7 @@ def ensure_test_resources(mdb): "client_secret": client_secret, }, "user": {"username": username, "password": password}, - "job": job.dict(exclude_unset=True), + "job": job.model_dump(mode="json", exclude_unset=True), } @@ -114,7 +124,7 @@ def get_token(): "POST", url=(base_url + "/users"), headers=headers, - json=user_in.dict(exclude_unset=True), + json=user_in.model_dump(mode="json", exclude_unset=True), ) try: @@ -181,3 +191,41 @@ def test_metadata_validate_json_with_unknown_collection(api_site_client): {"studi_set": []}, ) assert rv.json()["result"] == "errors" + + +def test_submit_changesheet(): + sheet_in = ChangesheetIn( + name="sheet", + content_type="text/tab-separated-values", + text="id\taction\tattribute\tvalue\nnmdc:bsm-12-7mysck21\tupdate\tpart_of\tnmdc:sty-11-pzmd0x14\n", + ) + mdb = get_mongo_db() + rs = ensure_test_resources(mdb) + if not mdb.biosample_set.find_one({"id": "nmdc:bsm-12-7mysck21"}): + mdb.biosample_set.insert_one( + json.loads( + ( + REPO_ROOT_DIR / "tests" / "files" / "nmdc_bsm-12-7mysck21.json" + ).read_text() + ) + ) + if not mdb.study_set.find_one({"id": "nmdc:sty-11-pzmd0x14"}): + mdb.study_set.insert_one( + json.loads( + ( + REPO_ROOT_DIR / "tests" / "files" / "nmdc_sty-11-pzmd0x14.json" + ).read_text() + ) + ) + df_change = df_from_sheet_in(sheet_in, mdb) + _ = _validate_changesheet(df_change, mdb) + drs_obj_doc = persist_content_and_get_drs_object( + content=sheet_in.text, + username=rs["user"]["username"], + filename=re.sub(r"[^A-Za-z0-9._\-]", "_", sheet_in.name), + content_type=sheet_in.content_type, + description="changesheet", + id_ns="changesheets", + ) + mdb.objects.delete_one({"id": drs_obj_doc["id"]}) + assert True diff --git a/tests/test_api/test_metadata.py b/tests/test_api/test_metadata.py index 9fc63254..82b6e70f 100644 --- a/tests/test_api/test_metadata.py +++ b/tests/test_api/test_metadata.py @@ -7,6 +7,8 @@ import pytest from nmdc_runtime.api.db.mongo import get_mongo_db +from nmdc_runtime.api.endpoints.util import persist_content_and_get_drs_object +from nmdc_runtime.api.models.metadata import ChangesheetIn from nmdc_runtime.util import get_nmdc_jsonschema_dict from toolz import dissoc @@ -15,6 +17,8 @@ update_mongo_db, mongo_update_command_for, copy_docs_in_update_cmd, + df_from_sheet_in, + _validate_changesheet, ) from nmdc_runtime.site.ops import ensure_data_object_type from nmdc_runtime.site.repository import run_config_frozen__normal_env diff --git a/util/mongorestore-nmdc.sh b/util/mongorestore-nmdc.sh index e0f5f253..aa23ccfb 100755 --- a/util/mongorestore-nmdc.sh +++ b/util/mongorestore-nmdc.sh @@ -4,4 +4,4 @@ # $ ./util/mongorestore-nmdc.sh mongorestore -h $MONGO_HOST -u $MONGO_USERNAME -p $MONGO_PASSWORD --authenticationDatabase=admin \ --gzip --drop \ - $HOME/nmdcdb-mongodump/nmdcdb/2023-05-24T11/ \ No newline at end of file + $HOME/nmdcdb-mongodump/nmdcdb/2023-11-02T11/ \ No newline at end of file