🐛 Fix duplicate counts in auth studies and add open access data use w…

…hen appropriate (#124)
kids-first · Mar 1, 2024 · c3a653b · c3a653b
1 parent f85b7a1
commit c3a653b
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 8 deletions.
diff --git a/admin/arrangerApi.mjs b/admin/arrangerApi.mjs
@@ -10,12 +10,12 @@ const createNewIndices = async (esClient, confIndices) => {
     }
 };
 
-const fixExtendedMapping = async (esClient, confExtendedMappingMutations) => {
+const fixExtendedMapping = async (esClient, mutations) => {
     const updateFieldExtendedMappingWithClient = updateFieldExtendedMapping(esClient);
-    for (const confExtendedMappingMutation of confExtendedMappingMutations) {
-        console.debug("fixing extendedMapping field =", confExtendedMappingMutation?.field)
+    for (const [index, mutation] of mutations.entries()) {
+        console.debug('updating field = ', mutation?.field, ` ${index + 1} of ${mutations.length}`);
         await updateFieldExtendedMappingWithClient({
-            ...confExtendedMappingMutation,
+            ...mutation,
         });
     }
 };

diff --git a/src/endpoints/authorizedStudies/computeAuthorizedStudies.test.ts b/src/endpoints/authorizedStudies/computeAuthorizedStudies.test.ts
@@ -64,6 +64,7 @@ describe('Compute Authorized Studies', () => {
                                 { key: 'SD_Z6MWD3H0', doc_count: 4 },
                                 { key: 'phs002330.c2', doc_count: 4 },
                                 { key: 'phs002330.c999', doc_count: 4 },
+                                { key: 'open_access', doc_count: 10 },
                             ],
                         },
                     },
@@ -84,6 +85,10 @@ describe('Compute Authorized Studies', () => {
                         hits: { total: { value: 0, relation: 'eq' }, max_score: null, hits: [] },
                         status: 200,
                     },
+                    {
+                        hits: { total: { value: 0, relation: 'eq' }, max_score: null, hits: [] },
+                        status: 200,
+                    },
                     {
                         hits: { total: { value: 100, relation: 'eq' }, max_score: null, hits: [] },
                         status: 200,
@@ -96,6 +101,10 @@ describe('Compute Authorized Studies', () => {
                         hits: { total: { value: 25, relation: 'eq' }, max_score: null, hits: [] },
                         status: 200,
                     },
+                    {
+                        hits: { total: { value: 10, relation: 'eq' }, max_score: null, hits: [] },
+                        status: 200,
+                    },
                 ]),
             );
 
@@ -121,7 +130,7 @@ describe('Compute Authorized Studies', () => {
                 },
                 {
                     study_id: 'SD_Z6MWD3H0',
-                    user_acl_in_study: ['phs002330.c2'],
+                    user_acl_in_study: ['phs002330.c2', 'open_access'],
                     study_code: 'KF-CHDALL',
                     title: 'Kids First: Genomic Analysis of Congenital...',
                     authorized_controlled_files_count: 4,

diff --git a/src/endpoints/authorizedStudies/computeAuthorizedStudies.ts b/src/endpoints/authorizedStudies/computeAuthorizedStudies.ts
@@ -35,10 +35,16 @@ export const computeAuthorizedStudiesForFence = async (
         indexOfTotalNOfFiles: 0,
         indexOfControlledNOfFiles: 1,
         indexOfRegisteredNOfFiles: 2,
+        indexOfStudyContainsOpenAccess: 3,
     };
 
     //Get all files count per candidate study
     const accessCounts = await multiSearchFilesAccessCounts(client, fence, allStudyIds);
+    const countFailures =
+        accessCounts?.filter(x => x._shards?.failures && x._shards.failures.length > 0)?.map(x => x._shards.failures) ||
+        [];
+    // eslint-disable-next-line no-console
+    console.assert(countFailures.length === 0, 'failures detected', countFailures);
     const size = Object.keys(M_SEARCH).length; //there are "size" response elements per study
     // eslint-disable-next-line no-console
     console.assert(
@@ -49,11 +55,13 @@ export const computeAuthorizedStudiesForFence = async (
         data: dataAggregations.map((x: StudyDataSpecific, i: number): StudyDataSpecific | StudyDataGlobal => {
             const extractMSearchHitsTotal = (index: number) =>
                 accessCounts.slice(i * size, i * size + size)[index].hits.total.value;
+            const containsOpenAccess = extractMSearchHitsTotal(M_SEARCH.indexOfStudyContainsOpenAccess) > 0;
             return {
                 ...x,
                 total_files_count: extractMSearchHitsTotal(M_SEARCH.indexOfTotalNOfFiles),
                 total_controlled_files_count: extractMSearchHitsTotal(M_SEARCH.indexOfControlledNOfFiles),
                 total_uncontrolled_files_count: extractMSearchHitsTotal(M_SEARCH.indexOfRegisteredNOfFiles),
+                user_acl_in_study: containsOpenAccess ? [...x.user_acl_in_study, 'open_access'] : x.user_acl_in_study,
             };
         }),
     };

diff --git a/src/endpoints/authorizedStudies/searchers.ts b/src/endpoints/authorizedStudies/searchers.ts
@@ -79,7 +79,7 @@ export const multiSearchFilesAccessCounts = async (
     const { body: bodyMSearch } = await client.msearch({
         body: studyIds
             .map((s: string) => [
-                {},
+                { index: esFileIndex },
                 {
                     track_total_hits: true,
                     size: 0,
@@ -89,7 +89,7 @@ export const multiSearchFilesAccessCounts = async (
                         },
                     },
                 },
-                {},
+                { index: esFileIndex },
                 {
                     track_total_hits: true,
                     size: 0,
@@ -103,7 +103,7 @@ export const multiSearchFilesAccessCounts = async (
                         },
                     },
                 },
-                {},
+                { index: esFileIndex },
                 {
                     track_total_hits: true,
                     size: 0,
@@ -117,6 +117,20 @@ export const multiSearchFilesAccessCounts = async (
                         },
                     },
                 },
+                { index: esFileIndex },
+                {
+                    track_total_hits: true,
+                    size: 0,
+                    query: {
+                        bool: {
+                            must: [
+                                { term: { study_id: { value: s } } },
+                                { term: { acl: { value: 'open_access' } } },
+                                { term: { repository: { value: fence } } },
+                            ],
+                        },
+                    },
+                },
             ])
             .flat(),
     });

diff --git a/src/endpoints/authorizedStudies/types.ts b/src/endpoints/authorizedStudies/types.ts
@@ -30,4 +30,24 @@ export type ResponseResult = {
 export type FileAccessCountsResponse = {
     hits: { total: { value: number } };
     status: number;
+    _shards: {
+        total: number;
+        successful: number;
+        skipped: number;
+        failed: number;
+        failures: {
+            shard: number;
+            index: string;
+            node: string;
+            reason: {
+                reason: string;
+                index: string;
+                index_uuid: string;
+                caused_by: {
+                    type: string;
+                    reason: string;
+                };
+            };
+        }[];
+    };
 };