TASK-5610 - Create the CVDB endpoint analysis/clinical/cvdb/variant/summary and overwrite analysis/clinical/variant

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java

@@ -63,7 +63,7 @@
  */
 public class OpenCGATestExternalResource extends ExternalResource {
 
-    private final CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource();
+    private final CatalogManagerExternalResource catalogManagerExternalResource;
     private Path opencgaHome;
     private String storageEngine;
     private boolean initiated = false;
@@ -72,21 +72,27 @@ public class OpenCGATestExternalResource extends ExternalResource {
     private StorageConfiguration storageConfiguration;
     private StorageEngineFactory storageEngineFactory;
     private ToolRunner toolRunner;
-
+    protected Path sourceAnalysisPath;
 
     public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource
             = new HadoopVariantStorageTest.HadoopExternalResource();
 
     public OpenCGATestExternalResource() {
-        this(false);
+        this(false, Paths.get("../opencga-app/app/analysis/"));
     }
 
     public OpenCGATestExternalResource(boolean storageHadoop) {
+        this(storageHadoop, Paths.get("../opencga-app/app/analysis/"));
+    }
+
+    public OpenCGATestExternalResource(boolean storageHadoop, Path sourceAnalysisPath) {
         if (storageHadoop) {
             this.storageEngine = HadoopVariantStorageEngine.STORAGE_ENGINE_ID;
         } else {
             this.storageEngine = DummyVariantStorageEngine.STORAGE_ENGINE_ID;
         }
+        this.sourceAnalysisPath = sourceAnalysisPath;
+        catalogManagerExternalResource = new CatalogManagerExternalResource(sourceAnalysisPath);
     }
 
     @Override

opencga-app/app/misc/clients/python_client_generator.py

@@ -161,10 +161,10 @@ def get_method_definition(self, category, endpoint):
         return '\n'.join(text)
 
     def get_file_name(self, category):
-        return self.to_snake_case(self.categories[self.get_category_name(category)]) + '_client.py' \
-            if self.categories[self.get_category_name(category)] != 'GA4GH' \
-            else self.categories[self.get_category_name(category)].lower() + '_client.py'
-
+        if self.categories[self.get_category_name(category)] == 'GA4GH' or self.categories[self.get_category_name(category)] == 'CVDB':
+            return self.categories[self.get_category_name(category)].lower() + '_client.py'
+        else:
+            return self.to_snake_case(self.categories[self.get_category_name(category)]) + '_client.py'
 
 def _setup_argparse():
     desc = 'This script creates automatically all Python RestClients files'

opencga-app/app/misc/clients/r_client_generator.py

@@ -27,7 +27,7 @@ def __init__(self, server_url, output_dir):
             'Analysis - Clinical': 'Clinical',
             'Operations - Variant Storage': 'Operation',
             'Meta': 'Meta',
-            'Cvdb': 'Cvdb',
+            'CVDB': 'CVDB',
             'GA4GH': 'GA4GH',
             'Admin': 'Admin'
         }

opencga-app/app/misc/clients/rest_client_generator.py

@@ -51,7 +51,7 @@ def __init__(self, rest_api_file, output_dir):
             'Analysis - Clinical': 'ClinicalAnalysis',
             'Operations - Variant Storage': 'VariantOperation',
             'Meta': 'Meta',
-            'Cvdb': 'Cvdb',
+            'CVDB': 'CVDB',
             'GA4GH': 'GA4GH',
             'Admin': 'Admin'
         }
@@ -219,6 +219,7 @@ def get_method_name(self, endpoint, category):
                 method_name = '_'.join([items[2], items[0]])
         elif len(items) == 4:
             # e.g. /{apiVersion}/operation/variant/sample/genotype/index
+            #analysis/clinical/cvdb/variant/{variantIds}/stats
             if not self.any_arg(items):
                 method_name = '_'.join([items[0], items[1], items[2], items[3]])
             # /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretationId}/merge
@@ -227,6 +228,9 @@ def get_method_name(self, endpoint, category):
             # /{apiVersion}/admin/users/{user}/groups/update
             elif self.all_arg([items[1]]) and not self.any_arg([items[0], items[2], items[3]]):
                 method_name = '_'.join([items[0], items[3], items[2]])
+            # /{apiVersion}/analysis/clinical/cvdb/variant/{variantIds}/stats
+            elif self.all_arg([items[2]]) and not self.any_arg([items[0], items[1], items[3]]):
+                method_name = '_'.join([items[0], items[1], items[3]])
         elif len(items) == 5:
             # e.g. /{apiVersion}/files/{file}/annotationSets/{annotationSet}/annotations/update
             if self.all_arg([items[0], items[2]]) and not self.any_arg([items[1], items[3], items[4]]):

opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java

@@ -0,0 +1,47 @@
+package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog;
+
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Updates;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor;
+import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory;
+import org.opencb.opencga.catalog.migration.Migration;
+import org.opencb.opencga.catalog.migration.MigrationTool;
+import org.opencb.opencga.core.models.clinical.ClinicalStatusValue;
+import org.opencb.opencga.core.models.clinical.CvdbIndexStatus;
+
+import java.util.Arrays;
+
+@Migration(id = "add_cvdb_index_to_clinical_analysis",
+        description = "Add CVDB index status to Clinical Analysis #TASK-5610", version = "4.0.0",
+        language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241118)
+public class ClinicalCvdbIndexMigration extends MigrationTool {
+
+    @Override
+    protected void run() throws Exception {
+        Bson closedCaseQuery = Filters.and(
+                Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false),
+                Filters.eq("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED)
+        );
+        Bson openCaseQuery = Filters.and(
+                Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false),
+                Filters.ne("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED)
+        );
+        CvdbIndexStatus pendingCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING);
+        Document pendingCvdbIndexDoc = convertToDocument(pendingCvdbIndexStatus);
+        Bson updateClosedCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), pendingCvdbIndexDoc);
+
+        CvdbIndexStatus noneCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.NONE);
+        Document noneCvdbIndexDoc = convertToDocument(noneCvdbIndexStatus);
+        Bson updateOpenCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), noneCvdbIndexDoc);
+
+        for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION,
+                OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION,
+                OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) {
+            getMongoCollection(collection).updateMany(closedCaseQuery, updateClosedCase);
+            getMongoCollection(collection).updateMany(openCaseQuery, updateOpenCase);
+        }
+    }
+
+}

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java

@@ -62,6 +62,7 @@ enum QueryParams implements QueryParam {
         INTERNAL_STATUS("internal.status", TEXT_ARRAY, ""),
         INTERNAL_STATUS_ID("internal.status.id", TEXT, ""),
         INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""),
+        INTERNAL_CVDB_INDEX("internal.cvdbIndex", OBJECT, ""),
         QUALITY_CONTROL("qualityControl", OBJECT, ""),
         QUALITY_CONTROL_SUMMARY("qualityControl.summary", TEXT, ""),
         CONSENT("consent", OBJECT, ""),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java

@@ -49,7 +49,10 @@
 import org.opencb.opencga.core.common.JacksonUtils;
 import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.config.Configuration;
-import org.opencb.opencga.core.models.clinical.*;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysisPermissions;
+import org.opencb.opencga.core.models.clinical.FlagValueParam;
+import org.opencb.opencga.core.models.clinical.Interpretation;
 import org.opencb.opencga.core.models.common.AnnotationSet;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.FlagAnnotation;
@@ -451,7 +454,7 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List<ClinicalA
         String[] acceptedObjectParams = {QueryParams.FAMILY.key(), QueryParams.DISORDER.key(), QUALITY_CONTROL.key(),
                 QueryParams.PROBAND.key(), QueryParams.ALERTS.key(), QueryParams.INTERNAL_STATUS.key(), QueryParams.PRIORITY.key(),
                 QueryParams.CONSENT.key(), QueryParams.STATUS.key(), QueryParams.INTERPRETATION.key(), REPORT.key(),
-                REQUEST.key(), RESPONSIBLE.key(), ATTRIBUTES.key(), };
+                INTERNAL_CVDB_INDEX.key(), REQUEST.key(), RESPONSIBLE.key(), ATTRIBUTES.key(), };
         filterObjectParams(parameters, document.getSet(), acceptedObjectParams);
 
         if (parameters.containsKey(INTERPRETATION.key()) && parameters.get(INTERPRETATION.key()) == null) {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java

@@ -597,9 +597,15 @@ public OpenCGAResult<ClinicalAnalysis> create(String studyStr, ClinicalAnalysis
                         if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) {
                             String msg = "Case '" + clinicalAnalysis.getId() + "' created with status '"
                                     + clinicalAnalysis.getStatus().getId() + "', which is of type CLOSED. Automatically locking "
-                                    + "ClinicalAnalysis.";
+                                    + "ClinicalAnalysis and setting CVDB index status to PENDING.";
                             logger.info(msg);
                             clinicalAnalysis.setLocked(true);
+
+                            CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING, "User '" + userId
+                                    + "' created case with status '" + clinicalAnalysis.getStatus().getId() + "', which is of type"
+                                    + " CLOSED. Automatically setting CVDB index status to " + CvdbIndexStatus.PENDING);
+                            clinicalAnalysis.getInternal().setCvdbIndex(cvdbIndexStatus);
+
                             events.add(new Event(Event.Type.INFO, clinicalAnalysis.getId(), msg));
                         }
                     }
@@ -761,7 +767,7 @@ private void load(ClinicalAnalysis clinicalAnalysis, String study, String token)
         }
     }
 
-        private void validateAndInitReport(String organizationId, Study study, ClinicalReport report, String userId)
+    private void validateAndInitReport(String organizationId, Study study, ClinicalReport report, String userId)
             throws CatalogException {
         if (report == null) {
             return;
@@ -1448,6 +1454,12 @@ private OpenCGAResult<ClinicalAnalysis> update(String organizationId, Study stud
                         throw new CatalogException("ClinicalAnalysis already have the status '" + clinicalAnalysis.getStatus().getId()
                                 + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED);
                     }
+                } else {
+                    // The user wants to remove the CLOSED status
+                    CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_REMOVE, "User '" + userId
+                            + "' requested to remove the status '" + clinicalAnalysis.getStatus().getId() + "' of type "
+                            + ClinicalStatusValue.ClinicalStatusType.CLOSED + " to set it to '" + updateParams.getStatus().getId() + "'");
+                    parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus);
                 }
             }
 
@@ -1733,10 +1745,28 @@ private OpenCGAResult<ClinicalAnalysis> update(String organizationId, Study stud
                         if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) {
                             String msg = "User '" + userId + "' changed case '" + clinicalAnalysis.getId() + "' to status '"
                                     + updateParamsClone.getStatus().getId() + "', which is of type CLOSED. Automatically locking "
-                                    + "ClinicalAnalysis";
+                                    + "ClinicalAnalysis and changing CVDB index status to be indexed";
                             logger.info(msg);
                             parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true);
                             events.add(new Event(Event.Type.INFO, clinicalAnalysis.getId(), msg));
+
+                            if (StringUtils.isEmpty(clinicalAnalysis.getInternal().getCvdbIndex().getId())
+                                    || clinicalAnalysis.getInternal().getCvdbIndex().getId().equals(CvdbIndexStatus.NONE)) {
+                                CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING, "User '" + userId
+                                        + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type"
+                                        + " CLOSED. Automatically changing CVDB index status to " + CvdbIndexStatus.PENDING);
+                                parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus);
+                            } else if (clinicalAnalysis.getInternal().getCvdbIndex().getId().equals(CvdbIndexStatus.PENDING_REMOVE)) {
+                                CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_OVERWRITE, "User '" + userId
+                                        + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type"
+                                        + " CLOSED. CVDB index was already in " + CvdbIndexStatus.PENDING_REMOVE + ", so automatically"
+                                        + " changing CVDB index status to " + CvdbIndexStatus.PENDING_OVERWRITE);
+                                parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus);
+                            } else {
+                                logger.warn("CVDB index status is unexpectedly set to '{}'. Although the user is closing the case, OpenCGA"
+                                        + " cannot automatically infer which should be the new CVDB index status.",
+                                        clinicalAnalysis.getInternal().getCvdbIndex().getId());
+                            }
                         }
                     }
                 }
@@ -1760,7 +1790,7 @@ private OpenCGAResult<ClinicalAnalysis> update(String organizationId, Study stud
     }
 
     public OpenCGAResult<ClinicalReport> updateReport(String studyStr, String clinicalAnalysisId, ClinicalReport report,
-                                                        QueryOptions options, String token) throws CatalogException {
+                                                      QueryOptions options, String token) throws CatalogException {
         JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token);
         CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload);
 
@@ -1847,6 +1877,44 @@ public OpenCGAResult<ClinicalReport> updateReport(String studyStr, String clinic
         }
     }
 
+    public OpenCGAResult<?> updateCvdbIndex(String studyFqn, ClinicalAnalysis clinical, CvdbIndexStatus index, String token)
+            throws CatalogException {
+        JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token);
+        CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload);
+        String organizationId = catalogFqn.getOrganizationId();
+        String userId = tokenPayload.getUserId(organizationId);
+
+        ObjectMap auditParams = new ObjectMap()
+                .append("studyFqn", studyFqn)
+                .append("clinical", clinical.getId())
+                .append("cvdbIndex", index)
+                .append("token", token);
+
+        String studyId = studyFqn;
+        String studyUuid = "";
+        try {
+            authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId);
+            Study study = studyManager.resolveId(catalogFqn, StudyManager.INCLUDE_STUDY_IDS, tokenPayload);
+            studyId = study.getFqn();
+            studyUuid = study.getUuid();
+
+            ObjectMap valueAsMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(index));
+            ObjectMap params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), valueAsMap);
+
+            OpenCGAResult<?> update = getClinicalAnalysisDBAdaptor(organizationId).update(clinical.getUid(), params,
+                    Collections.emptyList(), Collections.emptyList(), QueryOptions.empty());
+            auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.CLINICAL_ANALYSIS, clinical.getId(),
+                    clinical.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS));
+            return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1);
+        } catch (Exception e) {
+            auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.CLINICAL_ANALYSIS, clinical.getId(),
+                    clinical.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR,
+                            new Error().setName(e.getMessage())));
+            throw new CatalogException("Could not update CVDB Index status: " + e.getMessage(), e);
+        }
+    }
+
+
     /**
      * Sort the family members in the following order: proband, father, mother, others.
      *

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java

@@ -47,17 +47,25 @@
  */
 public class CatalogManagerExternalResource extends ExternalResource {
 
-    private CatalogManager catalogManager;
-    private Configuration configuration;
-    private Path opencgaHome;
-    private String adminToken;
+    protected CatalogManager catalogManager;
+    protected Configuration configuration;
+    protected Path opencgaHome;
+    protected String adminToken;
     public boolean initialized = false;
 
+    protected Path sourceAnalysisPath;
+
     public CatalogManagerExternalResource() {
+        this(Paths.get("../opencga-app/app/analysis/"));
+    }
+
+    public CatalogManagerExternalResource(Path sourceAnalysisPath) {
+        this.sourceAnalysisPath = sourceAnalysisPath;
         Configurator.setLevel("org.mongodb.driver.cluster", Level.WARN);
         Configurator.setLevel("org.mongodb.driver.connection", Level.WARN);
     }
 
+
     @Override
     public void before() throws Exception {
         initialized = true;
@@ -93,8 +101,9 @@ public Path clearOpenCGAHome(String testName) throws IOException {
 
         // Pedigree graph analysis
         Path analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/pedigree-graph")).toAbsolutePath();
-        FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R");
-        Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING);
+        Files.copy(sourceAnalysisPath.resolve("pedigree-graph/ped.R"),
+                analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING);
+
         return opencgaHome;
     }