diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java index ba31a2eb4a4..bdc9719fbff 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java @@ -16,7 +16,9 @@ package org.opencb.opencga.analysis.tools; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.core.config.Analysis; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.Tool; import org.reflections.Reflections; @@ -39,6 +41,19 @@ public class ToolFactory { public static final String DEFAULT_PACKAGE = "org.opencb.opencga"; +// public ToolFactory(Analysis analysisConf) { +// +// } + + private static void loadTools(Analysis analysisConf) { + if (analysisConf != null + && CollectionUtils.isNotEmpty(analysisConf.getPackages())) { + loadTools(analysisConf.getPackages()); + } else { + loadTools(Collections.singletonList(DEFAULT_PACKAGE)); + } + } + private static synchronized Map> loadTools(List packages) { if (toolsCache == null) { Reflections reflections = new Reflections(new ConfigurationBuilder() @@ -137,6 +152,10 @@ public final Class getToolClass(String toolId, List aClass) throws } } - public Collection> getTools() { - loadTools(Collections.singletonList(DEFAULT_PACKAGE)); - return toolsList; - } - - public Collection> getTools(List packages) { - loadTools(packages); + public Collection> getTools(Analysis analysisConf) { + loadTools(analysisConf); return toolsList; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 57e69ee4759..3b00d70c377 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -225,7 +225,7 @@ public VariantSearchLoadResult secondaryAnnotationIndex(String project, String r Query inputQuery = new Query(); inputQuery.putIfNotEmpty(VariantQueryParam.REGION.key(), region); VariantSearchLoadResult result = engine.secondaryIndex(inputQuery, new QueryOptions(params), overwrite); - getSynchronizer(engine).synchronizeCatalogFromStorage(token); + getSynchronizer(engine).synchronizeCatalogFromStorage(project, null, token); return result; }); } @@ -284,8 +284,7 @@ public void annotate(String projectStr, List studies, String region, boo public void saveAnnotation(String project, String annotationName, ObjectMap params, String token) throws CatalogException, StorageEngineException { secureOperationByProject(VariantAnnotationSaveOperationTool.ID, project, params, token, engine -> { - CatalogStorageMetadataSynchronizer - .updateProjectMetadata(catalogManager, engine.getMetadataManager(), project, token); + getSynchronizer(engine).synchronizeProjectMetadataFromCatalog(project, token); engine.saveAnnotation(annotationName, params); return null; }); @@ -294,8 +293,7 @@ public void saveAnnotation(String project, String annotationName, ObjectMap para public void deleteAnnotation(String project, String annotationName, ObjectMap params, String token) throws CatalogException, StorageEngineException { secureOperationByProject(VariantAnnotationDeleteOperationTool.ID, project, params, token, engine -> { - CatalogStorageMetadataSynchronizer - .updateProjectMetadata(catalogManager, engine.getMetadataManager(), project, token); + getSynchronizer(engine).synchronizeProjectMetadataFromCatalog(project, token); engine.deleteAnnotation(annotationName, params); return null; }); @@ -558,6 +556,7 @@ public OpenCGAResult configureSampleIndex(String studyStr, SampleIndexConfi String cellbaseVersion = engine.getCellBaseUtils().getVersionFromServer(); sampleIndexConfiguration.validate(cellbaseVersion); String studyFqn = getStudyFqn(studyStr, token); + int studyId; if (!engine.getMetadataManager().studyExists(studyFqn)) { studyId = engine.getMetadataManager().createStudy(studyFqn, cellbaseVersion).getId(); @@ -565,9 +564,10 @@ public OpenCGAResult configureSampleIndex(String studyStr, SampleIndexConfi studyId = engine.getMetadataManager().getStudyId(studyFqn); } engine.getMetadataManager().addSampleIndexConfiguration(studyId, sampleIndexConfiguration, true); + getSynchronizer(engine).synchronizeCatalogProjectFromStorageByStudy(studyFqn, token); catalogManager.getStudyManager() - .setVariantEngineConfigurationSampleIndex(studyStr, sampleIndexConfiguration, token); + .setVariantEngineConfigurationSampleIndex(studyFqn, sampleIndexConfiguration, token); if (skipRebuild) { return new OpenCGAResult<>(0, new ArrayList<>(), 0, new ArrayList<>(), 0); } else { @@ -600,6 +600,7 @@ public OpenCGAResult setCellbaseConfiguration(String project, CellBaseConfi .append("cellbaseConfiguration", cellbaseConfiguration) .append("annotate", annotate) .append("annotationSaveId", annotationSaveId), token, engine -> { + String projectFqn = getProjectFqn(project, token); OpenCGAResult result = new OpenCGAResult<>(); result.setResultType(Job.class.getCanonicalName()); result.setResults(new ArrayList<>()); @@ -617,6 +618,9 @@ public OpenCGAResult setCellbaseConfiguration(String project, CellBaseConfi engine.reloadCellbaseConfiguration(); if (engine.getMetadataManager().exists()) { + engine.getMetadataManager().invalidateCurrentVariantAnnotationIndex(); + logger.info("Invalidating current variant annotation index on project '{}'", projectFqn); + getSynchronizer(engine).synchronizeCatalogProjectFromStorage(projectFqn, token); List jobDependsOn = new ArrayList<>(1); if (StringUtils.isNotEmpty(annotationSaveId)) { VariantAnnotationSaveParams params = new VariantAnnotationSaveParams(annotationSaveId); @@ -1139,7 +1143,7 @@ public boolean synchronizeCatalogStudyFromStorage(String study, String token) String studySqn = getStudyFqn(study, token); return secureOperation("synchronizeCatalogStudyFromStorage", studySqn, new ObjectMap(), token, engine -> { CatalogStorageMetadataSynchronizer synchronizer = getSynchronizer(engine); - return synchronizer.synchronizeCatalogStudyFromStorage(studySqn, token); + return synchronizer.synchronizeCatalogFromStorage(studySqn, token); }); } @@ -1152,7 +1156,7 @@ public boolean synchronizeCatalogStudyFromStorage(String study, List fil return secureOperation("synchronizeCatalogStudyFromStorage", studySqn, new ObjectMap(), token, engine -> { List filesFromCatalog = catalogManager.getFileManager() .get(studySqn, files, FILE_GET_QUERY_OPTIONS, token).getResults(); - return getSynchronizer(engine).synchronizeCatalogFilesFromStorage(studySqn, filesFromCatalog, token); + return getSynchronizer(engine).synchronizeCatalogFromStorage(studySqn, filesFromCatalog, false, token); }); } @@ -1672,6 +1676,10 @@ private String getStudyFqn(String study, String token) throws CatalogException { return catalogManager.getStudyManager().get(study, StudyManager.INCLUDE_STUDY_IDS, token).first().getFqn(); } + private String getProjectFqn(String projectStr, String token) throws CatalogException { + return getProjectFqn(projectStr, ((List) null), token); + } + private String getProjectFqn(String projectStr, String study, String token) throws CatalogException { return getProjectFqn(projectStr, Arrays.asList(StringUtils.split(study, ",")), token); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/OperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/OperationManager.java index 539dac4ce44..03ea40b2880 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/OperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/OperationManager.java @@ -49,27 +49,21 @@ public OperationManager(VariantStorageManager variantStorageManager, VariantStor this.variantStorageEngine = variantStorageEngine; } - public final StudyMetadata synchronizeCatalogStudyFromStorage(String study, String token) - throws CatalogException, StorageEngineException { - return synchronizeCatalogStudyFromStorage(study, token, false); - } - public final StudyMetadata synchronizeCatalogStudyFromStorage(String study, String token, boolean failIfNotExist) throws CatalogException, StorageEngineException { VariantStorageMetadataManager metadataManager = variantStorageEngine.getMetadataManager(); CatalogStorageMetadataSynchronizer metadataSynchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, metadataManager); - StudyMetadata studyMetadata = metadataManager.getStudyMetadata(study); - if (studyMetadata == null) { + if (!metadataManager.studyExists(study)) { if (failIfNotExist) { throw new CatalogException("Study '" + study + "' does not exist on the VariantStorage"); } } else { // Update Catalog file and cohort status. - metadataSynchronizer.synchronizeCatalogStudyFromStorage(studyMetadata, token); + metadataSynchronizer.synchronizeCatalogFromStorage(study, token); } - return studyMetadata; + return metadataManager.getStudyMetadata(study); } protected final String getStudyFqn(String study, String token) throws CatalogException { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java index 7f117f66744..0ec5e097d98 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java @@ -27,8 +27,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; @@ -96,7 +94,7 @@ private void annotate(String projectStr, List studies, String loadFileSt if (StringUtils.isEmpty(loadFileStr)) { variantStorageEngine.annotate(outdir.toUri(), annotationQuery, annotationOptions); new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()) - .synchronizeCatalogFromStorage(token); + .synchronizeCatalogFromStorage(projectStr, null, token); } else { Path loadFilePath = Paths.get(loadFileStr); boolean fileExists = Files.exists(loadFilePath); @@ -123,11 +121,8 @@ private void annotate(String projectStr, List studies, String loadFileSt } private void synchronizeProjectMetadata(String projectStr, String token) throws CatalogException, StorageEngineException { - Project project = catalogManager.getProjectManager().get(projectStr, QueryOptions.empty(), token).first(); - ProjectOrganism organism = project.getOrganism(); - int currentRelease = project.getCurrentRelease(); - CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), organism, currentRelease, - project.getCellbase()); + new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()) + .synchronizeProjectMetadataFromCatalog(projectStr, token); } private String buildOutputFileName(String alias, String region) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index ad22b2fb8dc..65d07b08b27 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -102,7 +102,7 @@ public class VariantFileIndexerOperationManager extends OperationManager { private URI outDirUri; private int release; private List filesToIndex; - private CatalogStorageMetadataSynchronizer synchronizer; + private CatalogStorageMetadataSynchronizer _synchronizer; private boolean fullSynchronize = false; private boolean force; @@ -119,7 +119,7 @@ public List index(String study, List files, URI o updateProject(studyFqn, token); List fileUris = findFilesToIndex(params, token); - if (fileUris.size() == 0) { + if (fileUris.isEmpty()) { LOGGER.warn("Nothing to do."); return Collections.emptyList(); } @@ -130,6 +130,14 @@ public List index(String study, List files, URI o private void check(String study, ObjectMap params, String token) throws Exception { studyFqn = getStudyFqn(study, token); + String projectFqn = catalogManager.getStudyManager().getProjectFqn(studyFqn); + + Project project = catalogManager + .getProjectManager() + .get(projectFqn, + new QueryOptions(QueryOptions.INCLUDE, Collections.singletonList(CURRENT_RELEASE.key())), + token).first(); + release = project.getCurrentRelease(); JwtPayload jwtPayload = new JwtPayload(token); CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, jwtPayload); @@ -169,16 +177,9 @@ private void check(String study, ObjectMap params, String token) throws Exceptio private void updateProject(String studyFqn, String token) throws CatalogException, StorageEngineException { String projectFqn = catalogManager.getStudyManager().getProjectFqn(studyFqn); - Project project = catalogManager - .getProjectManager() - .get(projectFqn, - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(CURRENT_RELEASE.key(), ORGANISM.key(), CELLBASE.key())), - token).first(); - release = project.getCurrentRelease(); // Add species, assembly and release - CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), project.getOrganism(), release, - project.getCellbase()); + getSynchronizer().synchronizeProjectMetadataFromCatalog(projectFqn, token); } /** @@ -192,12 +193,10 @@ private void updateProject(String studyFqn, String token) throws CatalogExceptio * @throws StorageEngineException */ private List findFilesToIndex(ObjectMap params, String token) throws CatalogException, URISyntaxException, StorageEngineException { - synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); - List inputFiles = getInputFiles(catalogManager, studyFqn, files, token); // Update Catalog from the storage metadata. This may change the index status of the inputFiles . - synchronizer.synchronizeCatalogFilesFromStorage(studyFqn, inputFiles, token, FILE_GET_QUERY_OPTIONS); + getSynchronizer().synchronizeCatalogFromStorage(studyFqn, inputFiles, token, FILE_GET_QUERY_OPTIONS); LOGGER.debug("Index - Number of files to be indexed: {}, list of files: {}", inputFiles.size(), inputFiles.stream().map(File::getName).collect(Collectors.toList())); @@ -294,6 +293,7 @@ public static List getInputFiles(CatalogManager catalogManager, String stu } } } + return inputFiles; } @@ -340,13 +340,12 @@ private List indexFiles(List fileUris, String token, updateDefaultCohortStatus(studyFqn, prevDefaultCohortStatus, token); } if (fullSynchronize) { - synchronizer.synchronizeCatalogStudyFromStorage(studyFqn, token); + getSynchronizer().synchronizeCatalogFromStorage(studyFqn, token); } else { List inputFiles = catalogManager.getFileManager().search(studyFqn, new Query(FileDBAdaptor.QueryParams.URI.key(), fileUris), new QueryOptions(QueryOptions.INCLUDE, "id,name,path,uri"), token).getResults(); - synchronizer.synchronizeCatalogFilesFromStorage(studyFqn, inputFiles, token); - synchronizer.synchronizeCohorts(studyFqn, token); + getSynchronizer().synchronizeCatalogFromStorage(studyFqn, inputFiles, true, token); } } variantStorageEngine.close(); @@ -816,6 +815,14 @@ private String getTransformedFileIdFromOriginal(File file) throws CatalogExcepti return transformedFileId; } + private CatalogStorageMetadataSynchronizer getSynchronizer() throws StorageEngineException { + if (_synchronizer == null) { + _synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); + + } + return _synchronizer; + } + private enum Type { // AUTO, // TODO TRANSFORM, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantStatsOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantStatsOperationManager.java index 08629744157..58b6bb396b2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantStatsOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantStatsOperationManager.java @@ -70,7 +70,7 @@ public Collection stats(String study, List cohorts, String regio // Synchronize catalog with storage CatalogStorageMetadataSynchronizer synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); - synchronizer.synchronizeCatalogStudyFromStorage(study, token); + synchronizer.synchronizeCatalogFromStorage(study, token); Map> cohortsMap = checkCanCalculateCohorts(study, cohorts, overwriteStats, resume, token); @@ -108,7 +108,7 @@ public Collection delete(String study, List cohorts, ObjectMap p // Synchronize catalog with storage CatalogStorageMetadataSynchronizer synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); - synchronizer.synchronizeCatalogStudyFromStorage(study, token); + synchronizer.synchronizeCatalogFromStorage(study, token); try { // Modify cohort status to "INVALID" diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java index bb9b7fff211..dab8078c002 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java @@ -48,6 +48,9 @@ import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; +import org.opencb.opencga.core.models.variant.OperationIndexStatus; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -111,23 +114,21 @@ public CatalogStorageMetadataSynchronizer(CatalogManager catalogManager, Variant this.metadataManager = metadataManager; } - public static void updateProjectMetadata(CatalogManager catalog, VariantStorageMetadataManager scm, String project, String sessionId) + public void synchronizeProjectMetadataFromCatalog(String projectFqn, String token) throws CatalogException, StorageEngineException { - final Project p = catalog.getProjectManager().get(project, + final Project project = catalogManager.getProjectManager().get(projectFqn, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( ProjectDBAdaptor.QueryParams.ORGANISM.key(), ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key(), ProjectDBAdaptor.QueryParams.CELLBASE.key())), - sessionId) + token) .first(); - updateProjectMetadata(scm, p.getOrganism(), p.getCurrentRelease(), p.getCellbase()); - } - - public static void updateProjectMetadata(VariantStorageMetadataManager scm, ProjectOrganism organism, int release, CellBaseConfiguration cellbase) - throws StorageEngineException { + int release = project.getCurrentRelease(); + CellBaseConfiguration cellbase = project.getCellbase(); + ProjectOrganism organism = project.getOrganism(); String scientificName = CellBaseUtils.toCellBaseSpeciesName(organism.getScientificName()); - scm.updateProjectMetadata(projectMetadata -> { + metadataManager.updateProjectMetadata(projectMetadata -> { if (projectMetadata == null) { projectMetadata = new ProjectMetadata(); } @@ -139,8 +140,129 @@ public static void updateProjectMetadata(VariantStorageMetadataManager scm, Proj }); } - public StudyMetadata getStudyMetadata(String study) throws CatalogException { - return metadataManager.getStudyMetadata(study); + public void synchronizeCatalogProjectFromStorageByStudy(String studyFqn, String token) throws CatalogException { + String projectFqn = catalogManager.getStudyManager().getProjectFqn(studyFqn); + synchronizeCatalogProjectFromStorage(projectFqn, Collections.singletonList(studyFqn), token); + } + + public void synchronizeCatalogProjectFromStorage(String projectFqn, String token) throws CatalogException { + if (!metadataManager.exists()) { + return; + } + synchronizeCatalogProjectFromStorage(projectFqn, metadataManager.getStudyNames(), token); + } + + private void synchronizeCatalogProjectFromStorage(String projectFqn, List studies, String token) throws CatalogException { + + if (!metadataManager.exists()) { + return; + } + logger.info("Synchronize project '{}' from Storage", projectFqn); + ProjectMetadata projectMetadata = metadataManager.getProjectMetadata(); + + Project project = catalogManager.getProjectManager().get(projectFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + ProjectDBAdaptor.QueryParams.FQN.key(), + ProjectDBAdaptor.QueryParams.ORGANISM.key(), + ProjectDBAdaptor.QueryParams.INTERNAL.key(), + ProjectDBAdaptor.QueryParams.CELLBASE.key())), + token) + .first(); + projectFqn = project.getFqn(); + + String annotationIndexStatus = secureGet(() -> project.getInternal().getVariant().getAnnotationIndex().getStatus().getId(), null); + TaskMetadata.Status storageAnnotationIndexStatus = projectMetadata.getAnnotationIndexStatus(); + if (!storageAnnotationIndexStatus.name().equals(annotationIndexStatus)) { + OperationIndexStatus operationIndexStatus; + switch (storageAnnotationIndexStatus) { + case NONE: + operationIndexStatus = new OperationIndexStatus(OperationIndexStatus.PENDING, + "Variant annotation index operation pending. " + + " variantIndexTs = " + projectMetadata.getVariantIndexLastTimestamp() + + ", variantAnnotationIndexTs = " + projectMetadata.getAnnotationIndexLastTimestamp() + ); + break; + case READY: + operationIndexStatus = new OperationIndexStatus(storageAnnotationIndexStatus.name(), ""); + break; + default: + throw new IllegalStateException("Unexpected value: " + storageAnnotationIndexStatus); + } + logger.info("Update project '{}' annotation index status to {}", + projectFqn, operationIndexStatus.getId()); + catalogManager.getProjectManager().setProjectInternalVariantAnnotationIndex(projectFqn, + new InternalVariantOperationIndex(operationIndexStatus), + new QueryOptions(), token); + } + + String secondaryAnnotationIndexStatus = secureGet(() -> project.getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId(), null); + TaskMetadata.Status storageSecondaryAnnotationIndexStatus = projectMetadata.getSecondaryAnnotationIndexStatus(); + if (!storageSecondaryAnnotationIndexStatus.name().equals(secondaryAnnotationIndexStatus)) { + OperationIndexStatus operationIndexStatus; + switch (storageSecondaryAnnotationIndexStatus) { + case NONE: + operationIndexStatus = new OperationIndexStatus(OperationIndexStatus.PENDING, + "Variant secondary annotation index operation pending. " + + " variantIndexTs = " + projectMetadata.getVariantIndexLastTimestamp() + + ", variantSecondaryAnnotationIndexTs = " + projectMetadata.getSecondaryAnnotationIndexLastTimestamp() + + ", variantAnnotationIndexTs = " + projectMetadata.getAnnotationIndexLastTimestamp() + + ", variantIndexStatsTs = " + projectMetadata.getStatsLastTimestamp() + ); + break; + case READY: + operationIndexStatus = new OperationIndexStatus(storageAnnotationIndexStatus.name(), ""); + break; + default: + throw new IllegalStateException("Unexpected value: " + storageSecondaryAnnotationIndexStatus); + } + logger.info("Update project '{}' secondary annotation index status to {}", + projectFqn, operationIndexStatus.getId()); + catalogManager.getProjectManager().setProjectInternalVariantSecondaryAnnotationIndex(projectFqn, + new InternalVariantOperationIndex(operationIndexStatus), + new QueryOptions(), token); + } + + for (String studyName : studies) { + synchronizeStudyFromStorage(token, studyName); + } + } + + public void synchronizeStudyFromStorage(String token, String studyName) throws CatalogException { + StudyMetadata studyMetadata = metadataManager.getStudyMetadata(studyName); + if (studyMetadata == null) { + logger.info("Study '{}' not found in storage", studyName); + return; + } + Study study = catalogManager.getStudyManager().get(studyMetadata.getName(), new QueryOptions(), token).first(); + String status = secureGet(study, + s -> s.getInternal().getVariant().getSecondarySampleIndex().getStatus().getId(), OperationIndexStatus.PENDING); + StudyMetadata.SampleIndexConfigurationVersioned latest = studyMetadata.getSampleIndexConfigurationLatest(true); + + InternalVariantOperationIndex operationIndex = null; + switch (latest.getStatus()) { + case STAGING: + if (!status.equals(OperationIndexStatus.PENDING)) { + operationIndex = new InternalVariantOperationIndex(new OperationIndexStatus(OperationIndexStatus.PENDING, "")); + } + break; + case ACTIVE: + if (!status.equals(OperationIndexStatus.READY)) { + operationIndex = new InternalVariantOperationIndex(new OperationIndexStatus(OperationIndexStatus.READY, "")); + } + // What if there are some samples missing their family index? + break; + case DEPRECATED: + case REMOVED: + logger.warn("Study '{}' secondary sample index configuration is deprecated or removed. " + + "Please, update the configuration to the latest version.", studyMetadata.getName()); + break; + } + if (operationIndex != null) { + logger.info("Update study '{}' secondary sample index status to {}", + studyMetadata.getName(), operationIndex.getStatus().getId()); + catalogManager.getStudyManager().setStudyInternalVariantSecondarySampleIndex( + studyMetadata.getName(), operationIndex, new QueryOptions(), token); + } } /** @@ -152,10 +274,10 @@ public StudyMetadata getStudyMetadata(String study) throws CatalogException { * @return if there were modifications in catalog * @throws CatalogException if there is an error with catalog */ - public boolean synchronizeCatalogFilesFromStorage(String study, List files, String sessionId, QueryOptions fileQueryOptions) + public boolean synchronizeCatalogFromStorage(String study, List files, String sessionId, QueryOptions fileQueryOptions) throws CatalogException, StorageEngineException { - boolean modified = synchronizeCatalogFilesFromStorage(study, files, sessionId); + boolean modified = synchronizeCatalogFromStorage(study, files, false, sessionId); if (modified) { // Files updated. Reload files from catalog for (int i = 0; i < files.size(); i++) { @@ -169,14 +291,17 @@ public boolean synchronizeCatalogFilesFromStorage(String study, List files /** * Updates catalog metadata from storage metadata. * - * @param studyFqn Study FQN - * @param files Files to update - * @param sessionId User session id + * @param studyFqn Study FQN + * @param files Files to update + * @param synchronizeCohorts Synchronize cohorts + * @param sessionId User session id * @return if there were modifications in catalog * @throws CatalogException if there is an error with catalog */ - public boolean synchronizeCatalogFilesFromStorage(String studyFqn, List files, String sessionId) + public boolean synchronizeCatalogFromStorage(String studyFqn, List files, boolean synchronizeCohorts, String sessionId) throws CatalogException { + synchronizeCatalogProjectFromStorageByStudy(studyFqn, sessionId); + StudyMetadata study = metadataManager.getStudyMetadata(studyFqn); if (study == null) { return false; @@ -187,8 +312,12 @@ public boolean synchronizeCatalogFilesFromStorage(String studyFqn, List fi if (files != null && files.isEmpty()) { files = null; } + boolean modified = synchronizeFiles(study, files, sessionId); - return synchronizeFiles(study, files, sessionId); + if (synchronizeCohorts) { + modified |= synchronizeCohorts(study, sessionId); + } + return modified; } /** @@ -206,6 +335,8 @@ public boolean synchronizeCatalogSamplesFromStorage(String studyFqn, List sampleIds; @@ -232,18 +363,24 @@ public boolean synchronizeCatalogSamplesFromStorage(String studyFqn, List studies, String token) throws CatalogException { boolean modified = false; - for (String study : metadataManager.getStudyNames()) { - modified |= synchronizeCatalogStudyFromStorage(study, token); + synchronizeCatalogProjectFromStorage(project, token); + if (CollectionUtils.isEmpty(studies)) { + studies = metadataManager.getStudyNames(); + } + for (String study : studies) { + StudyMetadata studyMetadata = metadataManager.getStudyMetadata(study); + modified |= synchronizeCatalogStudyFromStorage(studyMetadata, token); } return modified; } - public boolean synchronizeCatalogStudyFromStorage(String study, String sessionId) + public boolean synchronizeCatalogFromStorage(String study, String sessionId) throws CatalogException { StudyMetadata studyMetadata = metadataManager.getStudyMetadata(study); if (studyMetadata != null) { + synchronizeCatalogProjectFromStorageByStudy(studyMetadata.getName(), sessionId); // Update Catalog file and cohort status. return synchronizeCatalogStudyFromStorage(studyMetadata, sessionId); } else { @@ -263,7 +400,7 @@ public boolean synchronizeCatalogStudyFromStorage(String study, String sessionId * @return if there were modifications in catalog * @throws CatalogException if there is an error with catalog */ - public boolean synchronizeCatalogStudyFromStorage(StudyMetadata study, String sessionId) + private boolean synchronizeCatalogStudyFromStorage(StudyMetadata study, String sessionId) throws CatalogException { logger.info("Synchronizing study " + study.getName()); @@ -274,16 +411,7 @@ public boolean synchronizeCatalogStudyFromStorage(StudyMetadata study, String se return modified; } - public boolean synchronizeCohorts(String study, String sessionId) throws CatalogException { - StudyMetadata studyMetadata = getStudyMetadata(study); - if (studyMetadata == null) { - return false; - } else { - return synchronizeCohorts(studyMetadata, sessionId); - } - } - - protected boolean synchronizeCohorts(StudyMetadata study, String sessionId) throws CatalogException { + private boolean synchronizeCohorts(StudyMetadata study, String sessionId) throws CatalogException { boolean modified = false; // ------------------------------------------------------------------- @@ -421,7 +549,7 @@ protected boolean synchronizeCohorts(StudyMetadata study, String sessionId) thro return modified; } - protected boolean synchronizeFiles(StudyMetadata study, List files, String token) throws CatalogException { + private boolean synchronizeFiles(StudyMetadata study, List files, String token) throws CatalogException { boolean modified = false; Map fileNameMap = new HashMap<>(); Map filePathMap = new HashMap<>(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index ef6b79bd153..eb48e265b8d 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -55,7 +55,9 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; +import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.OperationIndexStatus; import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; @@ -431,9 +433,10 @@ public void testVariantFileReload() throws Exception { @Test public void testVariantSecondaryAnnotationIndex() throws Exception { Assume.assumeTrue(HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()); - for (String sample : samples) { - SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex(); - assertEquals(IndexStatus.NONE, index.getStatus().getId()); + for (String sampleName : samples) { + Sample sample = catalogManager.getSampleManager().get(STUDY, sampleName, new QueryOptions(), token).first(); + assertEquals(IndexStatus.NONE, sample.getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId()); + assertEquals(IndexStatus.READY, sample.getInternal().getVariant().getAnnotationIndex().getStatus().getId()); } assertEquals(IndexStatus.NONE, catalogManager.getFileManager().get(STUDY, file.getId(), new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId()); @@ -441,6 +444,11 @@ public void testVariantSecondaryAnnotationIndex() throws Exception { new VariantSecondaryAnnotationIndexParams(), Paths.get(opencga.createTmpOutdir()), "annotation_index", false, token); + assertEquals(OperationIndexStatus.READY, + catalogManager.getProjectManager().get(PROJECT, new QueryOptions(), token).first().getInternal().getVariant().getAnnotationIndex().getStatus().getId()); + assertEquals(OperationIndexStatus.READY, + catalogManager.getProjectManager().get(PROJECT, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId()); + for (String sample : samples) { SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex(); assertEquals(IndexStatus.READY, index.getStatus().getId()); @@ -491,15 +499,24 @@ public void testVariantSecondarySampleIndex() throws Exception { assertEquals(sample, 1, sampleIndex.getVersion().intValue()); } + Study study = catalogManager.getStudyManager().get(STUDY, new QueryOptions(), token).first(); + assertEquals(OperationIndexStatus.READY, study.getInternal().getVariant().getSecondarySampleIndex().getStatus().getId()); + // Change the sample index configuration. OpenCGAResult jobs = variantStorageManager.configureSampleIndex(STUDY, SampleIndexConfiguration.defaultConfiguration() .addPopulation(new SampleIndexConfiguration.Population("1000G", "SAS")), false, token); + study = catalogManager.getStudyManager().get(STUDY, new QueryOptions(), token).first(); + assertEquals(OperationIndexStatus.PENDING, study.getInternal().getVariant().getSecondarySampleIndex().getStatus().getId()); + // Initially nothing should change, even after running a manual synchronization toolRunner.execute(VariantStorageMetadataSynchronizeOperationTool.class, new VariantStorageMetadataSynchronizeParams().setStudy(STUDY_FQN), Paths.get(opencga.createTmpOutdir()), "", false, catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken()); + study = catalogManager.getStudyManager().get(STUDY, new QueryOptions(), token).first(); + assertEquals(OperationIndexStatus.PENDING, study.getInternal().getVariant().getSecondarySampleIndex().getStatus().getId()); + for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token) .first().getInternal().getVariant().getSecondarySampleIndex(); @@ -525,6 +542,9 @@ public void testVariantSecondarySampleIndex() throws Exception { assertEquals(2, sampleIndex.getVersion().intValue()); } + study = catalogManager.getStudyManager().get(STUDY, new QueryOptions(), token).first(); + assertEquals(OperationIndexStatus.READY, study.getInternal().getVariant().getSecondarySampleIndex().getStatus().getId()); + // Same. Rerun configuration and change version. jobs = variantStorageManager.configureSampleIndex(STUDY, SampleIndexConfiguration.defaultConfiguration(), false, token); for (Job job : jobs.getResults()) { @@ -622,6 +642,31 @@ public void testCellbaseConfigure() throws Exception { String project = "Project_test_cellbase_configure"; catalogManager.getProjectManager().create(new ProjectCreateParams(project, project, "", "", "", new ProjectOrganism("hsapiens", "GRCh38"), null, null), QueryOptions.empty(), token); + assertEquals(OperationIndexStatus.NONE, + catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getInternal().getVariant().getAnnotationIndex().getStatus().getId()); + + testCellbaseConfigure(project); + + assertEquals(OperationIndexStatus.NONE, + catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getInternal().getVariant().getAnnotationIndex().getStatus().getId()); + + } + + @Test + public void testCellbaseConfigureAnnotated() throws Exception { + String project = PROJECT; + + assertEquals(OperationIndexStatus.READY, + catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getInternal().getVariant().getAnnotationIndex().getStatus().getId()); + + testCellbaseConfigure(project); + + assertEquals(OperationIndexStatus.PENDING, + catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getInternal().getVariant().getAnnotationIndex().getStatus().getId()); + + } + + private static void testCellbaseConfigure(String project) throws StorageEngineException, CatalogException { CellBaseUtils cellBaseUtils = variantStorageManager.getVariantStorageEngineByProject(project, null, token).getCellBaseUtils(); assertEquals(ParamConstants.CELLBASE_URL, cellBaseUtils.getURL()); assertEquals(ParamConstants.CELLBASE_VERSION, cellBaseUtils.getVersion()); @@ -643,7 +688,6 @@ public void testCellbaseConfigure() throws Exception { assertEquals(newCellbaseVersion, cellbaseConfiguration.getVersion()); assertEquals(newCellbaseDataRelease, cellbaseConfiguration.getDataRelease()); -// assertTrue(family.getPedigreeGraph() != null); } public void checkExecutionResult(ExecutionResult er) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java index 8833884c5a4..91e9ee640d3 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java @@ -83,7 +83,7 @@ public class CatalogStorageMetadataSynchronizerTest { static private LinkedHashSet indexedFiles = new LinkedHashSet<>(); private static String cohortId; private VariantStorageMetadataManager metadataManager; - private CatalogStorageMetadataSynchronizer studyConfigurationFactory; + private CatalogStorageMetadataSynchronizer synchronizer; @BeforeClass public static void beforeClass() throws Exception { @@ -137,7 +137,7 @@ public void setUp() throws Exception { metadataManager.addIndexedFiles(studyMetadata.getId(), indexedFiles.stream() .map(f -> metadataManager.getFileId(studyMetadata.getId(), f)) .collect(Collectors.toList())); - studyConfigurationFactory = new CatalogStorageMetadataSynchronizer(catalogManager, metadataManager); + synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, metadataManager); } @After @@ -172,7 +172,7 @@ public static File create(String resourceName, boolean indexed) throws IOExcepti @Test public void updateCatalogFromStorageTest() throws Exception { - StudyMetadata sm = studyConfigurationFactory.getStudyMetadata(studyId); + StudyMetadata sm = metadataManager.getStudyMetadata(studyId); List samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, sessionId) .getResults() @@ -189,7 +189,7 @@ public void updateCatalogFromStorageTest() throws Exception { assertNotNull(nonIndexedFile); metadataManager.addIndexedFiles(sm.getId(), Collections.singletonList(metadataManager.getFileId(sm.getId(), nonIndexedFile.getName()))); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); nonIndexedFile = catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first(); assertEquals(VariantIndexStatus.READY, nonIndexedFile.getInternal().getVariant().getIndex().getStatus().getId()); @@ -199,7 +199,7 @@ public void updateCatalogFromStorageTest() throws Exception { assertNotNull(nonIndexedFile); metadataManager.addRunningTask(sm.getId(), "LOAD", Collections.singletonList(metadataManager.getFileId(sm.getId(), nonIndexedFile.getName())), false, TaskMetadata.Type.LOAD); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); nonIndexedFile = catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first(); assertEquals(VariantIndexStatus.INDEXING, nonIndexedFile.getInternal().getVariant().getIndex().getStatus().getId()); @@ -208,14 +208,14 @@ public void updateCatalogFromStorageTest() throws Exception { @Test public void testInternalSampleStatuses() throws Exception { - StudyMetadata sm = studyConfigurationFactory.getStudyMetadata(studyId); + StudyMetadata sm = metadataManager.getStudyMetadata(studyId); String fileName = indexedFiles.iterator().next(); String sampleName = catalogManager.getFileManager().get(studyId, fileName, null, sessionId).first().getSampleIds().get(0); int version = sm.getSampleIndexConfigurationLatest().getVersion(); Sample sample; //-------------- - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s -> s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -225,7 +225,7 @@ public void testInternalSampleStatuses() throws Exception { //-------------- metadataManager.updateSampleMetadata(sm.getId(), metadataManager.getSampleId(sm.getId(), sampleName), s -> s.setAnnotationStatus(TaskMetadata.Status.READY)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s->s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -235,7 +235,7 @@ public void testInternalSampleStatuses() throws Exception { //-------------- metadataManager.updateSampleMetadata(sm.getId(), metadataManager.getSampleId(sm.getId(), sampleName), s -> s.setSampleIndexStatus(TaskMetadata.Status.READY, version)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s->s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -246,7 +246,7 @@ public void testInternalSampleStatuses() throws Exception { //-------------- metadataManager.updateSampleMetadata(sm.getId(), metadataManager.getSampleId(sm.getId(), sampleName), s -> s.setSampleIndexAnnotationStatus(TaskMetadata.Status.READY, version)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s->s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -256,7 +256,7 @@ public void testInternalSampleStatuses() throws Exception { //-------------- metadataManager.updateSampleMetadata(sm.getId(), metadataManager.getSampleId(sm.getId(), sampleName), s -> s.setFamilyIndexStatus(TaskMetadata.Status.READY, version)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s->s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -271,7 +271,7 @@ public void testInternalSampleStatuses() throws Exception { s.setSampleIndexConfigurations(Collections.singletonList(c)); return s; }); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); sample = catalogManager.getSampleManager().get(studyId, sampleName, null, sessionId).first(); assertEquals(IndexStatus.READY, secureGet(sample, s -> s.getInternal().getVariant().getIndex().getStatus().getId(), null)); @@ -284,7 +284,7 @@ public void testInternalSampleStatuses() throws Exception { @Test public void testInternalFileStatuses() throws Exception { - StudyMetadata sm = studyConfigurationFactory.getStudyMetadata(studyId); + StudyMetadata sm = metadataManager.getStudyMetadata(studyId); String fileName = indexedFiles.iterator().next(); File file; @@ -298,7 +298,7 @@ public void testInternalFileStatuses() throws Exception { // -------------------- metadataManager.updateFileMetadata(sm.getId(), metadataManager.getFileId(sm.getId(), fileName), fm -> fm.setAnnotationStatus(TaskMetadata.Status.READY)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); file = catalogManager.getFileManager().get(studyId, fileName, null, sessionId).first(); assertEquals(IndexStatus.READY, file.getInternal().getVariant().getIndex().getStatus().getId()); @@ -308,7 +308,7 @@ public void testInternalFileStatuses() throws Exception { // -------------------- metadataManager.updateFileMetadata(sm.getId(), metadataManager.getFileId(sm.getId(), fileName), fm -> fm.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.READY)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); file = catalogManager.getFileManager().get(studyId, fileName, null, sessionId).first(); assertEquals(IndexStatus.READY, file.getInternal().getVariant().getIndex().getStatus().getId()); @@ -318,7 +318,7 @@ public void testInternalFileStatuses() throws Exception { // -------------------- metadataManager.updateFileMetadata(sm.getId(), metadataManager.getFileId(sm.getId(), fileName), fm -> fm.setAnnotationStatus(TaskMetadata.Status.NONE)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(sm, sessionId); + synchronizer.synchronizeCatalogFromStorage(sm.getName(), sessionId); file = catalogManager.getFileManager().get(studyId, fileName, null, sessionId).first(); assertEquals(IndexStatus.READY, file.getInternal().getVariant().getIndex().getStatus().getId()); @@ -338,7 +338,7 @@ public void testMissingSamples() throws CatalogException { assertEquals(0, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().size()); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(studyId, sessionId); + synchronizer.synchronizeCatalogFromStorage(studyId, sessionId); assertEquals(1, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().size()); } @@ -357,7 +357,7 @@ public void testWrongSamples() throws CatalogException { assertEquals(1, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().size()); assertEquals(wrongSampleId, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().get(0)); - studyConfigurationFactory.synchronizeCatalogStudyFromStorage(studyId, sessionId); + synchronizer.synchronizeCatalogFromStorage(studyId, sessionId); assertEquals(1, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().size()); assertEquals(correctSampleId, catalogManager.getFileManager().get(studyId, fileId, new QueryOptions(), sessionId).first().getSampleIds().get(0)); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java index ebcf46144db..350b7a89fb3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java @@ -62,13 +62,7 @@ private void executeJob() throws CatalogException, ToolException { } private void listTools() { - Collection> tools; - if (configuration != null && configuration.getAnalysis() != null - && CollectionUtils.isNotEmpty(configuration.getAnalysis().getPackages())) { - tools = new ToolFactory().getTools(configuration.getAnalysis().getPackages()); - } else { - tools = new ToolFactory().getTools(); - } + Collection> tools = new ToolFactory().getTools(configuration.getAnalysis()); int toolIdSize = tools.stream().mapToInt(c -> c.getAnnotation(Tool.class).id().length()).max().orElse(0) + 2; int toolTypeSize = Arrays.stream(Tool.Type.values()).mapToInt(e->e.toString().length()).max().orElse(0) + 2; int toolResourceSize = Arrays.stream(Enums.Resource.values()).mapToInt(e->e.toString().length()).max().orElse(0) + 2; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/InternalVariantOperationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/InternalVariantOperationMigration.java new file mode 100644 index 00000000000..f079a006d77 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/InternalVariantOperationMigration.java @@ -0,0 +1,44 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0; + +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.project.ProjectInternalVariant; +import org.opencb.opencga.core.models.study.StudyInternalVariant; + +import java.util.Arrays; + +@Migration(id = "internalVariant_3_2_0", description = "Add internal.variant fields to Project and Study #TASK-6219", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240703) +public class InternalVariantOperationMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + ProjectInternalVariant projectInternalVariant = new ProjectInternalVariant(); + StudyInternalVariant studyInternalVariant = new StudyInternalVariant(); + Document projectInternalVariantDoc = convertToDocument(projectInternalVariant); + Document studyInternalVariantDoc = convertToDocument(studyInternalVariant); + + Document query = new Document() + .append("internal.variant", new Document("$exists", false)); + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION)) { + Document update = new Document() + .append("$set", new Document() + .append("internal.variant", projectInternalVariantDoc) + ); + getMongoCollection(collection).updateMany(query, update); + } + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + Document update = new Document() + .append("$set", new Document() + .append("internal.variant", studyInternalVariantDoc) + ); + getMongoCollection(collection).updateMany(query, update); + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index 1ff667bdbac..d1031d5c138 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -61,6 +61,8 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_DATASTORES("internal.datastores", TEXT_ARRAY, ""), INTERNAL_DATASTORES_VARIANT("internal.datastores.variant", TEXT_ARRAY, ""), + INTERNAL_VARIANT_ANNOTATION_INDEX("internal.variant.annotationIndex", OBJECT, ""), + INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX("internal.variant.secondaryAnnotationIndex", OBJECT, ""), INTERNAL("internal", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" @@ -117,7 +119,6 @@ public static QueryParams getParam(String key) { } } - default boolean exists(long projectId) throws CatalogDBException { return count(new Query(QueryParams.UID.key(), projectId)).getNumMatches() > 0; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index 0afd3f255d4..bb0b08c1ae3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -435,6 +435,7 @@ enum QueryParams implements QueryParam { INTERNAL_CONFIGURATION_VARIANT_ENGINE("internal.configuration.variantEngine", Type.OBJECT, ""), INTERNAL_CONFIGURATION_VARIANT_ENGINE_OPTIONS("internal.configuration.variantEngine.options", Type.OBJECT, ""), INTERNAL_CONFIGURATION_VARIANT_ENGINE_SAMPLE_INDEX("internal.configuration.variantEngine.sampleIndex", Type.OBJECT, ""), + INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX("internal.variant.secondarySampleIndex", OBJECT, ""), DATASTORES("dataStores", TEXT_ARRAY, ""), SIZE("size", INTEGER_ARRAY, ""), URI("uri", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index ad51ec236bf..b805617f393 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -699,6 +699,8 @@ private QueryOptions fixOptions(QueryOptions queryOptions) { // The order (ascending or descending) should be in priorityArray[1] fixedSortList.add(PRIVATE_PRIORITY + ":" + priorityArray[1]); } + } else if (key.equals(QueryParams.CREATION_DATE.key())) { + fixedSortList.add(PRIVATE_CREATION_DATE); } else { fixedSortList.add(key); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 3a91bc26284..77e75554b2a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -292,7 +292,8 @@ UpdateDocument getDocumentUpdateParams(ObjectMap parameters) throws CatalogDBExc filterMapParams(parameters, document.getSet(), acceptedMapParams); final String[] acceptedObjectParams = {QueryParams.INTERNAL_STATUS.key(), QueryParams.CELLBASE.key(), - QueryParams.INTERNAL_DATASTORES_VARIANT.key(), QueryParams.INTERNAL_DATASTORES.key()}; + QueryParams.INTERNAL_DATASTORES_VARIANT.key(), QueryParams.INTERNAL_DATASTORES.key(), + QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key()}; filterObjectParams(parameters, document.getSet(), acceptedObjectParams); if (!document.toFinalUpdateDocument().isEmpty()) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index c91fd4c3665..bf8a689feb2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -117,7 +117,8 @@ static Document getDocumentUpdateParams(ObjectMap parameters) throws CatalogDBEx final String[] acceptedObjectParams = {QueryParams.TYPE.key(), QueryParams.SOURCES.key(), QueryParams.STATUS.key(), QueryParams.INTERNAL_CONFIGURATION_CLINICAL.key(), QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), - QueryParams.INTERNAL_INDEX_RECESSIVE_GENE.key(), QueryParams.ADDITIONAL_INFO.key(), QueryParams.INTERNAL_STATUS.key()}; + QueryParams.INTERNAL_INDEX_RECESSIVE_GENE.key(), QueryParams.ADDITIONAL_INFO.key(), QueryParams.INTERNAL_STATUS.key(), + QueryParams.INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX.key()}; filterObjectParams(parameters, studyParameters, acceptedObjectParams); if (studyParameters.containsKey(QueryParams.STATUS.key())) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index 7cb92deec37..1f3ba3e982b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -42,6 +42,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; import org.opencb.opencga.core.response.OpenCGAResult; import java.io.*; @@ -72,6 +73,8 @@ public class ProjectManager extends AbstractManager { ProjectDBAdaptor.QueryParams.ATTRIBUTES.key() )); private static final Set PROTECTED_UPDATABLE_FIELDS = new HashSet<>(Arrays.asList( + ProjectDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), + ProjectDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), ProjectDBAdaptor.QueryParams.INTERNAL_DATASTORES_VARIANT.key(), ProjectDBAdaptor.QueryParams.CELLBASE.key() )); @@ -493,6 +496,46 @@ private OpenCGAResult update(String projectId, ObjectMap parameters, Qu } } + public OpenCGAResult setProjectInternalVariantAnnotationIndex(String projectStr, InternalVariantOperationIndex status, + QueryOptions options, + String token) + throws CatalogException { + return setProjectInternalVariant(projectStr, new ProjectInternalVariant() + .setSecondaryAnnotationIndex(null) + .setAnnotationIndex(status), options, token); + } + + public OpenCGAResult setProjectInternalVariantSecondaryAnnotationIndex(String projectStr, InternalVariantOperationIndex status, + QueryOptions options, + String token) + throws CatalogException { + return setProjectInternalVariant(projectStr, new ProjectInternalVariant() + .setSecondaryAnnotationIndex(status) + .setAnnotationIndex(null), options, token); + } + + public OpenCGAResult setProjectInternalVariant(String projectStr, ProjectInternalVariant variant, QueryOptions options, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + + authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + ParamUtils.checkObj(variant, "ProjectInternalVariant"); + + ObjectMap parameters = new ObjectMap(); + if (variant.getAnnotationIndex() != null) { + parameters.put(ProjectDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), variant.getAnnotationIndex()); + } + if (variant.getSecondaryAnnotationIndex() != null) { + parameters.put(ProjectDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), + variant.getSecondaryAnnotationIndex()); + } + + return update(projectStr, parameters, options, true, token); + } + public OpenCGAResult setDatastoreVariant(String projectStr, DataStore dataStore, String token) throws CatalogException { return update(projectStr, new ObjectMap(ProjectDBAdaptor.QueryParams.INTERNAL_DATASTORES_VARIANT.key(), dataStore), new QueryOptions(), true, token); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index ff87047bcce..eb6d2183ee2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -68,6 +68,7 @@ import org.opencb.opencga.core.models.summaries.VariableSetSummary; import org.opencb.opencga.core.models.summaries.VariableSummary; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; import org.opencb.opencga.core.response.OpenCGAResult; import org.reflections.Reflections; import org.reflections.scanners.ResourcesScanner; @@ -1837,6 +1838,37 @@ public void setVariantEngineConfigurationOptions(String studyStr, ObjectMap opti getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); } + public OpenCGAResult setStudyInternalVariantSecondarySampleIndex( + String studyStr, InternalVariantOperationIndex variant, QueryOptions options, String token) throws CatalogException { + return setStudyInternalVariant(studyStr, new StudyInternalVariant().setSecondarySampleIndex(variant), options, token); + } + + public OpenCGAResult setStudyInternalVariant(String studyStr, StudyInternalVariant variant, QueryOptions options, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + + Study study = resolveId(catalogFqn, null, tokenPayload); + + authorizationManager.isAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + ParamUtils.checkObj(variant, "StudyInternalVariant"); + + ObjectMap parameters = new ObjectMap(); + if (variant.getSecondarySampleIndex() != null) { + parameters.put(StudyDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX.key(), variant.getSecondarySampleIndex()); + } + + OpenCGAResult updateResult = getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated study + OpenCGAResult result = getStudyDBAdaptor(organizationId).get(study.getUid(), options); + updateResult.setResults(result.getResults()); + } + return updateResult; + } + public void setVariantEngineSetupOptions(String studyStr, VariantSetupResult variantSetupResult, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index c10ef156497..634971ec494 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -390,6 +390,35 @@ public class FieldConstants { public static final String JOB_STUDY = "Job study."; public static final String JOB_STUDY_PARAM_OTHERS = "List of strings."; + // Analysis configuration + public static final String CONFIGURATION_ANALYSIS_OPERATIONS = "Object containing the configuration of several analysis operations."; + + // Analysis Operation configuration + public static final String VARIANT_ANNOTATION_INDEX_OPERATION_CONFIGURATION = "Operation execution configuration for the variant" + + " annotation index tool."; + public static final String VARIANT_SECONDARY_ANNOTATION_INDEX_OPERATION_CONFIGURATION = "Operation execution configuration for the" + + " variant secondary annotation index tool.";; + public static final String VARIANT_SECONDARY_SAMPLE_INDEX_OPERATION_CONFIGURATION = "Operation execution configuration for the variant" + + " secondary sample index tool."; + + // OperationExecutionConfig + public static final String OPERATION_EXECUTION_CONFIG_POLICY = "The policy parameter determines when an automatic tool will run." + + " There are three options for this setting:\n" + + "IMMEDIATE means the tool will run automatically as soon as it is needed, ensuring prompt execution.\n" + + "NIGHTLY means the tool will run automatically when necessary but only between 24:00 and 5:00, reducing disruptions during" + + " peak hours.\n" + + "NEVER means the tool will not run automatically and will require manual initiation, giving users full control over when it" + + " operates.\n" + + "By setting this parameter, users can choose the most appropriate timing for the tool's execution based on their needs."; + public static final String OPERATION_EXECUTION_CONFIG_MAX_ATTEMPTS = "The maxAttempts parameter establishes the maximum number of times" + + " the automatic process will attempt to launch the tool in the event of a failure. This setting ensures that the tool will" + + " make multiple attempts to run successfully, up to the specified limit, before ceasing further attempts." + + " By configuring this parameter, users can control the resilience of the tool's execution process in response to failures."; + public static final String OPERATION_EXECUTION_CONFIG_JOB_PARAMS = "The jobParams parameter contains the job parameters required for" + + " the tool to run. This parameter ensures that all necessary configurations and values are predefined, enabling the tool" + + " to operate correctly without needing additional input each time it is executed. By setting this parameter, users can" + + " streamline the tool's execution process and ensure consistency in its operation.."; + //Project public static final String PROJECT_FQN = "Full Qualified Name (organization@projectId)."; public static final String PROJECT_ORGANISM = "Organism to which the project belongs."; @@ -397,8 +426,19 @@ public class FieldConstants { //ProjectInternal public static final String PROJECT_INTERNAL_DATA_STORES = "Default value is VARIANT."; + public static final String PROJECT_INTERNAL_VARIANT = "Variant index statuses."; public static final String PROJECT_CELLBASE = "Cellbase configuration."; + // ProjectInternalVariant + public static final String INTERNAL_VARIANT_ANNOTATION_INDEX = "Project internal variant containing the current status of the" + + " variant annotation index."; + public static final String INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX = "Project internal variant containing the current status of the" + + " variant secondary annotation index."; + + // StudyInternalVariant + public static final String INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX = "Study internal variant containing the current status of the" + + " variant secondary sample index."; + //StudyNotification public static final String STUDY_NOTIFICATION_WEBHOOK = "Url of the study notification."; @@ -432,6 +472,7 @@ public class FieldConstants { //StudyInternal public static final String STUDY_INTERNAL_INDEX = "Study index."; public static final String STUDY_INTERNAL_CONFIGURATION = "Study configuration."; + public static final String STUDY_INTERNAL_VARIANT = "Variant index statuses."; //AdditionalInfo public static final String ADDITIONAL_INFO_TYPE = "Type of the additional info."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 9f1c7b32683..2f0bc17aeb0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -16,6 +16,9 @@ package org.opencb.opencga.core.config; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -33,12 +36,16 @@ public class Analysis { private Execution execution; + @DataField(id = "operations", description = FieldConstants.CONFIGURATION_ANALYSIS_OPERATIONS) + private OperationConfig operations; + private List frameworks; public Analysis() { packages = new ArrayList<>(); tools = new ArrayList<>(); execution = new Execution(); + operations = new OperationConfig(); frameworks = new ArrayList<>(); } @@ -51,6 +58,15 @@ public Analysis setPackages(List packages) { return this; } + public OperationConfig getOperations() { + return operations; + } + + public Analysis setOperations(OperationConfig operations) { + this.operations = operations; + return this; + } + public String getScratchDir() { return scratchDir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationConfig.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationConfig.java new file mode 100644 index 00000000000..8921a039d74 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationConfig.java @@ -0,0 +1,67 @@ +package org.opencb.opencga.core.config; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class OperationConfig { + + @DataField(id = "variantAnnotationIndex", description = FieldConstants.VARIANT_ANNOTATION_INDEX_OPERATION_CONFIGURATION) + private OperationExecutionConfig variantAnnotationIndex; + + @DataField(id = "variantSecondaryAnnotationIndex", + description = FieldConstants.VARIANT_SECONDARY_ANNOTATION_INDEX_OPERATION_CONFIGURATION) + private OperationExecutionConfig variantSecondaryAnnotationIndex; + + @DataField(id = "variantSecondarySampleIndex", description = FieldConstants.VARIANT_SECONDARY_SAMPLE_INDEX_OPERATION_CONFIGURATION) + private OperationExecutionConfig variantSecondarySampleIndex; + + public OperationConfig() { + this.variantAnnotationIndex = new OperationExecutionConfig(); + this.variantSecondaryAnnotationIndex = new OperationExecutionConfig(); + this.variantSecondarySampleIndex = new OperationExecutionConfig(); + } + + public OperationConfig(OperationExecutionConfig variantAnnotationIndex, OperationExecutionConfig variantSecondaryAnnotationIndex, + OperationExecutionConfig variantSecondarySampleIndex) { + this.variantAnnotationIndex = variantAnnotationIndex; + this.variantSecondaryAnnotationIndex = variantSecondaryAnnotationIndex; + this.variantSecondarySampleIndex = variantSecondarySampleIndex; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OperationConfig{"); + sb.append("variantAnnotationIndex=").append(variantAnnotationIndex); + sb.append(", variantSecondaryAnnotationIndex=").append(variantSecondaryAnnotationIndex); + sb.append(", variantSecondarySampleIndex=").append(variantSecondarySampleIndex); + sb.append('}'); + return sb.toString(); + } + + public OperationExecutionConfig getVariantAnnotationIndex() { + return variantAnnotationIndex; + } + + public OperationConfig setVariantAnnotationIndex(OperationExecutionConfig variantAnnotationIndex) { + this.variantAnnotationIndex = variantAnnotationIndex; + return this; + } + + public OperationExecutionConfig getVariantSecondaryAnnotationIndex() { + return variantSecondaryAnnotationIndex; + } + + public OperationConfig setVariantSecondaryAnnotationIndex(OperationExecutionConfig variantSecondaryAnnotationIndex) { + this.variantSecondaryAnnotationIndex = variantSecondaryAnnotationIndex; + return this; + } + + public OperationExecutionConfig getVariantSecondarySampleIndex() { + return variantSecondarySampleIndex; + } + + public OperationConfig setVariantSecondarySampleIndex(OperationExecutionConfig variantSecondarySampleIndex) { + this.variantSecondarySampleIndex = variantSecondarySampleIndex; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationExecutionConfig.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationExecutionConfig.java new file mode 100644 index 00000000000..f5d72d81271 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/OperationExecutionConfig.java @@ -0,0 +1,73 @@ +package org.opencb.opencga.core.config; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +import java.util.Collections; +import java.util.Map; + +public class OperationExecutionConfig { + + @DataField(id = "policy", description = FieldConstants.OPERATION_EXECUTION_CONFIG_POLICY) + private Policy policy; + + @DataField(id = "maxAttempts", description = FieldConstants.OPERATION_EXECUTION_CONFIG_MAX_ATTEMPTS) + private int maxAttempts; + + @DataField(id = "jobParams", description = FieldConstants.OPERATION_EXECUTION_CONFIG_JOB_PARAMS) + private Map jobParams; + + public enum Policy { + IMMEDIATE, + NIGHTLY, + WEEKLY, + NEVER + } + + public OperationExecutionConfig() { + this(Policy.NIGHTLY, 3, Collections.emptyMap()); + } + + public OperationExecutionConfig(Policy policy, int maxAttempts, Map jobParams) { + this.policy = policy; + this.maxAttempts = maxAttempts; + this.jobParams = jobParams; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OperationExecutionConfig{"); + sb.append("policy=").append(policy); + sb.append(", maxAttempts=").append(maxAttempts); + sb.append(", jobParams=").append(jobParams); + sb.append('}'); + return sb.toString(); + } + + public Policy getPolicy() { + return policy; + } + + public OperationExecutionConfig setPolicy(Policy policy) { + this.policy = policy; + return this; + } + + public int getMaxAttempts() { + return maxAttempts; + } + + public OperationExecutionConfig setMaxAttempts(int maxAttempts) { + this.maxAttempts = maxAttempts; + return this; + } + + public OperationExecutionConfig setJobParams(Map jobParams) { + this.jobParams = jobParams; + return this; + } + + public Map getJobParams() { + return jobParams; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternal.java index 4fcae0a9ffe..df87ac84865 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternal.java @@ -24,21 +24,26 @@ public class ProjectInternal extends Internal { - @DataField(id = "datastores", indexed = true, uncommentedClasses = {"Datastores"}, + @DataField(id = "datastores", uncommentedClasses = {"Datastores"}, description = FieldConstants.PROJECT_INTERNAL_DATA_STORES) private Datastores datastores; + @DataField(id = "variant", description = FieldConstants.PROJECT_INTERNAL_VARIANT) + private ProjectInternalVariant variant; public ProjectInternal() { } - public ProjectInternal(InternalStatus status, String registrationDate, String modificationDate, Datastores datastores) { + public ProjectInternal(InternalStatus status, String registrationDate, String modificationDate, Datastores datastores, + ProjectInternalVariant variant) { super(status, registrationDate, modificationDate); this.datastores = datastores; + this.variant = variant; } public static ProjectInternal init() { - return new ProjectInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), new Datastores()); + return new ProjectInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), new Datastores(), + new ProjectInternalVariant()); } @Override @@ -48,6 +53,7 @@ public String toString() { sb.append(", registrationDate='").append(registrationDate).append('\''); sb.append(", modificationDate='").append(lastModified).append('\''); sb.append(", datastores=").append(datastores); + sb.append(", variant=").append(variant); sb.append('}'); return sb.toString(); } @@ -61,6 +67,14 @@ public ProjectInternal setDatastores(Datastores datastores) { return this; } + public ProjectInternalVariant getVariant() { + return variant; + } + + public ProjectInternal setVariant(ProjectInternalVariant variant) { + this.variant = variant; + return this; + } public InternalStatus getStatus() { return status; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternalVariant.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternalVariant.java new file mode 100644 index 00000000000..1ab430eab0f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/ProjectInternalVariant.java @@ -0,0 +1,52 @@ +package org.opencb.opencga.core.models.project; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; + +public class ProjectInternalVariant { + + @DataField(id = "annotationIndex", uncommentedClasses = {"InternalVariantOperationIndex"}, + description = FieldConstants.INTERNAL_VARIANT_ANNOTATION_INDEX) + private InternalVariantOperationIndex annotationIndex; + + @DataField(id = "secondaryAnnotationIndex", uncommentedClasses = {"InternalVariantOperationIndex"}, + description = FieldConstants.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX) + private InternalVariantOperationIndex secondaryAnnotationIndex; + + public ProjectInternalVariant() { + this(new InternalVariantOperationIndex(), new InternalVariantOperationIndex()); + } + + public ProjectInternalVariant(InternalVariantOperationIndex annotationIndex, InternalVariantOperationIndex secondaryAnnotationIndex) { + this.annotationIndex = annotationIndex; + this.secondaryAnnotationIndex = secondaryAnnotationIndex; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ProjectInternalVariant{"); + sb.append("annotationIndex=").append(annotationIndex); + sb.append(", secondaryAnnotationIndex=").append(secondaryAnnotationIndex); + sb.append('}'); + return sb.toString(); + } + + public InternalVariantOperationIndex getAnnotationIndex() { + return annotationIndex; + } + + public ProjectInternalVariant setAnnotationIndex(InternalVariantOperationIndex annotationIndex) { + this.annotationIndex = annotationIndex; + return this; + } + + public InternalVariantOperationIndex getSecondaryAnnotationIndex() { + return secondaryAnnotationIndex; + } + + public ProjectInternalVariant setSecondaryAnnotationIndex(InternalVariantOperationIndex secondaryAnnotationIndex) { + this.secondaryAnnotationIndex = secondaryAnnotationIndex; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternal.java index c3c840f383e..b24de377a6a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternal.java @@ -39,19 +39,23 @@ public class StudyInternal extends Internal { description = FieldConstants.STUDY_INTERNAL_CONFIGURATION) private StudyConfiguration configuration; + @DataField(id = "variant", description = FieldConstants.STUDY_INTERNAL_VARIANT) + private StudyInternalVariant variant; + public StudyInternal() { } public StudyInternal(InternalStatus status, String registrationDate, String modificationDate, StudyIndex index, - StudyConfiguration configuration) { + StudyConfiguration configuration, StudyInternalVariant variant) { super(status, registrationDate, modificationDate); this.index = index; this.configuration = configuration; + this.variant = variant; } public static StudyInternal init(String cellbaseVersion) { return new StudyInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), StudyIndex.init(), - StudyConfiguration.init(cellbaseVersion)); + StudyConfiguration.init(cellbaseVersion), new StudyInternalVariant()); } @Override @@ -62,6 +66,7 @@ public String toString() { sb.append(", status=").append(status); sb.append(", index=").append(index); sb.append(", configuration=").append(configuration); + sb.append(", variant=").append(variant); sb.append('}'); return sb.toString(); } @@ -101,4 +106,13 @@ public StudyInternal setConfiguration(StudyConfiguration configuration) { this.configuration = configuration; return this; } + + public StudyInternalVariant getVariant() { + return variant; + } + + public StudyInternal setVariant(StudyInternalVariant variant) { + this.variant = variant; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternalVariant.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternalVariant.java new file mode 100644 index 00000000000..addf93484ae --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyInternalVariant.java @@ -0,0 +1,37 @@ +package org.opencb.opencga.core.models.study; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; + +public class StudyInternalVariant { + + @DataField(id = "secondarySampleIndex", uncommentedClasses = {"InternalVariantOperationIndex"}, + description = FieldConstants.INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX) + private InternalVariantOperationIndex secondarySampleIndex; + + public StudyInternalVariant() { + this(new InternalVariantOperationIndex()); + } + + public StudyInternalVariant(InternalVariantOperationIndex secondarySampleIndex) { + this.secondarySampleIndex = secondarySampleIndex; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("StudyInternalVariant{"); + sb.append("secondarySampleIndex=").append(secondarySampleIndex); + sb.append('}'); + return sb.toString(); + } + + public InternalVariantOperationIndex getSecondarySampleIndex() { + return secondarySampleIndex; + } + + public StudyInternalVariant setSecondarySampleIndex(InternalVariantOperationIndex secondarySampleIndex) { + this.secondarySampleIndex = secondarySampleIndex; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/InternalVariantOperationIndex.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/InternalVariantOperationIndex.java new file mode 100644 index 00000000000..3246e689c01 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/InternalVariantOperationIndex.java @@ -0,0 +1,31 @@ +package org.opencb.opencga.core.models.variant; + +public class InternalVariantOperationIndex { + + private OperationIndexStatus status; + + public InternalVariantOperationIndex() { + this(new OperationIndexStatus()); + } + + public InternalVariantOperationIndex(OperationIndexStatus status) { + this.status = status; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ProjectInternalAnnotationIndex{"); + sb.append(", status=").append(status); + sb.append('}'); + return sb.toString(); + } + + public OperationIndexStatus getStatus() { + return status; + } + + public InternalVariantOperationIndex setStatus(OperationIndexStatus status) { + this.status = status; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/OperationIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/OperationIndexStatus.java new file mode 100644 index 00000000000..edb71cbd5ce --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/OperationIndexStatus.java @@ -0,0 +1,51 @@ +package org.opencb.opencga.core.models.variant; + + +import org.opencb.opencga.core.models.common.IndexStatus; + +import java.util.Arrays; +import java.util.List; + +public class OperationIndexStatus extends IndexStatus { + + public static final String PENDING = "PENDING"; + + public static final List STATUS_LIST = Arrays.asList(READY, PENDING, NONE); + + public OperationIndexStatus(String status, String message) { + if (isValid(status)) { + init(status, message); + } else { + throw new IllegalArgumentException("Unknown status " + status); + } + } + + public OperationIndexStatus(String status) { + this(status, ""); + } + + public OperationIndexStatus() { + this(NONE, ""); + } + + @Override + public String getId() { + return super.getId(); + } + + @Override + public String getName() { + return super.getName(); + } + + public static OperationIndexStatus init() { + return new OperationIndexStatus(); + } + + public static boolean isValid(String status) { + return status != null + && (status.equals(READY) + || status.equals(PENDING) + || status.equals(NONE)); + } +} diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index f18e25a580e..30467378eed 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -99,6 +99,19 @@ analysis: HG19: "exomiser/2402_hg19.zip" HG38: "exomiser/2402_hg38.zip" PHENOTYPE: "exomiser/2402_phenotype.zip" + operations: + variantAnnotationIndex: + policy: "NIGHTLY" + maxAttempts: 3 + jobParams: + variantSecondaryAnnotationIndex: + policy: "NIGHTLY" + maxAttempts: 3 + jobParams: + variantSecondarySampleIndex: + policy: "NIGHTLY" + maxAttempts: 3 + jobParams: execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java index 86b801b0390..21ee7fd17b0 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java @@ -30,11 +30,15 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.master.monitor.daemons.ExecutionDaemon; +import org.opencb.opencga.master.monitor.daemons.MonitorParentDaemon; +import org.opencb.opencga.master.monitor.daemons.VariantOperationJanitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import static org.opencb.opencga.catalog.managers.AbstractManager.OPENCGA; @@ -51,14 +55,8 @@ public class MonitorService { private static Server server; private int port; - private ExecutionDaemon executionDaemon; -// private FileDaemon fileDaemon; -// private AuthorizationDaemon authorizationDaemon; - - private Thread executionThread; -// private Thread indexThread; -// private Thread fileThread; -// private Thread authorizationThread; + private List daemons = new ArrayList<>(); + private List threads = new ArrayList<>(); private volatile boolean exit; @@ -98,7 +96,7 @@ private void init(String token) throws CatalogException { String nonExpiringToken = this.catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, OPENCGA, Collections.emptyMap(), token); - executionDaemon = new ExecutionDaemon( + ExecutionDaemon executionDaemon = new ExecutionDaemon( configuration.getMonitor().getExecutionDaemonInterval(), nonExpiringToken, catalogManager, @@ -108,7 +106,15 @@ private void init(String token) throws CatalogException { // fileDaemon = new FileDaemon(configuration.getMonitor().getFileDaemonInterval(), // configuration.getMonitor().getDaysToRemove(), nonExpiringToken, catalogManager); - executionThread = new Thread(executionDaemon, "execution-thread"); + VariantOperationJanitor variantJanitor = new VariantOperationJanitor( + catalogManager, + nonExpiringToken); + + daemons.add(executionDaemon); + daemons.add(variantJanitor); + + threads.add(new Thread(executionDaemon, "execution-thread")); + threads.add(new Thread(variantJanitor, "janitor-thread")); // fileThread = new Thread(fileDaemon, "file-thread"); // authorizationThread = new Thread(authorizationDaemon, "authorization-thread"); @@ -118,10 +124,9 @@ private void init(String token) throws CatalogException { public void start() throws Exception { // Launching the two daemons in two different threads - executionThread.start(); -// indexThread.start(); -// authorizationThread.start(); -// fileThread.start(); + for (Thread thread : threads) { + thread.start(); + } // Preparing the REST server configuration ResourceConfig resourceConfig = new ResourceConfig(); @@ -174,10 +179,9 @@ public void start() throws Exception { } public void stop() throws Exception { - executionDaemon.setExit(true); -// fileDaemon.setExit(true); -// executionDaemon.setExit(true); -// authorizationDaemon.setExit(true); + for (MonitorParentDaemon daemon : daemons) { + daemon.setExit(true); + } // By setting exit to true the monitor thread will close the Jetty server exit = true; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index c2d5b9c1d51..9c351d59322 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -104,6 +104,8 @@ import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.core.tools.result.Status; +import org.opencb.opencga.master.monitor.executors.BatchExecutor; +import org.opencb.opencga.master.monitor.executors.ExecutorFactory; import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; import org.opencb.opencga.master.monitor.schedulers.JobScheduler; @@ -147,6 +149,7 @@ public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { private final FileManager fileManager; private final Map jobsCountByType = new HashMap<>(); private final Map retainedLogsTime = new HashMap<>(); + protected BatchExecutor batchExecutor; private List packages; @@ -251,6 +254,9 @@ public ExecutionDaemon(int interval, String token, CatalogManager catalogManager String appHome, List packages) throws CatalogDBException { super(interval, token, catalogManager); + ExecutorFactory executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); + this.batchExecutor = executorFactory.getExecutor(); + this.jobManager = catalogManager.getJobManager(); this.fileManager = catalogManager.getFileManager(); this.storageConfiguration = storageConfiguration; @@ -276,15 +282,19 @@ public ExecutionDaemon(int interval, String token, CatalogManager catalogManager logger.info("Packages where to find tools/analyses: " + StringUtils.join(this.packages, ", ")); } + public MonitorParentDaemon setBatchExecutor(BatchExecutor batchExecutor) { + this.batchExecutor = batchExecutor; + return this; + } + @Override - public void apply() throws Exception { + protected void apply() throws Exception { checkJobs(); } @Override public void close() throws IOException { batchExecutor.close(); - try { logger.info("Attempt to shutdown webhook executor"); executor.shutdown(); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 393cb2afdd6..b9b1f955c77 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -17,8 +17,6 @@ package org.opencb.opencga.master.monitor.daemons; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.master.monitor.executors.BatchExecutor; -import org.opencb.opencga.master.monitor.executors.ExecutorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,11 +28,10 @@ */ public abstract class MonitorParentDaemon implements Runnable, Closeable { - protected final int interval; - protected final CatalogManager catalogManager; - protected BatchExecutor batchExecutor; + protected int interval; + protected CatalogManager catalogManager; - private volatile boolean exit = false; + protected volatile boolean exit = false; protected final String token; protected final Logger logger; @@ -44,8 +41,6 @@ public MonitorParentDaemon(int interval, String token, CatalogManager catalogMan this.catalogManager = catalogManager; this.token = token; logger = LoggerFactory.getLogger(this.getClass()); - ExecutorFactory executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); - this.batchExecutor = executorFactory.getExecutor(); } public boolean isExit() { @@ -56,6 +51,8 @@ public void setExit(boolean exit) { this.exit = exit; } + protected abstract void apply() throws Exception; + public void run() { try { init(); @@ -65,6 +62,12 @@ public void run() { } while (!exit) { + try { + apply(); + } catch (Exception e) { + logger.error("Catch exception " + e.getMessage(), e); + } + try { Thread.sleep(interval); } catch (InterruptedException e) { @@ -74,12 +77,6 @@ public void run() { // If interrupted, stop the daemon break; } - - try { - apply(); - } catch (Exception e) { - logger.error("Catch exception " + e.getMessage(), e); - } } try { @@ -93,5 +90,7 @@ public void init() throws Exception { } - public abstract void apply() throws Exception; + @Override + public void close() throws IOException { + } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitor.java new file mode 100644 index 00000000000..d32aa4b3f56 --- /dev/null +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitor.java @@ -0,0 +1,448 @@ +package org.opencb.opencga.master.monitor.daemons; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.tools.ToolFactory; +import org.opencb.opencga.analysis.variant.operations.VariantAnnotationIndexOperationTool; +import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; +import org.opencb.opencga.analysis.variant.operations.VariantSecondaryAnnotationIndexOperationTool; +import org.opencb.opencga.analysis.variant.operations.VariantSecondarySampleIndexOperationTool; +import org.opencb.opencga.catalog.db.api.JobDBAdaptor; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.config.OperationConfig; +import org.opencb.opencga.core.config.OperationExecutionConfig; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.variant.OperationIndexStatus; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.tools.annotations.Tool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.*; +import java.util.stream.Collectors; + +public class VariantOperationJanitor extends MonitorParentDaemon { + + public static final String TAG = "VariantOperationJanitor"; + private final CatalogManager catalogManager; + private final OperationConfig operationConfig; + private final String token; + private final List chores; + + static final String ATTEMPT = "attempt"; + static final String FAILED_ATTEMPT_JOB_IDS = "failedAttemptJobIds"; + + protected static Logger logger = LoggerFactory.getLogger(VariantOperationJanitor.class); + + /** + * Initialize VariantOperationJanitor with the catalog manager, configuration and token. + * + * @param catalogManager Instance of a working CatalogManager. + * @param token Valid administrator token. + */ + public VariantOperationJanitor(CatalogManager catalogManager, String token) { + super(30000, token, catalogManager); + this.catalogManager = catalogManager; + if (catalogManager.getConfiguration().getAnalysis().getOperations() == null) { + this.operationConfig = new OperationConfig(); + } else { + this.operationConfig = catalogManager.getConfiguration().getAnalysis().getOperations(); + } + this.token = token; + + chores = Arrays.asList( + new VariantAnnotationIndexOperationChore(operationConfig), + new VariantSecondaryAnnotationIndexOperationChore(operationConfig), + new VariantSecondarySampleIndexOperationChore(operationConfig)); + } + + @Override + public void apply() { + try { + checkPendingVariantOperations(); + } catch (Exception e) { + logger.error("Error checking pending variant operations", e); + } + } + + public void checkPendingVariantOperations() throws CatalogException, ToolException { + for (OperationChore chore : chores) { + try { + checkPendingVariantOperations(chore); + } catch (Exception e) { + logger.error("Error checking variant operation chore '{}'", chore.getToolId(), e); + } + } + } + + private void checkPendingVariantOperations(OperationChore operationChore) + throws CatalogException, ToolException { + OperationExecutionConfig config = operationChore.getConfig(); + String toolId = operationChore.getToolId(); + if (config.getPolicy() == OperationExecutionConfig.Policy.NEVER) { + logger.info("Automatic operation chore '{}' is disabled. Nothing to do.", toolId); + return; + } + + if (!isNightTime() && config.getPolicy() == OperationExecutionConfig.Policy.NIGHTLY) { + logger.info("Waiting until night time to check for pending operation '{}'", toolId); + return; + } + + if (!isWeekend() && config.getPolicy() == OperationExecutionConfig.Policy.WEEKLY) { + logger.info("Waiting until weekend to check for pending operation '{}'", toolId); + return; + } + + List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, OrganizationDBAdaptor.QueryParams.PROJECTS.key()); + for (String organizationId : organizationIds) { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, options, token).first(); + for (Project project : organization.getProjects()) { + if (CollectionUtils.isEmpty(project.getStudies())) { + // Empty project. Skip + continue; + } + + Tool tool = new ToolFactory().getTool(toolId); + switch (tool.scope()) { + case PROJECT: + try { + checkPendingChore(operationChore, project); + } catch (Exception e) { + logger.error("Error checking pending chore '{}' in project '{}'. Ignore exception.", + toolId, project.getFqn(), e); + } + break; + case STUDY: + for (Study study : project.getStudies()) { + try { + checkPendingChore(operationChore, project, study); + } catch (Exception e) { + logger.error("Error checking pending chore '{}' in study '{}'. Ignore exception.", + toolId, study.getFqn(), e); + } + } + break; + default: + throw new IllegalArgumentException("Unexpected tools with scope " + tool.scope() + " for tool id '" + toolId + "'"); + } + } + } + } + + private void checkPendingChore(OperationChore operationChore, Project project) throws CatalogException { + OperationExecutionConfig config = operationChore.getConfig(); + String toolId = operationChore.getToolId(); + List studyFqns = project.getStudies().stream().map(Study::getFqn).collect(Collectors.toList()); + // 1. Check if operation is pending + operationChore.isOperationRequired(project, null); + + // 2. Check if the operation is already created on any study of the project + if (pendingJobs(studyFqns, toolId)) { + logger.info("There's already a pending job for tool '{}' in project '{}'. Skipping.", toolId, project.getFqn()); + return; + } + + // 3. Check general rules + if (noPendingJobs(studyFqns, operationChore.dependantTools())) { + // Get last execution of this job + Job lastJobExecution = findLastJobExecution(studyFqns.get(0), toolId); + ObjectMap attributes = getNewAttributes(lastJobExecution); + if (attributes.getInt(ATTEMPT) > config.getMaxAttempts()) { + logger.info("Max attempts reached for tool '{}' in project '{}'. Skipping.", toolId, project.getFqn()); + return; + } + + Map paramsMap; + if (config.getJobParams() == null) { + paramsMap = new HashMap<>(config.getJobParams()); + } else { + paramsMap = new HashMap<>(); + } + paramsMap.put(ParamConstants.PROJECT_PARAM, project.getFqn()); + catalogManager.getJobManager().submit(studyFqns.get(0), toolId, Enums.Priority.HIGH, paramsMap, null, + generateJobDescription(config, operationChore, attributes), null, + Collections.singletonList(TAG), null, null, null, attributes, token); + } + } + + private void checkPendingChore(OperationChore operationChore, Project project, Study study) throws CatalogException { + OperationExecutionConfig config = operationChore.getConfig(); + String toolId = operationChore.getToolId(); + // 1. Check if operation is pending + operationChore.isOperationRequired(project, study); + + // 2. Check if the operation is already created + if (pendingJobs(study.getFqn(), toolId)) { + // Pending jobs of its own type. Skip study + logger.info("There's already a pending job for tool '{}' in study '{}'. Skipping.", toolId, study.getFqn()); + return; + } + + // 3. Check general rules + if (noPendingJobs(study.getFqn(), operationChore.dependantTools())) { + // Get last execution of this job + Job lastJobExecution = findLastJobExecution(study.getFqn(), toolId); + ObjectMap attributes = getNewAttributes(lastJobExecution); + if (attributes.getInt(ATTEMPT) > config.getMaxAttempts()) { + logger.info("Max attempts reached for tool '{}' in study '{}'. Skipping.", toolId, study.getFqn()); + return; + } + + Map paramsMap; + if (config.getJobParams() == null) { + paramsMap = new HashMap<>(config.getJobParams()); + } else { + paramsMap = new HashMap<>(); + } + paramsMap.put(ParamConstants.STUDY_PARAM, study.getFqn()); + catalogManager.getJobManager().submit(study.getFqn(), toolId, Enums.Priority.HIGH, paramsMap, null, + generateJobDescription(config, operationChore, attributes), null, + Collections.singletonList(TAG), null, null, null, attributes, token); + } + } + + private String generateJobDescription(OperationExecutionConfig config, OperationChore operationChore, ObjectMap attributes) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("Job automatically launched by the variant operation janitor. "); + stringBuilder.append("Tool: ").append(operationChore.getToolId()).append("; "); + stringBuilder.append("Policy: ").append(config.getPolicy()).append("; "); + stringBuilder.append("Attempt number: ").append(attributes.getInt(ATTEMPT)).append(" out of ").append(config.getMaxAttempts()) + .append("; "); + String jobIds = StringUtils.join(attributes.getAsStringList(FAILED_ATTEMPT_JOB_IDS), ", "); + stringBuilder.append("Job ids from previous attempts: ").append(jobIds).append("."); + return stringBuilder.toString(); + } + + private interface OperationChore { + + String getToolId(); + + boolean isOperationRequired(Project project, Study study); + + List dependantTools(); + + OperationExecutionConfig getConfig(); + + } + + private static class VariantSecondarySampleIndexOperationChore implements OperationChore { + + private final OperationExecutionConfig operationConfig; + + VariantSecondarySampleIndexOperationChore(OperationConfig operationConfig) { + this.operationConfig = operationConfig.getVariantSecondarySampleIndex(); + } + + @Override + public String getToolId() { + return VariantSecondarySampleIndexOperationTool.ID; + } + + @Override + public boolean isOperationRequired(Project project, Study study) { + if (study.getInternal() + .getVariant() + .getSecondarySampleIndex() + .getStatus().getId().equals(OperationIndexStatus.PENDING)) { + return true; + } +// try (DBIterator iterator = catalogManager.getFamilyManager().iterator(study.getFqn(), new Query(), +// new QueryOptions(QueryOptions.INCLUDE, FamilyDBAdaptor.QueryParams.MEMBERS), token)) { +// while (iterator.hasNext()) { +// Family family = iterator.next(); +// List trios = variantStorageManager.getTriosFromFamily(study.getFqn(), family, true, token); +// for (Trio trio : trios) { +// String childSample = trio.getChild(); +// Sample sample = catalogManager.getSampleManager().get(study.getFqn(), childSample, +// new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT), token).first(); +// if (!sample.getInternal().getVariant().getSecondarySampleIndex().getFamilyStatus().getId() +// .equals(IndexStatus.READY)) { +// return true; +// } +// } +// } +// } catch (Exception e) { +// logger.error("Error checking if secondary sample index is required", e); +// } + + return false; + } + + @Override + public List dependantTools() { + return Collections.unmodifiableList(Arrays.asList(VariantIndexOperationTool.ID, VariantAnnotationIndexOperationTool.ID, + VariantSecondaryAnnotationIndexOperationTool.ID)); + } + + @Override + public OperationExecutionConfig getConfig() { + return operationConfig; + } + } + + private static class VariantSecondaryAnnotationIndexOperationChore implements OperationChore { + + private final OperationExecutionConfig operationConfig; + + VariantSecondaryAnnotationIndexOperationChore(OperationConfig operationConfig) { + this.operationConfig = operationConfig.getVariantSecondaryAnnotationIndex(); + } + + @Override + public String getToolId() { + return VariantSecondaryAnnotationIndexOperationTool.ID; + } + + @Override + public boolean isOperationRequired(Project project, Study study) { + return project.getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId() + .equals(OperationIndexStatus.PENDING); + } + + @Override + public List dependantTools() { + return Collections.unmodifiableList(Arrays.asList(VariantIndexOperationTool.ID, VariantAnnotationIndexOperationTool.ID)); + } + + @Override + public OperationExecutionConfig getConfig() { + return operationConfig; + } + } + + private static class VariantAnnotationIndexOperationChore implements OperationChore { + + private final OperationExecutionConfig operationConfig; + + VariantAnnotationIndexOperationChore(OperationConfig operationConfig) { + this.operationConfig = operationConfig.getVariantAnnotationIndex(); + } + + @Override + public String getToolId() { + return VariantAnnotationIndexOperationTool.ID; + } + + @Override + public boolean isOperationRequired(Project project, Study study) { + return project.getInternal().getVariant().getAnnotationIndex().getStatus().getId().equals(OperationIndexStatus.PENDING); + } + + @Override + public List dependantTools() { + return Collections.unmodifiableList(Collections.singletonList(VariantIndexOperationTool.ID)); + } + + @Override + public OperationExecutionConfig getConfig() { + return operationConfig; + } + } + + private static boolean isNightTime() { + boolean isNightTime = false; + // Check date time is between 00:00 and 05:00 + //TODO: Define night time in configuration + LocalTime now = LocalTime.now(); + int hour = now.getHour(); + if (hour < 5) { + isNightTime = true; + } + return isNightTime; + } + + private static boolean isWeekend() { + LocalDate today = LocalDate.now(); + DayOfWeek dayOfWeek = today.getDayOfWeek(); + return dayOfWeek == DayOfWeek.SATURDAY || dayOfWeek == DayOfWeek.SUNDAY; + } + + private boolean pendingJobs(String studyIds, String toolId) throws CatalogException { + return !noPendingJobs(Collections.singletonList(studyIds), Collections.singletonList(toolId)); + } + + private boolean pendingJobs(List studyIds, String toolId) throws CatalogException { + return !noPendingJobs(studyIds, Collections.singletonList(toolId)); + } + + private boolean noPendingJobs(List studyIds, String toolId) throws CatalogException { + return noPendingJobs(studyIds, Collections.singletonList(toolId)); + } + + private boolean noPendingJobs(String studyId, List toolIds) throws CatalogException { + return noPendingJobs(Collections.singletonList(studyId), toolIds); + } + + /** + * Check if there is any pending (or queueud or running) job in any of the listed studies, for any of the listed tools. + * @param studyIds List of studies where to check + * @param toolIds List of tools to check + * @return If there is any pending job + * @throws CatalogException on error + */ + private boolean noPendingJobs(List studyIds, List toolIds) throws CatalogException { + for (String studyId : studyIds) { + Query query = new Query() + .append(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Arrays.asList(Enums.ExecutionStatus.PENDING, + Enums.ExecutionStatus.QUEUED, Enums.ExecutionStatus.RUNNING)) + .append(JobDBAdaptor.QueryParams.TOOL_ID.key(), toolIds); + + long numMatches = catalogManager.getJobManager().count(studyId, query, token).getNumMatches(); + if (numMatches != 0) { + return false; + } + } + return true; + } + + Job findLastJobExecution(String studyId, String toolId) throws CatalogException { + Query query = new Query() + .append(JobDBAdaptor.QueryParams.TOOL_ID.key(), toolId); + QueryOptions queryOptions = new QueryOptions() + .append(QueryOptions.SORT, JobDBAdaptor.QueryParams.CREATION_DATE.key()) + .append(QueryOptions.ORDER, QueryOptions.DESCENDING) + .append(QueryOptions.LIMIT, 1); + + OpenCGAResult search = catalogManager.getJobManager().search(studyId, query, queryOptions, token); + return search.getNumResults() > 0 ? search.first() : null; + } + + private ObjectMap getNewAttributes(Job job) { + ObjectMap attributes = new ObjectMap(); + // If job was launched by the Janitor and failed, extract number of attempts + if (job != null && (Enums.ExecutionStatus.ERROR.equals(job.getInternal().getStatus().getId()) + || Enums.ExecutionStatus.ABORTED.equals(job.getInternal().getStatus().getId())) + && job.getTags().contains(TAG)) { + ObjectMap jobAttributes = new ObjectMap(job.getAttributes()); + int jobAttempt = jobAttributes.getInt(ATTEMPT); + + List jobIds = new ArrayList<>(jobAttributes.getAsStringList(FAILED_ATTEMPT_JOB_IDS)); + // Add last job id + jobIds.add(job.getId()); + + attributes.put(ATTEMPT, jobAttempt + 1); + attributes.put(FAILED_ATTEMPT_JOB_IDS, jobIds); + } else { + attributes.put(ATTEMPT, 1); + attributes.put(FAILED_ATTEMPT_JOB_IDS, Collections.emptyList()); + } + return attributes; + } + +} diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitorTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitorTest.java new file mode 100644 index 00000000000..628bc84ec5e --- /dev/null +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/VariantOperationJanitorTest.java @@ -0,0 +1,136 @@ +package org.opencb.opencga.master.monitor.daemons; + +import org.junit.Before; +import org.junit.Test; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.analysis.variant.operations.VariantAnnotationIndexOperationTool; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; +import org.opencb.opencga.core.config.OperationExecutionConfig; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.job.JobInternal; +import org.opencb.opencga.core.models.project.ProjectInternalVariant; +import org.opencb.opencga.core.models.study.StudyInternalVariant; +import org.opencb.opencga.core.models.variant.InternalVariantOperationIndex; +import org.opencb.opencga.core.models.variant.OperationIndexStatus; +import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; + +import java.util.List; + +import static org.junit.Assert.*; + +public class VariantOperationJanitorTest extends AbstractManagerTest { + + private VariantOperationJanitor janitor; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + String expiringToken = this.catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + + // Change to immediate + catalogManager.getConfiguration().getAnalysis().getOperations().getVariantAnnotationIndex().setPolicy(OperationExecutionConfig.Policy.IMMEDIATE).setMaxAttempts(2); + + janitor = new VariantOperationJanitor(catalogManager, expiringToken); + + // Set all operations to PENDING + catalogManager.getProjectManager().setProjectInternalVariant(projectFqn1, new ProjectInternalVariant( + new InternalVariantOperationIndex(new OperationIndexStatus(OperationIndexStatus.PENDING)), + new InternalVariantOperationIndex(new OperationIndexStatus(OperationIndexStatus.PENDING))), + QueryOptions.empty(), expiringToken); + + catalogManager.getStudyManager().setStudyInternalVariant(studyFqn, new StudyInternalVariant( + new InternalVariantOperationIndex(new OperationIndexStatus(OperationIndexStatus.PENDING))), + QueryOptions.empty(), expiringToken); + } + + @Test + public void checkPendingVariantOperations() throws Exception { + janitor.checkPendingVariantOperations(); + Job lastJobExecution = janitor.findLastJobExecution(studyFqn, VariantAnnotationIndexOperationTool.ID); + assertNotNull(lastJobExecution); + assertEquals(Enums.ExecutionStatus.PENDING, lastJobExecution.getInternal().getStatus().getId()); + assertTrue(lastJobExecution.getTags().contains(VariantOperationJanitor.TAG)); + assertEquals(1, lastJobExecution.getAttributes().get(VariantOperationJanitor.ATTEMPT)); + String jobId = lastJobExecution.getId(); + // Set status to ERROR + PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams().setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR))); + catalogManager.getJobManager().update(studyFqn, lastJobExecution.getId(), updateParams, QueryOptions.empty(), ownerToken); + + + janitor.checkPendingVariantOperations(); + lastJobExecution = janitor.findLastJobExecution(studyFqn, VariantAnnotationIndexOperationTool.ID); + assertNotNull(lastJobExecution); + assertEquals(Enums.ExecutionStatus.PENDING, lastJobExecution.getInternal().getStatus().getId()); + assertTrue(lastJobExecution.getTags().contains(VariantOperationJanitor.TAG)); + assertEquals(2, lastJobExecution.getAttributes().get(VariantOperationJanitor.ATTEMPT)); + assertEquals(1, ((List) lastJobExecution.getAttributes().get(VariantOperationJanitor.FAILED_ATTEMPT_JOB_IDS)).size()); + assertEquals(jobId, ((List) lastJobExecution.getAttributes().get(VariantOperationJanitor.FAILED_ATTEMPT_JOB_IDS)).get(0)); + // Set status to ERROR + catalogManager.getJobManager().update(studyFqn, lastJobExecution.getId(), updateParams, QueryOptions.empty(), ownerToken); + + // Check a third time (This time it shouldn't create a new job) + janitor.checkPendingVariantOperations(); + lastJobExecution = janitor.findLastJobExecution(studyFqn, VariantAnnotationIndexOperationTool.ID); + assertNotNull(lastJobExecution); + assertEquals(Enums.ExecutionStatus.ERROR, lastJobExecution.getInternal().getStatus().getId()); + assertEquals(2, lastJobExecution.getAttributes().get(VariantOperationJanitor.ATTEMPT)); + assertEquals(1, ((List) lastJobExecution.getAttributes().get(VariantOperationJanitor.FAILED_ATTEMPT_JOB_IDS)).size()); + assertEquals(jobId, ((List) lastJobExecution.getAttributes().get(VariantOperationJanitor.FAILED_ATTEMPT_JOB_IDS)).get(0)); + assertTrue(lastJobExecution.getTags().contains(VariantOperationJanitor.TAG)); + + } + +// @Mock +// private CatalogManager catalogManager; +// +// @Mock +// private Configuration configuration; +// +// private VariantOperationOrchestrator variantOperationOrchestrator; +// +// @BeforeEach +// public void setUp() { +// MockitoAnnotations.initMocks(this); +// variantOperationOrchestrator = new VariantOperationOrchestrator(catalogManager, configuration, "token"); +// } +// +// @Test +// public void checkPendingVariantOperationsNeverPolicy() throws Exception { +// when(configuration.getAnalysis().getOperations().getVariantAnnotationIndex().getPolicy()) +// .thenReturn(OperationExecutionConfig.Policy.NEVER); +// +// variantOperationOrchestrator.checkPendingVariantOperations(); +// +// verify(catalogManager, never()).getOrganizationManager(); +// } +// +// @Test +// public void checkPendingVariantOperationsNightlyPolicyNotNightTime() throws Exception { +// when(configuration.getAnalysis().getOperations().getVariantAnnotationIndex().getPolicy()) +// .thenReturn(OperationExecutionConfig.Policy.NIGHTLY); +// +// variantOperationOrchestrator.checkPendingVariantOperations(); +// +// verify(catalogManager, never()).getOrganizationManager(); +// } +// +// @Test +// public void noPendingJobsReturnsTrueWhenNoJobs() throws Exception { +// when(catalogManager.getJobManager().count(anyString(), any(), anyString())) +// .thenReturn(new OpenCGAResult<>(0, Collections.emptyList(), 0, Collections.emptyList(), 0)); +// +// assertTrue(variantOperationOrchestrator.noPendingJobs(Collections.singletonList("studyId"), Collections.singletonList("toolId"))); +// } +// +// @Test +// public void noPendingJobsReturnsFalseWhenJobsExist() throws Exception { +// when(catalogManager.getJobManager().count(anyString(), any(), anyString())) +// .thenReturn(new OpenCGAResult<>(1, Collections.emptyList(), 1, Collections.emptyList(), 1)); +// +// assertFalse(variantOperationOrchestrator.noPendingJobs(Collections.singletonList("studyId"), Collections.singletonList("toolId"))); +// } +} \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index dfd0c0d2b5b..9ed5f4cde7c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -84,9 +84,7 @@ public Response retryJob( @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTagsStr, @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String jobScheduledStartTime, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, - @ApiParam(value = "job", required = true) JobRetryParams params - - ) { + @ApiParam(value = "job", required = true) JobRetryParams params) { try { List jobDependsOn; if (StringUtils.isNotEmpty(dependsOn)) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index c3457d5489b..286be46d386 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -575,6 +575,95 @@ public ProjectMetadata updateProjectMetadata(UpdateFunctio } } + /** + * Update the timestamp of the last variant index operation. + * @throws StorageEngineException if there is a problem updating the metadata + */ + public void updateVariantIndexTimestamp() throws StorageEngineException { + updateProjectMetadata(pm -> { + if (pm == null) { + pm = new ProjectMetadata(); + } + pm.setVariantIndexLastTimestamp(); + // As new variants have been added, the annotation and secondary-annotation are outdated. + pm.setAnnotationIndexStatus(TaskMetadata.Status.NONE); + pm.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); + return pm; + }); + } + + /** + * Update the timestamp of the last variant index operation. + * @throws StorageEngineException if there is a problem updating the metadata + */ + public void invalidateCurrentVariantAnnotationIndex() throws StorageEngineException { + updateProjectMetadata(pm -> { + pm.setAnnotationIndexLastTimestamp(0); + pm.setSecondaryAnnotationIndexLastTimestamp(0); + + pm.setAnnotationIndexStatus(TaskMetadata.Status.NONE); + pm.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); + return pm; + }); + } + + /** + * Update the timestamp with the last time that a variant annotation index was executed successfully. + * This must register the starting time of the operation, not the end time. + * This includes partial annotations. + * @param annotateAll If true, the annotation is being executed over all (pending) variants in the database. + * @param annotationStartTimestamp Starting time of the annotation index (not the end time) + * @throws StorageEngineException if there is a problem updating the metadata + */ + public void updateAnnotationIndexTimestamp(boolean annotateAll, long annotationStartTimestamp) throws StorageEngineException { + updateProjectMetadata(projectMetadata -> { + // Update annotation timestamp. + // This includes full and partial annotations + projectMetadata.setAnnotationIndexLastTimestamp(annotationStartTimestamp); + // Any time a new annotation is loaded, the secondary annotation index status is reset. + projectMetadata.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); + if (annotateAll) { + // The annotation would be marked as "READY" if all the variants from the database are annotated. + // This requires that no other variants where loaded while annotating, + // and for this process to be annotating all variants. + if (projectMetadata.getAnnotationIndexLastTimestamp() > projectMetadata.getVariantIndexLastTimestamp()) { + projectMetadata.setAnnotationIndexStatus(TaskMetadata.Status.READY); + } else { + projectMetadata.setAnnotationIndexStatus(TaskMetadata.Status.NONE); + } + } // else : partial annotations doesn't change the annotation status + }); + } + + /** + * Update the timestamp with the last time that a full secondary variant annotation was executed successfully. + * @param startTimestamp Starting time of the secondary annotation index (not the end time) + * @throws StorageEngineException if there is a problem updating the metadata + */ + public void updateSecondaryAnnotationIndexTimestamp(long startTimestamp) throws StorageEngineException { + updateProjectMetadata(projectMetadata -> { + projectMetadata.setSecondaryAnnotationIndexLastTimestamp(startTimestamp); + if (projectMetadata.getAnnotationIndexLastTimestamp() < startTimestamp + && projectMetadata.getStatsLastTimestamp() < startTimestamp) { + projectMetadata.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.READY); + } else { + projectMetadata.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); + } + }); + } + + /** + * Update the timestamp with the last time that a variant stats index was executed successfully. + * @throws StorageEngineException if there is a problem updating the metadata + */ + public void updateStatsIndexTimestamp() throws StorageEngineException { + updateProjectMetadata(project -> { + project.setStatsIndexLastTimestamp(System.currentTimeMillis()); + // As new variant stats have been added, the secondary-annotation-index is outdated. + project.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); + }); + } + public boolean exists() { return projectDBAdaptor.exists(); } @@ -799,6 +888,13 @@ public LinkedHashSet getIndexedFiles(int studyId, boolean includePartia return fileDBAdaptor.getIndexedFiles(studyId, includePartial); } + /** + * Register a set of files as indexed. This will update the index status of files associated samples. + * The ProjectMetadata "variantIndexTimestamp" will be updated to the current time. + * @param studyId Study id + * @param fileIds List of file ids to register as indexed + * @throws StorageEngineException if there is any error + */ public void addIndexedFiles(int studyId, List fileIds) throws StorageEngineException { // First update the samples Set samples = new HashSet<>(); @@ -820,8 +916,10 @@ public void addIndexedFiles(int studyId, List fileIds) throws StorageEn .getName(); logger.info("Register file " + name + " as INDEXED"); } + updateVariantIndexTimestamp(); fileIdsFromSampleIdCache.clear(); fileIdIndexedCache.clear(); + sampleIdIndexedCache.clear(); } public void removeIndexedFiles(int studyId, Collection fileIds) throws StorageEngineException { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java index e64deedff64..aca64d682f5 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java @@ -1,5 +1,6 @@ package org.opencb.opencga.storage.core.metadata.models; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.opencb.cellbase.core.models.DataRelease; @@ -13,7 +14,15 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public class ProjectMetadata { +public class ProjectMetadata extends ResourceMetadata { + + private static final String FILE_INDEX_LAST_TIMESTAMP = "file.index.last.timestamp"; + @Deprecated + // This value was used by opencga-storage-hadoop internally for the same purpose. It is not used anymore. + private static final String LAST_LOADED_FILE_TS = "lastLoadedFileTs"; + private static final String STATS_INDEX_LAST_TIMESTAMP = "stats.index.last.timestamp"; + private static final String ANNOTATION_INDEX_LAST_TIMESTAMP = "annotation.index.last.timestamp"; + private static final String SEARCH_INDEX_LAST_TIMESTAMP = "search.index.last.timestamp"; private String species; private String assembly; @@ -25,8 +34,6 @@ public class ProjectMetadata { private Map counters; - private ObjectMap attributes; - public static class VariantAnnotationSets { private VariantAnnotationMetadata current; private List saved; @@ -235,7 +242,7 @@ public ProjectMetadata() { dataRelease = ""; annotation = new VariantAnnotationSets(); counters = new HashMap<>(); - attributes = new ObjectMap(); + setAttributes(new ObjectMap()); } public ProjectMetadata(String species, String assembly, int release) { @@ -248,14 +255,15 @@ public ProjectMetadata(String species, String assembly, String dataRelease, int this.assembly = assembly; this.dataRelease = dataRelease; this.release = release; - this.attributes = attributes != null ? attributes : new ObjectMap(); + setAttributes(attributes != null ? attributes : new ObjectMap()); this.annotation = annotation != null ? annotation : new VariantAnnotationSets(); this.counters = counters != null ? counters : new HashMap<>(); } public ProjectMetadata copy() { - return new ProjectMetadata(species, assembly, dataRelease, release, new ObjectMap(attributes), new HashMap<>(counters), - annotation); + return new ProjectMetadata(species, assembly, dataRelease, release, new ObjectMap(getAttributes()), new HashMap<>(counters), + annotation) + .setStatus(new HashMap<>(getStatus())); } public String getSpecies() { @@ -312,13 +320,73 @@ public ProjectMetadata setCounters(Map counters) { return this; } - public ObjectMap getAttributes() { - return attributes; + @JsonIgnore + public TaskMetadata.Status getAnnotationIndexStatus() { + return getStatus("annotation"); + } + + @JsonIgnore + public ProjectMetadata setAnnotationIndexStatus(TaskMetadata.Status annotationStatus) { + return setStatus("annotation", annotationStatus); + } + + @JsonIgnore + public TaskMetadata.Status getSecondaryAnnotationIndexStatus() { + return getStatus("secondaryAnnotationIndex"); + } + + @JsonIgnore + public ProjectMetadata setSecondaryAnnotationIndexStatus(TaskMetadata.Status annotationStatus) { + return setStatus("secondaryAnnotationIndex", annotationStatus); + } + + @JsonIgnore + public ProjectMetadata setVariantIndexLastTimestamp() { + getAttributes().put(FILE_INDEX_LAST_TIMESTAMP, System.currentTimeMillis()); + return this; + } + + @JsonIgnore + public long getVariantIndexLastTimestamp() { + long ts = getAttributes().getLong(FILE_INDEX_LAST_TIMESTAMP, 0); + if (ts == 0) { + // Old versions of the metadata may still use the old field + return getAttributes().getLong(LAST_LOADED_FILE_TS, 0); + } + return ts; } - public ProjectMetadata setAttributes(ObjectMap attributes) { - this.attributes = attributes; + @JsonIgnore + public ProjectMetadata setAnnotationIndexLastTimestamp(long annotationStartTimestamp) { + getAttributes().put(ANNOTATION_INDEX_LAST_TIMESTAMP, annotationStartTimestamp); return this; } + @JsonIgnore + public long getAnnotationIndexLastTimestamp() { + return getAttributes().getLong(ANNOTATION_INDEX_LAST_TIMESTAMP, 0); + } + + @JsonIgnore + public ProjectMetadata setSecondaryAnnotationIndexLastTimestamp(long timestamp) { + getAttributes().put(SEARCH_INDEX_LAST_TIMESTAMP, timestamp); + return this; + } + + @JsonIgnore + public long getSecondaryAnnotationIndexLastTimestamp() { + return getAttributes().getLong(SEARCH_INDEX_LAST_TIMESTAMP, 0); + } + + @JsonIgnore + public ProjectMetadata setStatsIndexLastTimestamp(long timeMillis) { + getAttributes().put(STATS_INDEX_LAST_TIMESTAMP, timeMillis); + return this; + } + + @JsonIgnore + public long getStatsLastTimestamp() { + return getAttributes().getLong(STATS_INDEX_LAST_TIMESTAMP, 0); + } + } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ResourceMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ResourceMetadata.java new file mode 100644 index 00000000000..b862728161e --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ResourceMetadata.java @@ -0,0 +1,99 @@ +package org.opencb.opencga.storage.core.metadata.models; + +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created on 14/01/19. + * + * @author Jacobo Coll <jacobo167@gmail.com> + */ +public abstract class ResourceMetadata> { + + private int id; + private String name; + + private Map status = new HashMap<>(); + + private ObjectMap attributes; + + ResourceMetadata() { + } + + ResourceMetadata(int id, String name) { + this.id = id; + this.name = name; + } + + public int getId() { + return id; + } + + public T setId(int id) { + this.id = id; + return getThis(); + } + + public String getName() { + return name; + } + + public T setName(String name) { + this.name = name; + return getThis(); + } + + public Map getStatus() { + return status; + } + + public TaskMetadata.Status getStatus(String statusName) { + return getStatus(statusName, TaskMetadata.Status.NONE); + } + + public TaskMetadata.Status getStatus(String statusName, TaskMetadata.Status defaultValue) { + return getStatus().getOrDefault(statusName, defaultValue); + } + + public T setStatus(String statusName, TaskMetadata.Status status) { + getStatus().put(statusName, status); + return getThis(); + } + + public boolean isReady(String statusName) { + return TaskMetadata.Status.READY.equals(getStatus(statusName)); + } + + public boolean isError(String statusName) { + return TaskMetadata.Status.ERROR.equals(getStatus(statusName)); + } + + public T setStatus(Map status) { + this.status = status; + return getThis(); + } + + public ObjectMap getAttributes() { + if (attributes == null) { + attributes = new ObjectMap(); + } + return attributes; + } + + public T setAttributes(ObjectMap attributes) { + this.attributes = attributes; + return getThis(); + } + + public T putAttribute(String key, Object value) { + getAttributes().put(key, value); + return getThis(); + } + + @SuppressWarnings("unchecked") + private T getThis() { + return (T) this; + } +} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java index 75a7c58b1d8..18903f38008 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java @@ -1,32 +1,20 @@ package org.opencb.opencga.storage.core.metadata.models; -import org.opencb.commons.datastore.core.ObjectMap; - -import java.util.HashMap; -import java.util.Map; - /** * Created on 14/01/19. * * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class StudyResourceMetadata> { +public abstract class StudyResourceMetadata> extends ResourceMetadata { private int studyId; - private int id; - private String name; - - private Map status = new HashMap<>(); - - private ObjectMap attributes; StudyResourceMetadata() { } StudyResourceMetadata(int studyId, int id, String name) { + super(id, name); this.studyId = studyId; - this.id = id; - this.name = name; } public int getStudyId() { @@ -38,71 +26,6 @@ public T setStudyId(int studyId) { return getThis(); } - public int getId() { - return id; - } - - public T setId(int id) { - this.id = id; - return getThis(); - } - - public String getName() { - return name; - } - - public T setName(String name) { - this.name = name; - return getThis(); - } - - public Map getStatus() { - return status; - } - - public TaskMetadata.Status getStatus(String statusName) { - return getStatus(statusName, TaskMetadata.Status.NONE); - } - - public TaskMetadata.Status getStatus(String statusName, TaskMetadata.Status defaultValue) { - return getStatus().getOrDefault(statusName, defaultValue); - } - - public T setStatus(String statusName, TaskMetadata.Status status) { - getStatus().put(statusName, status); - return getThis(); - } - - public boolean isReady(String statusName) { - return TaskMetadata.Status.READY.equals(getStatus(statusName)); - } - - public boolean isError(String statusName) { - return TaskMetadata.Status.ERROR.equals(getStatus(statusName)); - } - - public T setStatus(Map status) { - this.status = status; - return getThis(); - } - - public ObjectMap getAttributes() { - if (attributes == null) { - attributes = new ObjectMap(); - } - return attributes; - } - - public T setAttributes(ObjectMap attributes) { - this.attributes = attributes; - return getThis(); - } - - public T putAttribute(String key, Object value) { - getAttributes().put(key, value); - return getThis(); - } - @SuppressWarnings("unchecked") private T getThis() { return (T) this; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index d861c5ec95f..0aa0e06bb3e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -741,15 +741,12 @@ public VariantSearchLoadResult secondaryIndex(Query inputQuery, QueryOptions inp } if (partialLoad) { - logger.info("Partial secondary annotation index. Do not update {} timestamp", SEARCH_INDEX_LAST_TIMESTAMP.key()); + logger.info("Partial secondary annotation index. Do not update operation timestamp"); } else { logger.info("Update secondary annotation index status for {} new files and {} new samples", filesToBeUpdated.values().stream().mapToInt(Collection::size).sum(), samplesToBeUpdated.values().stream().mapToInt(Collection::size).sum()); - mm.updateProjectMetadata(projectMetadata -> { - projectMetadata.getAttributes().put(SEARCH_INDEX_LAST_TIMESTAMP.key(), newTimestamp); - return projectMetadata; - }); + mm.updateSecondaryAnnotationIndexTimestamp(newTimestamp); for (Map.Entry> entry : filesToBeUpdated.entrySet()) { Integer study = entry.getKey(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index 73abcbae61f..51ba8703169 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -128,8 +128,7 @@ public enum VariantStorageOptions implements ConfigurationOption { RELEASE("release", 1), - MERGE_MODE("merge.mode", VariantStorageEngine.MergeMode.ADVANCED), - SEARCH_INDEX_LAST_TIMESTAMP("search.index.last.timestamp", 0); + MERGE_MODE("merge.mode", VariantStorageEngine.MergeMode.ADVANCED); private final String key; private final Object value; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java index f8e7f4c4b90..9d0322a14e1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java @@ -102,6 +102,8 @@ public class DefaultVariantAnnotationManager extends VariantAnnotationManager { protected Map> alreadyAnnotatedFiles = new HashMap<>(); private final IOConnectorProvider ioConnectorProvider; private final VariantReaderUtils variantReaderUtils; + private boolean annotateAll; + private long annotationStartTimestamp; public DefaultVariantAnnotationManager(VariantAnnotator variantAnnotator, VariantDBAdaptor dbAdaptor, IOConnectorProvider ioConnectorProvider) { @@ -424,11 +426,11 @@ protected void preAnnotate(Query query, boolean doCreate, boolean doLoad, Object } VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); - boolean annotateAll; Set queryParams = VariantQueryUtils.validParams(query, true); Set filesFilter = Collections.emptySet(); queryParams.removeAll(Arrays.asList(VariantQueryParam.ANNOTATION_EXISTS, VariantQueryParam.STUDY)); + annotationStartTimestamp = System.currentTimeMillis(); if (queryParams.isEmpty()) { // There are no invalid filters. annotateAll = true; @@ -519,11 +521,10 @@ protected void postAnnotate(Query query, boolean doCreate, boolean doLoad, Objec if (doLoad && doCreate) { ProjectMetadata.VariantAnnotationMetadata newAnnotationMetadata = variantAnnotator.getVariantAnnotationMetadata(); - dbAdaptor.getMetadataManager().updateProjectMetadata(projectMetadata -> { updateCurrentAnnotation(variantAnnotator, projectMetadata, overwrite, newAnnotationMetadata); - return projectMetadata; }); + dbAdaptor.getMetadataManager().updateAnnotationIndexTimestamp(annotateAll, annotationStartTimestamp); } if (doLoad && filesToBeAnnotated != null) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java index db6ead71f58..502a00fa091 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Throwables; import org.apache.avro.generic.GenericRecord; +import org.apache.commons.lang3.time.StopWatch; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.biodata.models.variant.stats.VariantStats; @@ -34,7 +35,10 @@ import org.opencb.commons.io.DataReader; import org.opencb.commons.run.ParallelTaskRunner; import org.opencb.commons.run.Task; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; +import org.opencb.opencga.core.models.common.mixins.VariantStatsJsonMixin; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.json.JsonDataReader; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; @@ -47,8 +51,6 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.io.db.VariantDBReader; import org.opencb.opencga.storage.core.variant.io.db.VariantStatsDBWriter; -import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; -import org.opencb.opencga.core.models.common.mixins.VariantStatsJsonMixin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,9 +99,14 @@ public void calculateStatistics(String study, List cohorts, QueryOptions throw new IllegalArgumentException(e); } + // Start time of the operation before calculate step + long startTime = System.currentTimeMillis(); + URI stats = createStats(dbAdaptor, output, study, cohorts, options); - loadStats(stats, study, options); + + StudyMetadata studyMetadata = dbAdaptor.getMetadataManager().getStudyMetadata(study); + loadStats(stats, studyMetadata, options, startTime); } /** @@ -348,10 +355,10 @@ public void loadStats(URI uri, String study, QueryOptions options) throws IOException, StorageEngineException { VariantStorageMetadataManager variantStorageMetadataManager = dbAdaptor.getMetadataManager(); StudyMetadata studyMetadata = variantStorageMetadataManager.getStudyMetadata(study); - loadStats(uri, studyMetadata, options); + loadStats(uri, studyMetadata, options, -1); } - public void loadStats(URI uri, StudyMetadata studyMetadata, QueryOptions options) throws + public void loadStats(URI uri, StudyMetadata studyMetadata, QueryOptions options, long startTime) throws IOException, StorageEngineException { URI variantStatsUri = UriUtils.replacePath(uri, uri.getPath() + VARIANT_STATS_SUFFIX); @@ -364,17 +371,17 @@ public void loadStats(URI uri, StudyMetadata studyMetadata, QueryOptions options boolean error = false; try { logger.info("starting stats loading from {}", variantStatsUri); - long start = System.currentTimeMillis(); + StopWatch stopWatch = StopWatch.createStarted(); loadVariantStats(variantStatsUri, studyMetadata, options); // loadSourceStats(variantDBAdaptor, sourceStatsUri, studyMetadata, options); - logger.info("finishing stats loading, time: {}ms", System.currentTimeMillis() - start); + logger.info("finishing stats loading, time: {}", TimeUtils.durationToString(stopWatch)); } catch (Exception e) { error = true; throw e; } finally { - postCalculateStats(dbAdaptor.getMetadataManager(), studyMetadata, cohorts, error); + postCalculateStats(dbAdaptor.getMetadataManager(), studyMetadata, cohorts, startTime, error); } // variantDBAdaptor.getMetadataManager().updateStudyMetadata(studyMetadata, options); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java index d4fdc948622..eed42ca7e72 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java @@ -101,7 +101,8 @@ public void preCalculateStats( } public void postCalculateStats( - VariantStorageMetadataManager metadataManager, StudyMetadata studyMetadata, Collection cohorts, boolean error) + VariantStorageMetadataManager metadataManager, StudyMetadata studyMetadata, Collection cohorts, + long startTime, boolean error) throws StorageEngineException { TaskMetadata.Status status = error ? TaskMetadata.Status.ERROR : TaskMetadata.Status.READY; @@ -110,6 +111,9 @@ public void postCalculateStats( metadataManager.updateCohortMetadata(studyMetadata.getId(), cohortId, cohort -> cohort.setStatsStatus(status)); } + if (startTime > 0 && !error) { + metadataManager.updateStatsIndexTimestamp(); + } } @Deprecated diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java index e90b49cff4d..699b063b1a5 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java @@ -263,8 +263,9 @@ public static StudyMetadata stats(VariantStatisticsManager vsm, QueryOptions opt URI resolve) throws IOException, StorageEngineException { if (vsm instanceof DefaultVariantStatisticsManager) { DefaultVariantStatisticsManager dvsm = (DefaultVariantStatisticsManager) vsm; + long startTime = System.currentTimeMillis(); URI stats = dvsm.createStats(dbAdaptor, resolve, cohorts, cohortIds, studyMetadata, options); - dvsm.loadStats(stats, studyMetadata, options); + dvsm.loadStats(stats, studyMetadata, options, startTime); } else { dbAdaptor.getMetadataManager().registerCohorts(studyMetadata.getName(), cohorts); // studyMetadata.getCohortIds().putAll(cohortIds); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index c8c1bc74752..cf75349125b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -133,8 +133,6 @@ public class HadoopVariantStorageEngine extends VariantStorageEngine implements public static final String STUDY_ID = "studyId"; // Project attributes - // Last time (in millis from epoch) that a file was loaded - public static final String LAST_LOADED_FILE_TS = "lastLoadedFileTs"; // Last time (in millis from epoch) that the list of "pendingVariantsToAnnotate" was updated public static final String LAST_VARIANTS_TO_ANNOTATE_UPDATE_TS = "lastVariantsToAnnotateUpdateTs"; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java index c3f5ebf550e..fd37c0c4315 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java @@ -264,10 +264,6 @@ public URI postLoad(URI input, URI output) throws StorageEngineException { metadataManager.updateVariantFileMetadata(studyId, fileMetadata); registerLoadedFiles(Collections.singletonList(getFileId())); - metadataManager.updateProjectMetadata(projectMetadata -> { - projectMetadata.getAttributes().put(LAST_LOADED_FILE_TS, System.currentTimeMillis()); - return projectMetadata; - }); // This method checks the loaded variants (if possible) and adds the loaded files to the metadata super.postLoad(input, output); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java index d3c608c8b42..835823e9095 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java @@ -84,7 +84,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.SEARCH_INDEX_LAST_TIMESTAMP; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*; @@ -296,8 +295,7 @@ public Iterator annotationIterator(String name, Query query, throw VariantQueryException.internalException(e); } }).iterator(); - long ts = getMetadataManager().getProjectMetadata().getAttributes() - .getLong(SEARCH_INDEX_LAST_TIMESTAMP.key()); + long ts = getMetadataManager().getProjectMetadata().getSecondaryAnnotationIndexLastTimestamp(); HBaseToVariantAnnotationConverter converter = new HBaseToVariantAnnotationConverter(ts) .setAnnotationIds(getMetadataManager().getProjectMetadata().getAnnotation()) .setIncludeFields(selectElements.getFields()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/HadoopDefaultVariantAnnotationManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/HadoopDefaultVariantAnnotationManager.java index 525362bf50e..f9b59c47a43 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/HadoopDefaultVariantAnnotationManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/HadoopDefaultVariantAnnotationManager.java @@ -122,8 +122,7 @@ protected void preAnnotate(Query query, boolean doCreate, boolean doLoad, Object } else { AnnotationPendingVariantsManager pendingVariantsManager = new AnnotationPendingVariantsManager(dbAdaptor); ProjectMetadata projectMetadata = dbAdaptor.getMetadataManager().getProjectMetadata(); - long lastLoadedFileTs = projectMetadata.getAttributes() - .getLong(HadoopVariantStorageEngine.LAST_LOADED_FILE_TS); + long lastLoadedFileTs = projectMetadata.getVariantIndexLastTimestamp(); long lastVariantsToAnnotateUpdateTs = projectMetadata.getAttributes() .getLong(HadoopVariantStorageEngine.LAST_VARIANTS_TO_ANNOTATE_UPDATE_TS); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java index 4bfcb0011e1..4ac644f5cb7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java @@ -47,8 +47,8 @@ import java.util.*; import java.util.stream.Collectors; -import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.SEARCH_INDEX_LAST_TIMESTAMP; -import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.*; +import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.extractVariantFromResult; +import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.extractVariantFromResultSet; /** * Created on 20/11/15. @@ -61,11 +61,10 @@ public abstract class HBaseToVariantConverter implements Converter configure(Configuration configuration) { return configure(HBaseVariantConverterConfiguration.builder(configuration).build()); } - public static boolean isFailOnWrongVariants() { - return failOnWrongVariants; - } - - public static void setFailOnWrongVariants(boolean b) { - failOnWrongVariants = b; - } - public static HBaseToVariantConverter fromResult(VariantStorageMetadataManager scm) { return new ResultToVariantConverter(scm); } @@ -169,14 +160,6 @@ protected Variant convert(Variant variant, Map studies, return variant; } - private void wrongVariant(String message) { - if (configuration.getFailOnWrongVariants()) { - throw new IllegalStateException(message); - } else { - logger.warn(message); - } - } - private static class ResultSetToVariantConverter extends HBaseToVariantConverter { ResultSetToVariantConverter(VariantStorageMetadataManager scm) { super(scm); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseVariantConverterConfiguration.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseVariantConverterConfiguration.java index 86e220f70bc..2a926151ee4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseVariantConverterConfiguration.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseVariantConverterConfiguration.java @@ -20,7 +20,6 @@ public class HBaseVariantConverterConfiguration { private final VariantQueryProjection projection; - private final boolean failOnWrongVariants; private final boolean failOnEmptyVariants; private final boolean studyNameAsStudyId; private final boolean simpleGenotypes; @@ -30,12 +29,11 @@ public class HBaseVariantConverterConfiguration { private final boolean includeSampleId; private final boolean includeIndexStatus; - public HBaseVariantConverterConfiguration(VariantQueryProjection projection, boolean failOnWrongVariants, boolean failOnEmptyVariants, + public HBaseVariantConverterConfiguration(VariantQueryProjection projection, boolean failOnEmptyVariants, boolean studyNameAsStudyId, boolean simpleGenotypes, String unknownGenotype, boolean mutableSamplesPosition, List sampleDataKeys, boolean includeSampleId, boolean includeIndexStatus) { this.projection = projection; - this.failOnWrongVariants = failOnWrongVariants; this.failOnEmptyVariants = failOnEmptyVariants; this.studyNameAsStudyId = studyNameAsStudyId; this.simpleGenotypes = simpleGenotypes; @@ -97,10 +95,6 @@ public boolean getSimpleGenotypes() { return simpleGenotypes; } - public boolean getFailOnWrongVariants() { - return failOnWrongVariants; - } - public boolean getFailOnEmptyVariants() { return failOnEmptyVariants; } @@ -134,7 +128,6 @@ public static class Builder { private VariantQueryProjection projection; private boolean studyNameAsStudyId = false; private boolean simpleGenotypes = false; - private boolean failOnWrongVariants = HBaseToVariantConverter.isFailOnWrongVariants(); private boolean failOnEmptyVariants = false; private String unknownGenotype = UNKNOWN_GENOTYPE; private boolean mutableSamplesPosition = true; @@ -157,11 +150,6 @@ public Builder setSimpleGenotypes(boolean simpleGenotypes) { return this; } - public Builder setFailOnWrongVariants(boolean failOnWrongVariants) { - this.failOnWrongVariants = failOnWrongVariants; - return this; - } - public Builder setFailOnEmptyVariants(boolean failOnEmptyVariants) { this.failOnEmptyVariants = failOnEmptyVariants; return this; @@ -198,7 +186,7 @@ public Builder setIncludeIndexStatus(boolean includeIndexStatus) { public HBaseVariantConverterConfiguration build() { return new HBaseVariantConverterConfiguration( - projection, failOnWrongVariants, + projection, failOnEmptyVariants, studyNameAsStudyId, simpleGenotypes, diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/HBaseToStudyEntryConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/HBaseToStudyEntryConverter.java index 7acd417c78e..b33fca77090 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/HBaseToStudyEntryConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/HBaseToStudyEntryConverter.java @@ -738,14 +738,6 @@ private String getSimpleGenotype(String genotype) { } } - private void wrongVariant(String message) { - if (configuration.getFailOnWrongVariants()) { - throw new IllegalStateException(message); - } else { - logger.warn(message); - } - } - /** * Add secondary alternates to the StudyEntry. Merge secondary alternates if needed. * diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java index cc6323b2c0a..1c6df86d68d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.function.Function; -import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.SEARCH_INDEX_LAST_TIMESTAMP; import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema.STUDY_SUFIX_BYTES; import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema.VariantColumn.*; @@ -74,7 +73,7 @@ public Function getPendingEvaluatorMapper(VariantStorageMetada // When overwriting mark all variants as pending return (value) -> getMutation(value, true); } else { - long ts = metadataManager.getProjectMetadata().getAttributes().getLong(SEARCH_INDEX_LAST_TIMESTAMP.key()); + long ts = metadataManager.getProjectMetadata().getSecondaryAnnotationIndexLastTimestamp(); return (value) -> { VariantStorageEngine.SyncStatus syncStatus = HadoopVariantSearchIndexUtils.getSyncStatusCheckStudies(ts, value); boolean pending = syncStatus != VariantStorageEngine.SyncStatus.SYNCHRONIZED; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopMRVariantStatisticsManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopMRVariantStatisticsManager.java index 6efa3f3fadc..3d0656095a9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopMRVariantStatisticsManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopMRVariantStatisticsManager.java @@ -61,6 +61,8 @@ public void calculateStatistics(String study, List cohorts, QueryOptions // .collect(Collectors.toMap(c -> c, c -> Collections.emptySet())), null, updateStats, overwriteStats); // dbAdaptor.getStudyConfigurationManager().updateStudyConfiguration(sc, options); + // Start time of the operation before pre-calculate step + long startTime = System.currentTimeMillis(); preCalculateStats(metadataManager, sm, cohorts, overwriteStats, options); options.put(VariantStatsDriver.COHORTS, cohorts); @@ -77,7 +79,7 @@ public void calculateStatistics(String study, List cohorts, QueryOptions error = true; throw e; } finally { - postCalculateStats(metadataManager, sm, cohorts, error); + postCalculateStats(metadataManager, sm, cohorts, startTime, error); } dbAdaptor.updateStatsColumns(sm); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopDBWriterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopDBWriterTest.java index 289052ad00f..863d39cf035 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopDBWriterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopDBWriterTest.java @@ -62,7 +62,6 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.archive.VariantHBaseArchiveDataWriter; -import org.opencb.opencga.storage.hadoop.variant.converters.HBaseToVariantConverter; import org.opencb.opencga.storage.hadoop.variant.load.VariantHadoopDBWriter; import org.opencb.opencga.storage.hadoop.variant.mr.VariantTableHelper; import org.opencb.opencga.storage.hadoop.variant.transform.VariantSliceReader; @@ -101,8 +100,6 @@ public class VariantHadoopDBWriterTest extends VariantStorageBaseTest implements public void setUp() throws Exception { HadoopVariantStorageEngine variantStorageManager = getVariantStorageEngine(); clearDB(variantStorageManager.getDBName()); - //Force HBaseConverter to fail if something goes wrong - HBaseToVariantConverter.setFailOnWrongVariants(true); engine = getVariantStorageEngine(); dbAdaptor = engine.getDBAdaptor(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java index e9e99cdb884..feaad7ff668 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java @@ -56,7 +56,6 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveRowKeyFactory; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; -import org.opencb.opencga.storage.hadoop.variant.converters.HBaseToVariantConverter; import org.opencb.opencga.storage.hadoop.variant.gaps.FillGapsTest; import java.io.IOException; @@ -93,8 +92,6 @@ public class VariantHadoopMultiSampleTest extends VariantStorageBaseTest impleme public void setUp() throws Exception { HadoopVariantStorageEngine variantStorageManager = getVariantStorageEngine(); clearDB(variantStorageManager.getDBName()); - //Force HBaseConverter to fail if something goes wrong - HBaseToVariantConverter.setFailOnWrongVariants(true); } @After diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java index 7bb7fe0e10e..8d89a06d127 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java @@ -57,7 +57,6 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; -import org.opencb.opencga.storage.hadoop.variant.converters.HBaseToVariantConverter; import org.opencb.opencga.storage.hadoop.variant.index.IndexUtils; import org.opencb.opencga.storage.hadoop.variant.index.family.MendelianErrorSampleIndexConverter; import org.opencb.opencga.storage.hadoop.variant.index.sample.*; @@ -393,8 +392,6 @@ public static void printVariants(StudyMetadata studyMetadata, VariantHadoopDBAda public static void printVariants(Collection studies, VariantHadoopDBAdaptor dbAdaptor, Path outDir) throws Exception { - boolean old = HBaseToVariantConverter.isFailOnWrongVariants(); - HBaseToVariantConverter.setFailOnWrongVariants(false); printMetaTable(dbAdaptor, outDir); for (StudyMetadata studyMetadata : studies) { printVariantsFromArchiveTable(dbAdaptor, studyMetadata, outDir); @@ -406,7 +403,6 @@ public static void printVariants(Collection studies, VariantHadoo printSampleIndexTable(dbAdaptor, outDir); printVariantsFromVariantsTable(dbAdaptor, outDir); printVariantsFromDBAdaptor(dbAdaptor, outDir); - HBaseToVariantConverter.setFailOnWrongVariants(old); } private static void printVcf(StudyMetadata studyMetadata, VariantHadoopDBAdaptor dbAdaptor, Path outDir) throws IOException {