diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java index ec0206e8695..6d43d9dd8f7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java @@ -1,15 +1,21 @@ package org.opencb.opencga.analysis; +import org.apache.commons.collections4.CollectionUtils; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.managers.JobManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.response.OpenCGAResult; import java.io.BufferedReader; @@ -20,6 +26,13 @@ public class AnalysisUtils { + public static String REFERENCE_GENOME_GRCH38_FA = "REFERENCE_GENOME_GRCH38_FA"; + public static String REFERENCE_GENOME_GRCH38_FAI= "REFERENCE_GENOME_GRCH38_FAI"; + public static String REFERENCE_GENOME_GRCH38_GZI = "REFERENCE_GENOME_GRCH38_GZI"; + public static String REFERENCE_GENOME_GRCH37_FA= "REFERENCE_GENOME_GRCH37_FA"; + public static String REFERENCE_GENOME_GRCH37_FAI = "REFERENCE_GENOME_GRCH37_FAI"; + public static String REFERENCE_GENOME_GRCH37_GZI= "REFERENCE_GENOME_GRCH37_GZI"; + public static boolean isSupportedCommand(String commands) { Set commandSet = new HashSet<>(Arrays.asList(commands.replace(" ", "").split(","))); if (!commandSet.contains(commands)) { @@ -193,4 +206,22 @@ public static String getJobFileRelativePath(String path) { } return path.substring(index + 5); } + + public static String getAssembly(CatalogManager catalogManager, String studyId, String sessionId) throws CatalogException { + String assembly = ""; + OpenCGAResult projectQueryResult; + + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + + projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), + new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), sessionId); + if (CollectionUtils.isNotEmpty(projectQueryResult.getResults()) + && projectQueryResult.first().getOrganism() != null + && projectQueryResult.first().getOrganism().getAssembly() != null) { + assembly = projectQueryResult.first().getOrganism().getAssembly(); + } + return assembly; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java index c494f8db7ca..a69361558d0 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java @@ -21,6 +21,7 @@ import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.ResourceFile; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.slf4j.Logger; @@ -121,4 +122,39 @@ public static String getToolDefaultVersion(String toolId, Configuration configur } return defaultVersion; } + + + public static AnalysisTool getAnalysisTool(String toolId, String version, Configuration configuration) throws ToolException { + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (toolId.equals(tool.getId())) { + if (StringUtils.isEmpty(version) || version.equals(tool.getVersion())) { + return tool; + } + } + } + throw new ToolException("Missing analysis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); + } + + public static Path getToolResourcePath(String resourceId, Configuration configuration) throws ToolException { + // Get resources from the configuration file + for (ResourceFile resourceFile : configuration.getAnalysis().getResource().getFiles()) { + if (resourceFile.getId().equalsIgnoreCase(resourceId)) { + Path resourcePath = configuration.getAnalysis().getResource().getBasePath().resolve(resourceFile.getPath()); + if (!Files.exists(resourcePath)) { + throw new ToolException("Error getting resource '" + resourceId + "': it is not installed (i.e., missing file '" + + resourcePath + "')"); + } + return resourcePath; + } + } + throw new ToolException("Error getting resource '" + resourceId + "': it does not exist in the configuration file"); + } + + public static void checkAnalysisResources(String toolId, String version, Configuration configuration) throws ToolException { + AnalysisTool analysisTool = getAnalysisTool(toolId, version, configuration); + for (String resourceId : analysisTool.getResources()) { + getToolResourcePath(resourceId, configuration); + } + + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java deleted file mode 100644 index 161c07890df..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.utils.URLUtils; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.utils.CatalogFqn; -import org.opencb.opencga.core.models.JwtPayload; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.response.OpenCGAResult; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Path; -import java.util.LinkedList; -import java.util.List; - -public class ResourceUtils { - - public static final String URL = "http://resources.opencb.org/opencb/opencga/"; - - public static File downloadThirdParty(URL url, Path outDir) throws IOException { - return URLUtils.download(url, outDir); - } - - public static File downloadAnalysis(String analysisId, String resouceName, Path outDir, Path openCgaHome) throws IOException { - Path path = null; - String filename = "analysis/" + analysisId + "/" + resouceName; - if (openCgaHome != null) { - path = openCgaHome.resolve(filename); - } - if (path != null && path.toFile().exists()) { - File outFile = outDir.resolve(path.toFile().getName()).toFile(); - System.out.println("downloadAnalysis from path: " + path + " to " + outFile.getAbsolutePath()); - FileUtils.copyFile(path.toFile(), outFile); - - return outFile; - } else { - System.out.println("downloadAnalysis from URL: " + (URL + filename) + ", (path does not exist: " + path + ")"); - return URLUtils.download(new URL(URL + filename), outDir); - } - } - - public static DownloadedRefGenome downloadRefGenome(String assembly, Path outDir, Path openCgaHome) throws IOException { - - // Download files - File gzFile = null; - File faiFile = null; - File gziFile = null; - - // Get files to downloadAnalysis - List filenames = new LinkedList<>(); - filenames.add("Homo_sapiens." + assembly + ".dna.primary_assembly.fa.gz"); - filenames.add("Homo_sapiens." + assembly + ".dna.primary_assembly.fa.gz.fai"); - filenames.add("Homo_sapiens." + assembly + ".dna.primary_assembly.fa.gz.gzi"); - - Path path = null; - for (String filename : filenames) { - File file; - - if (openCgaHome != null) { - path = openCgaHome.resolve("analysis/commons/reference-genomes/" + filename); - } - if (path != null && path.toFile().exists()) { - File outFile = outDir.resolve(path.toFile().getName()).toFile(); - System.out.println("downloadRefGenome from path: " + path + " to " + outFile.getAbsolutePath()); - FileUtils.copyFile(path.toFile(), outFile); - file = outFile; - } else { - URL url = new URL(URL + "analysis/commons/reference-genomes/" + filename); - System.out.println("downloadAnalysis from URL: " + URL + ", (path does not exist: " + path + ")"); - file = URLUtils.download(url, outDir); - if (file == null) { - // Something wrong happened, remove downloaded files - cleanRefGenome(filenames, outDir); - return null; - } - } - if (filename.endsWith("gz")) { - gzFile = file; - } else if (filename.endsWith("fai")) { - faiFile = file; - } else if (filename.endsWith("gzi")) { - gziFile = file; - } - - // Reset path for the next iteration - path = null; - } - return new DownloadedRefGenome(assembly, gzFile, faiFile, gziFile); - } - - //------------------------------------------------------------------------- - // Support for downloading reference genomes - //------------------------------------------------------------------------- - - public static String getAssembly(CatalogManager catalogManager, String studyId, String sessionId) throws CatalogException { - String assembly = ""; - OpenCGAResult projectQueryResult; - - JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(sessionId); - CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); - String organizationId = studyFqn.getOrganizationId(); - - projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), - new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), sessionId); - if (CollectionUtils.isNotEmpty(projectQueryResult.getResults()) - && projectQueryResult.first().getOrganism() != null - && projectQueryResult.first().getOrganism().getAssembly() != null) { - assembly = projectQueryResult.first().getOrganism().getAssembly(); - } - return assembly; - } - - public static class DownloadedRefGenome { - private String assembly; - private File gzFile; - private File faiFile; - private File gziFile; - - public DownloadedRefGenome(String assembly, File gzFile, File faiFile, File gziFile) { - this.assembly = assembly; - this.gzFile = gzFile; - this.faiFile = faiFile; - this.gziFile = gziFile; - } - - public String getAssembly() { - return assembly; - } - - public DownloadedRefGenome setAssembly(String assembly) { - this.assembly = assembly; - return this; - } - - public File getGzFile() { - return gzFile; - } - - public DownloadedRefGenome setGzFile(File gzFile) { - this.gzFile = gzFile; - return this; - } - - public File getFaiFile() { - return faiFile; - } - - public DownloadedRefGenome setFaiFile(File faiFile) { - this.faiFile = faiFile; - return this; - } - - public File getGziFile() { - return gziFile; - } - - public DownloadedRefGenome setGziFile(File gziFile) { - this.gziFile = gziFile; - return this; - } - } - - //------------------------------------------------------------------------- - // P R I V A T E M E T H O D S - //------------------------------------------------------------------------- - - private static void cleanRefGenome(List links, Path outDir) { - for (String link : links) { - String name = new File(link).getName(); - File file = outDir.resolve(name).toFile(); - if (file.exists()) { - file.delete(); - } - } - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index bbf685d1a03..d1601e223dc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -33,10 +33,12 @@ import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserAnalysisUtils; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; @@ -57,10 +59,9 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; -import static org.opencb.opencga.core.tools.OpenCgaToolExecutor.EXECUTOR_ID; - @Tool(id = ExomiserInterpretationAnalysis.ID, resource = Enums.Resource.CLINICAL) public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { @@ -75,6 +76,13 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private ClinicalAnalysis clinicalAnalysis; + private String dockerImage; + private String dockerImageVersion; + + private static final String PREPARE_RESOURCES_STEP = "prepare-resources"; + private static final String EXPORT_VARIANTS_STEP = "export-variants"; + private static final String SAVE_INTERPRETATION_STEP = "save-interpretation"; + @Override protected InterpretationMethod getInterpretationMethod() { return getInterpretationMethod(ID); @@ -84,6 +92,9 @@ protected InterpretationMethod getInterpretationMethod() { protected void check() throws Exception { super.check(); + // Update executor params with OpenCGA home and session ID + setUpStorageEngineExecutor(studyId); + // Check study if (StringUtils.isEmpty(studyId)) { // Missing study @@ -135,30 +146,55 @@ protected void check() throws Exception { logger.warn("Missing exomiser version, using the default {}", exomiserVersion); } - // Update executor params with OpenCGA home and session ID - setUpStorageEngineExecutor(studyId); + ExomiserAnalysisUtils.checkResources(exomiserVersion, studyId, catalogManager, token, getOpencgaHome()); + } + + @Override + protected List getSteps() { + return Arrays.asList(PREPARE_RESOURCES_STEP, EXPORT_VARIANTS_STEP, ExomiserWrapperAnalysis.ID, SAVE_INTERPRETATION_STEP); } @Override - protected void run() throws ToolException { - step(() -> { + protected void run() throws Exception { + // Main steps + step(PREPARE_RESOURCES_STEP, this::prepareResources); + step(EXPORT_VARIANTS_STEP, this::exportVariants); + step(ExomiserWrapperAnalysis.ID, this::runExomiser); + step(SAVE_INTERPRETATION_STEP, this::saveInterpretation); + } + + private void prepareResources() throws ToolException { + ExomiserAnalysisUtils.prepareResources(sampleId, studyId, clinicalAnalysisType.name(), exomiserVersion, catalogManager, token, + getOutDir(), getOpencgaHome()); + } - executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID); - ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) - .setStudyId(studyId) - .setSampleId(sampleId) - .setClinicalAnalysisType(clinicalAnalysisType) - .setExomiserVersion(exomiserVersion); + private void exportVariants() throws ToolException { + ExomiserAnalysisUtils.exportVariants(sampleId, studyId, clinicalAnalysisType.name(), getOutDir(), variantStorageManager, token); + } + + private void runExomiser() throws ToolException, CatalogException { + ExomiserWrapperAnalysisExecutor executor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + .setExomiserVersion(exomiserVersion) + .setAssembly(ExomiserAnalysisUtils.checkAssembly(studyId, catalogManager, token)) + .setExomiserDataPath(Paths.get(getOpencgaHome().toAbsolutePath().toAbsolutePath().toString(), + ResourceManager.ANALYSIS_DIRNAME, ResourceManager.RESOURCES_DIRNAME, ExomiserWrapperAnalysis.ID)) + .setSampleFile(ExomiserAnalysisUtils.getSamplePath(sampleId, getOutDir())) + .setVcfFile(ExomiserAnalysisUtils.getVcfPath(sampleId, getOutDir())); + + for (File file : getOutDir().toFile().listFiles()) { + if (file.getName().endsWith(".ped")) { + executor.setPedigreeFile(file.toPath()); + break; + } + } - exomiserExecutor.execute(); + dockerImage = executor.getDockerImageName(); + dockerImageVersion = executor.getDockerImageVersion(); - saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion()); - }); + executor.execute(); } - protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion) - throws ToolException, StorageEngineException, - CatalogException, IOException { + private void saveInterpretation() throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( @@ -292,11 +328,12 @@ private List getPrimaryFindings() throws IOException, StorageEn if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) { exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple()))); } else { - logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(), - normalizedToTsv.get(variant.toStringSimple())); + logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", + variant != null ? variant.toStringSimple() : null, + variant != null ? normalizedToTsv.get(variant.toStringSimple()) : null); } } else { - logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple()); + logger.warn("Variant {} not found in map normalizedToTsv", variant != null ? variant.toStringSimple() : null); } for (String[] fields : variantTsvMap.get(variant.toStringSimple())) { ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java index febc214e9e7..c8ebfe34ad1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java @@ -19,12 +19,12 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.CatalogFqn; - import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; @@ -101,6 +101,9 @@ protected void check() throws Exception { Path thresholdsPath = getOpencgaHome().resolve("analysis").resolve(FamilyQcAnalysis.ID).resolve("relatedness_thresholds.csv"); relatednessThresholds = AnalysisUtils.parseRelatednessThresholds(thresholdsPath); + + // Check resources + ConfigurationUtils.checkAnalysisResources(RelatednessAnalysis.ID, null, configuration); } @Override @@ -125,7 +128,7 @@ protected void run() throws ToolException { .setRelatednessMethod(relatednessMethod) .setRelatednessMaf(relatednessMaf) .setRelatednessThresholds(relatednessThresholds) - .setRelatednesResourcePath(getOpencgaHome().resolve("analysis/resources").resolve(RelatednessAnalysis.ID)) + .setRelatednesResourcePath(configuration.getAnalysis().getResource().getBasePath().resolve(RelatednessAnalysis.ID)) .setQualityControl(qualityControl); // Step by step diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcLocalAnalysisExecutor.java index ff0316f42de..fa98af03322 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcLocalAnalysisExecutor.java @@ -23,7 +23,9 @@ import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.ResourceException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.sample.Sample; @@ -31,10 +33,16 @@ import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.FamilyQcAnalysisExecutor; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import static org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis.RELATEDNESS_VARIANTS_FRQ; +import static org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis.RELATEDNESS_VARIANTS_PRUNE_IN; + @ToolExecutor(id="opencga-local", tool = FamilyQcAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class FamilyQcLocalAnalysisExecutor extends FamilyQcAnalysisExecutor implements StorageToolExecutor { @@ -42,7 +50,7 @@ public class FamilyQcLocalAnalysisExecutor extends FamilyQcAnalysisExecutor impl private CatalogManager catalogManager; @Override - public void run() throws ToolException { + public void run() throws ToolException, ResourceException { // Sanity check: quality control to update must not be null if (qualityControl == null) { throw new ToolException("Family quality control metrics is null"); @@ -62,7 +70,7 @@ public void run() throws ToolException { } } - private void runRelatedness() throws ToolException { + private void runRelatedness() throws ToolException, ResourceException { if (CollectionUtils.isNotEmpty(qualityControl.getRelatedness())) { for (RelatednessReport relatedness : qualityControl.getRelatedness()) { @@ -112,8 +120,17 @@ private void runRelatedness() throws ToolException { } // Run IBD/IBS computation using PLINK in docker - RelatednessReport report = IBDComputation.compute(getStudyId(), getFamily(), sampleIds, getRelatednessMaf(), - getRelatednessThresholds(), getRelatednesResourcePath(), getOutDir(), getVariantStorageManager(), getToken()); + RelatednessReport report; + try { + ResourceManager resourceManager = new ResourceManager(Paths.get(getExecutorParams().getString("opencgaHome"))); + Path pruneInPath = resourceManager.checkResourcePath(RELATEDNESS_VARIANTS_PRUNE_IN); + Path freqPath = resourceManager.checkResourcePath(RELATEDNESS_VARIANTS_FRQ); + + report = IBDComputation.compute(getStudyId(), getFamily(), sampleIds, getRelatednessMaf(), getRelatednessThresholds(), + pruneInPath, freqPath, getOutDir(), getVariantStorageManager(), getToken()); + } catch (IOException e) { + throw new ToolException(e); + } // Sanity check if (report == null) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java index 7277c3d2429..f31c8b4896f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java @@ -26,10 +26,8 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.exceptions.ToolException; @@ -40,7 +38,6 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import java.io.*; -import java.net.URL; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -48,41 +45,12 @@ import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.TPED; public class IBDComputation { - private static final String BASENAME = "variants"; private static final String FILTERED_BASENAME = "variants.filtered"; - private static final String FREQ_FILENAME = BASENAME + ".frq"; - private static final String PRUNE_IN_FILENAME = BASENAME + ".prune.in"; public static RelatednessReport compute(String study, Family family, List samples, String maf, - Map> thresholds, Path resourcesPath, Path outDir, + Map> thresholds, Path pruneInPath, Path freqPath, Path outDir, VariantStorageManager storageManager, String token) throws ToolException { - // Check resource (variants.frq and variants.prune.in) and download if necessary - // TODO: download into the folder /analysis/relatedness - try { - URL url = new URL(ResourceUtils.URL + "analysis/" + RelatednessAnalysis.ID + "/" + PRUNE_IN_FILENAME); - ResourceUtils.downloadThirdParty(url, outDir); - } catch (IOException e) { - throw new ToolException("Something wrong happened when downloading resource files during the relatedness analysis execution"); - } - Path pruneInPath = outDir.resolve(PRUNE_IN_FILENAME); -// Path pruneInPath = resourcesPath.resolve(PRUNE_IN_FILENAME); - - // if (!resourcesPath.resolve(FREQ_FILENAME).toFile().exists()) { -// // Download freq file from resources -// } - try { - URL url = new URL(ResourceUtils.URL + "analysis/" + RelatednessAnalysis.ID + "/" + FREQ_FILENAME); - ResourceUtils.downloadThirdParty(url, outDir); - } catch (IOException e) { - throw new ToolException("Something wrong happened when downloading resource files during the relatedness analysis execution"); - } - Path freqPath = outDir.resolve(FREQ_FILENAME); -// Path freqPath = resourcesPath.resolve(FREQ_FILENAME); -// if (!resourcesPath.resolve(PRUNE_IN_FILENAME).toFile().exists()) { -// // Download prune-in variant file from resources -// } - // Export family VCF if (family != null) { List trio = getTrio(family); @@ -308,7 +276,7 @@ private static File runIBD(String basename, Path freqPath, Path outDir) throws T "/output"); // Run IBD using PLINK in docker - String plinkParams = "plink1.9 --tfile /output/" + basename + " --genome rel-check --read-freq /input/" + FREQ_FILENAME + String plinkParams = "plink1.9 --tfile /output/" + basename + " --genome rel-check --read-freq /input/" + freqPath.getFileName() + " --out /output/" + basename; try { PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java index 3dcd8079060..dd70818dcb0 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java @@ -90,9 +90,17 @@ protected void check() throws Exception { } String userId = jwtPayload.getUserId(catalogFqn.getOrganizationId()); - // Check write permissions over the path - catalogManager.getAuthorizationManager().checkFilePermission(catalogFqn.getOrganizationId(), study.getUid(), - parents.first().getUid(), userId, FilePermissions.WRITE); + + boolean isResource = toolParams.getResource() != null && toolParams.getResource(); + if (isResource) { + // Check it is a study admin + catalogManager.getAuthorizationManager().checkIsAtLeastStudyAdministrator(catalogFqn.getOrganizationId(), study.getUid(), + userId); + } else { + // Check write permissions over the path + catalogManager.getAuthorizationManager().checkFilePermission(catalogFqn.getOrganizationId(), study.getUid(), + parents.first().getUid(), userId, FilePermissions.WRITE); + } } catch (CatalogException e) { throw new ToolException(e); } @@ -122,7 +130,8 @@ protected void run() throws Exception { step("register", () -> { // Move downloaded file and register try { - moveFile(studyFqn, getOutDir().resolve(fileName), null, toolParams.getPath(), token); + boolean isResource = toolParams.getResource() != null && toolParams.getResource(); + moveFile(studyFqn, getOutDir().resolve(fileName), null, toolParams.getPath(), isResource, token); } catch (Exception e) { deleteTemporaryFile(); throw new CatalogException(e.getMessage(), e); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/resource/ResourceFetcherTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/resource/ResourceFetcherTool.java new file mode 100644 index 00000000000..f37388730c0 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/resource/ResourceFetcherTool.java @@ -0,0 +1,94 @@ +package org.opencb.opencga.analysis.resource; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.tools.OpenCgaTool; +import org.opencb.opencga.catalog.exceptions.ResourceException; +import org.opencb.opencga.catalog.utils.ResourceManager; +import org.opencb.opencga.core.config.Resource; +import org.opencb.opencga.core.config.ResourceFile; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.resource.ResourceFetcherToolParams; +import org.opencb.opencga.core.tools.annotations.Tool; +import org.opencb.opencga.core.tools.annotations.ToolParams; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + +import static org.opencb.opencga.catalog.utils.ResourceManager.RESOURCES_DIRNAME; + +@Tool(id= ResourceFetcherTool.ID, + resource = Enums.Resource.RESOURCE, + type = Tool.Type.OPERATION, + scope = Tool.Scope.GLOBAL, + description = ResourceFetcherTool.DESCRIPTION, + priority = Enums.Priority.HIGH) +public class ResourceFetcherTool extends OpenCgaTool { + + public static final String ID = "resource-fetcher"; + public static final String DESCRIPTION = "Fetch resources from the public server and save them into the OpenCGA local installation"; + + private Path resourcePath; + + @ToolParams + protected final ResourceFetcherToolParams analysisParams = new ResourceFetcherToolParams(); + + @Override + protected void check() throws Exception { + super.check(); + + resourcePath = getOutDir().resolve(RESOURCES_DIRNAME); + if (!Files.exists(Files.createDirectories(resourcePath))) { + throw new IOException("Error creating resource directory '" + resourcePath.toAbsolutePath() + "'"); + } + } + + @Override + protected void run() throws Exception { + // Download all resources + step(ID, this::fetchResources); + } + + private void fetchResources() throws ResourceException, ToolException, IOException { + String msg; + if (CollectionUtils.isEmpty(analysisParams.getResources())) { + msg = "There are no resources to fetch because the input resource list is empty."; + addInfo(msg); + logger.info(msg); + return; + } + ResourceManager resourceManager = new ResourceManager(getOpencgaHome()); + Resource resourceConfig = configuration.getAnalysis().getResource(); + + Set resourceIds = new HashSet<>(); + for (String inputPattern : analysisParams.getResources()) { + // Convert the input pattern to a regex and compile the pattern + String regex = inputPattern.replace("*", ".*"); + Pattern pattern = Pattern.compile(regex); + + // Filter the resource IDs using regex + for (ResourceFile resourceFile : resourceConfig.getFiles()) { + if (pattern.matcher(resourceFile.getId()).matches()) { + resourceIds.add(resourceFile.getId()); + } + } + } + + if (CollectionUtils.isEmpty(resourceIds)) { + msg = "No resources to fetch from the input resource list: " + StringUtils.join(analysisParams.getResources(), ", "); + addInfo(msg); + logger.info(msg); + } else { + msg = "Fetching resources: " + StringUtils.join(resourceIds, ", "); + addInfo(msg); + logger.info(msg); + resourceManager.fetchResources(new ArrayList<>(resourceIds), getOutDir().resolve(RESOURCES_DIRNAME), catalogManager, token); + } + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 6024761d596..78e73895e54 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -510,10 +510,11 @@ protected final void addAttribute(String key, Object value) throws ToolException erm.addAttribute(key, value); } - protected final void moveFile(String study, Path source, Path destiny, String catalogDirectoryPath, String token) throws ToolException { + protected final void moveFile(String study, Path source, Path destiny, String catalogDirectoryPath, boolean isResource, String token) + throws ToolException { File file; try { - file = catalogManager.getFileManager().moveAndRegister(study, source, destiny, catalogDirectoryPath, token).first(); + file = catalogManager.getFileManager().moveAndRegister(study, source, destiny, catalogDirectoryPath, isResource, token).first(); } catch (Exception e) { throw new ToolException("Error moving file from " + source + " to " + destiny, e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosAnalysis.java index 96bf18034c0..b453af65979 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosAnalysis.java @@ -16,7 +16,7 @@ package org.opencb.opencga.analysis.variant.circos; -import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.AnalysisUtils; import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; @@ -62,7 +62,7 @@ protected void check() throws Exception { // // addAttribute("sampleName", sampleName); - assembly = ResourceUtils.getAssembly(catalogManager, study, token); + assembly = AnalysisUtils.getAssembly(catalogManager, study, token); } @Override diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java index 356d53d1d84..0c2faf4390a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java @@ -28,7 +28,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.AnalysisUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -142,7 +142,7 @@ public void run() throws ToolException, IOException, CatalogException { // Get genome version String genomeVersion = "hg38"; - String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken()); + String assembly = AnalysisUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken()); if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) { genomeVersion = "hg19"; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java index 04a5241623f..f7f63f37706 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java @@ -21,7 +21,6 @@ import org.opencb.biodata.models.clinical.qc.GenomePlotConfig; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java index a86cc797287..0efa9a0dd6a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java @@ -30,7 +30,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.AnalysisUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -127,7 +127,7 @@ public void run() throws ToolException, IOException, CatalogException { // Get genome version String genomeVersion = "hg38"; - String assembly = ResourceUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken()); + String assembly = AnalysisUtils.getAssembly(storageManager.getCatalogManager(), getStudy(), getToken()); if (StringUtils.isNotEmpty(assembly) && assembly.toUpperCase(Locale.ROOT).equals("GRCH37")) { genomeVersion = "hg19"; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java index e808c33cd36..08c924e4368 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java @@ -26,7 +26,6 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -81,7 +80,7 @@ protected void check() throws Exception { throw new ToolException("Missing study"); } - assembly = ResourceUtils.getAssembly(catalogManager, study, token); + assembly = AnalysisUtils.getAssembly(catalogManager, study, token); if (StringUtils.isEmpty(assembly)) { throw new ToolException("Missing assembly for study '" + study + "'"); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java index 56a95cfa92e..142a3cf1ed2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java @@ -26,10 +26,11 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; -import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; @@ -50,6 +51,7 @@ import java.nio.charset.Charset; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @Tool(id = MutationalSignatureAnalysis.ID, resource = Enums.Resource.VARIANT) @@ -77,6 +79,13 @@ public class MutationalSignatureAnalysis extends OpenCgaToolScopeStudy { public static final String TYPE_INV = "inv"; public static final String TYPE_TRANS = "trans"; + public static final String GRCH38_FA = "GRCH38_FA"; + public static final String GRCH38_FAI = "GRCH38_FAI"; + public static final String GRCH38_GZI = "GRCH38_GZI"; + public static final String GRCH37_FA = "GRCH37_FA"; + public static final String GRCH37_FAI = "GRCH37_FAI"; + public static final String GRCH37_GZI = "GRCH37_GZI"; + @ToolParams private MutationalSignatureAnalysisParams signatureParams = new MutationalSignatureAnalysisParams(); @@ -184,6 +193,10 @@ protected void check() throws Exception { // Get assembly assembly = getAssembly(study, catalogManager, token); + // Check resources + ResourceManager resourceManager = new ResourceManager(getOpencgaHome()); + resourceManager.checkResourcePaths(MutationalSignatureAnalysis.ID); + // Log messages logger.info("Signagture id: {}", signatureParams.getId()); logger.info("Signagture description: {}", signatureParams.getDescription()); @@ -309,7 +322,7 @@ public static String getContextIndexFilename(String sample, String assembly) { } public static String getAssembly(String study, CatalogManager catalogManager, String token) throws CatalogException, ToolException { - String assembly = ResourceUtils.getAssembly(catalogManager, study, token); + String assembly = AnalysisUtils.getAssembly(catalogManager, study, token); if (StringUtils.isEmpty(assembly)) { throw new ToolException("Missing assembly for study '" + study + "'"); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index 5be8325e066..ae74817e318 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -32,10 +32,11 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; import org.opencb.commons.utils.FileUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.ResourceException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; @@ -43,12 +44,12 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +58,7 @@ import java.nio.file.Paths; import java.util.*; +import static org.opencb.opencga.analysis.AnalysisUtils.*; import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.*; @ToolExecutor(id="opencga-local", tool = MutationalSignatureAnalysis.ID, @@ -158,18 +160,26 @@ private File checkGenomeContextFile() throws ToolExecutorException { private void createGenomeContextFile(File indexFile) throws ToolExecutorException { try { // First, - ResourceUtils.DownloadedRefGenome refGenome = ResourceUtils.downloadRefGenome(getAssembly(), getOutDir(), - opencgaHome); - - if (refGenome == null) { - throw new ToolExecutorException("Something wrong happened accessing reference genome, check local path" - + " and public repository"); + String resourceName; + ResourceManager resourceManager = new ResourceManager(opencgaHome); + + Path gzRefGenomePath; + Path faiRefGenomePath; + Path gziRefGenomePath; + String assembly = getAssembly(); + if ("GRCh38".equalsIgnoreCase(assembly)) { + gzRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH38_FA); + faiRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH38_FAI); + gziRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH38_GZI); + } else if ("GRCh37".equalsIgnoreCase(assembly)) { + gzRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH37_FA); + faiRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH37_FAI); + gziRefGenomePath = resourceManager.checkResourcePath(REFERENCE_GENOME_GRCH37_GZI); + } else { + throw new ToolExecutorException("Invalid assembly '" + assembly + "'. Valid values: GRCh38 and GRCh37"); } - Path refGenomePath = refGenome.getGzFile().toPath(); - // Compute signature profile: contextual frequencies of each type of base substitution - Query query = new Query() .append(VariantQueryParam.STUDY.key(), getStudy()) .append(VariantQueryParam.SAMPLE.key(), getSample()) @@ -181,12 +191,9 @@ private void createGenomeContextFile(File indexFile) throws ToolExecutorExceptio VariantDBIterator iterator = getVariantStorageManager().iterator(query, queryOptions, getToken()); // Read mutation context from reference genome (.gz, .gz.fai and .gz.gzi files) - String base = refGenomePath.toAbsolutePath().toString(); - try (PrintWriter pw = new PrintWriter(indexFile); BlockCompressedIndexedFastaSequenceFile indexed = new BlockCompressedIndexedFastaSequenceFile( - refGenomePath, new FastaSequenceIndex(new File(base + ".fai")), - GZIIndex.loadIndex(Paths.get(base + ".gzi")))) { + gzRefGenomePath, new FastaSequenceIndex(faiRefGenomePath.toFile()), GZIIndex.loadIndex(gziRefGenomePath))) { while (iterator.hasNext()) { Variant variant = iterator.next(); @@ -204,7 +211,7 @@ refGenomePath, new FastaSequenceIndex(new File(base + ".fai")), } } } - } catch (IOException | CatalogException | ToolException | StorageEngineException e) { + } catch (IOException | CatalogException | ToolException | StorageEngineException | ResourceException e) { throw new ToolExecutorException(e); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java index 59ebe1467c2..8634c532a29 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java @@ -20,33 +20,47 @@ import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.family.qc.IBDComputation; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.exceptions.ResourceException; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.IBDRelatednessAnalysisExecutor; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; +import static org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis.RELATEDNESS_VARIANTS_FRQ; +import static org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis.RELATEDNESS_VARIANTS_PRUNE_IN; + @ToolExecutor(id="opencga-local", tool = RelatednessAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class IBDRelatednessLocalAnalysisExecutor extends IBDRelatednessAnalysisExecutor implements StorageToolExecutor { @Override - public void run() throws ToolException { + public void run() throws ToolException, ResourceException { // Get managers VariantStorageManager variantStorageManager = getVariantStorageManager(); // Sanity check to compute String opencgaHome = getExecutorParams().getString("opencgaHome"); if (!Paths.get(opencgaHome).toFile().exists()) { - + throw new ToolException("Internal error: OpenCGA home not setting!"); } // Run IBD/IBS computation using PLINK in docker - RelatednessReport report = IBDComputation.compute(getStudyId(), getFamily(), getSampleIds(), getMinorAlleleFreq(), getThresholds(), - getResourcePath(), getOutDir(), variantStorageManager, getToken()); + RelatednessReport report; + try { + ResourceManager resourceManager = new ResourceManager(Paths.get(getExecutorParams().getString("opencgaHome"))); + Path pruneInPath = resourceManager.checkResourcePath(RELATEDNESS_VARIANTS_PRUNE_IN); + Path freqPath = resourceManager.checkResourcePath(RELATEDNESS_VARIANTS_FRQ); + + report = IBDComputation.compute(getStudyId(), getFamily(), getSampleIds(), getMinorAlleleFreq(), getThresholds(), pruneInPath, + freqPath, getOutDir(), variantStorageManager, getToken()); + } catch (IOException e) { + throw new ToolException(e); + } // Sanity check if (report == null) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java index 6fedc83d6a8..51d37d11e1c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java @@ -23,6 +23,7 @@ import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.family.Family; @@ -43,6 +44,9 @@ public class RelatednessAnalysis extends OpenCgaTool { public static final String MAF_DEFAULT_VALUE = "1000G:ALL>0.3"; + public static final String RELATEDNESS_VARIANTS_PRUNE_IN = "RELATEDNESS_VARIANTS_PRUNE_IN"; + public static final String RELATEDNESS_VARIANTS_FRQ = "RELATEDNESS_VARIANTS_FRQ"; + private String studyId; private String familyId; private List individualIds; @@ -159,6 +163,10 @@ protected void check() throws Exception { Path thresholdsPath = getOpencgaHome().resolve("analysis").resolve(FamilyQcAnalysis.ID).resolve("relatedness_thresholds.csv"); thresholds = AnalysisUtils.parseRelatednessThresholds(thresholdsPath); + + // Check resources + ResourceManager resourceManager = new ResourceManager(getOpencgaHome()); + resourceManager.checkResourcePaths(ID); } @Override @@ -172,7 +180,7 @@ protected void run() throws ToolException { .setSampleIds(sampleIds) .setMinorAlleleFreq(minorAlleleFreq) .setThresholds(thresholds) - .setResourcePath(getOpencgaHome().resolve("analysis/resources").resolve(ID)) + .setResourcePath(configuration.getAnalysis().getResource().getBasePath().resolve(ID)) .execute(); }); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index 0f1fc2bec67..20dc7606c26 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,6 +7,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; @@ -34,7 +35,7 @@ public String getDockerImageName() throws ToolException { public static final String DOCKER_CLI_MSG = "Docker CLI: "; - public String getDockerImageVersion() throws ToolException { + public String getDockerImageVersion() { if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; } else { @@ -42,22 +43,13 @@ public String getDockerImageVersion() throws ToolException { } } - protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException { - for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) { - if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) { - return tool; - } - } - throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); - } - public String getDockerImageName(String toolId, String version) throws ToolException { - AnalysisTool tool = getAnalysisTool(toolId, version); + AnalysisTool tool = ConfigurationUtils.getAnalysisTool(toolId, version, getConfiguration()); return tool.getDockerId().split(":")[0]; } public String getDockerImageVersion(String toolId, String version) throws ToolException { - AnalysisTool tool = getAnalysisTool(toolId, version); + AnalysisTool tool = ConfigurationUtils.getAnalysisTool(toolId, version, getConfiguration()); if (tool.getDockerId().contains(":")) { return tool.getDockerId().split(":")[1]; } else { @@ -65,16 +57,6 @@ public String getDockerImageVersion(String toolId, String version) throws ToolEx } } - protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException { - // Get resources from the configuration file - AnalysisTool tool = getAnalysisTool(toolId, version); - if (!tool.getResources().containsKey(resourceKey)) { - throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = " - + version + "): it does not exist in the configuration file"); - } - return tool.getResources().get(resourceKey); - } - private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); public String getShortPrefix() { @@ -127,14 +109,11 @@ protected Map appendMounts(List> inputFilen } protected void appendCommand(String command, StringBuilder sb) throws ToolException { - // Docker image and version - sb.append(getDockerImageName()); - if (StringUtils.isNotEmpty(getDockerImageVersion())) { - sb.append(":").append(getDockerImageVersion()); - } + appendDockerAndCommand(command, getDockerImageName(), getDockerImageVersion(), sb); + } - // Append command - sb.append(" ").append(command); + protected void appendCommand(String command, String toolId, String version, StringBuilder sb) throws ToolException { + appendDockerAndCommand(command, getDockerImageName(toolId, version), getDockerImageVersion(toolId, version), sb); } protected void appendInputFiles(List> inputFilenames, Map srcTargetMap, StringBuilder sb) { @@ -268,4 +247,16 @@ protected static boolean skipParameter(String param) { } return false; } + + private void appendDockerAndCommand(String command, String dockerImage, String dockerImageVersion, StringBuilder sb) { + // Docker image and version + sb.append(dockerImage); + if (StringUtils.isNotEmpty(dockerImageVersion)) { + sb.append(":").append(dockerImageVersion); + } + + // Append command + sb.append(" ").append(command); + + } } \ No newline at end of file diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserAnalysisUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserAnalysisUtils.java new file mode 100644 index 00000000000..a7f3ab9a2c1 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserAnalysisUtils.java @@ -0,0 +1,389 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.analysis.wrappers.exomiser; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.clinical.Disorder; +import org.opencb.biodata.models.clinical.Phenotype; +import org.opencb.biodata.models.clinical.pedigree.Member; +import org.opencb.biodata.models.clinical.pedigree.Pedigree; +import org.opencb.biodata.models.pedigree.IndividualProperty; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.ConfigurationUtils; +import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.ResourceException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.managers.FamilyManager; +import org.opencb.opencga.catalog.utils.ResourceManager; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; +import org.opencb.opencga.core.models.family.Family; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +import static org.opencb.commons.utils.FileUtils.copyFile; +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor.*; + + +public class ExomiserAnalysisUtils { + // It must match the resources key in the exomiser/tool section in the configuration file + public static final String HG19_RESOURCE_KEY = "HG19"; + public static final String HG38_RESOURCE_KEY = "HG38"; + public static final String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + + private static Logger logger = LoggerFactory.getLogger(ExomiserAnalysisUtils.class); + + public static void prepareResources(String sampleId, String studyId, String clinicalAnalysisType, String exomiserVersion, + CatalogManager catalogManager, String token, Path outDir, Path openCgaHome) throws ToolException { + // Check HPOs, it will use a set to avoid duplicate HPOs, + // and it will check both phenotypes and disorders + logger.info("Checking individual for sample {} in study {}", sampleId, studyId); + Set hpos = new HashSet<>(); + Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, catalogManager, token); + + // Set father and mother if necessary (family ?) + if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { + individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), catalogManager, token)); + } + if (individual.getMother() != null && StringUtils.isNotEmpty(individual.getMother().getId())) { + individual.setMother(IndividualQcUtils.getIndividualById(studyId, individual.getMother().getId(), catalogManager, token)); + } + + logger.info("Individual found: {}", individual.getId()); + if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { + for (Phenotype phenotype : individual.getPhenotypes()) { + if (phenotype.getId().startsWith("HP:")) { + hpos.add(phenotype.getId()); + } + } + } + if (CollectionUtils.isNotEmpty(individual.getDisorders())) { + for (Disorder disorder : individual.getDisorders()) { + if (disorder.getId().startsWith("HP:")) { + hpos.add(disorder.getId()); + } + } + } + + if (CollectionUtils.isEmpty(hpos)) { + throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId + + ")"); + } + logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); + + List samples = new ArrayList<>(); + samples.add(sampleId); + + // Check multi-sample (family) analysis + Pedigree pedigree = null; + if (ClinicalAnalysis.Type.FAMILY.name().equals(clinicalAnalysisType)) { + if (individual.getMother() != null && individual.getMother().getId() != null + && individual.getFather() != null && individual.getFather().getId() != null) { + Family family = IndividualQcUtils.getFamilyByIndividualId(studyId, individual.getId(), catalogManager, token); + if (family != null) { + pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId()); + } + + if (pedigree != null) { + if (individual.getFather() != null) { + samples.add(individual.getFather().getSamples().get(0).getId()); + } + if (individual.getMother() != null) { + samples.add(individual.getMother().getSamples().get(0).getId()); + } + // Create the Exomiser pedigree file + createPedigreeFile(family, pedigree, outDir); + } + } + } + // Create the Exomiser sample file + createSampleFile(sampleId, individual, hpos, pedigree, outDir); + + // Copy the analysis template file + Path srcPath = Paths.get(openCgaHome.toAbsolutePath().toString(), ResourceManager.ANALYSIS_DIRNAME, ExomiserWrapperAnalysis.ID, + exomiserVersion, EXOMISER_ANALYSIS_TEMPLATE_FILENAME); + Path destPath = outDir.resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME); + try { + copyFile(srcPath.toFile(), destPath.toFile()); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser analysis file '" + srcPath + "' to '" + destPath + "'", e); + } + + // Copy the application.properties and update data according to Exomiser version + srcPath = Paths.get(openCgaHome.toAbsolutePath().toString(), ResourceManager.ANALYSIS_DIRNAME, ExomiserWrapperAnalysis.ID, + exomiserVersion, EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + destPath = outDir.resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + try { + copyFile(srcPath.toFile(), destPath.toFile()); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser properties file '" + srcPath + "' to '" + destPath + "'", e); + } + + // Copy the output options + srcPath = Paths.get(openCgaHome.toAbsolutePath().toString(), ResourceManager.ANALYSIS_DIRNAME, ExomiserWrapperAnalysis.ID, + exomiserVersion, EXOMISER_OUTPUT_OPTIONS_FILENAME); + destPath = outDir.resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME); + try { + copyFile(srcPath.toFile(), destPath.toFile()); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser output options file '" + srcPath + "' to '" + destPath + "'", e); + } + } + + public static void exportVariants(String sampleId, String studyId, String clinicalAnalysisType, Path outDir, + VariantStorageManager variantStorageManager, String token) throws ToolException { + CatalogManager catalogManager = variantStorageManager.getCatalogManager(); + Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, catalogManager, token); + + // Set father and mother if necessary (family ?) + if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { + individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), catalogManager, token)); + } + if (individual.getMother() != null && StringUtils.isNotEmpty(individual.getMother().getId())) { + individual.setMother(IndividualQcUtils.getIndividualById(studyId, individual.getMother().getId(), catalogManager, token)); + } + + List samples = new ArrayList<>(); + samples.add(sampleId); + + // Check multi-sample (family) analysis + if (ClinicalAnalysis.Type.FAMILY.name().equals(clinicalAnalysisType)) { + if (individual.getMother() != null && individual.getMother().getId() != null + && individual.getFather() != null && individual.getFather().getId() != null) { + Pedigree pedigree = null; + Family family = IndividualQcUtils.getFamilyByIndividualId(studyId, individual.getId(), catalogManager, token); + if (family != null) { + pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId()); + } + + if (pedigree != null) { + if (individual.getFather() != null) { + samples.add(individual.getFather().getSamples().get(0).getId()); + } + if (individual.getMother() != null) { + samples.add(individual.getMother().getSamples().get(0).getId()); + } + } + } + } + + // Export data into VCF file + Path vcfPath = getVcfPath(sampleId, outDir); + + VariantQuery query = new VariantQuery() + .study(studyId) + .sample(sampleId) + .includeSample(samples) + .includeSampleData("GT") + .unknownGenotype("./.") + .append("includeAllFromSampleIndex", true); + + QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); + + logger.info("Exomiser exports variants using the query: {}", query.toJson()); + logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); + + try { + variantStorageManager.exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, + queryOptions, token); + } catch (StorageEngineException | CatalogException e) { + throw new ToolException(e); + } + } + + public static void checkResources(String inputVersion, String study, CatalogManager catalogManager, String token, Path openCgaHome) + throws ToolException, CatalogException, ResourceException, IOException { + // Check assembly + checkAssembly(study, catalogManager, token); + + // Check exomiser version + String exomiserVersion = inputVersion; + if (StringUtils.isEmpty(exomiserVersion)) { + // Missing exomiser version use the default one + exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, catalogManager.getConfiguration()); + } else { + exomiserVersion = inputVersion; + } + + // Check resources + ResourceManager resourceManager = new ResourceManager(openCgaHome); + resourceManager.checkResourcePaths(ExomiserWrapperAnalysis.ID, exomiserVersion); + } + + public static Path getSamplePath(String sampleId, Path outDir) { + return outDir.resolve(sampleId + ".yml"); + } + + public static Path getVcfPath(String sampleId, Path outDir) { + return outDir.resolve(sampleId + ".vcf.gz"); + } + + public static String checkAssembly(String studyId, CatalogManager catalogManager, String token) throws CatalogException, ToolException { + // Check assembly + String assembly = IndividualQcUtils.getAssembly(studyId, catalogManager, token); + if (assembly.equalsIgnoreCase("GRCh38") || assembly.equalsIgnoreCase("HG38")) { + return "hg38"; +// } else if (assembly.equalsIgnoreCase("GRCh37") || assembly.equalsIgnoreCase("HG19")) { +// return "hg19"; + } + throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38 and HG38"); +// throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38, GRCh37, HG38 and HG19"); + } + + private static File createPedigreeFile(Family family, Pedigree pedigree, Path outDir) throws ToolException { + Map fromIndividualToSample = new HashMap<>(); + for (Individual member : family.getMembers()) { + if (CollectionUtils.isNotEmpty(member.getSamples())) { + fromIndividualToSample.put(member.getId(), member.getSamples().get(0).getId()); + } + } + + File pedigreeFile = outDir.resolve(fromIndividualToSample.get(pedigree.getProband().getId() + ".ped")).toFile(); + try { + PrintWriter pw = new PrintWriter(pedigreeFile); + + String probandId = fromIndividualToSample.get(pedigree.getProband().getId()); + + String fatherId = "0"; + if (pedigree.getProband().getFather() != null) { + fatherId = fromIndividualToSample.get(pedigree.getProband().getFather().getId()); + } + + String motherId = "0"; + if (pedigree.getProband().getMother() != null) { + motherId = fromIndividualToSample.get(pedigree.getProband().getMother().getId()); + } + + // Proband + pw.write(family.getId() + + "\t" + probandId + + "\t" + fatherId + + "\t" + motherId + + "\t" + getPedigreeSex(pedigree.getProband()) + + "\t2\n"); + + // Father + if (!fatherId.equals("0")) { + pw.write(family.getId() + "\t" + fatherId + "\t0\t0\t1\t0\n"); + } + + // Mother + if (!motherId.equals("0")) { + pw.write(family.getId() + "\t" + motherId + "\t0\t0\t2\t0\n"); + } + + // Close file + pw.close(); + } catch (IOException e) { + throw new ToolException("Error writing Exomiser pedigree file", e); + } + + return pedigreeFile; + } + + private static int getPedigreeSex(Member member) { + if (member.getSex() != null) { + if (member.getSex().getSex() == IndividualProperty.Sex.MALE) { + return 1; + } if (member.getSex().getSex() == IndividualProperty.Sex.FEMALE) { + return 2; + } + } + return 0; + } + + private static File createSampleFile(String sampleId, Individual individual, Set hpos, Pedigree pedigree, Path outDir) + throws ToolException { + File sampleFile = getSamplePath(sampleId, outDir).toFile(); + try { + PrintWriter pw = new PrintWriter(sampleFile); + pw.write("# a v1 phenopacket describing an individual https://phenopacket-schema.readthedocs.io/en/1.0.0/phenopacket.html\n"); + pw.write("---\n"); + pw.write("id: " + sampleId + "\n"); + String prefix = ""; + if (pedigree != null) { + prefix = " "; + pw.write("proband:\n"); + } + pw.write(prefix + "subject:\n"); + pw.write(prefix + " id: " + sampleId + "\n"); + if (individual.getSex() != null) { + pw.write(prefix + " sex: " + individual.getSex().getSex().name() + "\n"); + } + pw.write(prefix + "phenotypicFeatures:\n"); + for (String hpo : hpos) { + pw.write(prefix + " - type:\n"); + pw.write(prefix + " id: " + hpo + "\n"); + } + if (pedigree != null) { + pw.write("pedigree:\n"); + pw.write(" persons:\n"); + + // Proband + pw.write(" - individualId: " + sampleId + "\n"); + if (pedigree.getProband().getFather() != null) { + pw.write(" paternalId: " + individual.getFather().getSamples().get(0).getId() + "\n"); + } + if (pedigree.getProband().getMother() != null) { + pw.write(" maternalId: " + individual.getMother().getSamples().get(0).getId() + "\n"); + } + if (pedigree.getProband().getSex() != null) { + pw.write(" sex: " + pedigree.getProband().getSex().getSex().name() + "\n"); + } + pw.write(" affectedStatus: AFFECTED\n"); + + // Father + if (pedigree.getProband().getFather() != null) { + pw.write(" - individualId: " + individual.getFather().getSamples().get(0).getId() + "\n"); + if (pedigree.getProband().getFather().getSex() != null) { + pw.write(" sex: " + pedigree.getProband().getFather().getSex().getSex().name() + "\n"); + } +// pw.write(" - affectedStatus:" + AffectationStatus + "\n"); + } + + // Mother + if (pedigree.getProband().getMother() != null) { + pw.write(" - individualId: " + individual.getMother().getSamples().get(0).getId() + "\n"); + if (pedigree.getProband().getMother().getSex() != null) { + pw.write(" sex: " + pedigree.getProband().getMother().getSex().getSex().name() + "\n"); + } +// pw.write(" - affectedStatus:" + AffectationStatus + "\n"); + } + } + + // Close file + pw.close(); + } catch (IOException e) { + throw new ToolException("Error writing Exomiser sample file", e); + } + return sampleFile; + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index a6d34af3f37..d6b865c5707 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -19,40 +19,47 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import java.io.File; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + @Tool(id = ExomiserWrapperAnalysis.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, description = ExomiserWrapperAnalysis.DESCRIPTION) public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { - public final static String ID = "exomiser"; - public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + public static final String ID = "exomiser"; + public static final String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; // It must match the tool prefix in the tool keys for exomiser in the configuration file - public final static String EXOMISER_PREFIX = "exomiser-"; + public static final String EXOMISER_PREFIX = "exomiser-"; - // It must match the resources key in the exomiser/tool section in the configuration file - public final static String HG19_RESOURCE_KEY = "HG19"; - public final static String HG38_RESOURCE_KEY = "HG38"; - public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + private static final String PREPARE_RESOURCES_STEP = "prepare-resources"; + private static final String EXPORT_VARIANTS_STEP = "export-variants"; @ToolParams protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams(); + @Override protected void check() throws Exception { super.check(); - if (StringUtils.isEmpty(getStudy())) { + if (StringUtils.isEmpty(study)) { throw new ToolException("Missing study"); } + setUpStorageEngineExecutor(study); + // Check exomiser version if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) { // Missing exomiser version use the default one @@ -60,19 +67,50 @@ protected void check() throws Exception { logger.warn("Missing exomiser version, using the default {}", exomiserVersion); analysisParams.setExomiserVersion(exomiserVersion); } + + ExomiserAnalysisUtils.checkResources(analysisParams.getExomiserVersion(), study, catalogManager, token, getOpencgaHome()); + } + + @Override + protected List getSteps() { + return Arrays.asList(PREPARE_RESOURCES_STEP, EXPORT_VARIANTS_STEP, ExomiserWrapperAnalysis.ID); } @Override protected void run() throws Exception { - setUpStorageEngineExecutor(study); + // Main steps + step(PREPARE_RESOURCES_STEP, this::prepareResources); + step(EXPORT_VARIANTS_STEP, this::exportVariants); + step(ID, this::runExomiser); + } + + private void prepareResources() throws ToolException { + ExomiserAnalysisUtils.prepareResources(analysisParams.getSample(), study, analysisParams.getClinicalAnalysisType(), + analysisParams.getExomiserVersion(), catalogManager, token, getOutDir(), getOpencgaHome()); + } + + private void exportVariants() throws ToolException { + ExomiserAnalysisUtils.exportVariants(analysisParams.getSample(), study, analysisParams.getClinicalAnalysisType(), getOutDir(), + variantStorageManager, token); + } + + private void runExomiser() throws ToolException, CatalogException { + ExomiserWrapperAnalysisExecutor executor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + .setExomiserVersion(analysisParams.getExomiserVersion()) + .setAssembly(ExomiserAnalysisUtils.checkAssembly(study, catalogManager, token)) + .setExomiserDataPath(Paths.get(getOpencgaHome().toAbsolutePath().toAbsolutePath().toString(), + ResourceManager.ANALYSIS_DIRNAME, ResourceManager.RESOURCES_DIRNAME, ExomiserWrapperAnalysis.ID, + analysisParams.getExomiserVersion())) + .setSampleFile(ExomiserAnalysisUtils.getSamplePath(analysisParams.getSample(), getOutDir())) + .setVcfFile(ExomiserAnalysisUtils.getVcfPath(analysisParams.getSample(), getOutDir())); + + for (File file : getOutDir().toFile().listFiles()) { + if (file.getName().endsWith(".ped")) { + executor.setPedigreeFile(file.toPath()); + break; + } + } - step(() -> { - getToolExecutor(ExomiserWrapperAnalysisExecutor.class) - .setStudyId(study) - .setSampleId(analysisParams.getSample()) - .setExomiserVersion(analysisParams.getExomiserVersion()) - .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType())) - .execute(); - }); + executor.execute(); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 2d0d0dfb0ab..e688baed751 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -1,42 +1,17 @@ package org.opencb.opencga.analysis.wrappers.exomiser; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.clinical.Phenotype; -import org.opencb.biodata.models.clinical.pedigree.Member; -import org.opencb.biodata.models.clinical.pedigree.Pedigree; -import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.exec.Command; import org.opencb.commons.utils.FileUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; -import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.FamilyManager; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.exceptions.ToolExecutorException; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.tools.annotations.ToolExecutor; -import org.opencb.opencga.storage.core.exceptions.StorageEngineException; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; -import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.net.URL; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; - -import static org.opencb.commons.utils.FileUtils.copyFile; -import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, tool = ExomiserWrapperAnalysis.ID, @@ -44,150 +19,23 @@ framework = ToolExecutor.Framework.LOCAL) public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecutor implements StorageToolExecutor { - public final static String ID = ExomiserWrapperAnalysis.ID + "-local"; + public static final String ID = ExomiserWrapperAnalysis.ID + "-local"; - private final static String EXOMISER_ANALYSIS_TEMPLATE_FILENAME = "exomiser-analysis.yml"; - private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; - private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; + public static final String EXOMISER_ANALYSIS_TEMPLATE_FILENAME = "exomiser-analysis.yml"; + public static final String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; + public static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - private String studyId; - private String sampleId; + private Path exomiserDataPath; + private Path sampleFile; + private Path pedigreeFile; + private Path vcfFile; + private String assembly; private String exomiserVersion; - private ClinicalAnalysis.Type clinicalAnalysisType; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override public void run() throws ToolException, IOException, CatalogException { - // Check HPOs, it will use a set to avoid duplicate HPOs, - // and it will check both phenotypes and disorders - logger.info("Checking individual for sample {} in study {}", sampleId, studyId); - Set hpos = new HashSet<>(); - Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), - getToken()); - - // Check assembly - String assembly = IndividualQcUtils.getAssembly(studyId, getVariantStorageManager().getCatalogManager(), getToken()); - if (assembly.equalsIgnoreCase("GRCh38")) { - assembly = "hg38"; -// } else if (assembly.equalsIgnoreCase("GRCh37")) { -// assembly = "hg19"; - } else { - throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38"); - } - - // Set father and mother if necessary (family ?) - if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { - individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), - getVariantStorageManager().getCatalogManager(), getToken())); - } - if (individual.getMother() != null && StringUtils.isNotEmpty(individual.getMother().getId())) { - individual.setMother(IndividualQcUtils.getIndividualById(studyId, individual.getMother().getId(), - getVariantStorageManager().getCatalogManager(), getToken())); - } - - logger.info("Individual found: {}", individual.getId()); - if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { - for (Phenotype phenotype : individual.getPhenotypes()) { - if (phenotype.getId().startsWith("HP:")) { - hpos.add(phenotype.getId()); - } - } - } - if (CollectionUtils.isNotEmpty(individual.getDisorders())) { - for (Disorder disorder : individual.getDisorders()) { - if (disorder.getId().startsWith("HP:")) { - hpos.add(disorder.getId()); - } - } - } - - if (CollectionUtils.isEmpty(hpos)) { - throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId - + ")"); - } - logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); - - List samples = new ArrayList<>(); - samples.add(sampleId); - - // Check multi-sample (family) analysis - File pedigreeFile = null; - Pedigree pedigree = null; - if (clinicalAnalysisType == ClinicalAnalysis.Type.FAMILY) { - if (individual.getMother() != null && individual.getMother().getId() != null - && individual.getFather() != null && individual.getFather().getId() != null) { - Family family = IndividualQcUtils.getFamilyByIndividualId(getStudyId(), individual.getId(), - getVariantStorageManager().getCatalogManager(), getToken()); - if (family != null) { - pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId()); - } - - if (pedigree != null) { - if (individual.getFather() != null) { - samples.add(individual.getFather().getSamples().get(0).getId()); - } - if (individual.getMother() != null) { - samples.add(individual.getMother().getSamples().get(0).getId()); - } - pedigreeFile = createPedigreeFile(family, pedigree); - } - } - } - File sampleFile = createSampleFile(individual, hpos, pedigree); - - // Export data into VCF file - Path vcfPath = getOutDir().resolve(sampleId + ".vcf.gz"); - - VariantQuery query = new VariantQuery() - .study(studyId) - .sample(sampleId) - .includeSample(samples) - .includeSampleData("GT") - .unknownGenotype("./.") - .append("includeAllFromSampleIndex", true); - - QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); - - logger.info("Exomiser exports variants using the query: {}", query.toJson()); - logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); - - try { - getVariantStorageManager().exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, - queryOptions, getToken()); - } catch (StorageEngineException | CatalogException e) { - throw new ToolException(e); - } - - // Get Exomiser (external) data - Path openCgaHome = getOpencgaHomePath(); - Path exomiserDataPath = getAnalysisDataPath(ExomiserWrapperAnalysis.ID); - - // Copy the analysis - try { - copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME) - .toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); - } catch (IOException e) { - throw new ToolException("Error copying Exomiser analysis file", e); - } - - // Copy the application.properties and update data according to Exomiser version - try { - Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); - copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME) - .toFile(), target.toFile()); - } catch (IOException e) { - throw new ToolException("Error copying Exomiser properties file", e); - } - - // Copy the output options - try { - copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); - } catch (IOException e) { - throw new ToolException("Error copying Exomiser output options file", e); - } - // Build the docker command line to run Exomiser String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); String dockerUser = userAndGroup[0] + ":" + userAndGroup[1]; @@ -195,19 +43,19 @@ public void run() throws ToolException, IOException, CatalogException { StringBuilder sb = initCommandLine(dockerUser); // Append mounts - sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") + sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data,readonly") .append(" --mount type=bind,source=" + getOutDir() + ",target=/jobdir "); // Append docker image, version and command - appendCommand("", sb); + appendCommand("", ExomiserWrapperAnalysis.ID, exomiserVersion, sb); // Append input file params sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME); - sb.append(" --sample /jobdir/").append(sampleFile.getName()); - if (pedigreeFile != null && pedigreeFile.exists()) { - sb.append(" --ped /jobdir/").append(pedigreeFile.getName()); + sb.append(" --sample /jobdir/").append(sampleFile.getFileName()); + if (pedigreeFile != null && Files.exists(pedigreeFile)) { + sb.append(" --ped /jobdir/").append(pedigreeFile.getFileName()); } - sb.append(" --vcf /jobdir/" + vcfPath.getFileName()) + sb.append(" --vcf /jobdir/" + vcfFile.getFileName()) .append(" --assembly ").append(assembly) .append(" --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); @@ -219,323 +67,48 @@ public void run() throws ToolException, IOException, CatalogException { runCommandLine(sb.toString()); } - private File createSampleFile(Individual individual, Set hpos, Pedigree pedigree) throws ToolException { - File sampleFile = getOutDir().resolve(sampleId + ".yml").toFile(); - try { - PrintWriter pw = new PrintWriter(sampleFile); - pw.write("# a v1 phenopacket describing an individual https://phenopacket-schema.readthedocs.io/en/1.0.0/phenopacket.html\n"); - pw.write("---\n"); - pw.write("id: " + sampleId + "\n"); - String prefix = ""; - if (pedigree != null) { - prefix = " "; - pw.write("proband:\n"); - } - pw.write(prefix + "subject:\n"); - pw.write(prefix + " id: " + sampleId + "\n"); - if (individual.getSex() != null) { - pw.write(prefix + " sex: " + individual.getSex().getSex().name() + "\n"); - } - pw.write(prefix + "phenotypicFeatures:\n"); - for (String hpo : hpos) { - pw.write(prefix + " - type:\n"); - pw.write(prefix + " id: " + hpo + "\n"); - } - if (pedigree != null) { - pw.write("pedigree:\n"); - pw.write(" persons:\n"); - - // Proband - pw.write(" - individualId: " + sampleId + "\n"); - if (pedigree.getProband().getFather() != null) { - pw.write(" paternalId: " + individual.getFather().getSamples().get(0).getId() + "\n"); - } - if (pedigree.getProband().getMother() != null) { - pw.write(" maternalId: " + individual.getMother().getSamples().get(0).getId() + "\n"); - } - if (pedigree.getProband().getSex() != null) { - pw.write(" sex: " + pedigree.getProband().getSex().getSex().name() + "\n"); - } - pw.write(" affectedStatus: AFFECTED\n"); - - // Father - if (pedigree.getProband().getFather() != null) { - pw.write(" - individualId: " + individual.getFather().getSamples().get(0).getId() + "\n"); - if (pedigree.getProband().getFather().getSex() != null) { - pw.write(" sex: " + pedigree.getProband().getFather().getSex().getSex().name() + "\n"); - } -// pw.write(" - affectedStatus:" + AffectationStatus + "\n"); - } - - // Mother - if (pedigree.getProband().getMother() != null) { - pw.write(" - individualId: " + individual.getMother().getSamples().get(0).getId() + "\n"); - if (pedigree.getProband().getMother().getSex() != null) { - pw.write(" sex: " + pedigree.getProband().getMother().getSex().getSex().name() + "\n"); - } -// pw.write(" - affectedStatus:" + AffectationStatus + "\n"); - } - } - - // Close file - pw.close(); - } catch (IOException e) { - throw new ToolException("Error writing Exomiser sample file", e); - } - return sampleFile; - } - - private File createPedigreeFile(Family family, Pedigree pedigree) throws ToolException { - Map fromIndividualToSample = new HashMap<>(); - for (Individual member : family.getMembers()) { - if (CollectionUtils.isNotEmpty(member.getSamples())) { - fromIndividualToSample.put(member.getId(), member.getSamples().get(0).getId()); - } - } - - File pedigreeFile = getOutDir().resolve(fromIndividualToSample.get(pedigree.getProband().getId()) + ".ped").toFile(); - try { - PrintWriter pw = new PrintWriter(pedigreeFile); - - String probandId = fromIndividualToSample.get(pedigree.getProband().getId()); - - String fatherId = "0"; - if (pedigree.getProband().getFather() != null) { - fatherId = fromIndividualToSample.get(pedigree.getProband().getFather().getId()); - } - - String motherId = "0"; - if (pedigree.getProband().getMother() != null) { - motherId = fromIndividualToSample.get(pedigree.getProband().getMother().getId()); - } - - // Proband - pw.write(family.getId() - + "\t" + probandId - + "\t" + fatherId - + "\t" + motherId - + "\t" + getPedigreeSex(pedigree.getProband()) - + "\t2\n"); - - // Father - if (!fatherId.equals("0")) { - pw.write(family.getId() + "\t" + fatherId + "\t0\t0\t1\t0\n"); - } - - // Mother - if (!motherId.equals("0")) { - pw.write(family.getId() + "\t" + motherId + "\t0\t0\t2\t0\n"); - } - - // Close file - pw.close(); - } catch (IOException e) { - throw new ToolException("Error writing Exomiser pedigree file", e); - } - - return pedigreeFile; - } - - private String getPedigreePaternalId(Member member) { - if (member.getFather() != null) { - return member.getFather().getId(); - } - return "0"; - } - - private String getPedigreeMaternalId(Member member) { - if (member.getMother() != null) { - return member.getMother().getId(); - } - return "0"; - } - - private int getPedigreeSex(Member member) { - if (member.getSex() != null) { - if (member.getSex().getSex() == IndividualProperty.Sex.MALE) { - return 1; - } if (member.getSex().getSex() == IndividualProperty.Sex.FEMALE) { - return 2; - } - } - return 0; - } - - private Path getOpencgaHomePath() throws ToolExecutorException { - String value = getExecutorParams().getString("opencgaHome"); - if (StringUtils.isEmpty(value)) { - throw new ToolExecutorException("Missing OpenCGA home in executor parameters."); - } - - Path path = Paths.get(value); - if (!path.toFile().exists()) { - throw new ToolExecutorException("OpenCGA home path (" + value + ") does not exist."); - } - return path; - } - - private Path getAnalysisDataPath(String analysisId) throws ToolException { - if (ExomiserWrapperAnalysis.ID.equals(analysisId)) { - return getExomiserDataPath(getOpencgaHomePath()); - } - return getOpencgaHomePath().resolve("analysis/resources"); - } - - private Path getExomiserDataPath(Path openCgaHome) throws ToolException { - Path exomiserDataPath = openCgaHome.resolve("analysis/resources/" + ExomiserWrapperAnalysis.ID); - if (!exomiserDataPath.toFile().exists()) { - if (!exomiserDataPath.toFile().mkdirs()) { - throw new ToolException("Error creating the Exomiser data directory"); - } - } - - // Mutex management to avoid multiple downloadings at the same time - // the first to come, download data, others have to wait for - String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); - String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; - File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); - - // If all is ready, then return - if (readyFile.exists()) { - logger.info("Exomiser {} data {} is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion, - resourceVersion); - return exomiserDataPath; - } - - // If it is preparing, then wait for ready and then return - if (preparingFile.exists()) { - long startTime = System.currentTimeMillis(); - logger.info("Exomiser {} data {} is downloading, waiting for it...", exomiserVersion, resourceVersion); - while (!readyFile.exists()) { - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - // Nothing to do here - preparingFile.delete(); - throw new ToolException(e); - } - long elapsedTime = System.currentTimeMillis() - startTime; - if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data " - + resourceVersion + " is not ready yet: maximum waiting time exceeded"); - } - } - logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); - return exomiserDataPath; - } - - // Mark as preparing - try { - preparingFile.createNewFile(); - } catch (IOException e) { - preparingFile.delete(); - throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data " + resourceVersion + " directory"); - } - - // Download resources and unzip files - try { - downloadAndUnzip(exomiserDataPath, HG38_RESOURCE_KEY); - downloadAndUnzip(exomiserDataPath, PHENOTYPE_RESOURCE_KEY); - } catch (ToolException e) { - // If something wrong happened, the preparing file has to be deleted - preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); - } - - // Mutex management, signal exomiser data is ready - try { - readyFile.createNewFile(); - } catch (IOException e) { - throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); - } - preparingFile.delete(); - + public Path getExomiserDataPath() { return exomiserDataPath; } - private String getHg38DataVersion() throws ToolException { - String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); - return Paths.get(resource).getFileName().toString().split("_")[0]; - } - - private String getPhenotypeDataVersion() throws ToolException { - String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); - return Paths.get(resource).getFileName().toString().split("_")[0]; + public ExomiserWrapperAnalysisExecutor setExomiserDataPath(Path exomiserDataPath) { + this.exomiserDataPath = exomiserDataPath; + return this; } - @Override - public String getDockerImageName() throws ToolException { - return getDockerImageName(ExomiserWrapperAnalysis.ID, exomiserVersion); + public Path getSampleFile() { + return sampleFile; } - @Override - public String getDockerImageVersion() throws ToolException { - return getDockerImageVersion(ExomiserWrapperAnalysis.ID, exomiserVersion); + public ExomiserWrapperAnalysisExecutor setSampleFile(Path sampleFile) { + this.sampleFile = sampleFile; + return this; } - public String getStudyId() { - return studyId; + public Path getPedigreeFile() { + return pedigreeFile; } - public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { - this.studyId = studyId; + public ExomiserWrapperAnalysisExecutor setPedigreeFile(Path pedigreeFile) { + this.pedigreeFile = pedigreeFile; return this; } - private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { - String filename; - String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, resourceKey); - if (resource.startsWith("file://")) { - // Copy resouce - try { - Path sourcePath = Paths.get(resource); - filename = sourcePath.getFileName().toString(); - Files.copy(sourcePath, exomiserDataPath.resolve(filename)); - } catch (IOException e) { - throw new ToolException("Error copying Exomiser data from " + resource, e); - } - } else { - // Download resource - String url; - if (resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://")) { - url = resource; - } else { - url = getConfiguration().getAnalysis().getResourceUrl() + resource; - } - logger.info("Downloading Exomiser data: {} in {}", url, exomiserDataPath); - try { - ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); - filename = Paths.get(url).getFileName().toString(); - } catch (IOException e) { - throw new ToolException("Error downloading Exomiser data from " + url, e); - } - } - - // Unzip - try { - logger.info("Decompressing Exomiser {} data: {}", exomiserDataPath, filename); - new Command("unzip -o -d " + exomiserDataPath + " " + exomiserDataPath + "/" + filename) - .setOutputOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stdout_unzip_" - + filename + ".txt").toFile()))) - .setErrorOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stderr_unzip_" - + filename + ".txt").toFile()))) - .run(); - } catch (FileNotFoundException e) { - throw new ToolException("Error unzipping Exomiser " + exomiserVersion + " data: " + filename, e); - } + public Path getVcfFile() { + return vcfFile; + } - // Free disk space - logger.info("Deleting Exomiser data: {}", filename); - exomiserDataPath.resolve(filename).toFile().delete(); + public ExomiserWrapperAnalysisExecutor setVcfFile(Path vcfFile) { + this.vcfFile = vcfFile; + return this; } - public String getSampleId() { - return sampleId; + public String getAssembly() { + return assembly; } - public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { - this.sampleId = sampleId; + public ExomiserWrapperAnalysisExecutor setAssembly(String assembly) { + this.assembly = assembly; return this; } @@ -547,13 +120,4 @@ public ExomiserWrapperAnalysisExecutor setExomiserVersion(String exomiserVersion this.exomiserVersion = exomiserVersion; return this; } - - public ClinicalAnalysis.Type getClinicalAnalysisType() { - return clinicalAnalysisType; - } - - public ExomiserWrapperAnalysisExecutor setClinicalAnalysisType(ClinicalAnalysis.Type clinicalAnalysisType) { - this.clinicalAnalysisType = clinicalAnalysisType; - return this; - } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index a6fd99aa3bc..e8470e3337e 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -16,8 +16,10 @@ import org.opencb.opencga.analysis.clinical.ClinicalAnalysisUtilsTest; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.response.OpenCGAResult; @@ -33,11 +35,8 @@ import java.util.List; import static com.mongodb.assertions.Assertions.assertFalse; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeThat; -import static org.junit.Assume.assumeTrue; @Category(MediumTests.class) public class ExomiserInterpretationAnalysisTest { @@ -51,7 +50,7 @@ public class ExomiserInterpretationAnalysisTest { @BeforeClass public static void setUp() throws Exception { - opencga.clearStorageDB(); + opencga.clear(); clinicalTest = ClinicalAnalysisUtilsTest.getClinicalTest(opencga); } @@ -77,12 +76,11 @@ public void testNormalization() throws NonStandardCompliantSampleField { @Test public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { -// String exomiserVersion = "13.1"; +// String exomiserVersion = "13.1.0"; // String resourceVersion = "2109"; - String exomiserVersion = "14.0"; + String exomiserVersion = "14.0.0"; String resourceVersion = "2402"; - assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); - assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); + assertTrue(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ExomiserWrapperAnalysis.ID).resolve(exomiserVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -128,12 +126,11 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE @Test public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { -// String exomiserVersion = "13.1"; +// String exomiserVersion = "13.1.0"; // String resourceVersion = "2109"; - String exomiserVersion = "14.0"; + String exomiserVersion = "14.0.0"; String resourceVersion = "2402"; - assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); - assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); + assertTrue(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ExomiserWrapperAnalysis.ID).resolve(exomiserVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family")); @@ -178,12 +175,11 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE @Test public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { -// String exomiserVersion = "13.1"; +// String exomiserVersion = "13.1.0"; // String resourceVersion = "2109"; - String exomiserVersion = "14.0"; + String exomiserVersion = "14.0.0"; String resourceVersion = "2402"; - assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); - assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); + assertTrue(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ExomiserWrapperAnalysis.ID).resolve(exomiserVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single")); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index e3b69751771..b6f88ca490f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -269,7 +269,7 @@ public Path isolateOpenCGA() throws IOException { } // Exomiser analysis files - List exomiserVersions = Arrays.asList("13.1", "14.0"); + List exomiserVersions = Arrays.asList("13.1.0", "14.0.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); for (String exomiserVersion : exomiserVersions) { analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index f9e9392be80..2a87e0c022b 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -58,6 +58,7 @@ import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.ResourceManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.JacksonUtils; @@ -771,6 +772,8 @@ public void testSampleMultiVariantFilterAnalysis() throws Exception { @Test public void testMutationalSignatureFittingSNV() throws Exception { + Assume.assumeTrue(Files.exists(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ResourceManager.RESOURCES_DIRNAME).resolve(ResourceManager.REFERENCE_GENOMES))); + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_fitting_snv")); System.out.println("outDir = " + outDir); @@ -826,10 +829,12 @@ public void testMutationalSignatureFittingSNV() throws Exception { @Test public void testMutationalSignatureCatalogueSV() throws Exception { + Assume.assumeTrue(Files.exists(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ResourceManager.RESOURCES_DIRNAME).resolve(ResourceManager.REFERENCE_GENOMES))); + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_catalogue_sv")); System.out.println("outDir = " + outDir); - Path opencgaHome = opencga.getOpencgaHome(); + Path opencgaHome = opencga.getOpencgaHome().toAbsolutePath(); System.out.println("OpenCGA home = " + opencgaHome); MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); @@ -881,6 +886,8 @@ public void testMutationalSignatureCatalogueSV() throws Exception { @Test public void testMutationalSignatureFittingSV() throws Exception { + Assume.assumeTrue(Files.exists(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ResourceManager.RESOURCES_DIRNAME).resolve(ResourceManager.REFERENCE_GENOMES))); + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_fitting")); System.out.println("outDir = " + outDir); @@ -931,6 +938,8 @@ public void testMutationalSignatureFittingSV() throws Exception { @Test public void testHRDetect() throws Exception { + Assume.assumeTrue(Files.exists(opencga.getOpencgaHome().resolve(ResourceManager.ANALYSIS_DIRNAME).resolve(ResourceManager.RESOURCES_DIRNAME).resolve(ResourceManager.REFERENCE_GENOMES))); + Path snvFittingOutDir = Paths.get(opencga.createTmpOutdir("_snv_fitting")); Path svFittingOutDir = Paths.get(opencga.createTmpOutdir("_sv_fitting")); Path hrdetectOutDir = Paths.get(opencga.createTmpOutdir("_hrdetect")); diff --git a/opencga-app/app/analysis/exomiser/13.1/application.properties b/opencga-app/app/analysis/exomiser/13.1.0/application.properties similarity index 100% rename from opencga-app/app/analysis/exomiser/13.1/application.properties rename to opencga-app/app/analysis/exomiser/13.1.0/application.properties diff --git a/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/13.1.0/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml rename to opencga-app/app/analysis/exomiser/13.1.0/exomiser-analysis.yml diff --git a/opencga-app/app/analysis/exomiser/13.1/output.yml b/opencga-app/app/analysis/exomiser/13.1.0/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/13.1/output.yml rename to opencga-app/app/analysis/exomiser/13.1.0/output.yml diff --git a/opencga-app/app/analysis/exomiser/14.0/application.properties b/opencga-app/app/analysis/exomiser/14.0.0/application.properties similarity index 100% rename from opencga-app/app/analysis/exomiser/14.0/application.properties rename to opencga-app/app/analysis/exomiser/14.0.0/application.properties diff --git a/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/14.0.0/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml rename to opencga-app/app/analysis/exomiser/14.0.0/exomiser-analysis.yml diff --git a/opencga-app/app/analysis/exomiser/14.0/output.yml b/opencga-app/app/analysis/exomiser/14.0.0/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/14.0/output.yml rename to opencga-app/app/analysis/exomiser/14.0.0/output.yml diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 090802f3741..154d81e6696 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -138,7 +138,8 @@ private void status() throws CatalogException, JsonProcessingException { result.put("installed", true); } else { String oldDatabase = configuration.getDatabasePrefix() + "_catalog"; - MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory()); + MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory(), + catalogManager.getCatalogIOManager()); MongoDataStore oldDatastore = mongoDBAdaptorFactory.getMongoManager().get(oldDatabase, mongoDBAdaptorFactory.getMongoDbConfiguration()); try { if (oldDatastore.getCollectionNames().contains("metadata")) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java index 04bafdfaada..58b3c06eae9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java @@ -99,7 +99,8 @@ private void fetch() throws ToolException { Path outDir = Paths.get(options.outDir); - toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path), outDir, null, false, options.commonOptions.token); + toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path, options.resource), outDir, null, false, + options.commonOptions.token); } private void tsvLoad() throws ToolException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java index 4825ec26306..f556580c32e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java @@ -121,6 +121,9 @@ public class FetchCommandOptions { description = "Folder path where the downloaded file will be registered", required = true, arity = 1) public String path; + @Parameter(names = {"--resource"}, description = "File resource", arity = 1) + public Boolean resource; + @Parameter(names = {"--url"}, description = "External url where the file to be registered can be downloaded from", required = true, arity = 1) public String url; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index fa5b3482284..6b05986a98f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -54,7 +54,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List adminList = asList( "audit-group-by","catalog-install","catalog-jwt","users-create","users-import","users-permissions","users-search","users-sync","update-groups-users") + private List adminList = asList( "audit-group-by","catalog-install","catalog-jwt","resource-fetch","users-create","users-import","users-permissions","users-search","users-sync","update-groups-users") .stream() .map(Candidate::new) .collect(toList()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index cfd4cee9919..1cb35dd2d7f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -171,6 +171,7 @@ public OpencgaCliOptionsParser() { adminSubCommands.addCommand("audit-group-by", adminCommandOptions.groupByAuditCommandOptions); adminSubCommands.addCommand("catalog-install", adminCommandOptions.installCatalogCommandOptions); adminSubCommands.addCommand("catalog-jwt", adminCommandOptions.jwtCatalogCommandOptions); + adminSubCommands.addCommand("resource-fetch", adminCommandOptions.fetchResourceCommandOptions); adminSubCommands.addCommand("users-create", adminCommandOptions.createUsersCommandOptions); adminSubCommands.addCommand("users-import", adminCommandOptions.importUsersCommandOptions); adminSubCommands.addCommand("users-permissions", adminCommandOptions.permissionsUsersCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandOptions.java index 5bcd4ac27f5..14489dfe271 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandOptions.java @@ -72,6 +72,8 @@ public class UploadCommandOptions { + "removed", arity = 0) public boolean replace; + @Parameter(names = {"--resource"}, description = "File resource", arity = 1) + public Boolean resource; // @Parameter(names = {"-ch", "--checksum"}, description = "[PENDING] Calculate checksum", arity = 0) // public boolean checksum; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index 99135ede6b0..22b2bd9dc46 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -21,6 +21,8 @@ import org.opencb.opencga.core.models.admin.UserImportParams; import org.opencb.opencga.core.models.admin.UserUpdateGroup; import org.opencb.opencga.core.models.common.Enums.Resource; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.resource.ResourceFetcherToolParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.User; @@ -71,6 +73,9 @@ public void execute() throws Exception { case "catalog-jwt": queryResponse = jwtCatalog(); break; + case "resource-fetch": + queryResponse = fetchResource(); + break; case "users-create": queryResponse = createUsers(); break; @@ -170,6 +175,41 @@ private RestResponse jwtCatalog() throws Exception { return openCGAClient.getAdminClient().jwtCatalog(jWTParams, queryParams); } + private RestResponse fetchResource() throws Exception { + logger.debug("Executing fetchResource in Admin command line"); + + AdminCommandOptions.FetchResourceCommandOptions commandOptions = adminCommandOptions.fetchResourceCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + + + ResourceFetcherToolParams resourceFetcherToolParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/admin/resource/fetch")); + return res; + } else if (commandOptions.jsonFile != null) { + resourceFetcherToolParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ResourceFetcherToolParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "resources", commandOptions.resources, true); + + resourceFetcherToolParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), ResourceFetcherToolParams.class); + } + return openCGAClient.getAdminClient().fetchResource(resourceFetcherToolParams, queryParams); + } + private RestResponse createUsers() throws Exception { logger.debug("Executing createUsers in Admin command line"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index 873a16baac3..4284400b1be 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -287,6 +287,7 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "resource", commandOptions.resource, true); putNestedIfNotEmpty(beanParams, "jobId", commandOptions.jobId, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); @@ -318,6 +319,7 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("bioformat", commandOptions.bioformat); queryParams.putIfNotEmpty("format", commandOptions.format); queryParams.putIfNotNull("external", commandOptions.external); + queryParams.putIfNotNull("resource", commandOptions.resource); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("internalVariantIndexStatus", commandOptions.internalVariantIndexStatus); @@ -372,6 +374,7 @@ private RestResponse fetch() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotNull(beanParams, "resource", commandOptions.resource, true); putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileFetch = JacksonUtils.getDefaultObjectMapper().copy() @@ -536,6 +539,7 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("bioformat", commandOptions.bioformat); queryParams.putIfNotEmpty("format", commandOptions.format); queryParams.putIfNotNull("external", commandOptions.external); + queryParams.putIfNotNull("resource", commandOptions.resource); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("internalVariantIndexStatus", commandOptions.internalVariantIndexStatus); @@ -569,6 +573,7 @@ private RestResponse upload() throws Exception { queryParams.putIfNotNull("fileFormat", commandOptions.fileFormat); queryParams.putIfNotNull("bioformat", commandOptions.bioformat); queryParams.putIfNotEmpty("checksum", commandOptions.checksum); + queryParams.putIfNotNull("resource", commandOptions.resource); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("relativeFilePath", commandOptions.relativeFilePath); queryParams.putIfNotEmpty("description", commandOptions.description); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java index 60684639ca0..d10e85d0798 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java @@ -36,6 +36,7 @@ public class AdminCommandOptions { public GroupByAuditCommandOptions groupByAuditCommandOptions; public InstallCatalogCommandOptions installCatalogCommandOptions; public JwtCatalogCommandOptions jwtCatalogCommandOptions; + public FetchResourceCommandOptions fetchResourceCommandOptions; public CreateUsersCommandOptions createUsersCommandOptions; public ImportUsersCommandOptions importUsersCommandOptions; public PermissionsUsersCommandOptions permissionsUsersCommandOptions; @@ -51,6 +52,7 @@ public AdminCommandOptions(CommonCommandOptions commonCommandOptions, JCommander this.groupByAuditCommandOptions = new GroupByAuditCommandOptions(); this.installCatalogCommandOptions = new InstallCatalogCommandOptions(); this.jwtCatalogCommandOptions = new JwtCatalogCommandOptions(); + this.fetchResourceCommandOptions = new FetchResourceCommandOptions(); this.createUsersCommandOptions = new CreateUsersCommandOptions(); this.importUsersCommandOptions = new ImportUsersCommandOptions(); this.permissionsUsersCommandOptions = new PermissionsUsersCommandOptions(); @@ -135,6 +137,44 @@ public class JwtCatalogCommandOptions { } + @Parameters(commandNames = {"resource-fetch"}, commandDescription ="Fetch resources from the public server and save them into the OpenCGA local installation") + public class FetchResourceCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--resources"}, description = "List of resource IDs, separated by commas, to fetch (available resources are specified in the configuration file). The wildcard '*' can be used: for example, use '*' to fetch all resources, or 'EXOMISER_*' to fetch only Exomiser resources.", required = false, arity = 1) + public String resources; + + } + @Parameters(commandNames = {"users-create"}, commandDescription ="Create a new user") public class CreateUsersCommandOptions { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index b9dc5e1956a..2dd936e91be 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -679,10 +679,10 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) public Boolean jobDryRun; - @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID to be analysed.", required = false, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version, e.g.: 14.0.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) public String exomiserVersion; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 44a0ad64dd7..f443340fd61 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -428,10 +428,10 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) public Boolean jobDryRun; - @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) + @Parameter(names = {"--sample"}, description = "Sample ID to be analysed.", required = false, arity = 1) public String sample; - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version, e.g.: 14.0.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) public String exomiserVersion; @Parameter(names = {"--clinical-analysis-type"}, description = "Clinical analysis type: SINGLE or FAMILY.", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java index dfbd2cb0e5d..f8ff2d64d22 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java @@ -227,6 +227,9 @@ public class CreateCommandOptions { @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) public String tags; + @Parameter(names = {"--resource"}, description = "Indicates the file is treated as a resource.", required = false, arity = 1) + public Boolean resource; + @Parameter(names = {"--job-id"}, description = "The body web service jobId parameter", required = false, arity = 1) public String jobId; @@ -286,6 +289,9 @@ public class DistinctCommandOptions { @Parameter(names = {"--external"}, description = "Boolean field indicating whether to filter by external or non external files", required = false, arity = 1) public Boolean external; + @Parameter(names = {"--resource"}, description = "Boolean field indicating whether the file is a resource or not", required = false, arity = 1) + public Boolean resource; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; @@ -378,6 +384,9 @@ public class FetchCommandOptions { @Parameter(names = {"--url"}, description = "The body web service url parameter", required = true, arity = 1) public String url; + @Parameter(names = {"--resource"}, description = "Indicates the file is treated as a resource.", required = false, arity = 1) + public Boolean resource; + @Parameter(names = {"--path"}, description = "The body web service path parameter", required = false, arity = 1) public String path; @@ -595,6 +604,9 @@ public class SearchCommandOptions { @Parameter(names = {"--external"}, description = "Boolean field indicating whether to filter by external or non external files", required = false, arity = 1) public Boolean external; + @Parameter(names = {"--resource"}, description = "Boolean field indicating whether the file is a resource or not", required = false, arity = 1) + public Boolean resource; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java new file mode 100644 index 00000000000..4057ed778bb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java @@ -0,0 +1,80 @@ +package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogDBRuntimeException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogRuntimeException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternal; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import java.util.Collections; + +@Migration(id = "add_missing_resources", description = "Add missing RESOURCES folder and dependencies #TASK-6442", version = "4.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241016) +public class ResourcesMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + MongoCollection fileCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION); + + queryMongo(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, new Document(), + Projections.include("fqn", "uid", "uri"), studyDoc -> { + String studyFqn = studyDoc.getString("fqn"); + long studyUid = studyDoc.get("uid", Number.class).longValue(); + String studyUriStr = studyDoc.getString("uri"); + + // Check if the resources folder already exists + Bson fileQuery = Filters.and( + Filters.eq("studyUid", studyUid), + Filters.eq("path", "RESOURCES/") + ); + if (fileCollection.countDocuments(fileQuery) > 0) { + // Nothing to do. Resources folder already exists. + return; + } + + // Obtain JOBS folder to get the release number that should be associated to the RESOURCES folder + fileQuery = Filters.and( + Filters.eq("studyUid", studyUid), + Filters.eq("path", "JOBS/") + ); + Bson projection = Projections.include("release"); + try { + URI studyUri = new URI(studyUriStr); + + queryMongo(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, fileQuery, projection, fileDoc -> { + int release = fileDoc.get("release", Number.class).intValue(); + + // Create the RESOURCES folder + File file = new File("RESOURCES", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, + "RESOURCES/", Paths.get(studyUri).resolve("RESOURCES").toUri(), "Default resources folder", + FileInternal.init(), true, 0, release); + try { + dbAdaptorFactory.getCatalogFileDBAdaptor(organizationId).insert(studyUid, file, Collections.emptyList(), + Collections.emptyList(), Collections.emptyList(), QueryOptions.empty()); + logger.info("Missing RESOURCES folder created for study '{}'", studyFqn); + } catch (CatalogException e) { + throw new CatalogRuntimeException("Could not create missing RESOURCES folder for study '" + studyFqn + "'.", + e); + } + }); + } catch (CatalogDBException | URISyntaxException e) { + throw new CatalogDBRuntimeException(e); + } + + }); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 811354e0a82..cd71772a90d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -58,6 +58,7 @@ enum QueryParams implements QueryParam { MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), DESCRIPTION("description", TEXT_ARRAY, ""), EXTERNAL("external", BOOLEAN, ""), + RESOURCE("resource", BOOLEAN, ""), RELEASE("release", INTEGER, ""), STATUS("status", TEXT_ARRAY, ""), STATUS_ID("status.id", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index 0afd3f255d4..25dd369d8c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -29,7 +29,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -191,7 +190,7 @@ default void checkId(long studyId) throws CatalogDBException { OpenCGAResult nativeInsert(Map study) throws CatalogDBException; - OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException; + OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException; boolean hasStudyPermission(long studyId, String user, StudyPermissions.Permissions permission) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 896e20764a0..409829fd978 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -1840,6 +1840,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) break; case UUID: case EXTERNAL: + case RESOURCE: case TYPE: case URI: case ID: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 2ff8ed5332e..62e490f37b9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -29,6 +29,7 @@ import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; @@ -54,6 +55,7 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory { private final IOManagerFactory ioManagerFactory; private final MongoDataStoreManager mongoManager; private final MongoDBConfiguration mongoDbConfiguration; + private final CatalogIOManager catalogIOManager; private final Configuration configuration; private static final String ORGANIZATION_PREFIX = "ORG_"; @@ -68,7 +70,8 @@ private enum OrganizationTag { private final Logger logger; - public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory) throws CatalogDBException { + public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager) + throws CatalogDBException { List dataStoreServerAddresses = new LinkedList<>(); for (String host : catalogConfiguration.getCatalog().getDatabase().getHosts()) { if (host.contains(":")) { @@ -92,6 +95,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactor this.mongoDbConfiguration = mongoDBConfiguration; this.configuration = catalogConfiguration; this.ioManagerFactory = ioManagerFactory; + this.catalogIOManager = catalogIOManager; logger = LoggerFactory.getLogger(this.getClass()); connect(catalogConfiguration); @@ -208,7 +212,8 @@ private OrganizationSummary getOrganizationSummary(Note note) { private OrganizationMongoDBAdaptorFactory configureOrganizationMongoDBAdaptorFactory(String organizationId, Configuration configuration) throws CatalogDBException { - return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, catalogIOManager, + ioManagerFactory); } public OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(String organization) throws CatalogDBException { @@ -232,7 +237,7 @@ private OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(S // Organization is present, so create new OrganizationMongoDBAdaptorFactory for the organization OrganizationMongoDBAdaptorFactory organizationMongoDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, - ioManagerFactory); + catalogIOManager, ioManagerFactory); organizationDBAdaptorMap.put(organizationId, organizationMongoDBAdaptorFactory); return organizationMongoDBAdaptorFactory; } @@ -269,7 +274,7 @@ public OpenCGAResult createOrganization(Organization organization, try { // Create organization OrganizationMongoDBAdaptorFactory organizationDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organization.getId(), - mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + mongoManager, mongoDbConfiguration, configuration, catalogIOManager, ioManagerFactory); organizationDBAdaptorMap.put(organization.getId(), organizationDBAdaptorFactory); OrganizationSummary organizationSummary = new OrganizationSummary(organization.getId(), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index 9884bbccc6c..95e60382fa3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -13,6 +13,7 @@ import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; @@ -145,7 +146,8 @@ public class OrganizationMongoDBAdaptorFactory { public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreManager mongoDataStoreManager, MongoDBConfiguration mongoDBConfiguration, Configuration configuration, - IOManagerFactory ioManagerFactory) throws CatalogDBException { + CatalogIOManager catalogIOManager, IOManagerFactory ioManagerFactory) + throws CatalogDBException { logger = LoggerFactory.getLogger(OrganizationMongoDBAdaptorFactory.class); this.mongoManager = mongoDataStoreManager; this.organizationId = organizationId; @@ -210,7 +212,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa sampleDBAdaptor = new SampleMongoDBAdaptor(sampleCollection, sampleArchivedCollection, deletedSampleCollection, configuration, this); - studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, configuration, this); + studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, catalogIOManager, configuration, this); userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this); cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this); panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index c91fd4c3665..cbdbd34cace 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -36,10 +36,8 @@ import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; import org.opencb.opencga.catalog.db.mongodb.converters.VariableSetConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -51,6 +49,7 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternal; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -58,6 +57,7 @@ import javax.annotation.Nullable; import java.net.URI; +import java.nio.file.Paths; import java.util.*; import java.util.function.Consumer; import java.util.function.Function; @@ -80,11 +80,15 @@ public class StudyMongoDBAdaptor extends CatalogMongoDBAdaptor implements StudyD private StudyConverter studyConverter; private VariableSetConverter variableSetConverter; - public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, Configuration configuration, + private final CatalogIOManager ioManager; + + public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, + CatalogIOManager catalogIOManager, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(StudyMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.studyCollection = studyCollection; + this.ioManager = catalogIOManager; this.deletedStudyCollection = deletedStudyCollection; this.studyConverter = new StudyConverter(); this.variableSetConverter = new VariableSetConverter(); @@ -199,23 +203,22 @@ public OpenCGAResult nativeInsert(Map study) throws Catal // } @Override - public OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException { + public OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting study insert transaction for study id '{}'", study.getId()); - insert(clientSession, project, study, files); + insert(clientSession, project, study); return endWrite(tmpStartTime, 1, 1, 0, 0, null); }); } catch (Exception e) { - logger.error("Could not create study {}: {}", study.getId(), e.getMessage()); - throw new CatalogDBException(e); + throw new CatalogDBException("Could not create study '" + study.getFqn() + "'", e); } } - Study insert(ClientSession clientSession, Project project, Study study, List files) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Study insert(ClientSession clientSession, Project project, Study study) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException, CatalogIOException { if (project.getUid() < 0) { throw CatalogDBException.uidNotFound("Project", project.getUid()); } @@ -243,12 +246,23 @@ Study insert(ClientSession clientSession, Project project, Study study, List(variableSets.size()); for (VariableSet variableSet : variableSets) { + long vsetUid = getNewUid(clientSession); + variableSet.setUid(vsetUid); variableSetDocuments.add(variableSetConverter.convertToStorageType(variableSet)); } } study.setVariableSets(null); - //Create DBObject + FqnUtils.FQN fqn = FqnUtils.parse(study.getFqn()); + String organization = fqn.getOrganization(); + try { + URI studyUri = ioManager.getStudyUri(organization, Long.toString(project.getUid()), Long.toString(study.getUid())); + study.setUri(studyUri); + } catch (CatalogIOException e) { + throw new CatalogIOException("Could not obtain study uri.", e); + } + + //Create Document and store in the database Document studyObject = studyConverter.convertToStorageType(study); studyObject.put(PRIVATE_UID, studyUid); studyObject.put(QueryParams.VARIABLE_SET.key(), variableSetDocuments); @@ -264,14 +278,29 @@ Study insert(ClientSession clientSession, Project project, Study study, List files = Arrays.asList( + new File(".", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "", study.getUri(), + "study root folder", FileInternal.init(), false, 0, project.getCurrentRelease()), + new File("JOBS", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "JOBS/", + ioManager.getJobsUri(), "Default jobs folder", FileInternal.init(), false, 0, project.getCurrentRelease()), + new File(ParamConstants.RESOURCES_FOLDER, File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, + ParamConstants.RESOURCES_FOLDER + "/", Paths.get(study.getUri()).resolve(ParamConstants.RESOURCES_FOLDER).toUri(), + "Default resources folder", FileInternal.init(), true, 0, project.getCurrentRelease()) + ); + + // Create default folders + for (File file : files) { + dbAdaptorFactory.getCatalogFileDBAdaptor().insert(clientSession, study.getUid(), file, Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); + } + + try { + ioManager.createStudy(organization, Long.toString(project.getUid()), Long.toString(study.getUid())); + } catch (CatalogIOException e) { + throw new CatalogIOException("Could not create study folder '" + study.getUri() + "' in file system.", e); } return study; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java index 81dc17b16af..ab0289ad14b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java @@ -8,6 +8,10 @@ public CatalogRuntimeException(String message) { super(message); } + public CatalogRuntimeException(Throwable cause) { + super(cause); + } + public CatalogRuntimeException(String message, Throwable cause) { super(message, cause); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/ResourceException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/ResourceException.java new file mode 100644 index 00000000000..29e46cf9c11 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/ResourceException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.catalog.exceptions; + +public class ResourceException extends Exception { + public ResourceException(String message) { + super(message); + } + + public ResourceException(String message, Throwable cause) { + super(message, cause); + } + + public ResourceException(Throwable cause) { + super(cause); + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java index 699bc9c99da..1d51ce61351 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java @@ -16,6 +16,7 @@ package org.opencb.opencga.catalog.io; +import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.Configuration; @@ -85,7 +86,7 @@ public void createDefaultOpenCGAFolders() throws CatalogIOException { } protected void checkParam(String param) throws CatalogIOException { - if (param == null || param.equals("")) { + if (StringUtils.isEmpty(param)) { throw new CatalogIOException("Parameter '" + param + "' not valid"); } } @@ -107,7 +108,7 @@ public URI getProjectUri(String organizationId, String projectId) throws Catalog return Paths.get(getProjectsUri(organizationId)).resolve(projectId.endsWith("/") ? projectId : (projectId + "/")).toUri(); } - private URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException { + public URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException { checkParam(studyId); return Paths.get(getProjectUri(organizationId, projectId)).resolve(studyId.endsWith("/") ? studyId : (studyId + "/")).toUri(); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 030505395d0..eabad72ea28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -105,7 +105,7 @@ private void init() throws CatalogException { logger.debug("CatalogManager configureIOManager"); configureIOManager(configuration); logger.debug("CatalogManager configureDBAdaptorFactory"); - catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); + catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory, catalogIOManager); authorizationDBAdaptorFactory = new AuthorizationMongoDBAdaptorFactory((MongoDBAdaptorFactory) catalogDBAdaptorFactory, configuration); authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory, configuration); @@ -365,6 +365,10 @@ public IOManagerFactory getIoManagerFactory() { return ioManagerFactory; } + public CatalogIOManager getCatalogIOManager() { + return catalogIOManager; + } + private void configureIOManager(Configuration configuration) throws CatalogIOException { ioManagerFactory = new IOManagerFactory(); catalogIOManager = new CatalogIOManager(configuration); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index fcbb16c4e9d..568b2501469 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -100,15 +100,12 @@ public class FileManager extends AnnotationSetManager { static { INCLUDE_FILE_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.UUID.key(), - FileDBAdaptor.QueryParams.STUDY_UID.key(), FileDBAdaptor.QueryParams.TYPE.key())); - INCLUDE_FILE_URI = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), - FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.UUID.key(), - FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.STUDY_UID.key(), FileDBAdaptor.QueryParams.TYPE.key())); - INCLUDE_FILE_URI_PATH = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), - FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.UUID.key(), - FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), FileDBAdaptor.QueryParams.FORMAT.key(), - FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), - FileDBAdaptor.QueryParams.EXTERNAL.key(), FileDBAdaptor.QueryParams.STUDY_UID.key(), FileDBAdaptor.QueryParams.TYPE.key())); + FileDBAdaptor.QueryParams.RESOURCE.key(), FileDBAdaptor.QueryParams.STUDY_UID.key(), + FileDBAdaptor.QueryParams.TYPE.key())); + INCLUDE_FILE_URI = keepFieldInQueryOptions(INCLUDE_FILE_IDS, FileDBAdaptor.QueryParams.URI.key()); + INCLUDE_FILE_URI_PATH = keepFieldsInQueryOptions(INCLUDE_FILE_URI, Arrays.asList(FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), + FileDBAdaptor.QueryParams.FORMAT.key(), FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), + FileDBAdaptor.QueryParams.EXTERNAL.key())); EXCLUDE_FILE_ATTRIBUTES = new QueryOptions(QueryOptions.EXCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ATTRIBUTES.key(), FileDBAdaptor.QueryParams.ANNOTATION_SETS.key(), FileDBAdaptor.QueryParams.STATS.key())); INCLUDE_STUDY_URI = new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.URI.key()); @@ -588,10 +585,16 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams ParamUtils.checkParameter(createParams.getPath(), "path"); ParamUtils.checkObj(createParams.getType(), "type"); + boolean isResource = createParams.getResource() != null && createParams.getResource(); + String path = createParams.getPath(); if (path.startsWith("/")) { path = path.substring(1); } + if (isResource) { + // Final path should be one from the RESOURCES folder + path = getResourcesPath(path); + } if (createParams.getType().equals(File.Type.FILE)) { if (path.endsWith("/")) { @@ -618,9 +621,15 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams } OpenCGAResult parentResult = getParents(organizationId, study.getUid(), path, false, FileManager.INCLUDE_FILE_URI_PATH); - // Check user can write in path - authorizationManager.checkFilePermission(organizationId, study.getUid(), parentResult.first().getUid(), userId, - FilePermissions.WRITE); + + if (isResource) { + // Check if the user is an study admin + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } else { + // Check user can write in path + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentResult.first().getUid(), userId, + FilePermissions.WRITE); + } // Check available path Query pathQuery = new Query() @@ -656,7 +665,7 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams } } File file = new File("", createParams.getType(), createParams.getFormat(), createParams.getBioformat(), null, path, "", - createParams.getCreationDate(), createParams.getModificationDate(), createParams.getDescription(), false, 0, + createParams.getCreationDate(), createParams.getModificationDate(), createParams.getDescription(), false, isResource, 0, createParams.getSoftware(), null, createParams.getSampleIds(), null, createParams.getJobId(), 1, createParams.getTags(), null, null, null, createParams.getStatus() != null ? createParams.getStatus().toStatus() : null, null, null); List eventList = validateNewFile(organizationId, study, file, false); @@ -710,6 +719,23 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams } + private String getResourcesPath(String path) { + Path resourcesPath = Paths.get(ParamConstants.RESOURCES_FOLDER + "/"); + if (StringUtils.isEmpty(path)) { + return resourcesPath.toString(); + } else if (path.startsWith(ParamConstants.RESOURCES_FOLDER + "/")) { + return path; + } else if (path.startsWith(ParamConstants.RESOURCES_FOLDER.toLowerCase() + "/")) { + return path.replaceFirst(ParamConstants.RESOURCES_FOLDER.toLowerCase() + "/", ParamConstants.RESOURCES_FOLDER + "/"); + } else { + return resourcesPath.resolve(path).toString(); + } + } + + private boolean isResourcesPath(String path) { + return path.toUpperCase().startsWith(ParamConstants.RESOURCES_FOLDER + "/"); + } + List validateNewFile(String organizationId, Study study, File file, boolean overwrite) throws CatalogException { /** Check and set all the params and create a File object **/ ParamUtils.checkObj(file, "File"); @@ -774,6 +800,12 @@ List validateNewFile(String organizationId, Study study, File file, boole if (file.getType() == File.Type.FILE && file.getPath().endsWith("/")) { file.setPath(file.getPath().substring(0, file.getPath().length() - 1)); } + if (file.isResource()) { + file.setPath(getResourcesPath(file.getPath())); + } else if (isResourcesPath(file.getPath())) { + throw new CatalogException("RESOURCES path can only be used for resource files."); + } + file.setName(Paths.get(file.getPath()).getFileName().toString()); file.setId(file.getPath().replace("/", ":")); @@ -839,7 +871,8 @@ OpenCGAResult register(String organizationId, Study study, File file, List if (parents) { newParent = true; File parentFile = new File(File.Type.DIRECTORY, File.Format.NONE, File.Bioformat.NONE, parentPath, "", FileInternal.init(), - 0, Collections.emptyList(), null, "", new FileQualityControl(), Collections.emptyMap(), Collections.emptyMap()); + file.isResource(), 0, Collections.emptyList(), null, "", new FileQualityControl(), Collections.emptyMap(), + Collections.emptyMap()); validateNewFile(organizationId, study, parentFile, false); parentFileId = register(organizationId, study, parentFile, Collections.emptyList(), Collections.emptyList(), parents, options, tokenPayload).first().getUid(); @@ -854,7 +887,11 @@ OpenCGAResult register(String organizationId, Study study, File file, List } else { if (!newParent) { //If parent has been created, for sure we have permissions to create the new file. - authorizationManager.checkFilePermission(organizationId, studyId, parentFileId, userId, FilePermissions.WRITE); + if (file.isResource()) { + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); + } else { + authorizationManager.checkFilePermission(organizationId, studyId, parentFileId, userId, FilePermissions.WRITE); + } } } @@ -1148,12 +1185,13 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, * @param fileSource Current location of the file (file system). * @param folderDestiny Directory where the file needs to be moved (file system). * @param path Directory in catalog where the file will be registered (catalog). + * @param isResource Flag indicating whether the file to be moved is a resource or not. * @param token Token of the user. * @return An OpenCGAResult with the file registry after moving it to the final destination. * @throws CatalogException CatalogException. */ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nullable Path folderDestiny, @Nullable String path, - String token) throws CatalogException { + boolean isResource, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String organizationId = studyFqn.getOrganizationId(); @@ -1165,6 +1203,7 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu .append("fileSource", fileSource) .append("folderDestiny", folderDestiny) .append("path", path) + .append("isResource", isResource) .append("token", token); try { @@ -1184,6 +1223,9 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu if (path.startsWith("/")) { path = path.substring(1); } + if (isResource) { + path = getResourcesPath(path); + } File parentFolder = getParents(organizationId, study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); // We get the relative path @@ -1208,9 +1250,14 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu path = Paths.get(study.getUri().getPath()).relativize(folderDestiny).toString(); } - File parentFolder = getParents(organizationId, study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); - authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, - FilePermissions.WRITE); + if (isResource) { + // Check if the user is an study admin + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } else { + File parentFolder = getParents(organizationId, study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, + FilePermissions.WRITE); + } } else { // It will be moved to an external folder. Only admins can move to that directory long studyId = study.getUid(); @@ -1264,6 +1311,7 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu File file = new File() .setPath(filePath) + .setResource(isResource) .setType(File.Type.FILE); validateNewFile(organizationId, study, file, false); @@ -2417,13 +2465,27 @@ public OpenCGAResult move(String studyStr, String entryStr, String targetP File file = internalGet(organizationId, study.getUid(), entryStr, QueryOptions.empty(), userId).first(); fileId = file.getId(); fileUuid = file.getUuid(); - // Check user has write permissions on file/folder - authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + + if (file.isResource()) { + authorizationManager.isAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } else { + // Check user has write permissions on file/folder + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + } OpenCGAResult parents = getParents(organizationId, study.getUid(), targetPathStr, false, INCLUDE_FILE_IDS); // Check user can write in target path File parentFolder = parents.first(); - authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, FilePermissions.WRITE); + if (file.isResource() && !parentFolder.isResource()) { + throw new CatalogException("Cannot move RESOURCE file to a non RESOURCE folder."); + } else if (!file.isResource() && parentFolder.isResource()) { + throw new CatalogException("Cannot move non RESOURCE file to a RESOURCE folder."); + } + if (!parentFolder.isResource()) { + // If it is RESOURCE, it was already checked that the user is at least a study administrator + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, + FilePermissions.WRITE); + } ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.PATH.key(), targetPathStr); OpenCGAResult update = getFileDBAdaptor(organizationId) @@ -3315,6 +3377,14 @@ void checkValidStatusForDeletion(File file, List expectedStatus) throws throw new CatalogException("Cannot delete file: " + file.getName() + ". The status is " + file.getInternal().getStatus().getId()); } + private OpenCGAResult getRootFile(String organizationId, long studyUid) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Query query = new Query() + .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(FileDBAdaptor.QueryParams.PATH.key(), ""); + return getFileDBAdaptor(organizationId).get(query, QueryOptions.empty()); + } + /** * Create the parent directories that are needed. * @@ -3326,20 +3396,21 @@ void checkValidStatusForDeletion(File file, List expectedStatus) throws * catalog) * @param checkPermissions Boolean indicating whether to check if the user has permissions to create a folder in the first directory * that is available in catalog. + * @return the parent folder. * @throws CatalogDBException */ - private void createParents(String organizationId, Study study, String userId, URI studyURI, Path path, boolean checkPermissions) - throws CatalogException { + private OpenCGAResult createParents(String organizationId, Study study, String userId, URI studyURI, Path path, + boolean checkPermissions) throws CatalogException { if (path == null) { if (checkPermissions) { authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); } - return; + return getRootFile(organizationId, study.getUid()); } String stringPath = path.toString(); if (("/").equals(stringPath)) { - return; + return getRootFile(organizationId, study.getUid()); } logger.debug("Path: {}", stringPath); @@ -3357,14 +3428,22 @@ private void createParents(String organizationId, Study study, String userId, UR .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), stringPath); + boolean isResource = false; if (getFileDBAdaptor(organizationId).count(query).getNumMatches() == 0) { - createParents(organizationId, study, userId, studyURI, path.getParent(), checkPermissions); + OpenCGAResult parents = createParents(organizationId, study, userId, studyURI, path.getParent(), checkPermissions); + isResource = parents.first().isResource(); } else { + OpenCGAResult result = getFileDBAdaptor(organizationId).get(query, INCLUDE_FILE_IDS); if (checkPermissions) { - long fileId = getFileDBAdaptor(organizationId).getId(study.getUid(), stringPath); - authorizationManager.checkFilePermission(organizationId, study.getUid(), fileId, userId, FilePermissions.WRITE); + File tmpFile = result.first(); + if (tmpFile.isResource()) { + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } else { + authorizationManager.checkFilePermission(organizationId, study.getUid(), tmpFile.getUid(), userId, + FilePermissions.WRITE); + } } - return; + return result; } String parentPath = getParentPath(stringPath); @@ -3377,7 +3456,7 @@ private void createParents(String organizationId, Study study, String userId, UR // Create the folder in catalog File folder = new File(path.getFileName().toString(), File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, completeURI, - stringPath, null, TimeUtils.getTime(), TimeUtils.getTime(), "", false, 0, null, new FileExperiment(), + stringPath, null, TimeUtils.getTime(), TimeUtils.getTime(), "", false, isResource, 0, null, new FileExperiment(), Collections.emptyList(), Collections.emptyList(), "", studyManager.getCurrentRelease(study), Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), null, new Status(), FileInternal.init(), null); folder.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); @@ -3390,6 +3469,7 @@ private void createParents(String organizationId, Study study, String userId, UR authorizationManager.replicateAcls(organizationId, Collections.singletonList(queryResult.first().getUid()), allFileAcls.getResults().get(0), Enums.Resource.FILE); } + return queryResult; } private OpenCGAResult privateLink(String organizationId, Study study, FileLinkParams params, boolean parents, @@ -3443,6 +3523,10 @@ private OpenCGAResult privateLink(String organizationId, Study study, File params.setPath(params.getPath() + "/"); } } + if (isResourcesPath(params.getPath())) { + throw new CatalogException("Linked files cannot be stored in the RESOURCES folder."); + } + String externalPathDestinyStr; if (Paths.get(normalizedUri).toFile().isDirectory()) { externalPathDestinyStr = Paths.get(params.getPath()).resolve(Paths.get(normalizedUri).getFileName()).toString() + "/"; @@ -3586,7 +3670,7 @@ public FileVisitResult preVisitDirectory(URI dir, BasicFileAttributes attrs) thr File folder = new File(Paths.get(dir).getFileName().toString(), File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, dir, destinyPath, null, creationDate, modificationDate, - params.getDescription(), true, 0, new Software(), new FileExperiment(), + params.getDescription(), true, false, 0, new Software(), new FileExperiment(), Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), params.getStatus() != null ? params.getStatus().toStatus() : new Status(), @@ -3646,7 +3730,7 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws File subfile = new File(Paths.get(fileUri).getFileName().toString(), File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.NONE, fileUri, destinyPath, null, creationDate, modificationDate, - params.getDescription(), true, size, new Software(), new FileExperiment(), + params.getDescription(), true, false, size, new Software(), new FileExperiment(), Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), params.getStatus() != null ? params.getStatus().toStatus() : new Status(), internal, @@ -3805,8 +3889,10 @@ OpenCGAResult registerFile(String organizationId, Study study, String file File.Format format = org.opencb.opencga.catalog.managers.FileUtils.detectFormat(fileUri); File.Bioformat bioformat = org.opencb.opencga.catalog.managers.FileUtils.detectBioformat(fileUri); + boolean isExternal = isExternal(study, filePath, fileUri); + boolean isResource = isResourcesPath(filePath); File subfile = new File(Paths.get(filePath).getFileName().toString(), type, format, bioformat, fileUri, filePath, "", - TimeUtils.getTime(), TimeUtils.getTime(), "", isExternal(study, filePath, fileUri), size, new Software(), + TimeUtils.getTime(), TimeUtils.getTime(), "", isExternal, isResource, size, new Software(), new FileExperiment(), Collections.emptyList(), Collections.emptyList(), jobId, studyManager.getCurrentRelease(study), Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), new Status(), FileInternal.init(), Collections.emptyMap()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 70f933f6430..6c397914599 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -54,7 +54,6 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.family.FamilyPermissions; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; import org.opencb.opencga.core.models.file.FilePermissions; import org.opencb.opencga.core.models.file.FileStatus; import org.opencb.opencga.core.models.individual.IndividualPermissions; @@ -439,15 +438,9 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions study.setAdditionalInfo(ParamUtils.defaultObject(study.getAdditionalInfo(), Collections::emptyList)); study.setAttributes(ParamUtils.defaultObject(study.getAttributes(), HashMap::new)); - study.setVariableSets(ParamUtils.defaultObject(study.getVariableSets(), ArrayList::new)); - - LinkedList files = new LinkedList<>(); - File rootFile = new File(".", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "", null, "study root folder", - FileInternal.init(), 0, project.getCurrentRelease()); - File jobsFile = new File("JOBS", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "JOBS/", - catalogIOManager.getJobsUri(), "Default jobs folder", FileInternal.init(), 0, project.getCurrentRelease()); - files.add(rootFile); - files.add(jobsFile); + // Scan and add default VariableSets + List variableSetList = scanDefaultVariableSets(study.getRelease()); + study.setVariableSets(variableSetList); // Get organization owner and admins Organization organization = catalogManager.getOrganizationManager().get(organizationId, @@ -463,37 +456,14 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions study.setGroups(groups); /* CreateStudy */ - getStudyDBAdaptor(organizationId).insert(project, study, files, options); - OpenCGAResult result = getStudy(organizationId, projectUid, study.getUuid(), options); - study = result.getResults().get(0); - - URI uri; - try { - uri = catalogIOManager.createStudy(organizationId, Long.toString(project.getUid()), Long.toString(study.getUid())); - } catch (CatalogIOException e) { - try { - getStudyDBAdaptor(organizationId).delete(study); - } catch (Exception e1) { - logger.error("Can't delete study after failure creating study", e1); - } - throw e; - } - - // Update uri of study - getStudyDBAdaptor(organizationId).update(study.getUid(), new ObjectMap("uri", uri), QueryOptions.empty()); - study.setUri(uri); - - long rootFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), ""); //Set studyUri to the root folder too - getFileDBAdaptor(organizationId).update(rootFileId, new ObjectMap("uri", uri), QueryOptions.empty()); - - // Read and process installation variable sets - createDefaultVariableSets(organizationId, study, token); + OpenCGAResult result = getStudyDBAdaptor(organizationId).insert(project, study, options); auditManager.auditCreate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { - result.setResults(Arrays.asList(study)); + OpenCGAResult tmpResult = getStudy(organizationId, projectUid, study.getUuid(), options); + result.setResults(tmpResult.getResults()); } return result; } catch (CatalogException e) { @@ -503,36 +473,47 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions } } - public void createDefaultVariableSets(String studyStr, String token) throws CatalogException { - JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); - CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); - String organizationId = catalogFqn.getOrganizationId(); - Study study = resolveId(catalogFqn, null, tokenPayload); - createDefaultVariableSets(organizationId, study, token); - } - - private void createDefaultVariableSets(String organizationId, Study study, String token) throws CatalogException { + public List scanDefaultVariableSets(int release) throws CatalogException { Set variablesets = new Reflections(new ResourcesScanner(), "variablesets/").getResources(Pattern.compile(".*\\.json")); + List variableSetList = new ArrayList<>(variablesets.size()); for (String variableSetFile : variablesets) { VariableSet vs; try { vs = JacksonUtils.getDefaultNonNullObjectMapper().readValue( getClass().getClassLoader().getResourceAsStream(variableSetFile), VariableSet.class); } catch (IOException e) { - logger.error("Could not parse variable set '{}'", variableSetFile, e); - continue; + throw new CatalogException("Could not parse default variable set '" + variableSetFile + "'.", e); } if (vs != null) { if (vs.getAttributes() == null) { vs.setAttributes(new HashMap<>()); } vs.getAttributes().put("resource", variableSetFile); + validateNewVariableSet(vs, release); + variableSetList.add(vs); + } else { + throw new CatalogException("Default VariableSet object is null after parsing file '" + variableSetFile + "'"); + } + } + return variableSetList; + } - if (study.getVariableSets().stream().anyMatch(tvs -> tvs.getId().equals(vs.getId()))) { - logger.debug("Skip already existing variable set " + vs.getId()); - } else { - createVariableSet(organizationId, study, vs, token); - } + public void createDefaultVariableSets(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + Study study = resolveId(catalogFqn, null, tokenPayload); + createDefaultVariableSets(organizationId, study, token); + } + + private void createDefaultVariableSets(String organizationId, Study study, String token) throws CatalogException { + List variableSetList = scanDefaultVariableSets(study.getRelease()); + + for (VariableSet vs : variableSetList) { + if (study.getVariableSets().stream().anyMatch(tvs -> tvs.getId().equals(vs.getId()))) { + logger.debug("Skip already existing variable set " + vs.getId()); + } else { + createVariableSet(organizationId, study, vs, token); } } } @@ -1408,6 +1389,28 @@ public OpenCGAResult getVariableSetSummary(String studyStr, return new OpenCGAResult<>(dbTime, Collections.emptyList(), 1, Collections.singletonList(variableSetSummary), 1); } + private void validateNewVariableSet(VariableSet variableSet, int release) throws CatalogException { + ParamUtils.checkParameter(variableSet.getId(), "id"); + ParamUtils.checkObj(variableSet.getVariables(), "Variables from VariableSet"); + + variableSet.setDescription(ParamUtils.defaultString(variableSet.getDescription(), "")); + variableSet.setAttributes(ParamUtils.defaultObject(variableSet.getAttributes(), new HashMap<>())); + variableSet.setEntities(ParamUtils.defaultObject(variableSet.getEntities(), Collections.emptyList())); + variableSet.setName(ParamUtils.defaultString(variableSet.getName(), variableSet.getId())); + variableSet.setRelease(release); + + for (Variable variable : variableSet.getVariables()) { + ParamUtils.checkParameter(variable.getId(), "variable ID"); + ParamUtils.checkObj(variable.getType(), "variable Type"); + variable.setAllowedValues(ParamUtils.defaultObject(variable.getAllowedValues(), Collections.emptyList())); + variable.setAttributes(ParamUtils.defaultObject(variable.getAttributes(), Collections.emptyMap())); + variable.setCategory(ParamUtils.defaultString(variable.getCategory(), "")); + variable.setDependsOn(ParamUtils.defaultString(variable.getDependsOn(), "")); + variable.setDescription(ParamUtils.defaultString(variable.getDescription(), "")); + variable.setName(ParamUtils.defaultString(variable.getName(), variable.getId())); +// variable.setRank(defaultString(variable.getDescription(), "")); + } + } /* * Variables Methods diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index 595aecb61d5..fae8ea4f2f3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -19,6 +19,7 @@ import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; @@ -127,8 +128,10 @@ public void runMigration(String version, Collection d String token) throws CatalogException, IOException { runMigration(ParamConstants.ADMIN_ORGANIZATION, version, domains, languages, offline, appHome, params, token); + CatalogIOManager catalogIOManager = new CatalogIOManager(configuration); // ***** Starts code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ - try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory())) { + try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, + catalogManager.getIoManagerFactory(), catalogIOManager)) { for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { // ***** Finish code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ResourceManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ResourceManager.java new file mode 100644 index 00000000000..9f943b212d6 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ResourceManager.java @@ -0,0 +1,408 @@ +package org.opencb.opencga.catalog.utils; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.FileUtils; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.ResourceException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.config.AnalysisTool; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Resource; +import org.opencb.opencga.core.config.ResourceFile; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URL; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.InputMismatchException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class ResourceManager { + + public static final String REFERENCE_GENOMES = "reference-genomes"; + + public static final String OK = "Ok"; + public static final String MD5_EXT = ".md5"; + public static final String RESOURCE_MSG = "Resource '"; + public static final String FOR_ANALYSIS_MSG = "' for analysis '"; + + private Path openCgaHome; + private Configuration configuration; + + // Flag to track if all resources are fetching + private static final AtomicBoolean IS_FETCHING = new AtomicBoolean(false); + + public static final String CONFIGURATION_FILENAME = "configuration.yml"; + public static final String CONF_DIRNAME = "conf"; + public static final String ANALYSIS_DIRNAME = "analysis"; + public static final String RESOURCES_DIRNAME = "resources"; +// public static final String RELEASES_DIRNAME = "releases"; + + protected static Logger logger = LoggerFactory.getLogger(ResourceManager.class); + + public ResourceManager(Path openCgaHome) { + this.openCgaHome = openCgaHome; + } + + public synchronized void fetchAllResources(Path tmpPath, CatalogManager catalogManager, String token) + throws ResourceException, IOException { + loadConfiguration(); + + // Get all resources from the configuration file + List resources = configuration.getAnalysis().getResource().getFiles().stream().map(ResourceFile::getId) + .collect(Collectors.toList()); + fetchResources(resources, tmpPath, catalogManager, token); + } + + public synchronized void fetchResources(List resources, Path tmpPath, CatalogManager catalogManager, String token) + throws ResourceException, IOException { + loadConfiguration(); + + // Check if the resource is already being downloaded + if (IS_FETCHING.compareAndSet(false, true)) { + try { + // Only installation administrators can fetch all resources + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + catalogManager.getAuthorizationManager().checkIsOpencgaAdministrator(jwtPayload, "fetch all resources"); + + // Check resources before fetching + List resourceFiles = new ArrayList<>(); + for (String resource : resources) { + boolean found = false; + for (ResourceFile resourceFile : configuration.getAnalysis().getResource().getFiles()) { + if (resource.equalsIgnoreCase(resourceFile.getId())) { + resourceFiles.add(resourceFile); + found = true; + break; + } + } + if (!found) { + throw new ResourceException("Unknown resource ID '" + resource + "'. Check the configuration file."); + } + } + + // Download resources + for (ResourceFile resourceFile: resourceFiles) { + fetchResourceFile(resourceFile, tmpPath); + } + + // Move resources to installation folder + move(tmpPath, openCgaHome.resolve(ANALYSIS_DIRNAME).resolve(RESOURCES_DIRNAME)); + } catch (CatalogException | NoSuchAlgorithmException | ToolException e) { + throw new ResourceException(e); + } finally { + // Ensure the flag is reset after fetching is done + IS_FETCHING.set(false); + } + } else { + throw new ResourceException("Resources are already being fetched."); + } + } + + public List checkResourcePaths(String analysisId) throws ResourceException, ToolException, IOException { + return checkResourcePaths(analysisId, null); + } + + public List checkResourcePaths(String analysisId, String version) throws ResourceException, ToolException, IOException { + loadConfiguration(); + + if (IS_FETCHING.get()) { + throw new ResourceException("Resources are not ready yet; they are currently being fetched."); + } + + // Sanity check + if (StringUtils.isEmpty(analysisId)) { + throw new ResourceException("Analysis ID is empty."); + } + + List files = new ArrayList<>(); + Resource resourceConfig = configuration.getAnalysis().getResource(); + + AnalysisTool analysisTool = null; + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (analysisId.equals(tool.getId())) { + if (StringUtils.isEmpty(version) || version.equals(tool.getVersion())) { + analysisTool = tool; + break; + } + } + } + if (analysisTool == null) { + String msg = "Missing analysis tool (ID = " + analysisId + (!StringUtils.isEmpty(version) ? (", version = " + version) : "") + + ") in the configuration file"; + throw new ToolException(msg); + } + + for (String resourceId : analysisTool.getResources()) { + boolean found = false; + for (ResourceFile resourceFile : resourceConfig.getFiles()) { + if (resourceId.equalsIgnoreCase(resourceFile.getId())) { + // Found + Path analysisResourcePath = resourceConfig.getBasePath().resolve(resourceFile.getPath()); + if (!Files.exists(analysisResourcePath)) { + throw new ResourceException(RESOURCE_MSG + resourceFile.getId() + "' is not fetched yet (file '" + + analysisResourcePath + "' is missing). Please fetch it first."); + } + files.add(analysisResourcePath.toFile()); + found = true; + break; + } + } + // Sanity check + if (!found) { + throw new ResourceException("Unmatched configuration: resource '" + resourceId + "' of analysis '" + analysisId + "' not" + + " found in the configuration file."); + } + } + + if (CollectionUtils.isEmpty(files)) { + throw new ResourceException("No resources found for analysis ID '" + analysisId + "'."); + } + return files; + } + + public Path checkResourcePath(String resourceId) throws ResourceException, IOException { + loadConfiguration(); + + if (IS_FETCHING.get()) { + throw new ResourceException("Resources are not ready yet; they are currently being fetched."); + } + + // Sanity check + if (StringUtils.isEmpty(resourceId)) { + throw new ResourceException("Resource ID is empty."); + } + + Resource resourceConfig = configuration.getAnalysis().getResource(); + for (ResourceFile resourceFile : resourceConfig.getFiles()) { + if (resourceId.equalsIgnoreCase(resourceFile.getId())) { + // Resource found, exists? + Path analysisResourcePath = resourceConfig.getBasePath().resolve(resourceFile.getPath()); + if (!Files.exists(analysisResourcePath)) { + throw new ResourceException(RESOURCE_MSG + resourceFile.getId() + "' is not fetched yet (file '" + analysisResourcePath + + "' is missing). Please fetch it first."); + } + return analysisResourcePath; + } + } + + // Resource not found !! + throw new ResourceException("Unmatched configuration: resource '" + resourceId + "' not found in the configuration file."); + } + + //------------------------------------------------------------------------- + // P R I V A T E M E T H O D S + //------------------------------------------------------------------------- + + private void loadConfiguration() throws IOException { + if (configuration == null) { + this.configuration = Configuration.load(new FileInputStream(openCgaHome.resolve(CONF_DIRNAME) + .resolve(CONFIGURATION_FILENAME).toFile())); + } + } + + private Path fetchResourceFile(ResourceFile resourceFile, Path downloadedPath) + throws IOException, NoSuchAlgorithmException, ResourceException, ToolException { + Resource resourceConfig = configuration.getAnalysis().getResource(); + + + boolean isExomiserResource = false; + Path fetchedFile = downloadedPath.resolve(resourceFile.getPath()); + if (resourceFile.getId().startsWith("EXOMISER_")) { + isExomiserResource = true; + fetchedFile = downloadedPath.resolve(resourceFile.getPath() + ".zip"); + } + + // First check installation directory, and check MD5 (if it exists) + Path installationFile = resourceConfig.getBasePath().resolve(resourceFile.getPath()); + if (Files.exists(installationFile)) { + if (isExomiserResource && Files.isDirectory(installationFile)) { + logger.info("Resource '{}' has already been downloaded: skipping download", resourceFile.getId()); + return installationFile; + } else { + try { + validateMD5(installationFile, resourceFile.getMd5()); + logger.info("Resource '{}' has already been downloaded and MD5 validation passed: skipping download", + resourceFile.getId()); + return installationFile; + } catch (Exception e) { + logger.warn("Resource '{}' has already been downloaded but MD5 validation failed: downloading again", + resourceFile.getId()); + } + } + } + + // Download resource file + String fileUrl = resourceConfig.getBaseUrl() + resourceFile.getUrl(); + logger.info("Downloading resource '{}' to '{}' ...", fileUrl, fetchedFile.toAbsolutePath()); + donwloadFile(new URL(fileUrl), fetchedFile); + logger.info(OK); + + // Checking MD5 + validateMD5(fetchedFile, resourceFile.getMd5()); + + // Any action to perform, e.g.: Exomiser resources need to be unzipped + if (isExomiserResource) { + unzip(fetchedFile, "exomiser"); + // Delete Exomiser files .zip, .sha256 + List exts = Arrays.asList(".zip", ".sha256"); + for (String ext : exts) { + if (Files.exists(downloadedPath.resolve(resourceFile.getPath() + ext))) { + logger.info("Deleting Exomiser file {} after unzipping", downloadedPath.resolve(resourceFile.getPath() + ext)); + Files.delete(downloadedPath.resolve(resourceFile.getPath() + ext)); + } + } + return downloadedPath.resolve(resourceFile.getPath()); + } + return fetchedFile; + } + + private Path donwloadFile(URL url, Path resourcePath) throws ResourceException, IOException { + Path parentPath = resourcePath.getParent(); + if (!Files.exists(parentPath)) { + Files.createDirectories(parentPath); + } + + try (BufferedInputStream in = new BufferedInputStream(url.openStream()); + FileOutputStream fileOutputStream = new FileOutputStream(resourcePath.toFile())) { + byte[] dataBuffer = new byte[1024]; + int bytesRead; + while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { + fileOutputStream.write(dataBuffer, 0, bytesRead); + } + } catch (Exception e) { + throw new ResourceException("Error downloading " + url, e); + } + + if (!Files.exists(resourcePath)) { + String msg = "Something wrong happened downloading '" + url + "'; it could not be found after downloading"; + logger.error(msg); + throw new ResourceException(msg); + } + + return resourcePath; + } + + public void validateMD5(Path filePath, Path md5filePath) throws IOException, NoSuchAlgorithmException, InputMismatchException { + validateMD5(filePath, new String(Files.readAllBytes(md5filePath))); + } + + public void validateMD5(Path filePath, String md5) throws IOException, NoSuchAlgorithmException, InputMismatchException { + String expectedMD5 = md5.trim(); + String actualMD5 = computeMD5(filePath); + if (!expectedMD5.equals(actualMD5)) { + throw new InputMismatchException("MD5 checksum mismatch! File may be corrupted."); + } + } + + private static String computeMD5(Path filePath) throws IOException, NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("MD5"); + try (FileInputStream fis = new FileInputStream(filePath.toString())) { + byte[] dataBytes = new byte[1024]; + int bytesRead; + + while ((bytesRead = fis.read(dataBytes)) != -1) { + md.update(dataBytes, 0, bytesRead); + } + } + byte[] mdBytes = md.digest(); + + // Convert byte array to hex string + StringBuilder sb = new StringBuilder(); + for (byte b : mdBytes) { + sb.append(String.format("%02x", b)); + } + + return sb.toString(); + } + + private void move(Path sourceDir, Path targetDir) throws IOException { + // Ensure the target directory exists + if (!Files.exists(targetDir)) { + logger.info("Creating directory {} ...", targetDir.toAbsolutePath()); + Files.createDirectories(targetDir); + logger.info(OK); + } + + // Walk through the directory tree at sourceDir + Files.walkFileTree(sourceDir, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + // Create corresponding subdirectory in targetDir + Path targetSubDir = targetDir.resolve(sourceDir.relativize(dir)); + if (!Files.exists(targetSubDir)) { + logger.info("Creating directory {} ...", targetSubDir.toAbsolutePath()); + Files.createDirectory(targetSubDir); + logger.info(OK); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + // Move each file to target directory + Path targetFile = targetDir.resolve(sourceDir.relativize(file)); + logger.info("Moving {} to {} ...", file.toAbsolutePath(), targetFile.toAbsolutePath()); + FileUtils.copyFile(file.toFile(), targetFile.toFile()); + Files.delete(file); +// Files.move(file, targetFile, StandardCopyOption.REPLACE_EXISTING); + logger.info(OK); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + // Delete the original directory after moving all its content + logger.info("Deleting source directory {} ...", dir.toAbsolutePath()); + Files.delete(dir); + logger.info(OK); + return FileVisitResult.CONTINUE; + } + }); + } + + private void unzip(Path zipPath, String analysisId) throws ToolException, IOException { + // Unzip + String filename = zipPath.getFileName().toString(); + Path stdoutPath = zipPath.getParent().resolve("stdout_unzip_" + filename + ".txt"); + Path stderrPath = zipPath.getParent().resolve("stderr_unzip_" + filename + ".txt"); + try { + logger.info("Unzipping resource file '{}' for analysis '{}'.", filename, analysisId); + new Command("unzip -o -d " + zipPath.getParent() + " " + zipPath) + .setOutputOutputStream(new DataOutputStream(new FileOutputStream(stdoutPath.toFile()))) + .setErrorOutputStream(new DataOutputStream(new FileOutputStream(stderrPath.toFile()))) + .run(); + } catch (FileNotFoundException e) { + throw new ToolException("Error unzipping resource file '" + filename + FOR_ANALYSIS_MSG + analysisId + "'. Check log files: " + + stdoutPath + ", " + stderrPath, e); + } + + // Deleting stdout and stderr after unzipping + if (Files.exists(stdoutPath)) { + Files.delete(stdoutPath); + } + if (Files.exists(stderrPath)) { + Files.delete(stderrPath); + } + } + + //------------------------------------------------------------------------- + // G E T T E R S & S E T T E R S + //------------------------------------------------------------------------- + + public Configuration getConfiguration() { + return configuration; + } +} diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 2beb12d1c2d..11174af762a 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -57,6 +57,7 @@ {"collections": ["file"], "fields": {"type": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"format": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"bioformat": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"resource": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"size": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"software.name": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java index 632b13694cc..8fb999325bb 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java @@ -31,7 +31,8 @@ public class AbstractMongoDBAdaptorTest extends AbstractManagerTest { @Before public void setUp() throws Exception { super.setUp(); - dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory()); + dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory(), + catalogManager.getCatalogIOManager()); catalogUserDBAdaptor = (UserMongoDBAdaptor) dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId); catalogStudyDBAdaptor = (StudyMongoDBAdaptor) dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId); catalogProjectDBAdaptor = (ProjectMongoDBAdaptor) dbAdaptorFactory.getCatalogProjectDbAdaptor(organizationId); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java index 252085c353b..d6c11dd6568 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -43,7 +43,7 @@ public class MongoBackupUtils { public static void dump(CatalogManager catalogManager, Path opencgaHome) throws CatalogDBException { StopWatch stopWatch = StopWatch.createStarted(); try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(dbAdaptorFactory.getOrganizationIds().get(0)) .getMongoDataStore().getMongoClient(); MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); @@ -91,7 +91,7 @@ public static void dump(CatalogManager catalogManager, Path opencgaHome) throws public static void restore(CatalogManager catalogManager, Path opencgaHome) throws Exception { try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getMongoClient(); @@ -147,7 +147,7 @@ private static void restore(CatalogManager catalogManager, Path opencgaHome, Obj logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); StopWatch stopWatch = StopWatch.createStarted(); try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { for (String existingOrganizationId : dbAdaptorFactory.getOrganizationIds()) { // We need to completely remove databases that were not backed up so tests that attempt to create them again don't fail diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java index 50f710ae3eb..dd5c18ab8ce 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java @@ -26,6 +26,7 @@ import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.project.Project; @@ -137,8 +138,12 @@ public void renameProjectTest() throws CatalogException { catalogProjectDBAdaptor.insert(new Project("myp2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()).setFqn(FqnUtils.buildFqn(organizationId, "myp2")), null); Project p2 = getProject("myp2"); + // Create project folder so the study insert doesn't fail + CatalogIOManager catalogIOManager = new CatalogIOManager(catalogManager.getConfiguration()); + catalogIOManager.createProject(organizationId, Long.toString(p1.getUid())); + // Add study - catalogStudyDBAdaptor.insert(p1, new Study().setId("study").setFqn(FqnUtils.buildFqn(organizationId, "myp1", "study")), null, null); + catalogStudyDBAdaptor.insert(p1, new Study().setId("study").setFqn(FqnUtils.buildFqn(organizationId, "myp1", "study")), null); catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap(ProjectDBAdaptor.QueryParams.ID.key(), "newpmp"), QueryOptions.empty()); Project project = getProject("newpmp"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index f81f28b37a6..2e8fd510215 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -69,13 +69,13 @@ private Study getMinimalStudyInstance(String projectId, String id) { @Test public void createStudyWithSameIdInDifferentProject() throws CatalogException { Project project = getProject(project1); - OpenCGAResult create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project1, "studyId"), null, null); + OpenCGAResult create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project1, "studyId"), null); assertEquals(1, create.getNumInserted()); Study ph1 = getStudy(project.getUid(), "studyId"); assertNotNull(ph1); project = getProject(project2); - create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project2, "studyId"), null, null); + create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project2, "studyId"), null); assertEquals(1, create.getNumInserted()); ph1 = getStudy(project.getUid(), "studyId"); assertNotNull(ph1); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java index 2de27beb3da..36eac42482b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java @@ -35,8 +35,12 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyCreateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -67,6 +71,8 @@ public class AbstractClinicalManagerTest extends GenericTest { public final static String CA_ID4 = "clinical-analysis-4"; public final static String PROBAND_ID4 = "HG105"; + private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); + @Rule public ExpectedException thrown = ExpectedException.none(); @@ -101,11 +107,13 @@ public void setUpCatalogManager() throws IOException, CatalogException, URISynta token = catalogManager.getUserManager().login(organizationId, "user", PASSWORD).getToken(); - catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", null, "GRCh38", - new QueryOptions(), token); + ProjectCreateParams projectCreateParams = new ProjectCreateParams() + .setId("1000G") + .setOrganism(new ProjectOrganism("hsapiens", "grch38")); + Project project = catalogManager.getProjectManager().create(projectCreateParams, INCLUDE_RESULT, token).first(); + System.out.println("project.toString() = " + project.toString()); - Study study = catalogManager.getStudyManager().create("1000G", "phase1", null, "Phase 1", "Done", null, null, - null, null, null, token).first(); + Study study = catalogManager.getStudyManager().create("1000G", new Study().setId("phase1"), INCLUDE_RESULT, token).first(); studyFqn = study.getFqn(); family = catalogManager.getFamilyManager().create(studyFqn, getFamily(), QueryOptions.empty(), token).first(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index 09c94ee33d3..59cf2745aee 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -24,6 +24,7 @@ import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; @@ -40,6 +41,9 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import static org.opencb.opencga.catalog.utils.ResourceManager.ANALYSIS_DIRNAME; +import static org.opencb.opencga.catalog.utils.ResourceManager.RESOURCES_DIRNAME; + /** * Created on 05/05/16 * @@ -88,13 +92,20 @@ public Path clearOpenCGAHome(String testName) throws IOException { } while (opencgaHome.toFile().exists()); Files.createDirectories(opencgaHome); configuration = Configuration.load(getClass().getResource("/configuration-test.yml").openStream()); + Path confPath = Files.createDirectories(opencgaHome.resolve("conf")); + Files.copy(getClass().getResource("/configuration-test.yml").openStream(), confPath.resolve("configuration.yml"), StandardCopyOption.REPLACE_EXISTING); + configuration.setWorkspace(opencgaHome.resolve("sessions").toAbsolutePath().toString()); configuration.setJobDir(opencgaHome.resolve("JOBS").toAbsolutePath().toString()); + Path analysisPath = opencgaHome.resolve(ANALYSIS_DIRNAME); + Files.createDirectories(analysisPath.resolve(RESOURCES_DIRNAME)); + // Pedigree graph analysis - Path analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/pedigree-graph")).toAbsolutePath(); + Path pedAnalysisPath = Files.createDirectories(analysisPath.resolve("pedigree-graph")).toAbsolutePath(); FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R"); - Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + Files.copy(inputStream, pedAnalysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + return opencgaHome; } @@ -137,7 +148,8 @@ public Path getOpencgaHome() { } public static void clearCatalog(Configuration configuration) throws CatalogException, URISyntaxException { - try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory())) { + CatalogIOManager catalogIOManager = new CatalogIOManager(configuration); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory(), catalogIOManager)) { dbAdaptorFactory.deleteCatalogDB(); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 8f0bce1401e..5a8bff6ffe7 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -360,7 +360,7 @@ public void testGetAllStudies() throws CatalogException { Query query = new Query(); String projectId = catalogManager.getProjectManager().search(organizationId, query, null, ownerToken).first().getFqn(); Study study_1 = catalogManager.getStudyManager().create(projectId, new Study().setId("study_1").setCreationDate("20150101120000") - , null, ownerToken).first(); + , INCLUDE_RESULT, ownerToken).first(); assertEquals("20150101120000", study_1.getCreationDate()); catalogManager.getStudyManager().create(projectId, "study_2", null, "study_2", "description", null, null, null, null, null, ownerToken); @@ -368,7 +368,7 @@ public void testGetAllStudies() throws CatalogException { catalogManager.getStudyManager().create(projectId, "study_3", null, "study_3", "description", null, null, null, null, null, ownerToken); String study_4 = catalogManager.getStudyManager().create(projectId, "study_4", null, "study_4", "description", null, null, null, - null, null, ownerToken).first().getId(); + null, INCLUDE_RESULT, ownerToken).first().getId(); assertEquals(new HashSet<>(Collections.singletonList(studyId)), catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(StudyDBAdaptor.QueryParams.GROUP_USER_IDS.key(), normalUserId1), null, ownerToken) @@ -484,7 +484,7 @@ public void testGetOnlyStudyUserAnonymousCanSee() throws CatalogException { // Create another study with alias phase3 DataResult study = catalogManager.getStudyManager().create(project.getFqn(), "phase3", null, "Phase 3", "d", null, - null, null, null, null, ownerToken); + null, null, null, INCLUDE_RESULT, ownerToken); try { studyManager.resolveIds(Collections.emptyList(), ParamConstants.ANONYMOUS_USER_ID, otherOrg); fail("This should throw an exception. No studies should be found for user anonymous"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 81c28e9daa8..ccdf2d9331c 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -112,6 +112,34 @@ public void testCreateFileFromUnsharedStudy() throws CatalogException { } } + @Test + public void testCreateResourcesFile() throws CatalogException { + FileCreateParams fileCreateParams = new FileCreateParams() + .setType(File.Type.FILE) + .setPath("data/test/folder/file.txt") + .setResource(true) + .setDescription("My description") + .setContent("blabla"); + File file = fileManager.create(studyFqn, fileCreateParams, true, studyAdminToken1).first(); + assertTrue(file.isResource()); + assertNotEquals(fileCreateParams.getPath(), file.getPath()); + assertTrue(file.getPath().startsWith("RESOURCES/")); + + // Only the study admin can create a resources file. Will try with some other users with generic WRITE file access + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId1, + new StudyAclParams(StudyPermissions.Permissions.WRITE_FILES.name(), ""), ParamUtils.AclAction.ADD, ownerToken); + + FileCreateParams fileCreateParams2 = new FileCreateParams() + .setType(File.Type.FILE) + .setPath("data/test/folder/file2.txt") + .setResource(true) + .setDescription("My description") + .setContent("blabla"); + CatalogAuthorizationException exception = assertThrows(CatalogAuthorizationException.class, + () -> fileManager.create(studyFqn, fileCreateParams2, true, normalToken1)); + assertTrue(exception.getMessage().contains("study administrator")); + } + @Test public void testCreateFileFromSharedStudy() throws CatalogException { StudyAclParams aclParams = new StudyAclParams("", "analyst"); @@ -1149,7 +1177,7 @@ public void testCreateFile() throws CatalogException, IOException { public void testCreateFolder() throws Exception { Set paths = fileManager.search(studyFqn, new Query("type", File.Type.DIRECTORY), new QueryOptions(), orgAdminToken1) .getResults().stream().map(File::getPath).collect(Collectors.toSet()); - assertEquals(9, paths.size()); + assertEquals(10, paths.size()); assertTrue(paths.containsAll(Arrays.asList("", "JOBS/", "data/", "data/test/", "data/test/folder/", "data/d1/", "data/d1/d2/", "data/d1/d2/d3/", "data/d1/d2/d3/d4/"))); @@ -1161,7 +1189,7 @@ public void testCreateFolder() throws Exception { paths = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), File.Type.DIRECTORY), new QueryOptions(), orgAdminToken1).getResults().stream().map(File::getPath).collect(Collectors.toSet()); - assertEquals(11, paths.size()); + assertEquals(12, paths.size()); assertTrue(paths.containsAll(Arrays.asList("", "JOBS/", "data/", "data/test/", "data/test/folder/", "data/d1/", "data/d1/d2/", "data/d1/d2/d3/", "data/d1/d2/d3/d4/", "data/new/", "data/new/folder/"))); @@ -1170,7 +1198,8 @@ public void testCreateFolder() throws Exception { fileManager.createFolder(studyFqn, Paths.get("WOLOLO").toString(), true, null, QueryOptions.empty(), orgAdminToken1); - String newStudy = catalogManager.getStudyManager().create(project2, "alias", null, "name", "", null, null, null, null, null, orgAdminToken1).first().getFqn(); + Study study = new Study().setId("newStudy"); + String newStudy = catalogManager.getStudyManager().create(project2, study, INCLUDE_RESULT, orgAdminToken1).first().getFqn(); folder = fileManager.createFolder(newStudy, Paths.get("WOLOLO").toString(), true, null, QueryOptions.empty(), orgAdminToken1).first(); @@ -1180,7 +1209,7 @@ public void testCreateFolder() throws Exception { @Test public void testCreateFolderAlreadyExists() throws Exception { Set paths = fileManager.search(studyFqn3, new Query("type", File.Type.DIRECTORY), new QueryOptions(), orgAdminToken1).getResults().stream().map(File::getPath).collect(Collectors.toSet()); - assertEquals(2, paths.size()); + assertEquals(3, paths.size()); assertTrue(paths.contains("")); //root // assertTrue(paths.contains("data/")); //data // assertTrue(paths.contains("analysis/")); //analysis @@ -1533,11 +1562,11 @@ public void testGetTreeView() throws CatalogException { .setContent("content"), true, ownerToken); DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()), ownerToken); - assertEquals(27, fileTree.getNumResults()); - assertEquals(27, countElementsInTree(fileTree.first())); + assertEquals(28, fileTree.getNumResults()); + assertEquals(28, countElementsInTree(fileTree.first())); fileTree = fileManager.getTree(studyFqn, "/", 2, new QueryOptions(), ownerToken); - assertEquals(17, fileTree.getNumResults()); + assertEquals(18, fileTree.getNumResults()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()); fileTree = fileManager.getTree(studyFqn, "/", 2, options, ownerToken); @@ -1558,10 +1587,10 @@ public void testGetTreeViewMoreThanOneFile() throws CatalogException { catalogManager.getStudyManager().create(project1, "phase2", null, "Phase 2", "Done", null, null, null, null, null, ownerToken); DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(), ownerToken); - assertEquals(12, fileTree.getNumResults()); + assertEquals(13, fileTree.getNumResults()); fileTree = fileManager.getTree("phase2", ".", 5, new QueryOptions(), ownerToken); - assertEquals(2, fileTree.getNumResults()); + assertEquals(3, fileTree.getNumResults()); } @Test @@ -1644,7 +1673,7 @@ public void searchFileTest() throws CatalogException { result = fileManager.search(studyFqn, query, null, ownerToken); result.getResults().forEach(f -> assertEquals(File.Type.DIRECTORY, f.getType())); int numFolders = result.getNumResults(); - assertEquals(9, numFolders); + assertEquals(10, numFolders); query = new Query(FileDBAdaptor.QueryParams.PATH.key(), ""); result = fileManager.search(studyFqn, query, null, ownerToken); @@ -1654,7 +1683,7 @@ public void searchFileTest() throws CatalogException { query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE,DIRECTORY"); result = fileManager.search(studyFqn, query, null, ownerToken); - assertEquals(13, result.getNumResults()); + assertEquals(14, result.getNumResults()); assertEquals(numFiles + numFolders, result.getNumResults()); query = new Query("type", "FILE"); @@ -1682,7 +1711,7 @@ public void searchFileTest() throws CatalogException { QueryOptions options = new QueryOptions(QueryOptions.LIMIT, 2).append(QueryOptions.COUNT, true); result = fileManager.search(studyFqn, new Query(), options, ownerToken); assertEquals(2, result.getNumResults()); - assertEquals(13, result.getNumMatches()); + assertEquals(14, result.getNumMatches()); } // // @Test @@ -2370,7 +2399,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I Path studyPath = Paths.get(study.getUri()); // Register in workspace folder - OpenCGAResult result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "myFolder", ownerToken); + OpenCGAResult result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "myFolder", false, ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); @@ -2384,7 +2413,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I Files.copy(sourcePath, copy); // Register without passing the path - result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), null, ownerToken); + result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), null, false, ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz"))); @@ -2397,7 +2426,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I Files.copy(sourcePath, copy); // Register without passing the destiny path - result = fileManager.moveAndRegister(studyFqn, copy, null, "myFolder", ownerToken); + result = fileManager.moveAndRegister(studyFqn, copy, null, "myFolder", false, ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz"))); @@ -2411,7 +2440,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I // Register to an incorrect path try { - fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "otherFolder", ownerToken); + fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "otherFolder", false, ownerToken); fail("The method should have raised an error saying the path does not match the one corresponding to the uri. It should both " + "point to myFolder or to otherFolder, but not to different paths."); } catch (CatalogException e) { @@ -2424,7 +2453,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I // Now, instead of moving it to the user's workspace, we will move it to an external path try { - fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", normalToken2); + fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", false, normalToken2); fail("user2 should not have permissions to move to an external folder"); } catch (CatalogAuthorizationException e) { assertTrue(e.getMessage().contains("owners or administrative users")); @@ -2435,7 +2464,7 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I new GroupUpdateParams(Collections.singletonList(normalUserId2)), ownerToken); // and try the same action again - result = fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", normalToken2); + result = fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", false, normalToken2); assertEquals("a/b/c/variant-test-file.vcf.gz", result.first().getPath()); assertEquals("/tmp/other/variant-test-file.vcf.gz", Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(Paths.get("/tmp/other/variant-test-file.vcf.gz"))); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ResourceManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ResourceManagerTest.java new file mode 100644 index 00000000000..6db4b10f6b6 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ResourceManagerTest.java @@ -0,0 +1,113 @@ +package org.opencb.opencga.catalog.managers; + +import org.junit.Before; +import org.junit.Test; +import org.opencb.opencga.catalog.exceptions.ResourceException; +import org.opencb.opencga.catalog.utils.ResourceManager; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.exceptions.ToolException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; + +import static org.opencb.opencga.catalog.utils.ResourceManager.ANALYSIS_DIRNAME; +import static org.opencb.opencga.catalog.utils.ResourceManager.RESOURCES_DIRNAME; + +public class ResourceManagerTest extends AbstractManagerTest { + + private Path openCgaHome; + private Path scratchDir; + private Path analysisPath; + private Path analysisResourcePath; + private ResourceManager resourceManager; + + @Before + public void setUp() throws Exception { + super.setUp(); + + openCgaHome = catalogManagerResource.getOpencgaHome(); + scratchDir = createDir("scratchdir"); + + analysisPath = openCgaHome.resolve(ANALYSIS_DIRNAME); + analysisResourcePath = analysisPath.resolve(RESOURCES_DIRNAME); + + Configuration configuration = Configuration.load(Files.newInputStream(openCgaHome.resolve("conf").resolve("configuration.yml").toFile().toPath())); + configuration.getAnalysis().getResource().setBasePath(openCgaHome.resolve("analysis/resources").toAbsolutePath()); + configuration.serialize(Files.newOutputStream(openCgaHome.resolve("conf").resolve("configuration.yml").toFile().toPath())); + resourceManager = new ResourceManager(openCgaHome); + } + + @Test(expected = ResourceException.class) + public void testNotFetchedResource() throws ResourceException, ToolException, IOException { + resourceManager.checkResourcePaths("relatedness"); + } + + // @Test +// public void testFetchAllResources() throws IOException, ResourceException { +// System.out.println("analysisResourcePath = " + analysisResourcePath.toAbsolutePath()); +// Path outDir = createDir("jobdir").resolve(RESOURCES_DIRNAME); +// Files.createDirectories(outDir); +// System.out.println("outDir = " + outDir.toAbsolutePath()); +// resourceManager.fetchAllResources(version, outDir, catalogManagerResource.getCatalogManager(), catalogManagerResource.getAdminToken()); +// resourceMetadata = JacksonUtils.getDefaultObjectMapper().readerFor(ResourceMetadata.class).readValue(openCgaHome.resolve(ANALYSIS_DIRNAME).resolve(RESOURCES_DIRNAME).resolve(resourceManager.getResourceMetaFilename(version)).toFile()); +// +// for (AnalysisResourceList list : resourceMetadata.getAnalysisResourceLists()) { +// for (AnalysisResource resource : list.getResources()) { +// Assert.assertTrue(Files.exists(analysisResourcePath.resolve(resource.getLocalRelativePath()))); +// } +// } +// } +// + @Test(expected = ResourceException.class) + public void testFetchAllResourcesNoAdmin() throws ResourceException, IOException { + resourceManager.fetchAllResources(null, catalogManagerResource.getCatalogManager(), normalToken1); + } + + @Test + public void testFetchResources() throws IOException, ResourceException, ToolException { + System.out.println("analysisResourcePath = " + analysisResourcePath.toAbsolutePath()); + Path outDir = createDir("jobdir").resolve(RESOURCES_DIRNAME); + Files.createDirectories(outDir); + System.out.println("outDir = " + outDir.toAbsolutePath()); + + resourceManager.fetchResources(Arrays.asList("RELATEDNESS_VARIANTS_PRUNE_IN", "RELATEDNESS_VARIANTS_FRQ"), outDir, + catalogManagerResource.getCatalogManager(), catalogManagerResource.getAdminToken()); + + resourceManager.checkResourcePaths("relatedness"); + } + + @Test + public void testFetchAGivenResource() throws IOException, ResourceException { + System.out.println("analysisResourcePath = " + analysisResourcePath.toAbsolutePath()); + Path outDir = createDir("jobdir").resolve(RESOURCES_DIRNAME); + Files.createDirectories(outDir); + System.out.println("outDir = " + outDir.toAbsolutePath()); + + resourceManager.fetchResources(Arrays.asList("RELATEDNESS_VARIANTS_PRUNE_IN"), outDir, + catalogManagerResource.getCatalogManager(), catalogManagerResource.getAdminToken()); + + resourceManager.checkResourcePath("RELATEDNESS_VARIANTS_PRUNE_IN"); + } + + //------------------------------------------------------------------------- + // P R I V A T E + //------------------------------------------------------------------------- + + private Path createDir(String name) throws IOException { + Path path; + int c = 0; + do { + path = Paths.get("target/test-data").resolve("junit_opencga_" + name + "_"+ TimeUtils.getTimeMillis() + (c > 0 ? "_" + c : "")); + c++; + } while (path.toFile().exists()); + + Files.createDirectories(path); + return path; + } +} \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java index 694f2af3f3f..bebb51c8558 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java @@ -50,8 +50,8 @@ public class StudyManagerTest extends AbstractManagerTest { @Test public void testDefaultVariableSets() throws Exception { - String fqn = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), ownerToken).first().getFqn(); + String fqn = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, null, null, null, + INCLUDE_RESULT, ownerToken).first().getFqn(); Study study = catalogManager.getStudyManager().get(fqn, null, ownerToken).first(); @@ -120,7 +120,7 @@ public void testCreateDuplicatedVariableSets() throws Exception { @Test public void internalVariableSetTest() throws CatalogException { Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), ownerToken).first(); + null, null, null, INCLUDE_RESULT, ownerToken).first(); Set variables = new HashSet<>(); variables.add(new Variable().setId("a").setType(Variable.VariableType.STRING)); @@ -147,8 +147,8 @@ public void internalVariableSetTest() throws CatalogException { @Test public void updateInternalRecessiveGene() throws CatalogException { - Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), ownerToken).first(); + Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, null, null, null, + INCLUDE_RESULT, ownerToken).first(); assertEquals(RecessiveGeneSummaryIndex.Status.NOT_INDEXED, study.getInternal().getIndex().getRecessiveGene().getStatus()); String date = TimeUtils.getTime(); @@ -170,8 +170,8 @@ public void updateInternalRecessiveGene() throws CatalogException { @Test public void updateClinicalConfiguration() throws CatalogException { - Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), ownerToken).first(); + Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, null, null, null, + INCLUDE_RESULT, ownerToken).first(); assertNotNull(study.getInternal().getConfiguration()); assertNotNull(study.getInternal().getConfiguration().getClinical()); assertFalse(study.getInternal().getConfiguration().getClinical().getPriorities().isEmpty()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java index b7d6b72f62f..12508962113 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java @@ -135,7 +135,7 @@ protected void run() throws Exception { public void setUp() throws Exception { super.setUp(); try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { mongoDBAdaptorFactory.getMongoDataStore(organizationId) .getCollection(OrganizationMongoDBAdaptorFactory.MIGRATION_COLLECTION) diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index d0ebcf32ba6..38c371c7a6e 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -17,25 +17,101 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. - # Default URL for downloading analysis resources. - resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + resource: + # Base URL from fetching/downloading analysis resources. + baseUrl: "http://resources.opencb.org/opencb/opencga/analysis/resources-test/" + # Base path where to download the analysis resources. + basePath: "${OPENCGA.INSTALLATION.DIR}/analysis/resources" + # List of resources to fetch during initialization; the willcard '*' can be used, e.g.: to fetch all resources, use '*'; + # to fetch only Exomiser resources, use 'EXOMISER_*' + fetchOnInit: [] + # List of resources + files: + # Exomiser resources + - id: "EXOMISER_13_1_0_HG19" + url: "exomiser/13.1.0/2109_hg19.zip" + md5: "" + path: "exomiser/13.1.0/2109_hg19" + - id: "EXOMISER_13_1_0_HG38" + url: "exomiser/13.1.0/2109_hg38.zip" + md5: "" + path: "exomiser/13.1.0/2109_hg38" + - id: "EXOMISER_13_1_0_PHENOTYPE" + url: "exomiser/13.1.0/2109_phenotype.zip" + md5: "" + path: "exomiser/13.1.0/2109_phenotype" + - id: "EXOMISER_14_0_0_HG19" + url: "exomiser/14.0.0/2402_hg19.zip" + md5: "" + path: "exomiser/2402_hg19" + - id: "EXOMISER_14_0_0_HG38" + url: "exomiser/14.0.0/2402_hg38.zip" + md5: "" + path: "exomiser/2402_hg38" + - id: "EXOMISER_14_0_0_PHENOTYPE" + url: "exomiser/14.0.0/2402_phenotype.zip" + md5: "" + path: "exomiser/14.0.0/2402_phenotype" + + # Reference genomes + - id: "REFERENCE_GENOME_GRCH38_FA" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz" + - id: "REFERENCE_GENOME_GRCH38_FAI" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.fai" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.fai" + - id: "REFERENCE_GENOME_GRCH38_GZI" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.gzi" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.gzi" + - id: "REFERENCE_GENOME_GRCH37_FA" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz" + - id: "REFERENCE_GENOME_GRCH37_FAI" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.fai" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.fai" + - id: "REFERENCE_GENOME_GRCH37_GZI" + url: "reference-genome/20240812_091434/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.gzi" + md5: "" + path: "reference-genome/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.gzi" + + # Relatedness resources + - id: "RELATEDNESS_VARIANTS_PRUNE_IN" + url: "data/relatedness/relatedness_thresholds.tsv" + md5: "97175d1574f14fe45bae13ffe277f295" + path: "relatedness/variants.prune.in" + - id: "RELATEDNESS_VARIANTS_FRQ" + url: "data/ibd/prune-20241015.out" + md5: "0d7ce55d5c48870698633ca7694e95d3" + path: "relatedness/variants.frq" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used opencgaExtTools: "opencb/opencga-ext-tools" tools: - id: "exomiser" - version: "13.1" + version: "13.1.0" dockerId: "exomiser/exomiser-cli:13.1.0" - resources: - HG38: "exomiser/2109_hg38.zip" - PHENOTYPE: "exomiser/2109_phenotype.zip" + resources: ["EXOMISER_13_1_0_HG19", "EXOMISER_13_1_0_HG38", "EXOMISER_13_1_0_PHENOTYPE"] - id: "exomiser" - version: "14.0" + version: "14.0.0" defaultVersion: true dockerId: "exomiser/exomiser-cli:14.0.0" + resources: ["EXOMISER_14_0_0_HG19", "EXOMISER_14_0_0_HG38", "EXOMISER_14_0_0_PHENOTYPE"] + - id: "mutational-signature" resources: - HG38: "exomiser/2402_hg38.zip" - PHENOTYPE: "exomiser/2402_phenotype.zip" + - "REFERENCE_GENOME_GRCH38_FA" + - "REFERENCE_GENOME_GRCH38_FAI" + - "REFERENCE_GENOME_GRCH38_GZI" + - "REFERENCE_GENOME_GRCH37_FA" + - "REFERENCE_GENOME_GRCH37_FAI" + - "REFERENCE_GENOME_GRCH37_GZI" + - id: "relatedness" + resources: ["RELATEDNESS_VARIANTS_PRUNE_IN", "RELATEDNESS_VARIANTS_FRQ"] execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index d63a1ad3478..a723ea8f7a6 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -22,6 +22,7 @@ #' | groupByAudit | /{apiVersion}/admin/audit/groupBy | count, limit, fields[*], entity[*], action, before, after, date | #' | installCatalog | /{apiVersion}/admin/catalog/install | body[*] | #' | jwtCatalog | /{apiVersion}/admin/catalog/jwt | organization, body[*] | +#' | fetchResource | /{apiVersion}/admin/resource/fetch | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | createUsers | /{apiVersion}/admin/users/create | body[*] | #' | importUsers | /{apiVersion}/admin/users/import | organization, body[*] | #' | permissionsUsers | /{apiVersion}/admin/users/permissions | study, entryIds, permissions, category | @@ -43,7 +44,7 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @param count Count the number of elements matching the group. #' @param limit Maximum number of documents (groups) to be returned. #' @param fields Comma separated list of fields by which to group by. - #' @param entity Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] + #' @param entity Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] #' @param action Action performed. #' @param before Object before update. #' @param after Object after update. @@ -65,6 +66,19 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para jwtCatalog=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="catalog", subcategoryId=NULL, action="jwt", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/admin/resource/fetch: + #' Fetch resources from the public server and save them into the OpenCGA local installation. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data Download-resources tool parameters. + fetchResource=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="resource", + subcategoryId=NULL, action="fetch", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/admin/users/create: #' Create a new user. #' @param data JSON containing the parameters. diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index da3baba3e62..7ba43e5128d 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -23,14 +23,14 @@ #' | loadAnnotationSets | /{apiVersion}/files/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | bioformats | /{apiVersion}/files/bioformats | | #' | create | /{apiVersion}/files/create | study, parents, body[*] | -#' | distinct | /{apiVersion}/files/distinct | study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release, field[*] | +#' | distinct | /{apiVersion}/files/distinct | study, id, uuid, name, path, uri, type, bioformat, format, external, resource, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release, field[*] | #' | fetch | /{apiVersion}/files/fetch | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body[*] | #' | formats | /{apiVersion}/files/formats | | #' | link | /{apiVersion}/files/link | study, parents, body[*] | #' | runLink | /{apiVersion}/files/link/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | runPostlink | /{apiVersion}/files/postlink/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | -#' | search | /{apiVersion}/files/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release | -#' | upload | /{apiVersion}/files/upload | file, fileName, fileFormat, bioformat, checksum, study, relativeFilePath, description, parents | +#' | search | /{apiVersion}/files/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, name, path, uri, type, bioformat, format, external, resource, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release | +#' | upload | /{apiVersion}/files/upload | file, fileName, fileFormat, bioformat, checksum, resource, study, relativeFilePath, description, parents | #' | acl | /{apiVersion}/files/{files}/acl | files[*], study, member, silent | #' | delete | /{apiVersion}/files/{files}/delete | study, files[*], skipTrash | #' | info | /{apiVersion}/files/{files}/info | include, exclude, flattenAnnotations, files[*], study, deleted | @@ -104,6 +104,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param bioformat Comma separated Bioformat values. For existing Bioformats see files/bioformats. #' @param format Comma separated Format values. For existing Formats see files/formats. #' @param external Boolean field indicating whether to filter by external or non external files. + #' @param resource Boolean field indicating whether the file is a resource or not. #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param internalVariantIndexStatus Filter by internal variant index status. @@ -198,6 +199,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param bioformat Comma separated Bioformat values. For existing Bioformats see files/bioformats. #' @param format Comma separated Format values. For existing Formats see files/formats. #' @param external Boolean field indicating whether to filter by external or non external files. + #' @param resource Boolean field indicating whether the file is a resource or not. #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param internalVariantIndexStatus Filter by internal variant index status. @@ -224,6 +226,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param fileFormat File format. Allowed values: ['VCF BCF GVCF TBI BIGWIG SAM BAM BAI CRAM CRAI FASTQ FASTA PED TAB_SEPARATED_VALUES COMMA_SEPARATED_VALUES XML PROTOCOL_BUFFER JSON AVRO PARQUET PDF IMAGE PLAIN BINARY NONE UNKNOWN'] #' @param bioformat File bioformat. Allowed values: ['MICROARRAY_EXPRESSION_ONECHANNEL_AGILENT MICROARRAY_EXPRESSION_ONECHANNEL_AFFYMETRIX MICROARRAY_EXPRESSION_ONECHANNEL_GENEPIX MICROARRAY_EXPRESSION_TWOCHANNELS_AGILENT MICROARRAY_EXPRESSION_TWOCHANNELS_GENEPIX DATAMATRIX_EXPRESSION IDLIST IDLIST_RANKED ANNOTATION_GENEVSANNOTATION OTHER_NEWICK OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN'] #' @param checksum Expected MD5 file checksum. + #' @param resource Boolean field indicating whether the file is a resource or not. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param relativeFilePath Path within catalog where the file will be located (default: root folder). #' @param description description. diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index bd20437eb98..6bef965d132 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -118,7 +118,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param operationId Audit operation UUID. #' @param userId User ID. #' @param action Action performed by the user. - #' @param resource Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] + #' @param resource Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] #' @param resourceId Resource ID. #' @param resourceUuid resource UUID. #' @param status Filter by status. Allowed values: ['SUCCESS ERROR'] diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 928fd361341..316eb361144 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -26,6 +26,8 @@ import org.opencb.opencga.core.models.admin.JWTParams; import org.opencb.opencga.core.models.admin.UserImportParams; import org.opencb.opencga.core.models.admin.UserUpdateGroup; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.resource.ResourceFetcherToolParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.User; @@ -100,6 +102,27 @@ public RestResponse jwtCatalog(JWTParams data, ObjectMap params) thro return execute("admin", null, "catalog", null, "jwt", params, POST, ObjectMap.class); } + /** + * Fetch resources from the public server and save them into the OpenCGA local installation. + * @param data Download-resources tool parameters. + * @param params Map containing any of the following optional parameters. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse fetchResource(ResourceFetcherToolParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("admin", null, "resource", null, "fetch", params, POST, Job.class); + } + /** * Create a new user. * @param data JSON containing the parameters. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index dcf3ae18ef8..6e4165e9172 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -142,6 +142,7 @@ public RestResponse create(FileCreateParams data, ObjectMap params) throws * bioformat: Comma separated Bioformat values. For existing Bioformats see files/bioformats. * format: Comma separated Format values. For existing Formats see files/formats. * external: Boolean field indicating whether to filter by external or non external files. + * resource: Boolean field indicating whether the file is a resource or not. * status: Filter by status. * internalStatus: Filter by internal status. * internalVariantIndexStatus: Filter by internal variant index status. @@ -285,6 +286,7 @@ public RestResponse runPostlink(PostLinkToolParams data, ObjectMap params) * bioformat: Comma separated Bioformat values. For existing Bioformats see files/bioformats. * format: Comma separated Format values. For existing Formats see files/formats. * external: Boolean field indicating whether to filter by external or non external files. + * resource: Boolean field indicating whether the file is a resource or not. * status: Filter by status. * internalStatus: Filter by internal status. * internalVariantIndexStatus: Filter by internal variant index status. @@ -321,6 +323,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * fileFormat: File format. * bioformat: File bioformat. * checksum: Expected MD5 file checksum. + * resource: Boolean field indicating whether the file is a resource or not. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * relativeFilePath: Path within catalog where the file will be located (default: root folder). * description: description. diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 3f6204d4165..84eaeb04938 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -37,7 +37,7 @@ export default class Admin extends OpenCGAParentClass { /** Group by operation * @param {String} fields - Comma separated list of fields by which to group by. * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS - * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} entity - Entity to be grouped by. + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE"} entity - Entity to be grouped by. * @param {Object} [params] - The Object containing the following optional parameters: * @param {Boolean} [params.count] - Count the number of elements matching the group. * @param {Number} [params.limit = "50"] - Maximum number of documents (groups) to be returned. The default value is 50. @@ -69,6 +69,24 @@ export default class Admin extends OpenCGAParentClass { return this._post("admin", null, "catalog", null, "jwt", data, params); } + /** Fetch resources from the public server and save them into the OpenCGA local installation + * @param {Object} data - Download-resources tool parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + fetchResource(data, params) { + return this._post("admin", null, "resource", null, "fetch", data, params); + } + /** Create a new user * @param {Object} data - JSON containing the parameters. * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index cfedee4d933..114b650cf26 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -97,6 +97,7 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.bioformat] - Comma separated Bioformat values. For existing Bioformats see files/bioformats. * @param {String} [params.format] - Comma separated Format values. For existing Formats see files/formats. * @param {Boolean} [params.external] - Boolean field indicating whether to filter by external or non external files. + * @param {Boolean} [params.resource] - Boolean field indicating whether the file is a resource or not. * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.internalVariantIndexStatus] - Filter by internal variant index status. @@ -222,6 +223,7 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.bioformat] - Comma separated Bioformat values. For existing Bioformats see files/bioformats. * @param {String} [params.format] - Comma separated Format values. For existing Formats see files/formats. * @param {Boolean} [params.external] - Boolean field indicating whether to filter by external or non external files. + * @param {Boolean} [params.resource] - Boolean field indicating whether the file is a resource or not. * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.internalVariantIndexStatus] - Filter by internal variant index status. @@ -259,6 +261,7 @@ export default class File extends OpenCGAParentClass { * ANNOTATION_GENEVSANNOTATION OTHER_NEWICK OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF OTHER_PED VCF4 VARIANT * ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN"} [params.bioformat] - File bioformat. * @param {String} [params.checksum] - Expected MD5 file checksum. + * @param {Boolean} [params.resource] - Boolean field indicating whether the file is a resource or not. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.relativeFilePath] - Path within catalog where the file will be located (default: root folder). * @param {String} [params.description] - description. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index fe51a747d05..a4139fa1813 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -119,7 +119,7 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.userId] - User ID. * @param {String} [params.action] - Action performed by the user. * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS - * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} [params.resource] - Resource involved. + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE"} [params.resource] - Resource involved. * @param {String} [params.resourceId] - Resource ID. * @param {String} [params.resourceUuid] - resource UUID. * @param {"SUCCESS ERROR"} [params.status] - Filter by status. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 7311fa7e5dd..1926c986c75 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -27,7 +27,8 @@ def group_by_audit(self, fields, entity, **options): :param str entity: Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION - VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] (REQUIRED) + VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW + RESOURCE'] (REQUIRED) :param str fields: Comma separated list of fields by which to group by. (REQUIRED) :param bool count: Count the number of elements matching the group. @@ -64,6 +65,30 @@ def jwt_catalog(self, data=None, **options): return self._post(category='admin', resource='jwt', subcategory='catalog', data=data, **options) + def fetch_resource(self, data=None, **options): + """ + Fetch resources from the public server and save them into the OpenCGA + local installation. + PATH: /{apiVersion}/admin/resource/fetch + + :param dict data: Download-resources tool parameters. (REQUIRED) + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='admin', resource='fetch', subcategory='resource', data=data, **options) + def create_users(self, data=None, **options): """ Create a new user. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index 505d21464b2..a5fe14dcf2b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -116,6 +116,8 @@ def distinct(self, field, **options): see files/formats. :param bool external: Boolean field indicating whether to filter by external or non external files. + :param bool resource: Boolean field indicating whether the file is a + resource or not. :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str internal_variant_index_status: Filter by internal variant @@ -290,6 +292,8 @@ def search(self, **options): see files/formats. :param bool external: Boolean field indicating whether to filter by external or non external files. + :param bool resource: Boolean field indicating whether the file is a + resource or not. :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str internal_variant_index_status: Filter by internal variant @@ -345,6 +349,8 @@ def upload(self, **options): OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN'] :param str checksum: Expected MD5 file checksum. + :param bool resource: Boolean field indicating whether the file is a + resource or not. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str relative_file_path: Path within catalog where the file will diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 4aa1518fdf5..2ab53b7c02a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -139,7 +139,7 @@ def search_audit(self, study, **options): :param str resource: Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT - ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] + ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] :param str resource_id: Resource ID. :param str resource_uuid: resource UUID. :param str status: Filter by status. Allowed values: ['SUCCESS ERROR'] diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index c10ef156497..5a2f1f6e02f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -291,7 +291,8 @@ public class FieldConstants { public static final String FILE_CHECKSUM = "The checksum of the file."; public static final String FILE_PATH = "The path of the file."; public static final String FILE_URI = "The uri of the file."; - public static final String FILE_EXTERNAL = "Indicates the file is external or not."; + public static final String FILE_EXTERNAL = "Indicates whether the file comes from an external path or not."; + public static final String FILE_RESOURCE = "Indicates the file is treated as a resource."; public static final String FILE_SIZE = "The size of the file."; public static final String FILE_SOFTWARE = "Software related with file."; public static final String FILE_EXPERIMENT = "File experiment."; @@ -549,10 +550,15 @@ public class FieldConstants { public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; // Exomiser - public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; - public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; + public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID to be analysed."; + public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID to be analysed."; public static final String EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION = "Clinical analysis type: SINGLE or FAMILY."; - public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version in the format X.Y where X is the major version and Y the" - + " minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration" - + " file to view all installed Exomiser versions and identify the default version."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version, e.g.: 14.0.0. If the version is not specified," + + " the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the" + + " default version."; + + // Fetch resources + public static final String FETCH_RESOURCES_DESCRIPTION = "List of resource IDs, separated by commas, to fetch (available resources are" + + " specified in the configuration file). The wildcard '*' can be used: for example, use '*' to fetch all resources, or" + + " 'EXOMISER_*' to fetch only Exomiser resources."; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 765a1c5cf25..b907e3b07ff 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -44,6 +44,7 @@ public class ParamConstants { public static final String PERMISSION_LIST = "permissions"; private static final String REGEX_SUPPORT = ". Also admits basic regular expressions using the operator '~', " + "i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search."; + public static final String RESOURCES_FOLDER = "RESOURCES"; @Deprecated // Use INTERNAL_VARIANT_INDEX_STATUS_PARAM public static final String INTERNAL_INDEX_STATUS_PARAM = "internalIndexStatus"; public static final String INTERNAL_VARIANT_INDEX_STATUS_PARAM = "internalVariantIndexStatus"; @@ -194,6 +195,7 @@ public class ParamConstants { public static final String FILE_TYPE_DESCRIPTION = "File type, either FILE or DIRECTORY"; public static final String FILE_FORMAT_DESCRIPTION = "Comma separated Format values. For existing Formats see files/formats"; public static final String FILE_EXTERNAL_DESCRIPTION = "Boolean field indicating whether to filter by external or non external files"; + public static final String FILE_RESOURCE_DESCRIPTION = "Boolean field indicating whether the file is a resource or not"; public static final String FILE_BIOFORMAT_DESCRIPTION = "Comma separated Bioformat values. For existing Bioformats see " + "files/bioformats"; public static final String FILE_STATUS_DESCRIPTION = "File status"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 9f1c7b32683..fc7c7b01049 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -17,16 +17,14 @@ package org.opencb.opencga.core.config; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class Analysis { private List packages; private String scratchDir; - private String resourceUrl; + private Resource resource; private String opencgaExtTools; private List tools; @@ -42,6 +40,20 @@ public Analysis() { frameworks = new ArrayList<>(); } + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Analysis{"); + sb.append("packages=").append(packages); + sb.append(", scratchDir='").append(scratchDir).append('\''); + sb.append(", resource=").append(resource); + sb.append(", opencgaExtTools='").append(opencgaExtTools).append('\''); + sb.append(", tools=").append(tools); + sb.append(", execution=").append(execution); + sb.append(", frameworks=").append(frameworks); + sb.append('}'); + return sb.toString(); + } + public List getPackages() { return packages; } @@ -60,12 +72,12 @@ public Analysis setScratchDir(String scratchDir) { return this; } - public String getResourceUrl() { - return resourceUrl; + public Resource getResource() { + return resource; } - public Analysis setResourceUrl(String resourceUrl) { - this.resourceUrl = resourceUrl; + public Analysis setResource(Resource resource) { + this.resource = resource; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java index 3744915b6b9..b7fd30dbccb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -16,8 +16,8 @@ package org.opencb.opencga.core.config; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; public class AnalysisTool { @@ -26,13 +26,13 @@ public class AnalysisTool { private boolean defaultVersion; private String dockerId; private String params; - private Map resources; + private List resources; public AnalysisTool() { - resources = new HashMap<>(); + resources = new ArrayList<>(); } - public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, Map resources) { + public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, List resources) { this.id = id; this.version = version; this.defaultVersion = defaultVersion; @@ -99,11 +99,11 @@ public AnalysisTool setParams(String params) { return this; } - public Map getResources() { + public List getResources() { return resources; } - public AnalysisTool setResources(Map resources) { + public AnalysisTool setResources(List resources) { this.resources = resources; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Resource.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Resource.java new file mode 100644 index 00000000000..f567b8fd5d1 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Resource.java @@ -0,0 +1,88 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public class Resource { + + private String baseUrl; + private Path basePath; + private List fetchOnInit; + private List files; + + public Resource() { + this.fetchOnInit = new ArrayList<>(); + this.files = new ArrayList<>(); + } + + public Resource(String baseUrl, Path basePath, List fetchOnInit, List files) { + this.baseUrl = baseUrl; + this.basePath = basePath; + this.fetchOnInit = fetchOnInit; + this.files = files; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Resource{"); + sb.append("baseUrl='").append(baseUrl).append('\''); + sb.append(", basePath=").append(basePath); + sb.append(", fetchOnInit=").append(fetchOnInit); + sb.append(", files=").append(files); + sb.append('}'); + return sb.toString(); + } + + public String getBaseUrl() { + return baseUrl; + } + + public Resource setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + public Path getBasePath() { + return basePath; + } + + public Resource setBasePath(Path basePath) { + this.basePath = basePath; + return this; + } + + public List getFetchOnInit() { + return fetchOnInit; + } + + public Resource setFetchOnInit(List fetchOnInit) { + this.fetchOnInit = fetchOnInit; + return this; + } + + public List getFiles() { + return files; + } + + public Resource setFiles(List files) { + this.files = files; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ResourceFile.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ResourceFile.java new file mode 100644 index 00000000000..aad7da96857 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ResourceFile.java @@ -0,0 +1,82 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +public class ResourceFile { + + private String id; + private String url; + private String md5; + private String path; + + public ResourceFile() { + } + + public ResourceFile(String id, String url, String md5, String path) { + this.id = id; + this.url = url; + this.md5 = md5; + this.path = path; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ResourceFile{"); + sb.append("id='").append(id).append('\''); + sb.append(", url='").append(url).append('\''); + sb.append(", md5='").append(md5).append('\''); + sb.append(", path='").append(path).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public ResourceFile setId(String id) { + this.id = id; + return this; + } + + public String getUrl() { + return url; + } + + public ResourceFile setUrl(String url) { + this.url = url; + return this; + } + + public String getMd5() { + return md5; + } + + public ResourceFile setMd5(String md5) { + this.md5 = md5; + return this; + } + + public String getPath() { + return path; + } + + public ResourceFile setPath(String path) { + this.path = path; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index 2777222715b..d06aafd174a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -79,7 +79,9 @@ public enum Resource { CLINICAL, EXPRESSION, RGA, - FUNCTIONAL; + FUNCTIONAL, + WORKFLOW, + RESOURCE; public List getFullPermissionList() { switch (this) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java index 581cc6d6dd1..47cad53aeca 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java @@ -121,10 +121,12 @@ public class File extends Annotable { description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "external", - description = FieldConstants.FILE_EXTERNAL) + @DataField(id = "external", description = FieldConstants.FILE_EXTERNAL) private boolean external; + @DataField(id = "resource", description = FieldConstants.FILE_RESOURCE) + private boolean resource; + @DataField(id = "size", description = FieldConstants.FILE_SIZE) private long size; @@ -197,26 +199,26 @@ public File() { } public File(String name, Type type, Format format, Bioformat bioformat, String path, URI uri, String description, FileInternal internal, - long size, int release) { + boolean resource, long size, int release) { this(name, type, format, bioformat, uri, path, null, TimeUtils.getTime(), TimeUtils.getTime(), description, - false, size, new Software(), new FileExperiment(), Collections.emptyList(), Collections.emptyList(), "", release, + false, resource, size, new Software(), new FileExperiment(), Collections.emptyList(), Collections.emptyList(), "", release, Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), new Status(), internal, Collections.emptyMap()); } - public File(Type type, Format format, Bioformat bioformat, String path, String description, FileInternal internal, long size, - List sampleIds, Software software, String jobId, FileQualityControl qualityControl, Map stats, - Map attributes) { + public File(Type type, Format format, Bioformat bioformat, String path, String description, FileInternal internal, boolean resource, + long size, List sampleIds, Software software, String jobId, FileQualityControl qualityControl, + Map stats, Map attributes) { this("", type, format, bioformat, null, path, null, TimeUtils.getTime(), TimeUtils.getTime(), description, - false, size, software, new FileExperiment(), sampleIds, Collections.emptyList(), jobId, -1, Collections.emptyList(), - Collections.emptyList(), qualityControl, stats, new Status(), internal, attributes); + false, resource, size, software, new FileExperiment(), sampleIds, Collections.emptyList(), jobId, -1, + Collections.emptyList(), Collections.emptyList(), qualityControl, stats, new Status(), internal, attributes); } public File(String name, Type type, Format format, Bioformat bioformat, URI uri, String path, String checksum, String creationDate, - String modificationDate, String description, boolean external, long size, Software software, FileExperiment experiment, - List sampleIds, List relatedFiles, String jobId, int release, List tags, - List annotationSets, FileQualityControl qualityControl, Map stats, Status status, - FileInternal internal, Map attributes) { + String modificationDate, String description, boolean external, boolean resource, long size, Software software, + FileExperiment experiment, List sampleIds, List relatedFiles, String jobId, int release, + List tags, List annotationSets, FileQualityControl qualityControl, Map stats, + Status status, FileInternal internal, Map attributes) { id = StringUtils.isNotEmpty(path) ? StringUtils.replace(path, "/", ":") : path; this.name = name; this.type = type; @@ -230,6 +232,7 @@ public File(String name, Type type, Format format, Bioformat bioformat, URI uri, this.description = description; this.release = release; this.external = external; + this.resource = resource; this.internal = internal; this.size = size; this.software = software; @@ -262,6 +265,7 @@ public String toString() { sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", external=").append(external); + sb.append(", resource=").append(resource); sb.append(", size=").append(size); sb.append(", software=").append(software); sb.append(", experiment=").append(experiment); @@ -419,6 +423,15 @@ public File setExternal(boolean external) { return this; } + public boolean isResource() { + return resource; + } + + public File setResource(boolean resource) { + this.resource = resource; + return this; + } + public long getSize() { return size; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParams.java index 0c097756bf4..d33825cf040 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParams.java @@ -1,6 +1,8 @@ package org.opencb.opencga.core.models.file; import org.opencb.biodata.models.clinical.interpretation.Software; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.models.common.StatusParams; import java.util.List; @@ -17,6 +19,10 @@ public class FileCreateParams { private List sampleIds; private Software software; private List tags; + + @DataField(id = "resource", description = FieldConstants.FILE_RESOURCE) + private Boolean resource; + private String jobId; private String creationDate; private String modificationDate; @@ -27,8 +33,8 @@ public FileCreateParams() { } public FileCreateParams(String content, String path, String description, File.Type type, File.Format format, File.Bioformat bioformat, - List sampleIds, Software software, List tags, String jobId, String creationDate, - String modificationDate, StatusParams status, Map attributes) { + List sampleIds, Software software, List tags, Boolean resource, String jobId, + String creationDate, String modificationDate, StatusParams status, Map attributes) { this.content = content; this.path = path; this.description = description; @@ -38,6 +44,7 @@ public FileCreateParams(String content, String path, String description, File.Ty this.sampleIds = sampleIds; this.software = software; this.tags = tags; + this.resource = resource; this.jobId = jobId; this.creationDate = creationDate; this.modificationDate = modificationDate; @@ -57,6 +64,7 @@ public String toString() { sb.append(", sampleIds=").append(sampleIds); sb.append(", software=").append(software); sb.append(", tags=").append(tags); + sb.append(", resource=").append(resource); sb.append(", jobId='").append(jobId).append('\''); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); @@ -147,6 +155,15 @@ public FileCreateParams setTags(List tags) { return this; } + public Boolean getResource() { + return resource; + } + + public FileCreateParams setResource(Boolean resource) { + this.resource = resource; + return this; + } + public String getJobId() { return jobId; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParamsOld.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParamsOld.java deleted file mode 100644 index 5f5727eefbb..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileCreateParamsOld.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.file; - -import com.fasterxml.jackson.annotation.JsonProperty; - -@Deprecated -public class FileCreateParamsOld { - - @JsonProperty(required = true) - private String path; - private String content; - private String description; - - @JsonProperty(defaultValue = "false") - private boolean parents; - - @JsonProperty(defaultValue = "false") - private boolean directory; - - public FileCreateParamsOld() { - } - - public FileCreateParamsOld(String path, String content, String description, boolean parents, boolean directory) { - this.path = path; - this.content = content; - this.description = description; - this.parents = parents; - this.directory = directory; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("FileCreateParams{"); - sb.append("path='").append(path).append('\''); - sb.append(", content='").append(content).append('\''); - sb.append(", description='").append(description).append('\''); - sb.append(", parents=").append(parents); - sb.append(", directory=").append(directory); - sb.append('}'); - return sb.toString(); - } - - public String getPath() { - return path; - } - - public FileCreateParamsOld setPath(String path) { - this.path = path; - return this; - } - - public String getContent() { - return content; - } - - public FileCreateParamsOld setContent(String content) { - this.content = content; - return this; - } - - public String getDescription() { - return description; - } - - public FileCreateParamsOld setDescription(String description) { - this.description = description; - return this; - } - - public boolean isParents() { - return parents; - } - - public FileCreateParamsOld setParents(boolean parents) { - this.parents = parents; - return this; - } - - public boolean isDirectory() { - return directory; - } - - public FileCreateParamsOld setDirectory(boolean directory) { - this.directory = directory; - return this; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileFetch.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileFetch.java index ea2bd28bce7..511be68529c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileFetch.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileFetch.java @@ -16,6 +16,8 @@ package org.opencb.opencga.core.models.file; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.CliParam; @@ -27,6 +29,9 @@ public class FileFetch extends ToolParams { @CliParam(required = true) private String url; + @DataField(id = "resource", description = FieldConstants.FILE_RESOURCE) + private Boolean resource; + /** * Folder path where the file will be downloaded. */ @@ -35,15 +40,17 @@ public class FileFetch extends ToolParams { public FileFetch() { } - public FileFetch(String url, String path) { + public FileFetch(String url, String path, Boolean resource) { this.path = path; this.url = url; + this.resource = resource; } @Override public String toString() { final StringBuilder sb = new StringBuilder("FileFetch{"); sb.append("url='").append(url).append('\''); + sb.append(", resource=").append(resource); sb.append(", path='").append(path).append('\''); sb.append('}'); return sb.toString(); @@ -66,4 +73,13 @@ public FileFetch setPath(String path) { this.path = path; return this; } + + public Boolean getResource() { + return resource; + } + + public FileFetch setResource(Boolean resource) { + this.resource = resource; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/resource/ResourceFetcherToolParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/resource/ResourceFetcherToolParams.java new file mode 100644 index 00000000000..713a789e9ae --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/resource/ResourceFetcherToolParams.java @@ -0,0 +1,57 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.models.resource; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.tools.ToolParams; + +import java.util.ArrayList; +import java.util.List; + +public class ResourceFetcherToolParams extends ToolParams { + + public static final String DESCRIPTION = "Download-resources tool parameters"; + + @DataField(id = "resources", description = FieldConstants.FETCH_RESOURCES_DESCRIPTION) + private List resources; + + public ResourceFetcherToolParams() { + this.resources = new ArrayList<>(); + } + + public ResourceFetcherToolParams(List resources) { + this.resources = resources; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ResourceFetcherToolParams{"); + sb.append(", resources=").append(resources); + sb.append('}'); + return sb.toString(); + } + + public List getResources() { + return resources; + } + + public ResourceFetcherToolParams setResources(List resources) { + this.resources = resources; + return this; + } +} diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 4eb3543f791..d0bf998c540 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -72,27 +72,93 @@ analysis: - "org.opencb.opencga" # Scratch folder for the analysis. scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" - # Default URL for downloading analysis resources. - resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + resource: + # Base URL from fetching/downloading analysis resources. + baseUrl: "http://resources.opencb.org/opencb/opencga/analysis/resources/" + # Base path where to download the analysis resources. + basePath: "${OPENCGA.INSTALLATION.DIR}/analysis/resources" + # List of resources to fetch during initialization; the willcard '*' can be used, e.g.: to fetch all resources, use '*'; + # to fetch only Exomiser resources, use 'EXOMISER_*' + fetchOnInit: ["*"] + # List of resources + files: + # Exomiser resources + - id: "EXOMISER_13_1_0_HG38" + url: "exomiser/13.1.0/2109_hg38.zip" + md5: "6d4011146705ca7a28eba9df0bfdacd2" + path: "exomiser/13.1.0/2109_hg38" + - id: "EXOMISER_13_1_0_PHENOTYPE" + url: "exomiser/13.1.0/2109_phenotype.zip" + md5: "197d126e1b441674f5d5979ac2027591" + path: "exomiser/13.1.0/2109_phenotype" + - id: "EXOMISER_14_0_0_HG38" + url: "exomiser/14.0.0/2402_hg38.zip" + md5: "7f89dcf9221c7aa5d5471a2ffdb58303" + path: "exomiser/14.0.0/2402_hg38" + - id: "EXOMISER_14_0_0_PHENOTYPE" + url: "exomiser/14.0.0/2402_phenotype.zip" + md5: "3e7092c852c5f23373ccd465bf2fcd85" + path: "exomiser/14.0.0/2402_phenotype" + + # Reference genomes + - id: "REFERENCE_GENOME_GRCH38_FA" + url: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz" + md5: "9732aa3c64469b25b73930db9c96a89f" + path: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz" + - id: "REFERENCE_GENOME_GRCH38_FAI" + url: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.fai" + md5: "d527f3eb6b664020cf4d882b5820056f" + path: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.fai" + - id: "REFERENCE_GENOME_GRCH38_GZI" + url: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.gzi" + md5: "e904772fc5dfa2d69690aafda36332a4" + path: "reference-genomes/grch38/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz.gzi" + - id: "REFERENCE_GENOME_GRCH37_FA" + url: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz" + md5: "39a3fff4234a1268c937ec012b3cabb4" + path: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz" + - id: "REFERENCE_GENOME_GRCH37_FAI" + url: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.fai" + md5: "15d081d2bdfe37d0fff6b753a4dcfef0" + path: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.fai" + - id: "REFERENCE_GENOME_GRCH37_GZI" + url: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.gzi" + md5: "bed234b55a30d624159e8577def7005c" + path: "reference-genomes/grch37/Homo_sapiens.GRCh37.dna.primary_assembly.fa.gz.gzi" + + # Relatedness resources + - id: "RELATEDNESS_VARIANTS_PRUNE_IN" + url: "relatedness/20221026_152242/variants.prune.in" + md5: "9d04ef8199108cfa1223f5217f96d144" + path: "relatedness/variants.prune.in" + - id: "RELATEDNESS_VARIANTS_FRQ" + url: "relatedness/20221026_152242/variants.frq" + md5: "30c42b1a87b5c028f1d8201fd47935e6" + path: "relatedness/variants.frq" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used opencgaExtTools: "opencb/opencga-ext-tools" tools: - id: "exomiser" - version: "13.1" + version: "13.1.0" dockerId: "exomiser/exomiser-cli:13.1.0" - resources: - HG19: "exomiser/2109_hg19.zip" - HG38: "exomiser/2109_hg38.zip" - PHENOTYPE: "exomiser/2109_phenotype.zip" + resources: ["EXOMISER_13_1_0_HG38", "EXOMISER_13_1_0_PHENOTYPE"] - id: "exomiser" - version: "14.0" + version: "14.0.0" defaultVersion: true dockerId: "exomiser/exomiser-cli:14.0.0" + resources: ["EXOMISER_14_0_0_HG38", "EXOMISER_14_0_0_PHENOTYPE"] + - id: "mutational-signature" resources: - HG19: "exomiser/2402_hg19.zip" - HG38: "exomiser/2402_hg38.zip" - PHENOTYPE: "exomiser/2402_phenotype.zip" + - "REFERENCE_GENOME_GRCH38_FA" + - "REFERENCE_GENOME_GRCH38_FAI" + - "REFERENCE_GENOME_GRCH38_GZI" + - "REFERENCE_GENOME_GRCH37_FA" + - "REFERENCE_GENOME_GRCH37_FAI" + - "REFERENCE_GENOME_GRCH37_GZI" + - id: "relatedness" + resources: ["RELATEDNESS_VARIANTS_PRUNE_IN", "RELATEDNESS_VARIANTS_FRQ"] execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java index 86b801b0390..2c89ec2ce99 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java @@ -16,6 +16,7 @@ package org.opencb.opencga.master.monitor; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -24,15 +25,21 @@ import org.eclipse.jetty.servlet.ServletHolder; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; +import org.opencb.opencga.analysis.resource.ResourceFetcherTool; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Resource; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.resource.ResourceFetcherToolParams; import org.opencb.opencga.master.monitor.daemons.ExecutionDaemon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Paths; import java.util.Collections; @@ -65,7 +72,7 @@ public class MonitorService { protected static Logger logger; public MonitorService(Configuration configuration, StorageConfiguration storageConfiguration, String appHome, String token) - throws CatalogException { + throws CatalogException, ToolException, IOException { this.configuration = configuration; this.storageConfiguration = storageConfiguration; this.appHome = appHome; @@ -73,7 +80,7 @@ public MonitorService(Configuration configuration, StorageConfiguration storageC init(token); } - private void init(String token) throws CatalogException { + private void init(String token) throws CatalogException, ToolException, IOException { String logDir = configuration.getLogDir(); boolean logFileEnabled; @@ -113,6 +120,8 @@ private void init(String token) throws CatalogException { // authorizationThread = new Thread(authorizationDaemon, "authorization-thread"); this.port = configuration.getMonitor().getPort(); + + fetchResources(token); } public void start() throws Exception { @@ -198,4 +207,25 @@ private void stopRestServer() throws Exception { logger.info("REST server shut down"); logger.info("*********************************"); } + + private void fetchResources(String token) { + if (CollectionUtils.isEmpty(configuration.getAnalysis().getResource().getFetchOnInit())) { + // Nothing to do + logger.info("There are no resources to fetch because the configuration parameter 'fetchOnInit' is empty."); + return; + } + + try { + Resource resourceConfig = configuration.getAnalysis().getResource(); + + ResourceFetcherToolParams params = new ResourceFetcherToolParams(); + params.setResources(resourceConfig.getFetchOnInit()); + + catalogManager.getJobManager() + .submit(ParamConstants.ADMIN_STUDY_FQN, ResourceFetcherTool.ID, Enums.Priority.URGENT, params.toParams(), null, null, + null, null, null, null, false, token); + } catch (CatalogException e) { + logger.error("Error submitting job '" + ResourceFetcherTool.ID + "'", e); + } + } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java index 01be67e5a16..2819c3f7952 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java @@ -177,6 +177,7 @@ public Response upload( @ApiParam(value = "File format") @DefaultValue("") @FormDataParam("fileFormat") File.Format fileFormat, @ApiParam(value = "File bioformat") @DefaultValue("") @FormDataParam("bioformat") File.Bioformat bioformat, @ApiParam(value = "Expected MD5 file checksum") @DefaultValue("") @FormDataParam("checksum") String expectedChecksum, + @ApiParam(value = ParamConstants.FILE_RESOURCE_DESCRIPTION) @FormDataParam("resource") Boolean resource, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @FormDataParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Path within catalog where the file will be located (default: root folder)") @DefaultValue("") @FormDataParam("relativeFilePath") String relativeFilePath, @ApiParam(value = "description") @DefaultValue("") @FormDataParam("description") @@ -205,9 +206,11 @@ public Response upload( } long expectedSize = fileMetaData.getSize(); + boolean isResource = resource != null && resource; File file = new File() .setName(fileName) .setPath(relativeFilePath + fileName) + .setResource(isResource) .setFormat(fileFormat) .setBioformat(bioformat); return createOkResponse(fileManager.upload(studyStr, fileInputStream, file, false, parents, true, @@ -319,6 +322,7 @@ public Response search( @ApiParam(value = ParamConstants.FILE_BIOFORMAT_DESCRIPTION) @QueryParam("bioformat") String bioformat, @ApiParam(value = ParamConstants.FILE_FORMAT_DESCRIPTION) @QueryParam("format") String formats, @ApiParam(value = ParamConstants.FILE_EXTERNAL_DESCRIPTION) @QueryParam("external") Boolean external, + @ApiParam(value = ParamConstants.FILE_RESOURCE_DESCRIPTION) @QueryParam("resource") Boolean resource, @ApiParam(value = ParamConstants.STATUS_DESCRIPTION) @QueryParam(ParamConstants.STATUS_PARAM) String status, @ApiParam(value = ParamConstants.INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.INTERNAL_VARIANT_INDEX_STATUS_DESCRIPTION) @QueryParam(ParamConstants.INTERNAL_VARIANT_INDEX_STATUS_PARAM) String internalVariantIndexStatus, @@ -357,6 +361,7 @@ public Response distinct( @ApiParam(value = ParamConstants.FILE_BIOFORMAT_DESCRIPTION) @QueryParam("bioformat") String bioformat, @ApiParam(value = ParamConstants.FILE_FORMAT_DESCRIPTION) @QueryParam("format") String formats, @ApiParam(value = ParamConstants.FILE_EXTERNAL_DESCRIPTION) @QueryParam("external") Boolean external, + @ApiParam(value = ParamConstants.FILE_RESOURCE_DESCRIPTION) @QueryParam("resource") Boolean resource, @ApiParam(value = ParamConstants.STATUS_DESCRIPTION) @QueryParam(ParamConstants.STATUS_PARAM) String status, @ApiParam(value = ParamConstants.INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.INTERNAL_VARIANT_INDEX_STATUS_DESCRIPTION) @QueryParam(ParamConstants.INTERNAL_VARIANT_INDEX_STATUS_PARAM) String internalIndexStatus, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java index c0345d421f5..9bb2a20def6 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java @@ -20,6 +20,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.ListUtils; +import org.opencb.opencga.analysis.resource.ResourceFetcherTool; import org.opencb.opencga.catalog.db.api.MetaDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -28,6 +29,8 @@ import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.admin.*; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.resource.ResourceFetcherToolParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -35,7 +38,7 @@ import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; -import org.opencb.opencga.server.rest.OpenCGAWSServer; +import org.opencb.opencga.server.rest.analysis.AnalysisWebService; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -44,12 +47,13 @@ import java.util.Collections; import java.util.Map; +import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; import static org.opencb.opencga.core.models.admin.UserImportParams.ResourceType.*; @Path("/{apiVersion}/admin") @Produces(MediaType.APPLICATION_JSON) @Api(value = "Admin", position = 4, description = "Administrator webservices") -public class AdminWSServer extends OpenCGAWSServer { +public class AdminWSServer extends AnalysisWebService { //OpenCGAWSServer { public AdminWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { @@ -354,4 +358,20 @@ public Response token( } } + //******************************** RESOURCES **********************************// + + @POST + @Path("/resource/fetch") + @ApiOperation(value = ResourceFetcherTool.DESCRIPTION, response = Job.class) + public Response fetchResources( + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = ResourceFetcherToolParams.DESCRIPTION, required = true) ResourceFetcherToolParams params) { + return submitJob(ResourceFetcherTool.ID, null, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index 3dd6fa53aa4..2c7df87663e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -32,8 +32,10 @@ import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationAnalysis; import org.opencb.opencga.analysis.rga.RgaManager; import org.opencb.opencga.analysis.rga.RgaQueryParams; +import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserAnalysisUtils; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; @@ -1324,7 +1326,13 @@ public Response interpretationExomiserRun( @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserInterpretationAnalysisParams.DESCRIPTION, required = true) ExomiserInterpretationAnalysisParams params) { - return submitJob(ExomiserInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + return run(() -> { + // Check before submitting the job + ExomiserAnalysisUtils.checkResources(params.getExomiserVersion(), study, catalogManager, token, opencgaHome); + + // Submit the exomiser interpretation analysis + return submitJobRaw(ExomiserInterpretationAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + }); } @POST diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index 70748fffb0c..15a2cb19977 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -51,6 +51,7 @@ import org.opencb.opencga.analysis.variant.stats.CohortVariantStatsAnalysis; import org.opencb.opencga.analysis.variant.stats.SampleVariantStatsAnalysis; import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserAnalysisUtils; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.gatk.GatkWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysis; @@ -609,7 +610,13 @@ public Response exomiserRun( @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserWrapperParams.DESCRIPTION, required = true) ExomiserWrapperParams params) { - return submitJob(ExomiserWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + return run(() -> { + // Check before submitting the job + ExomiserAnalysisUtils.checkResources(params.getExomiserVersion(), study, catalogManager, token, opencgaHome); + + // Submit the exomiser analysis + return submitJobRaw(ExomiserWrapperAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + }); } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/fileupload/FileUploadServlet.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/fileupload/FileUploadServlet.java index 2aefadef49b..14cbc0414b6 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/fileupload/FileUploadServlet.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/fileupload/FileUploadServlet.java @@ -101,7 +101,7 @@ protected void doPost(HttpServletRequest request, copyInputStreamToFile(fileContent, file); logger.info("File uploaded " + file.getAbsolutePath()); result = catalogManager.getFileManager().moveAndRegister(studyId, - file.toPath(), null, path != null ? path : "/", token); + file.toPath(), null, path != null ? path : "/", false, token); logger.info("Registered file " + file.getName() + " to " + studyId); }