diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageToolExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageToolExecutor.java index 2221a4e23bf..2152f8cc91b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageToolExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageToolExecutor.java @@ -23,6 +23,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.storage.core.StorageEngineFactory; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisExecutor.java index b93de5d90ea..3394f84f104 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisExecutor.java @@ -17,7 +17,7 @@ package org.opencb.opencga.analysis.clinical; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.ConfigurationUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.config.Configuration; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java index 81e4502625f..4b4f0ef0a67 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java @@ -23,7 +23,7 @@ import org.opencb.biodata.models.clinical.interpretation.Software; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.ConfigurationUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 39de4806eb0..245a55f3742 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -38,6 +38,8 @@ import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.Interpretation; @@ -136,13 +138,21 @@ protected void run() throws ToolException { protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, CatalogException, IOException { + // Get Docker image name and version + String dockerImage = ConfigurationUtils.getDockerImage(Docker.EXOMISER_IMAGE_KEY, configuration); + String dockerVersion = ""; + String[] split = dockerImage.split(":"); + String dockerName = split[0]; + if (split.length > 1) { + dockerVersion = split[1]; + } // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") - .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) - .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION))); + .setRepository("Docker: " + dockerName) + .setVersion(dockerVersion))); // Analyst ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java index 2e99c99c2fd..7183f9f8d96 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java @@ -28,12 +28,9 @@ import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor; -@Tool(id = PedigreeGraphAnalysis.ID, resource = Enums.Resource.FAMILY) +@Tool(id = PedigreeGraphAnalysisExecutor.ID, resource = Enums.Resource.FAMILY) public class PedigreeGraphAnalysis extends OpenCgaToolScopeStudy { - public static final String ID = "pedigree-graph"; - public static final String DESCRIPTION = "Compute the family pedigree graph image."; - @ToolParams private PedigreeGraphAnalysisParams pedigreeParams = new PedigreeGraphAnalysisParams(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphInitAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphInitAnalysis.java index 619c7f3490e..b5f47978ea5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphInitAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphInitAnalysis.java @@ -25,6 +25,8 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.PedigreeGraphUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.family.Family; @@ -59,11 +61,12 @@ public List getSteps() { @Override protected void run() throws ToolException { step("pull-docker", () -> { + String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, configuration); // Be sure docker is alive and that opencga-ext-tools is downloaded logger.info("Checking if docker daemon is alive"); DockerUtils.checkDockerDaemonAlive(); - logger.info("Pulling docker '{}'", PedigreeGraphUtils.R_DOCKER_IMAGE); - new Command("docker pull " + PedigreeGraphUtils.R_DOCKER_IMAGE).run(); + logger.info("Pulling docker '{}'", dockerImage); + new Command("docker pull " + dockerImage).run(); }); step(getId(), () -> { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java index 4dfd4267480..ef0b2d0c54b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java @@ -16,31 +16,22 @@ package org.opencb.opencga.analysis.family; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.analysis.StorageToolExecutor; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.PedigreeGraphUtils; -import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; -@ToolExecutor(id="opencga-local", tool = PedigreeGraphAnalysis.ID, +@ToolExecutor(id="opencga-local", tool = PedigreeGraphAnalysisExecutor.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class PedigreeGraphLocalAnalysisExecutor extends PedigreeGraphAnalysisExecutor { @@ -51,8 +42,10 @@ public class PedigreeGraphLocalAnalysisExecutor extends PedigreeGraphAnalysisExe @Override public void run() throws ToolException, CatalogException, IOException, StorageEngineException { opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); + Configuration configuration = ConfigurationUtils.loadConfiguration(opencgaHome.toAbsolutePath().toString()); // Run R script for fitting signature - PedigreeGraphUtils.createPedigreeGraph(getFamily(), opencgaHome.resolve("analysis/" + PedigreeGraphAnalysis.ID), getOutDir()); + PedigreeGraphUtils.createPedigreeGraph(getFamily(), opencgaHome.resolve("analysis/" + PedigreeGraphAnalysisExecutor.ID), + getOutDir(), configuration); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java index 7277c3d2429..a73082f5270 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java @@ -30,8 +30,9 @@ import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; -import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.individual.Individual; @@ -117,7 +118,9 @@ public static RelatednessReport compute(String study, Family family, List> inputBindings = new ArrayList<>(); inputBindings.add(new AbstractMap.SimpleEntry<>(freqPath.getParent().toString(), "/input")); @@ -311,8 +314,7 @@ private static File runIBD(String basename, Path freqPath, Path outDir) throws T String plinkParams = "plink1.9 --tfile /output/" + basename + " --genome rel-check --read-freq /input/" + FREQ_FILENAME + " --out /output/" + basename; try { - PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor(); - DockerUtils.run(plinkExecutor.getDockerImageName(), inputBindings, outputBinding, plinkParams, null); + DockerUtils.run(dockerImage, inputBindings, outputBinding, plinkParams, null); } catch (IOException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java index 6c1f4b94ed7..c20117ca595 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java @@ -24,12 +24,13 @@ import org.opencb.commons.exec.Command; import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.InternalStatus; @@ -85,7 +86,9 @@ public static void selectMarkers(String basename, String study, List sam exportData(tpedFile, tfamFile, query, storageManager, token); if (tpedFile.exists() && tpedFile.length() > 0) { - pruneVariants(basename, outputBinding); + String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, + storageManager.getCatalogManager().getConfiguration()); + pruneVariants(dockerImage, basename, outputBinding); } if (!tpedFile.exists() || tpedFile.length() == 0) { @@ -362,12 +365,11 @@ private static void exportData(File tpedFile, File tfamFile, Query query, Varian } } - private static void pruneVariants(String basename, AbstractMap.SimpleEntry outputBinding) throws ToolException { + private static void pruneVariants(String dockerImage, String basename, AbstractMap.SimpleEntry outputBinding) throws ToolException { // Variant pruning using PLINK in docker String plinkParams = "plink1.9 --tfile /data/output/" + basename + " --indep 50 5 2 --out /data/output/" + basename; try { - PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor(); - DockerUtils.run(plinkExecutor.getDockerImageName(), null, outputBinding, plinkParams, null); + DockerUtils.run(dockerImage, null, outputBinding, plinkParams, null); } catch (IOException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/InferredSexComputation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/InferredSexComputation.java index 770b75e9c68..72c44fcb08c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/InferredSexComputation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/InferredSexComputation.java @@ -20,7 +20,6 @@ import org.opencb.biodata.models.core.Region; import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.analysis.alignment.AlignmentStorageManager; -import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureLocalAnalysisExecutor; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.file.File; @@ -88,13 +87,13 @@ public static double[] computeRatios(String study, File bwFile, String assembly, return new double[]{1.0d * means[1] / means[0], 1.0d * means[2] / means[0]}; } - public static java.io.File plot(File inputFile, Path outDir) throws ToolException { + public static java.io.File plot(File inputFile, Path outDir, String dockerImage) throws ToolException { // Execute R script in docker AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(outDir.toAbsolutePath().toString(), "/data/output"); String rParams = "R CMD Rscript --vanilla /data/input/" + inputFile.getName(); try { - DockerUtils.run(MutationalSignatureLocalAnalysisExecutor.R_DOCKER_IMAGE, null, outputBinding, rParams, null); + DockerUtils.run(dockerImage, null, outputBinding, rParams, null); } catch (IOException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 7769315137b..9e802d8df59 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -22,7 +22,6 @@ import org.apache.commons.lang3.time.StopWatch; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; @@ -31,6 +30,7 @@ import org.opencb.opencga.core.common.MemoryUsageMonitor; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java index 356d53d1d84..67a70127dcf 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java @@ -32,8 +32,9 @@ import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.variant.CircosAnalysisParams; @@ -59,9 +60,8 @@ framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class CircosLocalAnalysisExecutor extends CircosAnalysisExecutor implements StorageToolExecutor { - public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" - + GitRepositoryState.getInstance().getBuildVersion(); private VariantStorageManager storageManager; + private String dockerImage; private File snvsFile; private File rearrsFile; @@ -96,6 +96,8 @@ public VariantStorageManager getVariantStorageManager() throws ToolExecutorExcep @Override public void run() throws ToolException, IOException, CatalogException { + dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, + storageManager.getCatalogManager().getConfiguration()); // Create query Query query = new Query(); @@ -160,7 +162,7 @@ public void run() throws ToolException, IOException, CatalogException { + " " + getCircosParams().getTitle(); StopWatch stopWatch = StopWatch.createStarted(); - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams, null); + String cmdline = DockerUtils.run(dockerImage, inputBindings, outputBinding, scriptParams, null); logger.info("Docker command line: " + cmdline); logger.info("Execution time: " + TimeUtils.durationToString(stopWatch)); } else { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java index a86cc797287..e9478a0708b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java @@ -34,10 +34,12 @@ import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.GenomePlotAnalysisExecutor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; @@ -59,9 +61,7 @@ framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor implements StorageToolExecutor { - public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" - + GitRepositoryState.getInstance().getBuildVersion(); - + private String dockerImage; private GenomePlotConfig plotConfig; private File snvsFile; @@ -79,7 +79,8 @@ public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor @Override public void run() throws ToolException, IOException, CatalogException { - + dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, + getVariantStorageManager().getCatalogManager().getConfiguration()); plotConfig = JacksonUtils.getDefaultObjectMapper().readerFor(GenomePlotConfig.class).readValue(getConfigFile()); // Create query @@ -145,7 +146,7 @@ public void run() throws ToolException, IOException, CatalogException { + " " + plotConfig.getTitle(); StopWatch stopWatch = StopWatch.createStarted(); - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams, null); + String cmdline = DockerUtils.run(dockerImage, inputBindings, outputBinding, scriptParams, null); logger.info("Docker command line: " + cmdline); logger.info("Execution time: " + TimeUtils.durationToString(stopWatch)); } else { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java index f85e0a1873a..a93d4a9d2d1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java @@ -26,7 +26,8 @@ import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; @@ -38,18 +39,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; @ToolExecutor(id="opencga-local", tool = HRDetectAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class HRDetectLocalAnalysisExecutor extends HRDetectAnalysisExecutor implements StorageToolExecutor { - public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); - private final static String CNV_FILENAME = "cnv.tsv"; private final static String INDEL_FILENAME = "indel.vcf"; private final static String INDEL_SORTED_FILENAME = "indel.sorted.vcf"; @@ -60,13 +62,14 @@ public class HRDetectLocalAnalysisExecutor extends HRDetectAnalysisExecutor private final static String VIRTUAL_VOLUMEN_SNV = "/snv/"; private final static String VIRTUAL_VOLUMEN_SV = "/sv/"; - private Path opencgaHome; + private String dockerImage; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override public void run() throws ToolException, CatalogException, IOException, StorageEngineException { - opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); + dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, + getVariantStorageManager().getCatalogManager().getConfiguration()); // Prepare CNV data prepareCNVData(); @@ -161,12 +164,12 @@ private void prepareINDELData() throws ToolExecutorException, StorageEngineExcep // BGZIP AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(getOutDir() .toAbsolutePath().toString(), VIRTUAL_VOLUMEN_DATA); - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, null, outputBinding, "bgzip " + VIRTUAL_VOLUMEN_DATA + INDEL_SORTED_FILENAME, + String cmdline = DockerUtils.run(dockerImage, null, outputBinding, "bgzip " + VIRTUAL_VOLUMEN_DATA + INDEL_SORTED_FILENAME, null); logger.info("Docker command line: " + cmdline); // TABIX - cmdline = DockerUtils.run(R_DOCKER_IMAGE, null, outputBinding, "tabix -p vcf " + VIRTUAL_VOLUMEN_DATA + INDEL_GZ_FILENAME, null); + cmdline = DockerUtils.run(dockerImage, null, outputBinding, "tabix -p vcf " + VIRTUAL_VOLUMEN_DATA + INDEL_GZ_FILENAME, null); logger.info("Docker command line: " + cmdline); } @@ -177,7 +180,7 @@ private void prepareInputTable() throws FileNotFoundException { pw.close(); } - private void executeRScript() throws IOException { + private void executeRScript() throws IOException, ToolExecutorException { // Input List> inputBindings = new ArrayList<>(); inputBindings.add(new AbstractMap.SimpleEntry<>(getSnvRDataPath().toFile().getParent(), VIRTUAL_VOLUMEN_SNV)); @@ -222,7 +225,7 @@ private void executeRScript() throws IOException { } } - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams.toString(), null); + String cmdline = DockerUtils.run(dockerImage, inputBindings, outputBinding, scriptParams.toString(), null); logger.info("Docker command line: " + cmdline); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageToolExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageToolExecutor.java index 64018d34947..107165484f5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageToolExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageToolExecutor.java @@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.ConfigurationUtils; -import org.opencb.opencga.analysis.alignment.AlignmentStorageManager; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.config.Configuration; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index 042784b844a..e116a7a6a0a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -36,8 +36,9 @@ import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.sample.Sample; @@ -67,15 +68,15 @@ public class MutationalSignatureLocalAnalysisExecutor extends MutationalSignatur private static final String SVCLASS = "SVCLASS"; private static final String EXT_SVTYPE = "EXT_SVTYPE"; - public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" - + GitRepositoryState.getInstance().getBuildVersion(); - + private String dockerImage; private Path opencgaHome; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override public void run() throws ToolException, CatalogException, IOException, StorageEngineException { + dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, + getVariantStorageManager().getCatalogManager().getConfiguration()); opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); // Check genome context file for that sample, and create it if necessary @@ -457,7 +458,7 @@ private File computeClusteredFile(Query query, QueryOptions queryOptions) throws + " /jobdir/" + outputFile.getName(); // Execute R script in docker - DockerUtils.run(MutationalSignatureLocalAnalysisExecutor.R_DOCKER_IMAGE, inputBindings, outputBinding, rParams, null); + DockerUtils.run(dockerImage, inputBindings, outputBinding, rParams, null); } catch (Exception e) { throw new ToolException(e); } @@ -634,7 +635,7 @@ private void computeSignatureFitting() throws IOException, ToolException, Catalo } } - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams.toString(), + String cmdline = DockerUtils.run(dockerImage, inputBindings, outputBinding, scriptParams.toString(), null); logger.info("Docker command line: {}", cmdline); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java index 59ebe1467c2..2a64a7a3e0a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/IBDRelatednessLocalAnalysisExecutor.java @@ -38,12 +38,6 @@ public void run() throws ToolException { // Get managers VariantStorageManager variantStorageManager = getVariantStorageManager(); - // Sanity check to compute - String opencgaHome = getExecutorParams().getString("opencgaHome"); - if (!Paths.get(opencgaHome).toFile().exists()) { - - } - // Run IBD/IBS computation using PLINK in docker RelatednessReport report = IBDComputation.compute(getStudyId(), getFamily(), getSampleIds(), getMinorAlleleFreq(), getThresholds(), getResourcePath(), getOutDir(), variantStorageManager, getToken()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/bwa/BwaWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/bwa/BwaWrapperAnalysisExecutor.java index 9a607095339..0873cd4764e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/bwa/BwaWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/bwa/BwaWrapperAnalysisExecutor.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -50,7 +51,8 @@ private void runIndex() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("bwa " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "bwa " + command, sb); // Append other params appendOtherParams(null, sb); @@ -59,7 +61,7 @@ private void runIndex() throws ToolException { appendInputFiles(inputFilenames, mountMap, sb); // Execute command and redirect stdout and stderr to the files - logger.info("Docker command line: " + sb.toString()); + logger.info("Docker command line: " + sb); runCommandLine(sb.toString()); } @@ -85,7 +87,8 @@ private void runMem() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand(command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "bwa " + command, sb); // Append other params Set skipParams = new HashSet<>(Arrays.asList("o")); @@ -108,7 +111,7 @@ private void runMem() throws ToolException { appendInputFiles(inputFilenames, mountMap, sb); // Execute command and redirect stdout and stderr to the files - logger.info("Docker command line: " + sb.toString()); + logger.info("Docker command line: " + sb); runCommandLine(sb.toString()); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/deeptools/DeeptoolsWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/deeptools/DeeptoolsWrapperAnalysisExecutor.java index 1c730f10e7a..90ddb9734ae 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/deeptools/DeeptoolsWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/deeptools/DeeptoolsWrapperAnalysisExecutor.java @@ -6,6 +6,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.alignment.AlignmentConstants; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -31,16 +32,6 @@ public class DeeptoolsWrapperAnalysisExecutor extends DockerWrapperAnalysisExecu private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Override - public String getDockerImageName() { - return "dhspence/docker-deeptools"; - } - - @Override - public String getDockerImageVersion() { - return null; - } - @Override public void run() throws ToolException { switch (command) { @@ -103,7 +94,8 @@ private void runBamCommonCommand() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand(command, sb); + String dockerImage = getDockerImageName(Docker.DEEPTOOLS_IMAGE_KEY); + appendCommand(dockerImage, command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index ae2b5072473..f9f71d7e962 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,23 +7,18 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; -import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.ConfigurationUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.*; import java.util.*; -public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { +public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; public final static String DOCKER_OUTPUT_PATH = "/data/output"; @@ -31,16 +26,26 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; - public String getDockerImageName() { - return "opencb/opencga-ext-tools"; - } - - public String getDockerImageVersion() { - return GitRepositoryState.getInstance().getBuildVersion(); - } + private Configuration configuration = null; private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); + public String getDockerImageName(String key) throws ToolExecutorException { + if (configuration == null) { + String opencgaHome = getExecutorParams().getString("opencgaHome"); + if (StringUtils.isEmpty(opencgaHome)) { + throw new ToolExecutorException("Missing OpenCGA in Docker wrapper executor parameters"); + } + try { + configuration = ConfigurationUtils.loadConfiguration(opencgaHome); + } catch (IOException e) { + throw new ToolExecutorException("Error loading configuration file", e); + } + } + + return ConfigurationUtils.getDockerImage(key, configuration); + } + public String getShortPrefix() { return "-"; } @@ -82,15 +87,9 @@ protected Map appendMounts(List> inputFilen return mountMap; } - protected void appendCommand(String command, StringBuilder sb) { - // Docker image and version - sb.append(getDockerImageName()); - if (StringUtils.isNotEmpty(getDockerImageVersion())) { - sb.append(":").append(getDockerImageVersion()); - } - - // Append command - sb.append(" ").append(command); + protected void appendCommand(String dockerImage, String command, StringBuilder sb) { + // Append Docker image and version; and command + sb.append(dockerImage).append(" ").append(command); } protected void appendInputFiles(List> inputFilenames, Map srcTargetMap, StringBuilder sb) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 86bb3760b3c..8addceaafe5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -16,6 +16,7 @@ import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FamilyManager; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.family.Family; @@ -45,9 +46,6 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - public final static String DOCKER_IMAGE_NAME = "exomiser/exomiser-cli"; - public final static String DOCKER_IMAGE_VERSION = "13.1.0"; - private String studyId; private String sampleId; @@ -179,7 +177,7 @@ public void run() throws ToolException { .append(" --mount type=bind,source=" + getOutDir() + ",target=/jobdir "); // Append docker image, version and command - appendCommand("", sb); + appendCommand(getDockerImageName(Docker.EXOMISER_IMAGE_KEY), "", sb); // Append input file params sb.append(" --analysis /jobdir/").append(EXOMISER_ANALYSIS_TEMPLATE_FILENAME); @@ -426,16 +424,6 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { return exomiserDataPath; } - @Override - public String getDockerImageName() { - return DOCKER_IMAGE_NAME; - } - - @Override - public String getDockerImageVersion() { - return DOCKER_IMAGE_VERSION; - } - public String getStudyId() { return studyId; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/fastqc/FastqcWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/fastqc/FastqcWrapperAnalysisExecutor.java index e13d9468d13..dcabd236d6d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/fastqc/FastqcWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/fastqc/FastqcWrapperAnalysisExecutor.java @@ -3,6 +3,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +33,8 @@ protected void run() throws Exception { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("fastqc", sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "fastqc", sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/gatk/GatkWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/gatk/GatkWrapperAnalysisExecutor.java index 277d2bc725a..f7c0477baa3 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/gatk/GatkWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/gatk/GatkWrapperAnalysisExecutor.java @@ -4,7 +4,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; -import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -25,16 +25,6 @@ public class GatkWrapperAnalysisExecutor extends DockerWrapperAnalysisExecutor { private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Override - public String getDockerImageName() { - return "broadinstitute/gatk"; - } - - @Override - public String getDockerImageVersion() { - return ""; - } - @Override public void run() throws ToolException { switch (command) { @@ -56,7 +46,8 @@ private void runHaplotypeCaller() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("gatk " + command, sb); + String dockerImage = getDockerImageName(Docker.GATK_IMAGE_KEY); + appendCommand(dockerImage, "gatk " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/picard/PicardWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/picard/PicardWrapperAnalysisExecutor.java index 9f83d27b538..bcaac880feb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/picard/PicardWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/picard/PicardWrapperAnalysisExecutor.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -24,16 +25,6 @@ public class PicardWrapperAnalysisExecutor extends DockerWrapperAnalysisExecutor private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Override - public String getDockerImageName() { - return "broadinstitute/picard"; - } - - @Override - public String getDockerImageVersion() { - return ""; - } - @Override public String getShortPrefix() { return ""; @@ -72,7 +63,8 @@ private void runCommonCommand() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("java -jar /usr/picard/picard.jar " + command, sb); + String dockerImage = getDockerImageName(Docker.PICARD_IMAGE_KEY); + appendCommand(dockerImage, "java -jar /usr/picard/picard.jar " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/plink/PlinkWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/plink/PlinkWrapperAnalysisExecutor.java index 890475b40fc..9876e21beb3 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/plink/PlinkWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/plink/PlinkWrapperAnalysisExecutor.java @@ -3,6 +3,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +32,8 @@ protected void run() throws Exception { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("plink1.9", sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "plink1.9", sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/rvtests/RvtestsWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/rvtests/RvtestsWrapperAnalysisExecutor.java index f9e60c5b31c..d36b313a0e8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/rvtests/RvtestsWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/rvtests/RvtestsWrapperAnalysisExecutor.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -24,16 +25,6 @@ public class RvtestsWrapperAnalysisExecutor extends DockerWrapperAnalysisExecuto private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Override - public String getDockerImageName() { - return "zhanxw/rvtests-docker"; - } - - @Override - public String getDockerImageVersion() { - return ""; - } - @Override public void run() throws ToolException { switch (command) { @@ -56,7 +47,8 @@ private void runCommonCommand() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand(command, sb); + String dockerImage = getDockerImageName(Docker.RVTESTS_IMAGE_KEY); + appendCommand(dockerImage, command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/samtools/SamtoolsWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/samtools/SamtoolsWrapperAnalysisExecutor.java index 67a7743df87..86bcc43f1af 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/samtools/SamtoolsWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/samtools/SamtoolsWrapperAnalysisExecutor.java @@ -4,7 +4,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; -import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.Docker; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; @@ -71,7 +71,8 @@ private void runDepth() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -96,7 +97,8 @@ private void runDict() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -134,7 +136,8 @@ private void runView() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -170,7 +173,8 @@ private void runIndex() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -194,7 +198,8 @@ private void runSort() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -229,7 +234,8 @@ private void runFaidx() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -252,7 +258,8 @@ private void runStats() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -275,7 +282,8 @@ private void runPlotBamStats() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand(command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); @@ -304,7 +312,8 @@ private void runFlagstat() throws ToolException { Map mountMap = appendMounts(inputFilenames, sb); // Append docker image, version and command - appendCommand("samtools " + command, sb); + String dockerImage = getDockerImageName(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY); + appendCommand(dockerImage, "samtools " + command, sb); // Append input file params appendInputFiles(inputFilenames, mountMap, sb); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java index bdc744fc49c..b98e1793b6c 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java @@ -16,6 +16,7 @@ package org.opencb.opencga.analysis.alignment; +import org.apache.commons.collections4.CollectionUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.ClassRule; @@ -23,20 +24,29 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.opencb.biodata.formats.sequence.fastqc.FastQcMetrics; import org.opencb.biodata.models.clinical.Phenotype; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.analysis.alignment.qc.AlignmentFastQcMetricsAnalysis; +import org.opencb.opencga.analysis.alignment.qc.AlignmentFlagStatsAnalysis; import org.opencb.opencga.analysis.alignment.qc.AlignmentGeneCoverageStatsAnalysis; +import org.opencb.opencga.analysis.alignment.qc.AlignmentStatsAnalysis; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.analysis.wrappers.bwa.BwaWrapperAnalysis; +import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; +import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; +import org.opencb.opencga.analysis.wrappers.fastqc.FastqcWrapperAnalysis; +import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.alignment.AlignmentGeneCoverageStatsParams; +import org.opencb.opencga.core.models.alignment.*; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; @@ -49,12 +59,14 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Map; +import java.util.*; -import static org.junit.Assert.assertEquals; +import static org.apache.commons.io.FileUtils.readLines; +import static org.junit.Assert.*; @RunWith(Parameterized.class) @Category(MediumTests.class) @@ -106,7 +118,6 @@ public AlignmentAnalysisTest(String storageEngine) { private static String storageEngine; private static boolean indexed = false; private static String token; - private static File file; @Before public void setUp() throws Throwable { @@ -223,7 +234,7 @@ public static void afterClass() { opencga.after(); } - public void setUpCatalogManager() throws CatalogException { + public void setUpCatalogManager() throws CatalogException, IOException { catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("test"), null, opencga.getAdminToken()); catalogManager.getUserManager().create(new User().setId(USER).setName("User Name").setEmail("mail@ebi.ac.uk").setOrganization("test"), PASSWORD, opencga.getAdminToken()); @@ -259,30 +270,347 @@ public void setUpCatalogManager() throws CatalogException { // assertEquals(2, individual.getSamples().size()); } + //----------------------------------------- + // S A M T O O L S + //----------------------------------------- @Test - public void geneCoverageStatsTest() throws IOException, ToolException, CatalogException { - Path outdir = Paths.get(opencga.createTmpOutdir("_genecoveragestats")); + public void testSamtoolsSort() throws IOException, CatalogException, ToolException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testSamtoolsSort")); + File bamFile = copyBamFile("testSamtoolsSort", outPath); + + SamtoolsWrapperParams params = new SamtoolsWrapperParams(); + params.setInputFile(bamFile.getId()); + Map samtoolsParams = new HashMap<>(); + samtoolsParams.put("o", "sorted.bam"); + params.setSamtoolsParams(samtoolsParams); + params.setCommand("sort"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + assertTrue(outPath.resolve(samtoolsParams.get("o")).toFile().exists()); + } - // setup BAM files - String bamFilename = opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").toString(); - String baiFilename = opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam.bai").toString(); - //String bamFilename = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - File bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFilename, "", "", "", null, null, null, + @Test + public void testSamtoolsIndex() throws IOException, CatalogException, ToolException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testSamtoolsIndex")); + + // Prepare BAM file + Path bamPath = outPath.resolve("bam"); + Files.createDirectories(bamPath); + File bamFile = copyBamFile("testSamtoolsIndex", bamPath); + + SamtoolsWrapperParams params = new SamtoolsWrapperParams(); + params.setInputFile(bamFile.getId()); + Map samtoolsParams = new HashMap<>(); + params.setSamtoolsParams(samtoolsParams); + params.setCommand("index"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + assertTrue(outPath.resolve(bamFile.getName() + ".bai").toFile().exists()); + } + + @Test + public void testSamtoolsFlagstats() throws IOException, CatalogException, ToolException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testSamtoolsFlagstats")); + File bamFile = copyBamFile("testSamtoolsFlagstats", outPath); + + SamtoolsWrapperParams params = new SamtoolsWrapperParams(); + params.setInputFile(bamFile.getId()); + params.setCommand("flagstat"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + java.io.File stdoutFile = outPath.resolve(DockerWrapperAnalysisExecutor.STDOUT_FILENAME).toFile(); + assertTrue(stdoutFile.exists()); + List lines = readLines(stdoutFile, Charset.defaultCharset()); + if (CollectionUtils.isEmpty(lines) && !lines.get(0).contains("QC-passed")) { + fail(); + } + } + + @Test + public void testSamtoolsStats() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testSamtoolsStats")); + File bamFile = copyBamFile("testSamtoolsStats", outPath); + + SamtoolsWrapperParams params = new SamtoolsWrapperParams(); + params.setInputFile(bamFile.getId()); + params.setCommand("stats"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + java.io.File stdoutFile = outPath.resolve(DockerWrapperAnalysisExecutor.STDOUT_FILENAME).toFile(); + assertTrue(stdoutFile.exists()); + List lines = readLines(stdoutFile, Charset.defaultCharset()); + if (CollectionUtils.isEmpty(lines) && !lines.get(0).startsWith("# This file was produced by samtools stats")) { + fail(); + } + } + + @Test + public void testAlignmentFlagStats() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testAlignmentFlagStats")); + File bamFile = copyBamFile("testAlignmentFlagStats", outPath); + + AlignmentFlagStatsParams params = new AlignmentFlagStatsParams(); + params.setFile(bamFile.getId()); + + toolRunner.execute(AlignmentFlagStatsAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + assertTrue(AlignmentFlagStatsAnalysis.getResultPath(outPath.toAbsolutePath().toString(), bamFile.getName()).toFile().exists()); + } + + @Test + public void testAlignmentStats() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testAlignmentStats")); + File bamFile = copyBamFile("testAlignmentStats", outPath); + + AlignmentStatsParams params = new AlignmentStatsParams(); + params.setFile(bamFile.getId()); + + toolRunner.execute(AlignmentStatsAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + assertTrue(AlignmentStatsAnalysis.getResultPath(outPath.toAbsolutePath().toString(), bamFile.getName()).toFile().exists()); + } + + //----------------------------------------- + // F A S T Q C + //----------------------------------------- + + @Test + public void testFastQcZip() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testFastQcZip")); + File bamFile = copyBamFile("testFastQcZip", outPath); + + FastqcWrapperParams params = new FastqcWrapperParams(); + params.setInputFile(bamFile.getId()); + Map fastqcParams = new HashMap<>(); + params.setFastqcParams(fastqcParams); + + toolRunner.execute(FastqcWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check results + String basename = bamFile.getName().replace(".bam", ""); + assertTrue(outPath.resolve(basename + "_fastqc.zip").toFile().exists()); + assertTrue(outPath.resolve(basename + "_fastqc.html").toFile().exists()); + } + + @Test + public void testFastQc() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testFastQc")); + File bamFile = copyBamFile("testFastQc", outPath); + + FastqcWrapperParams params = new FastqcWrapperParams(); + params.setInputFile(bamFile.getId()); + Map fastqcParams = new HashMap<>(); + fastqcParams.put("extract", "true"); + params.setFastqcParams(fastqcParams); + + toolRunner.execute(FastqcWrapperAnalysis.class, params, new ObjectMap(), outPath, null, token); + + // Check restults + FastQcMetrics fastQcMetrics = AlignmentFastQcMetricsAnalysis.parseResults(outPath); + assertEquals("PASS", fastQcMetrics.getSummary().getBasicStatistics()); + assertEquals("46", fastQcMetrics.getBasicStats().get("%GC")); + assertEquals(8, fastQcMetrics.getFiles().size()); + } + + //----------------------------------------- + // D E E P T O O L S + //----------------------------------------- + + @Test + public void testDeeptoolsCoverage() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testDeeptoolsCoverage")); + File bamFile = copyBamFile("testDeeptoolsCoverage", outPath); + + // Execute samtools index + Path samtoolsIndexJobPath = outPath.resolve("samtools_index"); + Files.createDirectories(samtoolsIndexJobPath); + + SamtoolsWrapperParams samtoolsIndexParams = new SamtoolsWrapperParams(); + samtoolsIndexParams.setInputFile(bamFile.getId()); + Map samtoolsParams = new HashMap<>(); + samtoolsIndexParams.setSamtoolsParams(samtoolsParams); + samtoolsIndexParams.setCommand("index"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, samtoolsIndexParams, new ObjectMap(), samtoolsIndexJobPath, "testDeeptoolsCoverageSamtoolsIndexJobId", token); + assertTrue(samtoolsIndexJobPath.resolve(bamFile.getName() + ".bai").toFile().exists()); + + File baiFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(outPath.resolve(bamFile.getName() + ".bai").toString(), "testDeeptoolsCoverage", "", "", null, null, null, null, null), false, token).first(); - assertEquals(0, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().size()); + assertEquals(bamFile.getName() + ".bai", baiFile.getName()); + + // Execute bamCoverage + DeeptoolsWrapperParams params = new DeeptoolsWrapperParams(); + params.setCommand("bamCoverage"); + Map deeptoolsParams = new HashMap<>(); + deeptoolsParams.put("b", bamFile.getId()); + deeptoolsParams.put("o", bamFile.getName() + ".bw"); + deeptoolsParams.put("binSize", "500"); + deeptoolsParams.put("outFileFormat", "bigwig"); + deeptoolsParams.put("minMappingQuality", "20"); + params.setDeeptoolsParams(deeptoolsParams); + + toolRunner.execute(DeeptoolsWrapperAnalysis.class, params, new ObjectMap(), outPath, "testDeeptoolsCoverageDeeptoolsBamCoverageJobId", token); + assertTrue(outPath.resolve(bamFile.getName() + ".bw").toFile().exists()); + } + + //----------------------------------------- + // C O V E R A G E + //----------------------------------------- + + @Test + public void testCoverage() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testCoverage")); + File bamFile = copyBamFile("testCoverage", outPath); + + // Execute samtools index + Path samtoolsIndexJobPath = outPath.resolve("samtools_index"); + Files.createDirectories(samtoolsIndexJobPath); + + SamtoolsWrapperParams samtoolsIndexParams = new SamtoolsWrapperParams(); + samtoolsIndexParams.setInputFile(bamFile.getId()); + Map samtoolsParams = new HashMap<>(); + samtoolsIndexParams.setSamtoolsParams(samtoolsParams); + samtoolsIndexParams.setCommand("index"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, samtoolsIndexParams, new ObjectMap(), samtoolsIndexJobPath, "testCoverageSamtoolsIndexJobId", token); + assertTrue(samtoolsIndexJobPath.resolve(bamFile.getName() + ".bai").toFile().exists()); + + File baiFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(outPath.resolve(bamFile.getName() + ".bai").toString(), "testCoverage", "", "", null, null, null, + null, null), false, token).first(); + assertEquals(bamFile.getName() + ".bai", baiFile.getName()); + + // Execute coverage + CoverageIndexParams coverageIndexParams = new CoverageIndexParams(); + coverageIndexParams.setBamFileId(bamFile.getId()); + coverageIndexParams.setBaiFileId(baiFile.getId()); + coverageIndexParams.setWindowSize(500); + toolRunner.execute(AlignmentCoverageAnalysis.class, coverageIndexParams, new ObjectMap(), outPath, "testCoverageCoverageIndexJobId", token); + assertTrue(outPath.resolve(bamFile.getName() + ".bw").toFile().exists()); + } + + @Test + public void testGeneCoverageStats() throws IOException, ToolException, CatalogException { + // Create JUnit test path and copy BAM file there + Path outPath = Paths.get(opencga.createTmpOutdir("_testGeneCoverageStats")); + File bamFile = copyBamFile("testGeneCoverageStats", outPath); + + // Execute samtools index + Path samtoolsIndexJobPath = outPath.resolve("samtools_index"); + Files.createDirectories(samtoolsIndexJobPath); + + SamtoolsWrapperParams samtoolsIndexParams = new SamtoolsWrapperParams(); + samtoolsIndexParams.setInputFile(bamFile.getId()); + Map samtoolsParams = new HashMap<>(); + samtoolsIndexParams.setSamtoolsParams(samtoolsParams); + samtoolsIndexParams.setCommand("index"); + + toolRunner.execute(SamtoolsWrapperAnalysis.class, samtoolsIndexParams, new ObjectMap(), samtoolsIndexJobPath, "testGeneCoverageStatsSamtoolsIndexJobId", token); + + // Check results + assertTrue(samtoolsIndexJobPath.resolve(bamFile.getName() + ".bai").toFile().exists()); + + // Execute gene coverage stats AlignmentGeneCoverageStatsParams params = new AlignmentGeneCoverageStatsParams(); params.setBamFile(bamFile.getId()); String geneName = "BRCA2"; params.setGenes(Arrays.asList(geneName)); - toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outdir, null, token); + toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outPath, "test", token); + + // Check results + File file = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFile.getId()), null, null, + false, token).first(); + assertEquals(1, file.getQualityControl().getCoverage().getGeneCoverageStats().size()); + assertEquals(geneName, file.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getGeneName()); + assertEquals(10, file.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getStats().size()); + } - bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFilename, "", "", "", null, null, null, + //----------------------------------------- + // B W A + //----------------------------------------- + + @Test + public void testBwaIndex() throws IOException, CatalogException, ToolException { + Path outdir = Paths.get(opencga.createTmpOutdir("_bwa_index")); + System.out.println("outdir = " + outdir); + + String fastaFilename = opencga.getResourceUri("biofiles/cram/hg19mini.fasta").toString(); + File fastasFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(fastaFilename, "", "", "", null, null, null, + null, null), false, token).first(); + + BwaWrapperParams params = new BwaWrapperParams(); + params.setFastaFile(fastasFile.getId()); + params.setCommand("index"); + + toolRunner.execute(BwaWrapperAnalysis.class, params, new ObjectMap(), outdir, null, token); + assertTrue(outdir.resolve(fastasFile.getName() + ".sa").toFile().exists()); + assertTrue(outdir.resolve(fastasFile.getName() + ".pac").toFile().exists()); + assertTrue(outdir.resolve(fastasFile.getName() + ".ann").toFile().exists()); + assertTrue(outdir.resolve(fastasFile.getName() + ".amb").toFile().exists()); + assertTrue(outdir.resolve(fastasFile.getName() + ".bwt").toFile().exists()); + } + + + @Test + public void testBwaMem() throws IOException, CatalogException, ToolException { + Path outdir = Paths.get(opencga.createTmpOutdir("_bwa_index")); + System.out.println("outdir = " + outdir); + + String faFilename = opencga.getResourceUri("biofiles/cram/hg19mini.fasta").toString(); + File faFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(faFilename, "", "", "", null, null, null, + null, null), false, token).first(); + + BwaWrapperParams params = new BwaWrapperParams(); + params.setFastaFile(faFile.getId()); + params.setCommand("index"); + + toolRunner.execute(BwaWrapperAnalysis.class, params, new ObjectMap(), outdir, null, token); + assertTrue(outdir.resolve(faFile.getName() + ".sa").toFile().exists()); + assertTrue(outdir.resolve(faFile.getName() + ".pac").toFile().exists()); + assertTrue(outdir.resolve(faFile.getName() + ".ann").toFile().exists()); + assertTrue(outdir.resolve(faFile.getName() + ".amb").toFile().exists()); + assertTrue(outdir.resolve(faFile.getName() + ".bwt").toFile().exists()); + + String fqFilename = opencga.getResourceUri("biofiles/cram/reads.fq").toString(); + File fqFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(fqFilename, "", "", "", null, null, null, null, null), false, token).first(); - assertEquals(1, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().size()); - assertEquals(geneName, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getGeneName()); - assertEquals(10, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getStats().size()); + + params.setFastq1File(fqFile.getId()); + params.setCommand("mem"); + Map bwaParams = new HashMap<>(); + bwaParams.put("o", fqFile.getName() + ".sam"); + params.setBwaParams(bwaParams); + + toolRunner.execute(BwaWrapperAnalysis.class, params, new ObjectMap(), outdir, null, token); + assertTrue(outdir.resolve(fqFile.getName() + ".sam").toFile().exists()); + } + + private File copyBamFile(String catalogPathName, Path outPath) throws CatalogException, IOException { + // setup BAM files + Path sourcePath = Paths.get(opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").getPath()); + Path targetPath = outPath.resolve(sourcePath.getFileName()); + Files.copy(sourcePath, targetPath); + return catalogManager.getFileManager().link(STUDY, new FileLinkParams(targetPath.toAbsolutePath().toString(), catalogPathName, "", "", + null, null, null, null, null), true, token).first(); } } \ No newline at end of file diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 2b7ed748772..8f106e3b83f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -209,7 +209,7 @@ private Family insert(ClientSession clientSession, long studyUid, Family family, // Pedigree graph try { PedigreeGraph pedigreeGraph = PedigreeGraphUtils.getPedigreeGraph(family, Paths.get(configuration.getWorkspace()).getParent(), - Paths.get(configuration.getAnalysis().getScratchDir())); + configuration); family.setPedigreeGraph(pedigreeGraph); } catch (IOException e) { throw new CatalogDBException("Error computing pedigree graph for family " + family.getId(), e); @@ -549,7 +549,7 @@ private PedigreeGraph computePedigreeGraph(ClientSession clientSession, Family f try { return PedigreeGraphUtils.getPedigreeGraph(tmpFamily, Paths.get(configuration.getWorkspace()).getParent(), - Paths.get(configuration.getAnalysis().getScratchDir())); + configuration); } catch (IOException e) { String msg = "Error computing/updating the pedigree graph for the family " + family.getId(); logger.error(msg); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java index 480781b0e21..1946d991ce7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java @@ -5,11 +5,15 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.ConfigurationUtils; +import org.opencb.opencga.core.config.Docker; +import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.PedigreeGraph; import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor; import java.io.File; import java.io.FileNotFoundException; @@ -29,21 +33,20 @@ public class PedigreeGraphUtils { public static final String PEDIGREE_JSON_FILENAME = "ped_coords.json"; public static final String PEDIGREE_TSV_FILENAME = "ped_coords.tsv"; - public static final String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); - - public static PedigreeGraph getPedigreeGraph(Family family, Path openCgaHome, Path scratchDir) throws IOException { + public static PedigreeGraph getPedigreeGraph(Family family, Path openCgaHome, Configuration configuration) throws IOException { PedigreeGraph pedigreeGraph = new PedigreeGraph(); if (hasMinTwoGenerations(family)) { // Prepare R script and out paths - Path rScriptPath = openCgaHome.resolve("analysis/pedigree-graph"); + Path rScriptPath = openCgaHome.resolve("analysis/" + PedigreeGraphAnalysisExecutor.ID); - Path outDir = Paths.get(scratchDir + "/pedigree-graph-" + family.getUuid() + "-" + System.nanoTime()); + Path scratchDir = Paths.get(configuration.getAnalysis().getScratchDir()); + Path outDir = Paths.get(scratchDir + "/" + PedigreeGraphAnalysisExecutor.ID + "-" + family.getUuid() + "-" + System.nanoTime()); outDir.toFile().mkdir(); Runtime.getRuntime().exec("chmod 777 " + outDir.toAbsolutePath()); // Execute R script and get b64 image - createPedigreeGraph(family, rScriptPath, outDir); + createPedigreeGraph(family, rScriptPath, outDir, configuration); pedigreeGraph.setBase64(getB64Image(outDir)); pedigreeGraph.setJson(getJsonPedigreeGraph(outDir)); @@ -56,7 +59,7 @@ public static PedigreeGraph getPedigreeGraph(Family family, Path openCgaHome, Pa return pedigreeGraph; } - public static void createPedigreeGraph(Family family, Path rScriptPath, Path outDir) throws IOException { + public static void createPedigreeGraph(Family family, Path rScriptPath, Path outDir, Configuration configuration) throws IOException { File pedFile; try { pedFile = createPedFile(family, outDir); @@ -78,8 +81,9 @@ public static void createPedigreeGraph(Family family, Path rScriptPath, Path out .append(" --plot_format png"); try { - String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams.toString(), null); - } catch (IOException e) { + String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, configuration); + String cmdline = DockerUtils.run(dockerImage, inputBindings, outputBinding, scriptParams.toString(), null); + } catch (IOException | ToolExecutorException e) { throw new IOException("Error running the command line to create the family pedigree graph", e); } } diff --git a/opencga-catalog/src/test/resources/biofiles/cram/reads.fq b/opencga-catalog/src/test/resources/biofiles/cram/reads.fq new file mode 100644 index 00000000000..2adb64937fa --- /dev/null +++ b/opencga-catalog/src/test/resources/biofiles/cram/reads.fq @@ -0,0 +1,8 @@ +@read1 +AGATGGTGTGGCGTTTTCCTCAGGGATCTAGAACTAGAAACACCATTTGACCCACCCATCCCA ++ +IIIJAIAJDIFAJSDFIIIIIIAIAIIIAJJJJIJIIIIJAAJJISDFIIIIIIIIIIFIFII +@read2 +TGACAAACCTGACAAAAACAAGCAATGGGGAAAG ++ +JJJJJJJIIAJJJJJJTJHJJBHJHKSJJJJJJJ \ No newline at end of file diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 7b84c12702c..f4d43758b2c 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -19,6 +19,14 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. + docker: + images: + opecgaExtTools: "opencb/opencga-ext-tools" + gatk: "broadinstitute/gatk:4.5.0.0" + picard: "broadinstitute/picard:3.1.1" + deeptools: "dhspence/docker-deeptools" + exomiser: "exomiser/exomiser-cli:13.1.0" + rvtests: "zhanxw/rvtests-docker" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 6eb2beecb49..de52f64b7ab 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -24,15 +24,33 @@ public class Analysis { private List packages; private String scratchDir; - + private Docker docker; private Execution execution; - private List frameworks; public Analysis() { - packages = new ArrayList<>(); - execution = new Execution(); - frameworks = new ArrayList<>(); + this.packages = new ArrayList<>(); + this.docker = new Docker(); + this.execution = new Execution(); + this.frameworks = new ArrayList<>(); + } + + public Analysis(String scratchDir, Docker docker, Execution execution, List frameworks) { + this.scratchDir = scratchDir; + this.docker = docker; + this.execution = execution; + this.frameworks = frameworks; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Analysis{"); + sb.append("scratchDir='").append(scratchDir).append('\''); + sb.append(", docker=").append(docker); + sb.append(", execution=").append(execution); + sb.append(", frameworks=").append(frameworks); + sb.append('}'); + return sb.toString(); } public List getPackages() { @@ -53,6 +71,15 @@ public Analysis setScratchDir(String scratchDir) { return this; } + public Docker getDocker() { + return docker; + } + + public Analysis setDocker(Docker docker) { + this.docker = docker; + return this; + } + public Execution getExecution() { return execution; } @@ -70,5 +97,4 @@ public Analysis setFrameworks(List frameworks) { this.frameworks = frameworks; return this; } - } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index bb5057a76e2..081d30c7b1d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ConfigurationUtils.java similarity index 73% rename from opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java rename to opencga-core/src/main/java/org/opencb/opencga/core/config/ConfigurationUtils.java index bc6e1dc674b..faf5f9e4d8e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ConfigurationUtils.java @@ -14,11 +14,14 @@ * limitations under the License. */ -package org.opencb.opencga.analysis; +package org.opencb.opencga.core.config; +import org.apache.commons.collections4.CollectionUtils; import org.opencb.commons.utils.FileUtils; -import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,8 +31,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Collectors; public class ConfigurationUtils { + private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class); /** @@ -83,4 +89,21 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome) .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); } } + + public static String getDockerImage(String key, Configuration configuration) throws ToolExecutorException { + if (configuration.getAnalysis().getDocker() != null) { + Docker docker = configuration.getAnalysis().getDocker(); + if (docker.getImages().containsKey(key)) { + String imageName = docker.getImages().get(key); + if (Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY.equals(key) && !imageName.contains(":")) { + imageName += (":" + GitRepositoryState.getInstance().getBuildVersion()); + } + logger.debug("Using Docker image '{}'", imageName); + return imageName; + } + throw new ToolExecutorException("It could not find the Docker image " + key + " in the configuration analysis section"); + } else { + throw new ToolExecutorException("Missing docker in the configuration analysis section"); + } + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Docker.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Docker.java new file mode 100644 index 00000000000..8de48c716dc --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Docker.java @@ -0,0 +1,65 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by jtarraga on 24/01/24. + */ +public class Docker { + + // Important these keys have to match with the keys of the images map with the docker analysis configuration section + public static final String OPENCGA_EXT_TOOLS_IMAGE_KEY = "opecgaExtTools"; + public static final String GATK_IMAGE_KEY = "gatk"; + public static final String PICARD_IMAGE_KEY = "picard"; + public static final String DEEPTOOLS_IMAGE_KEY = "deeptools"; + public static final String EXOMISER_IMAGE_KEY = "exomiser"; + public static final String RVTESTS_IMAGE_KEY = "rvtests"; + + private Map images; + + public Docker() { + images = new HashMap<>(); + } + + public Docker(Map images) { + this.images = images; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Docker{"); + sb.append("images=").append(images); + sb.append('}'); + return sb.toString(); + } + + public Map getImages() { + return images; + } + + public Docker setImages(Map images) { + this.images = images; + return this; + } +} + diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/family/PedigreeGraphAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/family/PedigreeGraphAnalysisExecutor.java index d399c744156..2a246695c9b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/family/PedigreeGraphAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/family/PedigreeGraphAnalysisExecutor.java @@ -21,6 +21,9 @@ public abstract class PedigreeGraphAnalysisExecutor extends OpenCgaToolExecutor { + public static final String ID = "pedigree-graph"; + public static final String DESCRIPTION = "Compute the family pedigree graph image."; + private String study; private Family family; diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index ac4acf283d8..4b28acab5d0 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -110,6 +110,14 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" # Scratch folder for the analysis. + docker: + images: + opecgaExtTools: "opencb/opencga-ext-tools" + gatk: "broadinstitute/gatk:4.5.0.0" + picard: "broadinstitute/picard:3.1.1" + deeptools: "dhspence/docker-deeptools" + exomiser: "exomiser/exomiser-cli:13.1.0" + rvtests: "zhanxw/rvtests-docker" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory