Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TASK-5513 Fix the Docker image configuration for OpenCGA analysis #2389

Closed
wants to merge 7 commits into from
Closed

TASK-5513 Fix the Docker image configuration for OpenCGA analysis #2389

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c66d0e6
analysis: add docker image names in the analysis section within the c…
jtarraga Jan 24, 2024
7fef49b
Merge branch 'develop' into TASK-5513
jtarraga Feb 5, 2024
9608f1d
analysis: add Junit tests for samtools, fastqc, deeptools analyses, #…
jtarraga Feb 5, 2024
2dd26ed
analysis: fix 'bwa mem' and add Junit tests, #TASK-5513
jtarraga Feb 6, 2024
35088f6
test: add comment, #TASK-5513
jtarraga Feb 6, 2024
6452f2a
Merge branch 'develop' into TASK-5513
jtarraga Apr 30, 2024
d3636f7
analysis: fix alignment JUnit tests, #TASK-5513
jtarraga May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.opencb.opencga.analysis;

import org.opencb.biodata.models.clinical.qc.RelatednessScore;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
Expand All @@ -13,12 +12,12 @@
import org.opencb.opencga.core.models.job.Job;
import org.opencb.opencga.core.response.OpenCGAResult;

import java.io.*;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;

import static org.opencb.opencga.core.api.ParamConstants.SAMTOOLS_COMMANDS_SUPPORTED;

public class AnalysisUtils {

public static boolean isSupportedCommand(String commands) {
Expand Down
1 change: 1 addition & 0 deletions opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageToolExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.core.config.Configuration;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.exceptions.ToolExecutorException;
import org.opencb.opencga.storage.core.StorageEngineFactory;

Expand Down
16 changes: 13 additions & 3 deletions ...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.file.FileLinkParams;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.core.tools.annotations.ToolParams;
Expand Down Expand Up @@ -140,7 +141,16 @@ protected void run() throws Exception {

// And finally, link the BW file is necessary
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of manually "moving and linking" the file, there is a method OpenCgaTool#moveFile that you could use, which moves, links and associates the file with the job

boolean isLinked = true;
Path outputCatalogPath = Paths.get(bamCatalogFile.getPath()).getParent().resolve(outputPath.getFileName());
boolean isParentNull = false;
Path outputCatalogPath;
String pathDestiny;
if (Paths.get(bamCatalogFile.getPath()).getParent() == null) {
outputCatalogPath = outputPath.getFileName();
pathDestiny = outputPath.getFileName().toString();
} else {
outputCatalogPath = Paths.get(bamCatalogFile.getPath()).getParent().resolve(outputPath.getFileName());
pathDestiny = Paths.get(bamCatalogFile.getPath()).getParent().toString();
}
OpenCGAResult<File> fileResult;
try {
fileResult = catalogManager.getFileManager().get(getStudy(), outputCatalogPath.toString(), QueryOptions.empty(),
Expand All @@ -153,8 +163,8 @@ protected void run() throws Exception {
}
if (!isLinked) {
logger.info("{}: linking file {} in catalog", ID, bwCatalogPath.toFile().getName());
catalogManager.getFileManager().link(getStudy(), bwCatalogPath.toUri(), outputCatalogPath.getParent().toString(),
new ObjectMap("parents", true), getToken());
catalogManager.getFileManager().link(getStudy(), bwCatalogPath.toUri(), pathDestiny, new ObjectMap("parents", true),
getToken());
}
});
}
Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.opencb.opencga.analysis.clinical;

import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.opencga.analysis.ConfigurationUtils;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.core.config.Configuration;
Expand Down
2 changes: 1 addition & 1 deletion ...a-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.opencb.biodata.models.clinical.interpretation.Software;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.ConfigurationUtils;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.analysis.tools.OpenCgaTool;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
Expand Down
14 changes: 12 additions & 2 deletions ...in/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
import org.opencb.opencga.core.common.GitRepositoryState;
import org.opencb.opencga.core.common.JacksonUtils;
import org.opencb.opencga.core.common.TimeUtils;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.config.Docker;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
import org.opencb.opencga.core.models.clinical.Interpretation;
Expand Down Expand Up @@ -136,13 +138,21 @@ protected void run() throws ToolException {

protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
CatalogException, IOException {
// Get Docker image name and version
String dockerImage = ConfigurationUtils.getDockerImage(Docker.EXOMISER_IMAGE_KEY, configuration);
String dockerVersion = "";
String[] split = dockerImage.split(":");
String dockerName = split[0];
if (split.length > 1) {
dockerVersion = split[1];
}
// Interpretation method
InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
new Software()
.setName("Exomiser")
.setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)
.setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION)));
.setRepository("Docker: " + dockerName)
.setVersion(dockerVersion)));

// Analyst
ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(token);
Expand Down
5 changes: 1 addition & 4 deletions opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,9 @@
import org.opencb.opencga.core.tools.annotations.ToolParams;
import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor;

@Tool(id = PedigreeGraphAnalysis.ID, resource = Enums.Resource.FAMILY)
@Tool(id = PedigreeGraphAnalysisExecutor.ID, resource = Enums.Resource.FAMILY)
public class PedigreeGraphAnalysis extends OpenCgaToolScopeStudy {

public static final String ID = "pedigree-graph";
public static final String DESCRIPTION = "Compute the family pedigree graph image.";

@ToolParams
private PedigreeGraphAnalysisParams pedigreeParams = new PedigreeGraphAnalysisParams();

Expand Down
7 changes: 5 additions & 2 deletions ...-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphInitAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.utils.PedigreeGraphUtils;
import org.opencb.opencga.core.api.ParamConstants;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.config.Docker;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.family.Family;
Expand Down Expand Up @@ -59,11 +61,12 @@ public List<String> getSteps() {
@Override
protected void run() throws ToolException {
step("pull-docker", () -> {
String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY, configuration);
// Be sure docker is alive and that opencga-ext-tools is downloaded
logger.info("Checking if docker daemon is alive");
DockerUtils.checkDockerDaemonAlive();
logger.info("Pulling docker '{}'", PedigreeGraphUtils.R_DOCKER_IMAGE);
new Command("docker pull " + PedigreeGraphUtils.R_DOCKER_IMAGE).run();
logger.info("Pulling docker '{}'", dockerImage);
new Command("docker pull " + dockerImage).run();
});

step(getId(), () -> {
Expand Down
19 changes: 6 additions & 13 deletions .../src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,22 @@

package org.opencb.opencga.analysis.family;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.models.clinical.Disorder;
import org.opencb.commons.utils.DockerUtils;
import org.opencb.opencga.analysis.StorageToolExecutor;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.utils.PedigreeGraphUtils;
import org.opencb.opencga.core.common.GitRepositoryState;
import org.opencb.opencga.core.config.Configuration;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.individual.Individual;
import org.opencb.opencga.core.tools.annotations.ToolExecutor;
import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

@ToolExecutor(id="opencga-local", tool = PedigreeGraphAnalysis.ID,
@ToolExecutor(id="opencga-local", tool = PedigreeGraphAnalysisExecutor.ID,
framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE)
public class PedigreeGraphLocalAnalysisExecutor extends PedigreeGraphAnalysisExecutor {

Expand All @@ -51,8 +42,10 @@ public class PedigreeGraphLocalAnalysisExecutor extends PedigreeGraphAnalysisExe
@Override
public void run() throws ToolException, CatalogException, IOException, StorageEngineException {
opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome"));
Configuration configuration = ConfigurationUtils.loadConfiguration(opencgaHome.toAbsolutePath().toString());

// Run R script for fitting signature
PedigreeGraphUtils.createPedigreeGraph(getFamily(), opencgaHome.resolve("analysis/" + PedigreeGraphAnalysis.ID), getOutDir());
PedigreeGraphUtils.createPedigreeGraph(getFamily(), opencgaHome.resolve("analysis/" + PedigreeGraphAnalysisExecutor.ID),
getOutDir(), configuration);
}
}
17 changes: 7 additions & 10 deletions opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@

package org.opencb.opencga.analysis.family.qc;

import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.ode.nonstiff.RungeKuttaFieldIntegrator;
import org.opencb.biodata.models.clinical.qc.RelatednessReport;
import org.opencb.biodata.models.clinical.qc.RelatednessScore;
import org.opencb.biodata.models.variant.avro.VariantType;
Expand All @@ -32,21 +30,19 @@
import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis;
import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.common.JacksonUtils;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.config.Docker;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.family.Family;
import org.opencb.opencga.core.models.individual.Individual;
import org.opencb.opencga.core.models.sample.Sample;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;

import java.io.*;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -122,7 +118,9 @@ public static RelatednessReport compute(String study, Family family, List<String
} catch (IOException e) {
throw new ToolException("Something wrong happened when copying files during the relatedness analysis execution");
}
File outFile = runIBD(FILTERED_BASENAME, freqPath, outDir);
String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY,
storageManager.getCatalogManager().getConfiguration());
File outFile = runIBD(dockerImage, FILTERED_BASENAME, freqPath, outDir);

if (!outFile.exists()) {
throw new ToolException("Something wrong happened executing relatedness analysis");
Expand Down Expand Up @@ -303,7 +301,7 @@ public static void filterFamilyTpedFile(Path tPedPath, Path tPedFilteredPath, Pa
bw.close();
}

private static File runIBD(String basename, Path freqPath, Path outDir) throws ToolException {
private static File runIBD(String dockerImage, String basename, Path freqPath, Path outDir) throws ToolException {
// Input bindings
List<AbstractMap.SimpleEntry<String, String>> inputBindings = new ArrayList<>();
inputBindings.add(new AbstractMap.SimpleEntry<>(freqPath.getParent().toString(), "/input"));
Expand All @@ -316,8 +314,7 @@ private static File runIBD(String basename, Path freqPath, Path outDir) throws T
String plinkParams = "plink1.9 --tfile /output/" + basename + " --genome rel-check --read-freq /input/" + FREQ_FILENAME
+ " --out /output/" + basename;
try {
PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor();
DockerUtils.run(plinkExecutor.getDockerImageName(), inputBindings, outputBinding, plinkParams, null);
DockerUtils.run(dockerImage, inputBindings, outputBinding, plinkParams, null);
} catch (IOException e) {
throw new ToolException(e);
}
Expand Down
12 changes: 7 additions & 5 deletions ...a-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@
import org.opencb.commons.exec.Command;
import org.opencb.commons.utils.DockerUtils;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor;
import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor;
import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.config.Docker;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.InternalStatus;
import org.opencb.opencga.core.models.family.Family;
Expand Down Expand Up @@ -83,7 +84,9 @@ public static void selectMarkers(String basename, String study, List<String> sam

exportData(tpedFile, tfamFile, query, storageManager, token);
if (tpedFile.exists() && tpedFile.length() > 0) {
pruneVariants(basename, outputBinding);
String dockerImage = ConfigurationUtils.getDockerImage(Docker.OPENCGA_EXT_TOOLS_IMAGE_KEY,
storageManager.getCatalogManager().getConfiguration());
pruneVariants(dockerImage, basename, outputBinding);
}

if (!tpedFile.exists() || tpedFile.length() == 0) {
Expand Down Expand Up @@ -355,12 +358,11 @@ private static void exportData(File tpedFile, File tfamFile, Query query, Varian
}
}

private static void pruneVariants(String basename, AbstractMap.SimpleEntry<String, String> outputBinding) throws ToolException {
private static void pruneVariants(String dockerImage, String basename, AbstractMap.SimpleEntry<String, String> outputBinding) throws ToolException {
// Variant pruning using PLINK in docker
String plinkParams = "plink1.9 --tfile /data/output/" + basename + " --indep 50 5 2 --out /data/output/" + basename;
try {
PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor();
DockerUtils.run(plinkExecutor.getDockerImageName(), null, outputBinding, plinkParams, null);
DockerUtils.run(dockerImage, null, outputBinding, plinkParams, null);
} catch (IOException e) {
throw new ToolException(e);
}
Expand Down
16 changes: 5 additions & 11 deletions ...lysis/src/main/java/org/opencb/opencga/analysis/individual/qc/InferredSexComputation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,16 @@

import org.opencb.biodata.models.alignment.RegionCoverage;
import org.opencb.biodata.models.core.Region;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.DockerUtils;
import org.opencb.opencga.analysis.alignment.AlignmentStorageManager;
import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureLocalAnalysisExecutor;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.FileManager;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.response.OpenCGAResult;

import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.AbstractMap;
import java.util.List;
import java.util.Map;

import static org.opencb.opencga.core.tools.variant.InferredSexAnalysisExecutor.GRCH37_CHROMOSOMES;
import static org.opencb.opencga.core.tools.variant.InferredSexAnalysisExecutor.GRCH38_CHROMOSOMES;
Expand Down Expand Up @@ -92,14 +87,13 @@ public static double[] computeRatios(String study, File bamFile, String assembly
return new double[]{1.0d * means[1] / means[0], 1.0d * means[2] / means[0]};
}

public static java.io.File plot(File inputFile, Path outDir) throws ToolException {
public static java.io.File plot(File inputFile, Path outDir, String dockerImage) throws ToolException {
// Execute R script in docker
AbstractMap.SimpleEntry<String, String> outputBinding = new AbstractMap.SimpleEntry<>(outDir.toAbsolutePath().toString(),
"/data/output");
String rParams = "R CMD Rscript --vanilla /data/input/" + inputFile.getName();
try {
String cmdline = DockerUtils.run(MutationalSignatureLocalAnalysisExecutor.R_DOCKER_IMAGE, null, outputBinding, rParams, null);
System.out.println("Docker command line: " + cmdline);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😎

DockerUtils.run(dockerImage, null, outputBinding, rParams, null);
} catch (IOException e) {
throw new ToolException(e);
}
Expand Down
2 changes: 1 addition & 1 deletion opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.commons.lang3.time.StopWatch;
import org.opencb.commons.datastore.core.Event;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.opencga.analysis.ConfigurationUtils;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
Expand All @@ -31,6 +30,7 @@
import org.opencb.opencga.core.common.MemoryUsageMonitor;
import org.opencb.opencga.core.common.TimeUtils;
import org.opencb.opencga.core.config.Configuration;
import org.opencb.opencga.core.config.ConfigurationUtils;
import org.opencb.opencga.core.config.storage.StorageConfiguration;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.Enums;
Expand Down
Loading
Loading