diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/VariationDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/VariationDownloadManager.java index 0f0f96783..7f317d5f7 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/VariationDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/VariationDownloadManager.java @@ -46,7 +46,7 @@ public DownloadFile downloadDbSnp() throws IOException, InterruptedException { return null; } if (speciesConfiguration.getScientificName().equals("Homo sapiens")) { - logger.info("Downloading dbSNP scores information ..."); + logger.info("Downloading dbSNP information ..."); Path variation = downloadFolder.resolve(VARIATION_DATA); Files.createDirectories(variation); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java index cacf8457d..f1e664a50 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java @@ -85,11 +85,6 @@ public List> info(List ids, ProjectionQ return results; } - @Override - public CellBaseDataResult count(OntologyQuery query) { - return null; - } - @Override public CellBaseDataResult distinct(OntologyQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java index a279f0765..5c8fcb571 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java @@ -86,11 +86,6 @@ public List> info(List ids, Projection return results; } - @Override - public CellBaseDataResult count(PublicationQuery query) { - return null; - } - @Override public CellBaseDataResult distinct(PublicationQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java index 19f6f92aa..5a7eb2cb8 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.lib.impl.core; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Snp; @@ -36,6 +37,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import static org.opencb.cellbase.core.ParamConstants.API_KEY_PARAM; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; @@ -72,8 +74,8 @@ public CellBaseIterator iterator(SnpQuery query) throws CellBaseException { } @Override - public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, - String apiKey) throws CellBaseException { + public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) + throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); @@ -86,14 +88,12 @@ public List> info(List ids, ProjectionQueryOptio return results; } - @Override - public CellBaseDataResult count(SnpQuery query) { - return null; - } - @Override public CellBaseDataResult distinct(SnpQuery query) throws CellBaseException { - return null; + Bson bsonQuery = parseQuery(query); + logger.info("snpQuery distinct: {}", bsonQuery.toBsonDocument().toJson()); + MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonQuery, String.class)); } @Override @@ -103,7 +103,27 @@ public CellBaseDataResult aggregationStats(SnpQuery query) { @Override public CellBaseDataResult groupBy(SnpQuery query) throws CellBaseException { - return null; + Bson bsonQuery = parseQuery(query); + logger.info("snpQuery groupBy: {}", bsonQuery.toBsonDocument().toJson()); + MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + return groupBy(bsonQuery, query, "name", mongoDBCollection); + } + + public CellBaseDataResult startsWith(String id, QueryOptions options, int dataRelease) throws CellBaseException { + Bson regex = Filters.regex("id", Pattern.compile("^" + id)); + Bson projection; + if (options.containsKey(QueryOptions.INCLUDE)) { + projection = Projections.include(options.getAsStringList(QueryOptions.INCLUDE)); + } else { + if (options.containsKey(QueryOptions.EXCLUDE)) { + projection = Projections.exclude(options.getAsStringList(QueryOptions.EXCLUDE)); + } else { + projection = Projections.exclude("annotation"); + } + } + + MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + return new CellBaseDataResult<>(mongoDBCollection.find(regex, projection, CONVERTER, options)); } public Bson parseQuery(SnpQuery query) { @@ -132,9 +152,8 @@ public Bson parseQuery(SnpQuery query) { e.printStackTrace(); } - logger.info("SNP parsed query: " + andBsonList); - if (andBsonList.size() > 0) { - System.out.println("SnpMongoDBAdaptor, parse query = " + andBsonList); + logger.info("SnpMongoDBAdaptor parsed query: {}", andBsonList); + if (!andBsonList.isEmpty()) { return Filters.and(andBsonList); } else { return new Document(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java index c32e84f43..2b1a04c69 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java @@ -18,6 +18,7 @@ import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -247,7 +248,7 @@ private Bson parseQuery(Query query) throws CellBaseException { "annotation.consequenceTypes.sequenceOntologyTerms.name", andBsonList); createGeneOrQuery(query, ParamConstants.QueryParams.GENE.key(), andBsonList); - if (andBsonList.size() > 0) { + if (!andBsonList.isEmpty()) { return Filters.and(andBsonList); } else { return new Document(); @@ -262,6 +263,7 @@ public Bson parseQuery(VariantQuery query) throws CellBaseException { Object value = entry.getValue(); switch (dotNotationName) { case "id": + // Both variant IDs and dbSNP IDs are allowed List variantIds = getVariantIds(Arrays.asList(query.getId().split(",")), query.getDataRelease()); createAndOrQuery(variantIds, dotNotationName, QueryParam.Type.STRING, andBsonList); break; @@ -302,8 +304,8 @@ public Bson parseQuery(VariantQuery query) throws CellBaseException { throw new CellBaseException(e.getMessage()); } - logger.debug("variant parsed query: " + andBsonList.toString()); - if (andBsonList.size() > 0) { + logger.debug("variant parsed query: {}", andBsonList); + if (!andBsonList.isEmpty()) { return Filters.and(andBsonList); } else { return new Document(); @@ -585,21 +587,11 @@ public CellBaseDataResult getFunctionalScoreVariant(Variant variant, Quer if (position >= chunkStart && position <= chunkEnd) { int offset = (position - chunkStart); ArrayList basicDBList = dbObject.get("values", ArrayList.class); - -// long l1 = 0L; // TODO: delete -// try { // TODO: delete long l1 = Long.parseLong(basicDBList.get(offset).toString()); -// l1 = (Long) basicDBList.get(offset); -// } catch (Exception e) { // TODO: delete -// logger.error("problematic variant: {}", variant.toString()); -// throw e; -// } - if (dbObject.getString("source").equalsIgnoreCase("cadd_raw")) { float value = 0f; switch (alternate.toLowerCase()) { case "a": -// value = ((short) (l1 >> 48) - 10000) / DECIMAL_RESOLUTION; value = (((short) (l1 >> 48)) / DECIMAL_RESOLUTION) - 10; break; case "c": @@ -618,7 +610,6 @@ public CellBaseDataResult getFunctionalScoreVariant(Variant variant, Quer .setScore(value) .setSource(dbObject.getString("source")) .setDescription(null) - // .setDescription("") .build()); } @@ -795,8 +786,9 @@ public CellBaseDataResult getFunctionalScoreRegion(List getVariantIds(List ids, int dataRelease) throws CellBaseException { - List variantIds = new ArrayList<>(); + List variantIds = new ArrayList<>(ids.size()); List snpIds = new ArrayList<>(); + // Split dbSNP IDs and variant IDs for (String id : ids) { if (id.startsWith("rs")) { snpIds.add(id); @@ -804,19 +796,22 @@ private List getVariantIds(List ids, int dataRelease) throws Cel variantIds.add(id); } } + + // Get the variant ID for the dbSNP ID if (CollectionUtils.isNotEmpty(snpIds)) { + // 1. Prepare the query List orBsonList = new ArrayList<>(); for (String snpId : snpIds) { orBsonList.add(Filters.eq("id", snpId)); } Bson query = Filters.or(orBsonList); + // 2. We must exclude as much information as possible to improve performance MongoDBCollection mongoDBCollection = getCollectionByRelease(snpDBCollectionByRelease, dataRelease); - DataResult snpDataResult = mongoDBCollection.find(query, null, Snp.class, new QueryOptions()); + DataResult snpDataResult = mongoDBCollection.find(query, Projections.exclude("annotation"), Snp.class, new QueryOptions()); + // 3. Build the variant IDs Set results = new HashSet<>(); - - // Build the variant IDs if (snpDataResult.getNumResults() > 0) { for (Snp snp : snpDataResult.getResults()) { for (String allele : snp.getAlleles()) { @@ -825,7 +820,7 @@ private List getVariantIds(List ids, int dataRelease) throws Cel } } - // Add new variant IDs, if necessary + // 4. Add new variant IDs, if necessary if (CollectionUtils.isNotEmpty(results)) { variantIds.addAll(results); } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index 8d6983293..3bb57515a 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -52,9 +52,9 @@ public VariantWSServer(@PathParam("apiVersion") @ApiParam(name = "apiVersion", v defaultValue = DEFAULT_VERSION) String apiVersion, @PathParam("species") @ApiParam(name = "species", value = SPECIES_DESCRIPTION) String species, @ApiParam(name = "assembly", value = ASSEMBLY_DESCRIPTION) @DefaultValue("") @QueryParam("assembly") - String assembly, + String assembly, @ApiParam(name = "dataRelease", value = DATA_RELEASE_DESCRIPTION) @DefaultValue("0") @QueryParam("dataRelease") - int dataRelease, + int dataRelease, @ApiParam(name = "apiKey", value = API_KEY_DESCRIPTION) @DefaultValue("") @QueryParam("apiKey") String apiKey, @Context UriInfo uriInfo, @Context HttpServletRequest hsr) throws CellBaseServerException { @@ -119,20 +119,6 @@ public Response getNormalization(@PathParam("variants") @ApiParam(name = "varian } - // @GET -// @Path("/{phenotype}/phenotype") -// @ApiOperation(httpMethod = "GET", -// value = "Not implemented yet", -// response = CellBaseDataResponse.class, hidden = true) -// public Response getVariantsByPhenotype(@PathParam("phenotype") String phenotype) { -// try { -// parseQueryParams(); -// return Response.ok("Not implemented").build(); -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } - @POST @Consumes("text/plain") @Path("/annotation") @@ -291,7 +277,7 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") @QueryParam("checkAminoAcidChange") @ApiParam(name = "checkAminoAcidChange", value = "", allowableValues = "false,true", defaultValue = "false", required = false) - Boolean checkAminoAcidChange, + Boolean checkAminoAcidChange, @QueryParam("consequenceTypeSource") @ApiParam(name = "consequenceTypeSource", value = "Gene set, either ensembl (default) " + "or refseq", allowableValues = "ensembl,refseq", allowMultiple = true, @@ -360,29 +346,6 @@ private Response getAnnotationByVariant(String variants, } } -// @GET -// @Deprecated -// @Path("/{variants}/cadd") -// @ApiOperation(httpMethod = "GET", value = "Get CADD scores for a (list of) variant(s)", response = Score.class, -// responseContainer = "QueryResponse", hidden = true) -// public Response getCaddScoreByVariant(@PathParam("variants") -// @ApiParam(name = "variants", value = "Comma separated list of variants for" -// + "which CADD socores will be returned, e.g. " -// + "19:45411941:T:C,14:38679764:-:GATCTG,1:6635210:G:-," -// + "2:114340663:GCTGGGCATCCT:ACTGGGCATCCT", -// required = true) String variants) { -// try { -// parseQueryParams(); -// VariantDBAdaptor variantDBAdaptor = dbAdaptorFactory.getVariationDBAdaptor(this.species, this.assembly); -// -// List> functionalScoreVariant = -// variantDBAdaptor.getFunctionalScoreVariant(Variant.parseVariants(variants), queryOptions); -// return createOkResponse(functionalScoreVariant); -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } - // @GET // @Path("/stats") // @Override @@ -497,9 +460,8 @@ public Response getAllConsequenceTypes() { @GET @Path("/snp") - @ApiOperation(httpMethod = "GET", value = "Get SNPs", - response = Snp.class, responseContainer = "QueryResponse") - @ApiImplicitParams({ + @ApiOperation(httpMethod = "GET", value = "Get SNPs", response = Snp.class, responseContainer = "QueryResponse") + @ApiImplicitParams({ @ApiImplicitParam(name = "exclude", value = EXCLUDE_DESCRIPTION, required = false, dataType = "java.util.List", paramType = "query"), @ApiImplicitParam(name = "include", value = INCLUDE_DESCRIPTION, @@ -510,15 +472,14 @@ public Response getAllConsequenceTypes() { required = false, dataType = "java.util.List", paramType = "query", defaultValue = "", allowableValues="ASCENDING,DESCENDING"), @ApiImplicitParam(name = "limit", value = LIMIT_DESCRIPTION, - required = false, defaultValue = DEFAULT_LIMIT, dataType = "java.util.List", - paramType = "query"), + required = false, defaultValue = DEFAULT_LIMIT, dataType = "java.util.List", paramType = "query"), @ApiImplicitParam(name = "skip", value = SKIP_DESCRIPTION, - required = false, defaultValue = DEFAULT_SKIP, dataType = "java.util.List", - paramType = "query") + required = false, defaultValue = DEFAULT_SKIP, dataType = "java.util.List", paramType = "query") }) public Response getSnps(@QueryParam("id") @ApiParam(name = "id", value = "ID") String id, - @QueryParam("chromosome") @ApiParam(name = "chromosome", value = "Chromsome") String chromosome, - @QueryParam("position") @ApiParam(name = "position", value = "Position") Integer position) { + @QueryParam("chromosome") @ApiParam(name = "chromosome", value = "Chromosome") String chromosome, + @QueryParam("position") @ApiParam(name = "position", value = "Position") Integer position, + @QueryParam("reference") @ApiParam(name = "reference", value = "Reference") String reference) { try { SnpQuery query = new SnpQuery(uriParams); CellBaseDataResult queryResult = variantManager.getSnps(query); @@ -528,58 +489,4 @@ public Response getSnps(@QueryParam("id") @ApiParam(name = "id", value = "ID") S } } - // FIXME: 29/04/16 GET and POST web services to be fixed -// @GET -// @Path("/{variants}/consequenceType") -// @ApiOperation(httpMethod = "GET", value = "Get the biological impact of the variant(s)", response = String.class, -// responseContainer = "QueryResponse") -// public Response getConsequenceTypeByGetMethod(@PathParam("variants") String variants) { -// return getConsequenceType(variants); -// } -// -// private Response getConsequenceType(String variants) { -// try { -// parseQueryParams(); -// VariantDBAdaptor variationDBAdaptor = dbAdaptorFactory.getVariationDBAdaptor(this.species, this.assembly); -// query.put(VariantDBAdaptor.QueryParams.ID.key(), variants); -// queryOptions.put(QueryOptions.INCLUDE, "annotation.displayConsequenceType"); -// CellBaseDataResult queryResult = variationDBAdaptor.get(query, queryOptions); -// CellBaseDataResult queryResult1 = new CellBaseDataResult<>( -// queryResult.getId(), queryResult.getTime(), queryResult.getEvents(), queryResult.getNumResults(), -// Collections.singletonList(queryResult.getResults().get(0).getAnnotation().getDisplayConsequenceType()), 1); -// return createOkResponse(queryResult1); -// } catch (Exception e) { -// return createErrorResponse("getConsequenceTypeByPostMethod", e.toString()); -// } -// } - - // FIXME: 29/04/16 GET and POST methods to be fixed -// @GET -// @Path("/{variants}/regulatory") -// @ApiOperation(httpMethod = "GET", value = "Get the regulatory impact of the variant(s)", hidden = true) -// public Response getRegulatoryByGetMethod(@PathParam("variants") String variants) { -// return getRegulatoryType(variants); -// } -// -// private Response getRegulatoryType(String variants) { -// try { -// parseQueryParams(); -// VariantDBAdaptor variationDBAdaptor = dbAdaptorFactory.getVariationDBAdaptor(this.species, this.assembly); -// return null; -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } - -// @GET -// @Path("/{variants}/sequence") -// @ApiOperation(httpMethod = "GET", value = "Get the adjacent sequence to the SNP(s) - Not yet implemented", -// hidden = true) -// public Response getSequence(@PathParam("variants") String query) { -// try { -// return null; -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } }