Merge pull request #53 from cancogen-virus-seq/RC/0.2.0

Release 0.2.0

Jenkinsfile

@@ -95,20 +95,18 @@ spec:
             }
         }
 
-//        stage('deploy to cancogen-virus-seq-dev') {
-//            when {
-//                branch "develop"
-//            }
-//            steps {
-//                build(job: "/provision/helm", parameters: [
-//                    [$class: 'StringParameterValue', name: 'AP_RDPC_ENV', value: 'dev' ],
-//                    [$class: 'StringParameterValue', name: 'AP_CHART_NAME', value: 'muse'],
-//                    [$class: 'StringParameterValue', name: 'AP_RELEASE_NAME', value: 'muse'],
-//                    [$class: 'StringParameterValue', name: 'AP_HELM_CHART_VERSION', value: "${chartVersion}"],
-//                    [$class: 'StringParameterValue', name: 'AP_ARGS_LINE', value: "--set-string image.tag=${version}-${commit}" ]
-//                ])
-//            }
-//        }
+       stage('deploy to cancogen-virus-seq-dev') {
+           when {
+               branch "develop"
+           }
+           steps {
+               build(job: "virusseq/update-app-version", parameters: [
+                   [$class: 'StringParameterValue', name: 'CANCOGEN_ENV', value: 'dev' ],
+                   [$class: 'StringParameterValue', name: 'TARGET_RELEASE', value: 'muse'],
+                   [$class: 'StringParameterValue', name: 'NEW_APP_VERSION', value: "${version}-${commit}" ]
+               ])
+           }
+       }
 
         stage('Release & Tag') {
             when {

compose/docker-compose.yaml

@@ -10,7 +10,7 @@ services:
       SPRING_FLYWAY_ENABLED: "true"
       SPRING_FLYWAY_LOCATIONS: "classpath:flyway/sql,classpath:db/migration"
       SPRING_PROFILES: demo, auth
-      JWT_DURATIONMS: 1800000 # expire tokens in 30 min for local testing
+      JWT_DURATIONMS: 2147483647 # expire tokens in 24.855 days (max-int ms) for local testing
     expose:
       - "8080"
     ports:
@@ -44,7 +44,6 @@ services:
     ports:
       - "8085:9000"
   score-server:
-    network_mode: "host"
     image: overture/score-server:5.3.0
     environment:
       SPRING_PROFILES_ACTIVE: amazon,collaboratory,prod,secure,jwt
@@ -53,13 +52,13 @@ services:
       BUCKET_NAME_OBJECT: oicr.icgc.test
       BUCKET_NAME_STATE: oicr.icgc.test
       COLLABORATORY_DATA_DIRECTORY: data
-      METADATA_URL: http://localhost:8089
-      S3_ENDPOINT:  http://localhost:8085
+      METADATA_URL: http://song:8089
+      S3_ENDPOINT:  http://object-storage:8085
       S3_ACCESSKEY: minio
       S3_SECRETKEY: minio123
       S3_SIGV4ENABLED: "true"
-      AUTH_JWT_PUBLICKEYURL: http://localhost:8081/oauth/token/public_key
-      AUTH_SERVER_URL: http://localhost:8081/o/check_api_key/
+      AUTH_JWT_PUBLICKEYURL: http://ego-api:8081/oauth/token/public_key
+      AUTH_SERVER_URL: http://ego-api:8081/o/check_api_key/
       AUTH_SERVER_CLIENTID: score
       AUTH_SERVER_CLIENTSECRET: scoresecret
       AUTH_SERVER_TOKENNAME: apiKey
@@ -96,28 +95,27 @@ services:
     volumes:
       - "./song-db-init:/docker-entrypoint-initdb.d"
   song-server:
-    network_mode: "host"
     image: overture/song-server:4.5.0
     environment:
       SERVER_PORT: 8089
       SPRING_PROFILES_ACTIVE: "prod,secure,default,jwt,score-client-cred"
-      AUTH_JWT_PUBLICKEYURL: http://localhost:8081/oauth/token/public_key
-      AUTH_SERVER_URL: http://localhost:8081/o/check_api_key/
+      AUTH_JWT_PUBLICKEYURL: http://ego-api:8081/oauth/token/public_key
+      AUTH_SERVER_URL: http://ego-api:8081/o/check_api_key/
       AUTH_SERVER_CLIENTID: song
       AUTH_SERVER_TOKENNAME: apiKey
       AUTH_SERVER_CLIENTSECRET: songsecret
       AUTH_SERVER_SCOPE_STUDY_PREFIX: song.
       AUTH_SERVER_SCOPE_STUDY_SUFFIX: .WRITE
       AUTH_SERVER_SCOPE_SYSTEM: song.WRITE
-      SCORE_URL: http://localhost:8087
+      SCORE_URL: http://score:8087
       SCORE_CLIENTCREDENTIALS_ID: adminId
       SCORE_CLIENTCREDENTIALS_SECRET: adminSecret
-      SCORE_CLIENTCREDENTIALS_TOKENURL: http://localhost:8081/oauth/token
+      SCORE_CLIENTCREDENTIALS_TOKENURL: http://ego-api:8081/oauth/token
       SCORE_CLIENTCREDENTIALS_SYSTEMSCOPE: "score.WRITE"
       MANAGEMENT_SERVER_PORT: 8088
       SPRING_DATASOURCE_USERNAME: postgres
       SPRING_DATASOURCE_PASSWORD: password
-      SPRING_DATASOURCE_URL: jdbc:postgresql://localhost:8432/song?stringtype=unspecified
+      SPRING_DATASOURCE_URL: jdbc:postgresql://song-db:5432/song?stringtype=unspecified
       SPRING_FLYWAY_ENABLED: "true"
       SPRING_FLYWAY_LOCATIONS: "classpath:db/migration"
     ports:

pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>org.cancogen-virus-seq</groupId>
     <artifactId>muse</artifactId>
-    <version>0.1.3</version>
+    <version>0.2.0</version>
     <name>muse</name>
     <description>Service to submit, validate, and download virus-seq data</description>
     <properties>
@@ -46,6 +46,12 @@
             <artifactId>r2dbc-postgresql</artifactId>
             <version>0.8.7.RELEASE</version>
         </dependency>
+        <dependency>
+            <groupId>io.r2dbc</groupId>
+            <artifactId>r2dbc-pool</artifactId>
+            <version>0.8.6.RELEASE</version>
+        </dependency>
+
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
@@ -87,9 +93,9 @@
             <version>29.0-jre</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity</artifactId>
-            <version>1.7</version>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

sample-files/L00210314.fasta

@@ -1,4 +1,4 @@
->MuseTest/Qc-L00210314/2020 seq_method:Illumina_NexteraFlex|assemb_method:ivar|snv_call_method:ivar
+>Canada/Qc-L00210314/2020
 NNNNNNNNNNNNNNNNNTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCT
 AGATGATTACCAAGGTAAACCTTTGGAATTTGGTGCCACTTCTGCTGCTCTTCAACCTGA
 AGAAGAGCAAGAAGAAGATTGGTTAGATGATGATAGTCAACAAACTGTTGGTCAACAAGA

sample-files/L00212401.fasta

@@ -1,4 +1,4 @@
->MuseTest/Qc-L00212401/2020 seq_method:Illumina_NexteraFlex|assemb_method:ivar|snv_call_method:ivar
+>Canada/Qc-L00212401/2020
 NNNNNNNNNNNNNNNNNTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCT
 AGATGATTACCAAGGTAAACCTTTGGAATTTGGTGCCACTTCTGCTGCTCTTCAACCTGA
 AGAAGAGCAAGAAGAAGATTGGTTAGATGATGATAGTCAACAAACTGTTGGTCAACAAGA

sample-files/metadata.tsv

@@ -1,3 +1,3 @@
-specimen collector sample ID	sample collected by	sequence submitted by	sample collection date	sample collection date precision	geo_loc_name (country)	geo_loc_name (state/province/territory)	geo_loc_name (city)	organism	isolate	purpose of sampling	purpose of sampling details	NML submitted specimen type	anatomical material	anatomical part	body product	environmental material	environmental site	collection device	collection method	host (scientific name)	host disease	host age	host age unit	host age bin	host gender	purpose of sequencing	purpose of sequencing details	sequencing date	library ID	sequencing instrument	sequencing protocol name	raw sequence data processing method	dehosting method	consensus sequence software name	breadth of coverage value	depth of coverage value	consensus genome length	Ns per 100 kbp	reference genome accession	lineage/clade name	lineage/clade analysis software name	lineage/clade analysis software version	variant designation	variant evidence
-Qc-L00210314	Not Provided	Laboratoire de santé publique du Québec (LSPQ)	2021-03-01	month	Canada	Quebec	Montreal	Severe acute respiratory syndrome coronavirus 2	Canada/Qc-L00210314/2020	Cluster/Outbreak investigation	Not Provided	Not Applicable	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Homo sapiens	COVID-19	41	year	40 - 49	Female	Baseline surveillance (random sampling)	Not Provided	2020-05-01	exp_001	Oxford Nanopore	1D_DNA_MinION	Trimmomatic v. 0.38	Nanostripper	bcftools	95%	400x	38677	330	NC_045512.2	B.1.1.7	Pangolin	2.1.10	Not Applicable	Sequencing
-Qc-L00212401	Not Provided	Laboratoire de santé publique du Québec (LSPQ)	2021-03-04	month	Canada	Quebec	Montreal	Severe acute respiratory syndrome coronavirus 2	Canada/Qc-L00212401/2020	Cluster/Outbreak investigation	Not Provided	Not Applicable	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Homo sapiens	COVID-19	42	year	40 - 49	Male	Baseline surveillance (random sampling)	Not Provided	2020-08-01	exp_002	Oxford Nanopore	1D_DNA_MinION	Trimmomatic v. 0.38	Nanostripper	bcftools	98%	400x	28677	240	NC_045512.2	B.1.1.4	Pangolin	2.1.10	Not Provided	Sequencing
+study_id	specimen collector sample ID	sample collected by	sequence submitted by	sample collection date	sample collection date precision	geo_loc_name (country)	geo_loc_name (state/province/territory)	geo_loc_name (city)	organism	isolate	purpose of sampling	purpose of sampling details	NML submitted specimen type	anatomical material	anatomical part	body product	environmental material	environmental site	collection device	collection method	host (scientific name)	host disease	host age	host age unit	host age bin	host gender	purpose of sequencing	purpose of sequencing details	sequencing date	library ID	sequencing instrument	sequencing protocol name	raw sequence data processing method	dehosting method	consensus sequence software name	consensus sequence software version	breadth of coverage value	depth of coverage value	consensus genome length	Ns per 100 kbp	reference genome accession	lineage/clade name	lineage/clade analysis software name	lineage/clade analysis software version	variant designation	variant evidence	variant evidence details	bioinformatics protocol
+COVIDPR	Qc-L00210314	Not Provided	Laboratoire de santé publique du Québec (LSPQ)	2021-03-01	month	Canada	Quebec	Montreal	Severe acute respiratory syndrome coronavirus 2	Canada/Qc-L00210314/2020	Cluster/Outbreak investigation	Not Provided	Not Applicable	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Homo sapiens	COVID-19	41	year	40 - 49	Female	Baseline surveillance (random sampling)	Not Provided	2020-05-01	exp_001	Oxford Nanopore	1D_DNA_MinION	Trimmomatic v. 0.38	Nanostripper	bcftools	1	95%	400x	38677	330	NC_045512.2	B.1.1.7	Pangolin	2.1.10	Not Applicable	Sequencing		
+COVIDPR	Qc-L00212401	Not Provided	Laboratoire de santé publique du Québec (LSPQ)	2021-03-04	month	Canada	Quebec	Montreal	Severe acute respiratory syndrome coronavirus 2	Canada/Qc-L00212401/2020	Cluster/Outbreak investigation	Not Provided	Not Applicable	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Not Provided	Homo sapiens	COVID-19	42	year	40 - 49	Male	Baseline surveillance (random sampling)	Not Provided	2020-08-01	exp_002	Oxford Nanopore	1D_DNA_MinION	Trimmomatic v. 0.38	Nanostripper	bcftools	1	98%	400x	28677	240	NC_045512.2	B.1.1.4	Pangolin	2.1.10	Not Provided	Sequencing		

src/main/java/org/cancogenvirusseq/muse/api/ApiController.java

@@ -21,26 +21,30 @@
 import static java.lang.String.format;
 import static org.cancogenvirusseq.muse.components.FastaFileProcessor.FASTA_FILE_EXTENSION;
 
+import com.google.common.io.ByteStreams;
+import java.time.Instant;
 import java.util.List;
 import java.util.UUID;
-import javax.validation.Valid;
+import java.util.zip.GZIPOutputStream;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import lombok.val;
 import org.cancogenvirusseq.muse.api.model.*;
 import org.cancogenvirusseq.muse.exceptions.MuseBaseException;
+import org.cancogenvirusseq.muse.exceptions.download.UnknownException;
 import org.cancogenvirusseq.muse.service.DownloadsService;
 import org.cancogenvirusseq.muse.service.SubmissionService;
 import org.cancogenvirusseq.muse.service.UploadService;
 import org.cancogenvirusseq.muse.utils.SecurityContextWrapper;
 import org.springframework.core.io.buffer.DataBuffer;
+import org.springframework.core.io.buffer.DefaultDataBufferFactory;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.http.codec.multipart.FilePart;
 import org.springframework.web.bind.annotation.ExceptionHandler;
-import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import reactor.core.publisher.Flux;
@@ -57,6 +61,12 @@ public class ApiController implements ApiDefinition {
 
   private static final String CONTENT_DISPOSITION_HEADER = "Content-Disposition";
 
+  public Mono<SubmissionDTO> getSubmissionById(@NonNull UUID submissionId) {
+    return SecurityContextWrapper.forMono(submissionService::getSubmissionById)
+        .apply(submissionId)
+        .map(SubmissionDTO::fromDAO);
+  }
+
   public Mono<EntityListResponse<SubmissionDTO>> getSubmissions(
       Integer page, Integer size, Sort.Direction sortDirection, SubmissionSortField sortField) {
     return SecurityContextWrapper.forFlux(submissionService::getSubmissions)
@@ -87,16 +97,47 @@ public Mono<EntityListResponse<UploadDTO>> getUploads(
   public Flux<UploadDTO> streamUploads(String accessToken, UUID submissionId) {
     return SecurityContextWrapper.forFlux(uploadService::getUploadStream)
         .apply(submissionId)
-        .map(UploadDTO::fromDAO);
+        .map(UploadDTO::fromDAO)
+        .log("ApiController::streamUploads");
+  }
+
+  public ResponseEntity<Flux<DataBuffer>> download(List<UUID> objectIds) {
+    return ResponseEntity.ok()
+        .header(
+            CONTENT_DISPOSITION_HEADER,
+            format(
+                "attachment; filename=sample-bundle-%s%s",
+                Instant.now().toString(), FASTA_FILE_EXTENSION))
+        .body(downloadsService.download(objectIds));
   }
 
-  public ResponseEntity<Flux<DataBuffer>> download(
-      @NonNull @Valid @RequestBody DownloadRequest downloadRequest) {
+  public ResponseEntity<Flux<DataBuffer>> downloadGzip(List<UUID> objectIds) {
     return ResponseEntity.ok()
         .header(
             CONTENT_DISPOSITION_HEADER,
-            format("attachment; filename=%s", downloadRequest.getStudyId() + FASTA_FILE_EXTENSION))
-        .body(downloadsService.download(downloadRequest));
+            // convention for gzip is original file name with `.gz`
+            format(
+                "attachment; filename=sample-bundle-%s%s.gz",
+                Instant.now().toString(), FASTA_FILE_EXTENSION))
+        .body(downloadsService.download(objectIds).flatMap(this::gzipDataBuffer));
+  }
+
+  private Mono<DataBuffer> gzipDataBuffer(DataBuffer inputDataBuffer) {
+    // allocate gzipped data buffer
+    val gzipDataBuffer = new DefaultDataBufferFactory().allocateBuffer();
+    try {
+      // GZIPOutputStream basically decorates an OutputStream and allows writing bytes to it.
+      // Since a spring DataBuffer can be obtained as an OutputStream,
+      // we create a GZIPOutputStream around gzipDataBuffer and writes bytes from inputDataBuffer
+      val gzip = new GZIPOutputStream(gzipDataBuffer.asOutputStream());
+      val bytes = ByteStreams.toByteArray(inputDataBuffer.asInputStream());
+      gzip.write(bytes);
+      gzip.close();
+    } catch (Exception e) {
+      e.printStackTrace();
+      return Mono.error(new UnknownException());
+    }
+    return Mono.just(gzipDataBuffer);
   }
 
   @ExceptionHandler

src/main/java/org/cancogenvirusseq/muse/api/ApiDefinition.java

@@ -19,6 +19,7 @@
 package org.cancogenvirusseq.muse.api;
 
 import io.swagger.annotations.*;
+import java.util.List;
 import java.util.UUID;
 import javax.validation.Valid;
 import org.cancogenvirusseq.muse.api.model.*;
@@ -39,6 +40,25 @@ public interface ApiDefinition {
   String FORBIDDEN_MSG = "The requester is not authorized to perform this action.";
   String UNKNOWN_MSG = "An unexpected error occurred.";
 
+  @ApiOperation(
+      value = "Get Submission by submissionId",
+      nickname = "Get Submission by submissionId",
+      response = SubmissionDTO.class,
+      tags = "Muse")
+  @ApiResponses(
+      value = {
+        @ApiResponse(code = 200, message = "", response = SubmissionDTO.class),
+        @ApiResponse(code = 400, message = BAD_REQUEST, response = ErrorResponse.class),
+        @ApiResponse(code = 401, message = UNAUTHORIZED_MSG, response = ErrorResponse.class),
+        @ApiResponse(code = 403, message = FORBIDDEN_MSG, response = ErrorResponse.class),
+        @ApiResponse(code = 500, message = UNKNOWN_MSG, response = ErrorResponse.class)
+      })
+  @RequestMapping(
+      value = "/submissions/{submissionId}",
+      produces = MediaType.APPLICATION_JSON_VALUE,
+      method = RequestMethod.GET)
+  Mono<SubmissionDTO> getSubmissionById(@PathVariable("submissionId") UUID submissionId);
+
   @ApiOperation(
       value = "Get All Submissions",
       nickname = "Get Submissions",
@@ -188,14 +208,27 @@ Flux<UploadDTO> streamUploads(
   @ApiResponses(
       value = {
         @ApiResponse(code = 200, message = "", response = MultipartFile.class),
-        @ApiResponse(code = 401, message = UNAUTHORIZED_MSG, response = ErrorResponse.class),
-        @ApiResponse(code = 403, message = FORBIDDEN_MSG, response = ErrorResponse.class),
         @ApiResponse(code = 500, message = UNKNOWN_MSG, response = ErrorResponse.class)
       })
   @RequestMapping(
       value = "/download",
       produces = MediaType.APPLICATION_OCTET_STREAM_VALUE,
-      consumes = MediaType.APPLICATION_JSON_VALUE,
       method = RequestMethod.GET)
-  ResponseEntity<Flux<DataBuffer>> download(@Valid @RequestBody DownloadRequest downloadRequest);
+  ResponseEntity<Flux<DataBuffer>> download(@Valid @RequestParam List<UUID> objectIds);
+
+  @ApiOperation(
+      value = "Download molecular data as a single .fasta.gz gzip compressed file",
+      nickname = "Download Gzip",
+      response = MultipartFile.class,
+      tags = "Muse")
+  @ApiResponses(
+      value = {
+        @ApiResponse(code = 200, message = "", response = MultipartFile.class),
+        @ApiResponse(code = 500, message = UNKNOWN_MSG, response = ErrorResponse.class)
+      })
+  @RequestMapping(
+      value = "/download/gzip",
+      produces = MediaType.APPLICATION_OCTET_STREAM_VALUE,
+      method = RequestMethod.GET)
+  ResponseEntity<Flux<DataBuffer>> downloadGzip(@Valid @RequestParam List<UUID> objectIds);
 }

src/main/java/org/cancogenvirusseq/muse/api/model/SubmissionDTO.java

@@ -19,7 +19,7 @@
 package org.cancogenvirusseq.muse.api.model;
 
 import java.time.OffsetDateTime;
-import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import lombok.NonNull;
 import lombok.Value;
@@ -29,7 +29,7 @@
 public class SubmissionDTO {
   @NonNull UUID submissionId;
   @NonNull OffsetDateTime createdAt;
-  @NonNull List<String> originalFileNames;
+  @NonNull Set<String> originalFileNames;
   @NonNull Integer totalRecords;
 
   public static SubmissionDTO fromDAO(Submission submission) {

src/main/java/org/cancogenvirusseq/muse/api/model/UploadDTO.java

@@ -18,7 +18,7 @@
 
 package org.cancogenvirusseq.muse.api.model;
 
-import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import lombok.NonNull;
 import lombok.Value;
@@ -31,7 +31,7 @@ public class UploadDTO {
   @NonNull String studyId;
   @NonNull String submitterSampleId;
   @NonNull UploadStatus status;
-  @NonNull List<String> originalFilePair;
+  @NonNull Set<String> originalFilePair;
   UUID analysisId;
   String error;