diff --git a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextOutput.groovy b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextJSONOutput.groovy similarity index 56% rename from plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextOutput.groovy rename to plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextJSONOutput.groovy index 75e52d0..ffe80ee 100644 --- a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextOutput.groovy +++ b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextJSONOutput.groovy @@ -22,23 +22,60 @@ import java.nio.file.Paths import java.util.List import java.util.Map import java.nio.file.Path +import java.io.OutputStream +import java.util.zip.GZIPOutputStream + import groovy.transform.CompileStatic import groovy.json.JsonOutput import groovy.util.logging.Slf4j @Slf4j @CompileStatic -class IridaNextOutput { +class IridaNextJSONOutput { private Map files = ["global": [], "samples": [:]] private Map metadata = ["samples": [:]] + private Map> scopeIds = ["samples": [] as Set] // private final Map>> files = ["global": [], "samples": []] // private final Map> metadata = ["samples": []] + private Path relativizePath + private Boolean shouldRelativize + + public IridaNextJSONOutput(Path relativizePath) { + this.relativizePath = relativizePath + this.shouldRelativize = (this.relativizePath != null) + } + + public void appendMetadata(String scope, Map data) { + if (scope in metadata.keySet()) { + Map validMetadata = data.collectEntries { k, v -> + if (k in scopeIds[scope]) { + return [(k): v] + } else { + log.trace "scope=${scope}, id=${k} is not a valid identifier. Removing from metadata." + } + } + metadata[scope] = (metadata[scope] as Map) + validMetadata + } + } + + public void addId(String scope, String id) { + log.trace "Adding scope=${scope} id=${id}" + scopeIds[scope].add(id) + } + + public Boolean isValidId(String scope, String id) { + return id in scopeIds[scope] + } public void addFile(String scope, String subscope, Path path) { if (!(scope in files.keySet())) { throw new Exception("scope=${scope} not in valid set of scopes: ${files.keySet()}") } + if (shouldRelativize) { + path = relativizePath.relativize(path) + } + // Treat empty string and null as same if (subscope == "") { subscope = null @@ -48,6 +85,8 @@ class IridaNextOutput { if (scope == "samples" && subscope == null) { throw new Exception("scope=${scope} but subscope is null") } else if (scope == "samples" && subscope != null) { + assert isValidId(scope, subscope) + def files_scope_map = (Map)files_scope if (!files_scope_map.containsKey(subscope)) { files_scope_map[subscope] = [] @@ -68,4 +107,18 @@ class IridaNextOutput { public String toJson() { return JsonOutput.toJson(["files": files, "metadata": metadata]) } + + public void write(Path path) { + // Documentation for reading/writing to Nextflow files using this method is available at + // https://www.nextflow.io/docs/latest/script.html#reading-and-writing + path.withOutputStream { + OutputStream outputStream = it as OutputStream + if (path.extension == 'gz') { + outputStream = new GZIPOutputStream(outputStream) + } + + outputStream.write(JsonOutput.prettyPrint(toJson()).getBytes("utf-8")) + outputStream.close() + } + } } \ No newline at end of file diff --git a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserver.groovy b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserver.groovy index 7630682..00313d1 100644 --- a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserver.groovy +++ b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserver.groovy @@ -32,8 +32,9 @@ import nextflow.trace.TraceRecord import nextflow.processor.TaskRun import nextflow.script.params.FileOutParam import nextflow.script.params.ValueOutParam +import nextflow.Nextflow -import nextflow.iridanext.IridaNextOutput +import nextflow.iridanext.IridaNextJSONOutput /** * IridaNext workflow observer @@ -48,10 +49,17 @@ class IridaNextObserver implements TraceObserver { private Map publishedFiles = [:] private List tasks = [] private List traces = [] - private IridaNextOutput iridaNextOutput = new IridaNextOutput() + private IridaNextJSONOutput iridaNextJSONOutput private Map> pathMatchers private List samplesMatchers private List globalMatchers + private PathMatcher samplesMetadataMatcher + private String filesMetaId + private String samplesMetadataId + private Path iridaNextOutputPath + private Path outputFilesRootDir + private Boolean outputFileOverwrite + private Session session public IridaNextObserver() { pathMatchers = [:] @@ -76,26 +84,56 @@ class IridaNextObserver implements TraceObserver { @Override void onFlowCreate(Session session) { - def iridaNextFiles = session.config.navigate('iridanext.files') - if (iridaNextFiles != null) { - if (!iridaNextFiles instanceof Map) { - throw new Exception("Expected a map in config for iridanext.files=${iridaNextFiles}") + this.session = session + Path relativizePath = null + + Boolean relativizeOutputPaths = session.config.navigate('iridanext.output.relativize', true) + + iridaNextOutputPath = session.config.navigate('iridanext.output.path') as Path + if (iridaNextOutputPath != null) { + iridaNextOutputPath = Nextflow.file(iridaNextOutputPath) as Path + if (relativizeOutputPaths) { + relativizePath = iridaNextOutputPath.getParent() } + } + + outputFileOverwrite = session.config.navigate('iridanext.output.overwrite', false) + + Map iridaNextFiles = session.config.navigate('iridanext.output.files') as Map + if (iridaNextFiles != null) { + // Used for overriding the "meta.id" key used to define identifiers for a scope + // (e.g., by default meta.id is used for a sample identifier in a pipeline) + this.filesMetaId = iridaNextFiles?.idkey ?: "id" - iridaNextFiles = (Map)iridaNextFiles iridaNextFiles.each {scope, matchers -> - if (matchers instanceof String) { - matchers = [matchers] - } + // "id" is a special keyword and isn't used for file matchers + if (scope != "idkey") { + if (matchers instanceof String) { + matchers = [matchers] + } - if (!(matchers instanceof List)) { - throw new Exception("Invalid configuration for iridanext.files=${iridaNextFiles}") + if (!(matchers instanceof List)) { + throw new Exception("Invalid configuration for iridanext.files=${iridaNextFiles}") + } + + List matchersGlob = matchers.collect {FileSystems.getDefault().getPathMatcher("glob:${it}")} + addPathMatchers(scope, matchersGlob) } + } + } - List matchersGlob = matchers.collect {FileSystems.getDefault().getPathMatcher("glob:${it}")} - addPathMatchers(scope, matchersGlob) + def iridaNextMetadata = session.config.navigate('iridanext.output.metadata') + if (iridaNextMetadata != null) { + if (!iridaNextMetadata instanceof Map) { + throw new Exception("Expected a map in config for iridanext.metadata=${iridaNextMetadata}") } + + Map samplesMetadata = iridaNextMetadata["samples"] as Map + samplesMetadataMatcher = FileSystems.getDefault().getPathMatcher("glob:${samplesMetadata['path']}") + samplesMetadataId = samplesMetadata["id"] } + + iridaNextJSONOutput = new IridaNextJSONOutput(relativizePath) } @Override @@ -107,13 +145,46 @@ class IridaNextObserver implements TraceObserver { publishedFiles[source] = destination } + private Map csvToJsonById(Path path, String idColumn) { + path = Nextflow.file(path) as Path + List rowsList = path.splitCsv(header:true, strip:true, sep:',', quote:'\"') + + Map rowsMap = rowsList.collectEntries { row -> + if (idColumn !in row) { + throw new Exception("Error: column with idColumn=${idColumn} not in CSV ${path}") + } else { + return [(row[idColumn] as String): (row as Map).findAll { it.key != idColumn }] + } + } + + return rowsMap + } + + private void processOutputPath(Path outputPath, Map indexInfo) { + if (publishedFiles.containsKey(outputPath)) { + Path publishedPath = publishedFiles[outputPath] + def currScope = indexInfo["scope"] + + if (pathMatchers[currScope].any {it.matches(publishedPath)}) { + iridaNextJSONOutput.addFile(currScope, indexInfo["subscope"], publishedPath) + } + } else { + log.trace "Not match outputPath: ${outputPath}" + } + } + @Override void onFlowComplete() { + if (!session.isSuccess()) + return + + // Generate files section // Some of this code derived from https://github.com/nextflow-io/nf-prov/blob/master/plugins/nf-prov tasks.each { task -> Map> outParamInfo = [:] def currSubscope = null task.outputs.each { outParam, object -> + log.debug "task ${task}, outParam ${outParam}, object ${object}" Short paramIndex = outParam.getIndex() if (!outParamInfo.containsKey(paramIndex)) { Map currIndexInfo = [:] @@ -122,35 +193,55 @@ class IridaNextObserver implements TraceObserver { // case meta map if (outParam instanceof ValueOutParam && object instanceof Map) { Map objectMap = (Map)object - if (outParam.getName() == "meta" && "id" in objectMap) { + if (outParam.getName() == "meta" && this.filesMetaId in objectMap) { + log.trace "${this.filesMetaId} in ${objectMap}" currIndexInfo["scope"] = "samples" - currIndexInfo["subscope"] = objectMap["id"].toString() + currIndexInfo["subscope"] = objectMap[this.filesMetaId].toString() + iridaNextJSONOutput.addId(currIndexInfo["scope"], currIndexInfo["subscope"]) } else { - throw new Exception("Found value channel output that doesn't have meta.id: ${objectMap}") + throw new Exception("Found value channel output in task [${task.getName()}] that doesn't have meta.${this.filesMetaId}: ${objectMap}") } } else { currIndexInfo["scope"] = "global" currIndexInfo["subscope"] = "" } - } - Map currIndexInfo = outParamInfo[paramIndex] + log.debug "Setup info task [${task.getName()}], outParamInfo[${paramIndex}]: ${outParamInfo[paramIndex]}" + } if (object instanceof Path) { - Path processPath = (Path)object - - if (publishedFiles.containsKey(processPath)) { - Path publishedPath = publishedFiles[processPath] - def currScope = currIndexInfo["scope"] - - if (pathMatchers[currScope].any {it.matches(publishedPath)}) { - iridaNextOutput.addFile(currScope, currIndexInfo["subscope"], publishedPath) + log.debug "outParamInfo[${paramIndex}]: ${outParamInfo[paramIndex]}, object as Path: ${object as Path}" + processOutputPath(object as Path, outParamInfo[paramIndex]) + } else if (object instanceof List) { + log.debug "outParamInfo[${paramIndex}]: ${outParamInfo[paramIndex]}, object as List: ${object as List}" + (object as List).each { + if (it instanceof Path) { + processOutputPath(it as Path, outParamInfo[paramIndex]) } } } } } - log.info "${JsonOutput.prettyPrint(iridaNextOutput.toJson())}" + // Generate metadata section + // some code derived from https://github.com/nextflow-io/nf-validation + if (samplesMetadataMatcher != null && samplesMetadataId != null) { + List matchedFiles = new ArrayList(publishedFiles.values().findAll {samplesMetadataMatcher.matches(it)}) + + if (!matchedFiles.isEmpty()) { + log.trace "Matched metadata: ${matchedFiles}" + Map metadataSamplesMap = csvToJsonById(matchedFiles[0] as Path, samplesMetadataId) + iridaNextJSONOutput.appendMetadata("samples", metadataSamplesMap) + } + } + + if (iridaNextOutputPath != null) { + if (iridaNextOutputPath.exists() && !outputFileOverwrite) { + throw new Exception("Error: iridanext.output.path=${iridaNextOutputPath} exists and iridanext.output.overwrite=${outputFileOverwrite}") + } else { + iridaNextJSONOutput.write(iridaNextOutputPath) + log.debug "Wrote IRIDA Next output to ${iridaNextOutputPath}" + } + } } } diff --git a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextFactory.groovy b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserverFactory.groovy similarity index 83% rename from plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextFactory.groovy rename to plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserverFactory.groovy index a246f3e..d14626d 100644 --- a/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextFactory.groovy +++ b/plugins/nf-iridanext/src/main/nextflow/iridanext/IridaNextObserverFactory.groovy @@ -28,12 +28,17 @@ import nextflow.trace.TraceObserverFactory * @author Paolo Di Tommaso */ @CompileStatic -class IridaNextFactory implements TraceObserverFactory { +class IridaNextObserverFactory implements TraceObserverFactory { @Override Collection create(Session session) { final result = new ArrayList() - result.add( new IridaNextObserver() ) + final enabled = session.config.navigate('iridanext.enabled') + + if (enabled) { + result.add( new IridaNextObserver() ) + } + return result } } diff --git a/plugins/nf-iridanext/src/resources/META-INF/extensions.idx b/plugins/nf-iridanext/src/resources/META-INF/extensions.idx index d635867..34261d1 100644 --- a/plugins/nf-iridanext/src/resources/META-INF/extensions.idx +++ b/plugins/nf-iridanext/src/resources/META-INF/extensions.idx @@ -1 +1 @@ -nextflow.iridanext.IridaNextFactory \ No newline at end of file +nextflow.iridanext.IridaNextObserverFactory \ No newline at end of file diff --git a/plugins/nf-iridanext/src/test/nextflow/hello/HelloDslTest.groovy b/plugins/nf-iridanext/src/test/nextflow/hello/HelloDslTest.groovy deleted file mode 100644 index 56f8b19..0000000 --- a/plugins/nf-iridanext/src/test/nextflow/hello/HelloDslTest.groovy +++ /dev/null @@ -1,96 +0,0 @@ -package nextflow.hello - -import nextflow.Channel -import nextflow.plugin.Plugins -import nextflow.plugin.TestPluginDescriptorFinder -import nextflow.plugin.TestPluginManager -import nextflow.plugin.extension.PluginExtensionProvider -import org.pf4j.PluginDescriptorFinder -import spock.lang.Shared -import spock.lang.Timeout -import test.Dsl2Spec - -import java.nio.file.Path - - -/** - * Unit test for Hello DSL - * - * @author : jorge - */ -@Timeout(10) -class HelloDslTest extends Dsl2Spec{ - - @Shared String pluginsMode - - def setup() { - // reset previous instances - PluginExtensionProvider.reset() - // this need to be set *before* the plugin manager class is created - pluginsMode = System.getProperty('pf4j.mode') - System.setProperty('pf4j.mode', 'dev') - // the plugin root should - def root = Path.of('.').toAbsolutePath().normalize() - def manager = new TestPluginManager(root){ - @Override - protected PluginDescriptorFinder createPluginDescriptorFinder() { - return new TestPluginDescriptorFinder(){ - protected Path getManifestPath(Path pluginPath) { - return pluginPath.resolve('build/resources/main/META-INF/MANIFEST.MF') - } - } - } - } - Plugins.init(root, 'dev', manager) - } - - def cleanup() { - Plugins.stop() - PluginExtensionProvider.reset() - pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') - } - - def 'should perform a hi and create a channel' () { - when: - def SCRIPT = ''' - include {reverse} from 'plugin/nf-hello' - channel.reverse('hi!') - ''' - and: - def result = new MockScriptRunner([hello:[prefix:'>>']]).setScript(SCRIPT).execute() - then: - result.val == 'hi!'.reverse() - result.val == Channel.STOP - } - - def 'should store a goodbye' () { - when: - def SCRIPT = ''' - include {goodbye} from 'plugin/nf-hello' - channel - .of('folks') - .goodbye() - ''' - and: - def result = new MockScriptRunner([:]).setScript(SCRIPT).execute() - then: - result.val == 'Goodbye folks' - result.val == Channel.STOP - - } - - def 'can use an imported function' () { - when: - def SCRIPT = ''' - include {randomString} from 'plugin/nf-hello' - channel - .of( randomString(20) ) - ''' - and: - def result = new MockScriptRunner([:]).setScript(SCRIPT).execute() - then: - result.val.size() == 20 - result.val == Channel.STOP - } - -} diff --git a/plugins/nf-iridanext/src/test/nextflow/hello/HelloFactoryTest.groovy b/plugins/nf-iridanext/src/test/nextflow/hello/HelloFactoryTest.groovy deleted file mode 100644 index cc0ab15..0000000 --- a/plugins/nf-iridanext/src/test/nextflow/hello/HelloFactoryTest.groovy +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2021, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.hello - -import nextflow.Session -import spock.lang.Specification - -/** - * - * @author Paolo Di Tommaso - */ -class HelloFactoryTest extends Specification { - - def 'should return observer' () { - when: - def result = new HelloFactory().create(Mock(Session)) - then: - result.size()==1 - result[0] instanceof HelloObserver - } - -} diff --git a/plugins/nf-iridanext/src/test/nextflow/iridanext/IridaNextObserverFactoryTest.groovy b/plugins/nf-iridanext/src/test/nextflow/iridanext/IridaNextObserverFactoryTest.groovy new file mode 100644 index 0000000..ea0f036 --- /dev/null +++ b/plugins/nf-iridanext/src/test/nextflow/iridanext/IridaNextObserverFactoryTest.groovy @@ -0,0 +1,64 @@ +/* + * Original file Copyright 2021, Seqera Labs (from nf-hello plugin template) + * Modifications Copyright 2023, Government of Canada + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.iridanext + +import nextflow.iridanext.IridaNextObserverFactory +import nextflow.iridanext.IridaNextObserver + +import nextflow.Session +import spock.lang.Specification + +/** + * Test class for IridaNextObserverFactory + * @author Aaron Petkau + * @author Paolo Di Tommaso + */ +class IridaNextObserverFactoryTest extends Specification { + + def 'should not return observer' () { + when: + // How to set this up from https://github.com/nextflow-io/nf-prov tests + def config = [ + iridanext: [ + enabled: false + ] + ] + def session = Spy(Session) { + getConfig() >> config + } + def result = new IridaNextObserverFactory().create(session) + then: + result.size()==0 + } + + def 'should return observer' () { + when: + def config = [ + iridanext: [ + enabled: true + ] + ] + def session = Spy(Session) { + getConfig() >> config + } + def result = new IridaNextObserverFactory().create(session) + then: + result.size()==1 + result[0] instanceof IridaNextObserver + } +} diff --git a/plugins/nf-iridanext/src/test/nextflow/hello/MockHelpers.groovy b/plugins/nf-iridanext/src/test/nextflow/iridanext/MockHelpers.groovy similarity index 99% rename from plugins/nf-iridanext/src/test/nextflow/hello/MockHelpers.groovy rename to plugins/nf-iridanext/src/test/nextflow/iridanext/MockHelpers.groovy index ac81cb3..581bd0f 100644 --- a/plugins/nf-iridanext/src/test/nextflow/hello/MockHelpers.groovy +++ b/plugins/nf-iridanext/src/test/nextflow/iridanext/MockHelpers.groovy @@ -1,4 +1,4 @@ -package nextflow.hello +package nextflow.iridanext import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowBroadcast diff --git a/plugins/nf-iridanext/src/test/nextflow/hello/TestHelper.groovy b/plugins/nf-iridanext/src/test/nextflow/iridanext/TestHelper.groovy similarity index 95% rename from plugins/nf-iridanext/src/test/nextflow/hello/TestHelper.groovy rename to plugins/nf-iridanext/src/test/nextflow/iridanext/TestHelper.groovy index d63444b..1f759a5 100644 --- a/plugins/nf-iridanext/src/test/nextflow/hello/TestHelper.groovy +++ b/plugins/nf-iridanext/src/test/nextflow/iridanext/TestHelper.groovy @@ -1,4 +1,5 @@ /* + * Copyright 2023, Government of Canada * Copyright 2020-2022, Seqera Labs * Copyright 2013-2019, Centre for Genomic Regulation (CRG) * @@ -15,7 +16,7 @@ * limitations under the License. */ -package nextflow.hello +package nextflow.iridanext import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs diff --git a/settings.gradle b/settings.gradle index 6d35336..a25a9d5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,4 +18,3 @@ rootProject.name = 'nf-iridanext' include('plugins') include('plugins:nf-iridanext') -includeBuild('../nextflow')