From 481b5d00bdeb2830caed4a1225f311622cde40a6 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Wed, 18 Dec 2024 23:16:21 +0100 Subject: [PATCH] feat(scanner)!: Migrate to new plugin API Migrate the `ScannerWrapper` plugins to the new plugin API. Signed-off-by: Martin Nonnenmacher --- .../scanner/src/main/kotlin/ScannerCommand.kt | 17 +- plugins/scanners/askalono/build.gradle.kts | 4 +- .../src/funTest/kotlin/AskalonoFunTest.kt | 3 +- .../askalono/src/main/kotlin/Askalono.kt | 81 ++++-- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - plugins/scanners/boyterlc/build.gradle.kts | 4 +- .../src/funTest/kotlin/BoyterLcFunTest.kt | 3 +- .../boyterlc/src/main/kotlin/BoyterLc.kt | 79 ++++-- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - plugins/scanners/dos/build.gradle.kts | 4 +- .../dos/src/main/kotlin/DosScanner.kt | 45 ++-- .../dos/src/main/kotlin/DosScannerConfig.kt | 31 +-- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - .../dos/src/test/kotlin/DosScannerTest.kt | 5 +- plugins/scanners/fossid/build.gradle.kts | 4 +- .../scanners/fossid/src/main/kotlin/FossId.kt | 117 +++++---- .../fossid/src/main/kotlin/FossIdConfig.kt | 241 +++++------------- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - .../src/test/kotlin/FossIdConfigTest.kt | 185 +++----------- .../fossid/src/test/kotlin/FossIdTest.kt | 2 +- .../fossid/src/test/kotlin/TestUtils.kt | 11 +- plugins/scanners/licensee/build.gradle.kts | 4 +- .../src/funTest/kotlin/LicenseeFunTest.kt | 3 +- .../licensee/src/main/kotlin/Licensee.kt | 80 ++++-- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - .../licensee/src/test/kotlin/LicenseeTest.kt | 5 +- plugins/scanners/scancode/build.gradle.kts | 4 +- .../funTest/kotlin/ScanCodeScannerFunTest.kt | 3 +- .../scancode/src/main/kotlin/ScanCode.kt | 59 +++-- .../src/main/kotlin/ScanCodeConfig.kt | 96 ++++--- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - .../scancode/src/test/kotlin/ScanCodeTest.kt | 44 +--- plugins/scanners/scanoss/build.gradle.kts | 4 +- .../scanoss/src/main/kotlin/ScanOss.kt | 44 ++-- .../scanoss/src/main/kotlin/ScanOssConfig.kt | 64 ++--- ...eviewtoolkit.scanner.ScannerWrapperFactory | 1 - .../src/test/kotlin/ScanOssConfigTest.kt | 44 ---- .../kotlin/ScanOssScannerDirectoryTest.kt | 4 +- .../src/test/kotlin/ScanOssScannerFileTest.kt | 4 +- scanner/build.gradle.kts | 1 + .../scanners/ScannerIntegrationFunTest.kt | 4 +- .../main/kotlin/LocalPathScannerWrapper.kt | 2 +- scanner/src/main/kotlin/Scanner.kt | 37 +-- scanner/src/main/kotlin/ScannerWrapper.kt | 10 +- .../src/main/kotlin/ScannerWrapperFactory.kt | 29 +-- .../kotlin/storages/ClearlyDefinedStorage.kt | 7 +- scanner/src/test/kotlin/ScannerTest.kt | 12 +- 47 files changed, 630 insertions(+), 777 deletions(-) delete mode 100644 plugins/scanners/askalono/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/boyterlc/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/dos/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/fossid/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/licensee/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/scancode/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/scanoss/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory delete mode 100644 plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index d42f8bba3cc23..33c2d64d4d8d8 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -51,6 +51,7 @@ import org.ossreviewtoolkit.model.config.OrtConfiguration import org.ossreviewtoolkit.model.utils.DefaultResolutionProvider import org.ossreviewtoolkit.model.utils.mergeLabels import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginConfig import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.plugins.commands.api.OrtCommand import org.ossreviewtoolkit.plugins.commands.api.OrtCommandFactory @@ -167,34 +168,34 @@ class ScannerCommand(descriptor: PluginDescriptor = ScannerCommandFactory.descri @Suppress("ForbiddenMethodCall") private fun runScanners( - scannerWrapperFactories: List>, - projectScannerWrapperFactories: List>, + scannerWrapperFactories: List, + projectScannerWrapperFactories: List, ortConfig: OrtConfiguration ): OrtResult { val packageScannerWrappers = scannerWrapperFactories .takeIf { PackageType.PACKAGE in packageTypes }.orEmpty() .map { - val config = ortConfig.scanner.config?.get(it.type) - it.create(config?.options.orEmpty(), config?.secrets.orEmpty()) + val config = ortConfig.scanner.config?.get(it.descriptor.id) + it.create(PluginConfig(config?.options.orEmpty(), config?.secrets.orEmpty())) } val projectScannerWrappers = projectScannerWrapperFactories .takeIf { PackageType.PROJECT in packageTypes }.orEmpty() .map { - val config = ortConfig.scanner.config?.get(it.type) - it.create(config?.options.orEmpty(), config?.secrets.orEmpty()) + val config = ortConfig.scanner.config?.get(it.descriptor.id) + it.create(PluginConfig(config?.options.orEmpty(), config?.secrets.orEmpty())) } if (projectScannerWrappers.isNotEmpty()) { echo("Scanning projects with:") - echo(projectScannerWrappers.joinToString { "\t${it.name} (version ${it.version})" }) + echo(projectScannerWrappers.joinToString { "\t${it.descriptor.displayName} (version ${it.version})" }) } else { echo("Projects will not be scanned.") } if (packageScannerWrappers.isNotEmpty()) { echo("Scanning packages with:") - echo(packageScannerWrappers.joinToString { "\t${it.name} (version ${it.version})" }) + echo(packageScannerWrappers.joinToString { "\t${it.descriptor.displayName} (version ${it.version})" }) } else { echo("Packages will not be scanned.") } diff --git a/plugins/scanners/askalono/build.gradle.kts b/plugins/scanners/askalono/build.gradle.kts index 8e9a5d578624f..22cfe5e247f39 100644 --- a/plugins/scanners/askalono/build.gradle.kts +++ b/plugins/scanners/askalono/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") // Apply third-party plugins. alias(libs.plugins.kotlinSerialization) @@ -32,5 +32,7 @@ dependencies { implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.json) + ksp(projects.scanner) + funTestApi(testFixtures(projects.scanner)) } diff --git a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt index c945463350e87..6abdbcf09d996 100644 --- a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt +++ b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt @@ -22,10 +22,9 @@ package org.ossreviewtoolkit.plugins.scanners.askalono import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig class AskalonoFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = Askalono("Askalono", ScannerWrapperConfig.EMPTY) + override val scanner = AskalonoFactory.create() override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 1.0f) diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index 29f55414c1ac1..cc158784b83a6 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -32,14 +32,16 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for any known license" @@ -58,23 +60,70 @@ object AskalonoCommand : CommandLineTool { override fun displayName() = "Askalono" } -class Askalono internal constructor(name: String, private val wrapperConfig: ScannerWrapperConfig) : - LocalPathScannerWrapper(name) { - class Factory : ScannerWrapperFactory("Askalono") { - override fun create(config: Unit, wrapperConfig: ScannerWrapperConfig) = Askalono(type, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = Unit - } - +data class AskalonoConfig( + /** + * A regular expression to match the scanner name when looking up scan results in the storage. + */ + val regScannerName: String?, + + /** + * The minimum version of stored scan results to use. + */ + val minVersion: String?, + + /** + * The maximum version of stored scan results to use. + */ + val maxVersion: String?, + + /** + * The configuration to use for the scanner. Only scan results with the same configuration are used when looking up + * scan results in the storage. + */ + val configuration: String?, + + /** + * Whether to read scan results from the storage. + */ + @OrtPluginOption(defaultValue = "true") + val readFromStorage: Boolean, + + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) + +@OrtPlugin( + id = "askalono", + displayName = "askalono", + description = "askalono is a library and command-line tool to help detect license texts. It's designed to be " + + "fast, accurate, and to support a wide variety of license texts.", + factory = ScannerWrapperFactory::class +) +class Askalono( + override val descriptor: PluginDescriptor = AskalonoFactory.descriptor, + config: AskalonoConfig +) : LocalPathScannerWrapper() { override val configuration = "" - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + override val matcher by lazy { + ScannerMatcher.create( + details, + ScannerMatcherConfig( + config.regScannerName, + config.minVersion, + config.maxVersion, + config.configuration + ) + ) + } override val version by lazy { AskalonoCommand.getVersion() } - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val readFromStorage = config.readFromStorage + override val writeToStorage = config.writeToStorage override fun runScanner(path: File, context: ScanContext): String { val process = AskalonoCommand.run( @@ -97,7 +146,7 @@ class Askalono internal constructor(name: String, private val wrapperConfig: Sca val issues = mutableListOf( Issue( - source = name, + source = descriptor.id, message = "This scanner is not capable of detecting copyright statements.", severity = Severity.HINT ) @@ -114,7 +163,7 @@ class Askalono internal constructor(name: String, private val wrapperConfig: Sca if (it.error != null) { issues += Issue( - source = name, + source = descriptor.id, message = it.error, severity = if (it.error == CONFIDENCE_NOTICE) Severity.HINT else Severity.ERROR ) diff --git a/plugins/scanners/askalono/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/askalono/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 7f1e5b0590fda..0000000000000 --- a/plugins/scanners/askalono/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.askalono.Askalono$Factory diff --git a/plugins/scanners/boyterlc/build.gradle.kts b/plugins/scanners/boyterlc/build.gradle.kts index 8e9a5d578624f..22cfe5e247f39 100644 --- a/plugins/scanners/boyterlc/build.gradle.kts +++ b/plugins/scanners/boyterlc/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") // Apply third-party plugins. alias(libs.plugins.kotlinSerialization) @@ -32,5 +32,7 @@ dependencies { implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.json) + ksp(projects.scanner) + funTestApi(testFixtures(projects.scanner)) } diff --git a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt index b141958ea6946..0fef06baddaaa 100644 --- a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt +++ b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt @@ -22,10 +22,9 @@ package org.ossreviewtoolkit.plugins.scanners.boyterlc import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig class BoyterLcFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = BoyterLc("BoyterLc", ScannerWrapperConfig.EMPTY) + override val scanner = BoyterLcFactory.create() override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 0.98388565f), diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index fea21d44a84f8..1a930ced2ac9b 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -31,14 +31,16 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.safeDeleteRecursively import org.ossreviewtoolkit.utils.ort.createOrtTempDir @@ -57,8 +59,52 @@ object BoyterLcCommand : CommandLineTool { override fun displayName() = "BoyterLc" } -class BoyterLc internal constructor(name: String, private val wrapperConfig: ScannerWrapperConfig) : - LocalPathScannerWrapper(name) { +data class BoyterLcConfig( + /** + * A regular expression to match the scanner name when looking up scan results in the storage. + */ + val regScannerName: String?, + + /** + * The minimum version of stored scan results to use. + */ + val minVersion: String?, + + /** + * The maximum version of stored scan results to use. + */ + val maxVersion: String?, + + /** + * The configuration to use for the scanner. Only scan results with the same configuration are used when looking up + * scan results in the storage. + */ + val configuration: String?, + + /** + * Whether to read scan results from the storage. + */ + @OrtPluginOption(defaultValue = "true") + val readFromStorage: Boolean, + + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) + +@OrtPlugin( + id = "BoyterLc", + displayName = "BoyterLc", + description = "A command line application which scans directories and identifies what software license things " + + "are under.", + factory = ScannerWrapperFactory::class +) +class BoyterLc( + override val descriptor: PluginDescriptor = BoyterLcFactory.descriptor, + config: BoyterLcConfig +) : LocalPathScannerWrapper() { companion object { val CONFIGURATION_OPTIONS = listOf( "--confidence", "0.95", // Cut-off value to only get most relevant matches. @@ -66,21 +112,24 @@ class BoyterLc internal constructor(name: String, private val wrapperConfig: Sca ) } - class Factory : ScannerWrapperFactory("BoyterLc") { - override fun create(config: Unit, wrapperConfig: ScannerWrapperConfig) = BoyterLc(type, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = Unit - } - override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + override val matcher by lazy { + ScannerMatcher.create( + details, + ScannerMatcherConfig( + config.regScannerName, + config.minVersion, + config.maxVersion, + config.configuration + ) + ) + } override val version by lazy { BoyterLcCommand.getVersion() } - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val readFromStorage = config.readFromStorage + override val writeToStorage = config.writeToStorage override fun runScanner(path: File, context: ScanContext): String { val resultFile = createOrtTempDir().resolve("result.json") @@ -118,7 +167,7 @@ class BoyterLc internal constructor(name: String, private val wrapperConfig: Sca licenseFindings = licenseFindings, issues = listOf( Issue( - source = name, + source = descriptor.id, message = "This scanner is not capable of detecting copyright statements.", severity = Severity.HINT ) diff --git a/plugins/scanners/boyterlc/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/boyterlc/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 0dd3f0fd61ed1..0000000000000 --- a/plugins/scanners/boyterlc/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.boyterlc.BoyterLc$Factory diff --git a/plugins/scanners/dos/build.gradle.kts b/plugins/scanners/dos/build.gradle.kts index 9d939782e144c..7f47ef35df329 100644 --- a/plugins/scanners/dos/build.gradle.kts +++ b/plugins/scanners/dos/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") // Apply third-party plugins. alias(libs.plugins.kotlinSerialization) @@ -40,6 +40,8 @@ dependencies { implementation(libs.kotlinx.serialization.json) implementation(libs.log4j.api) + ksp(projects.scanner) + testImplementation(libs.mockk) testImplementation(libs.wiremock) } diff --git a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt index 54a4133b8757c..26bbbf8fe2441 100644 --- a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt +++ b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt @@ -47,14 +47,14 @@ import org.ossreviewtoolkit.model.createAndLogIssue import org.ossreviewtoolkit.model.utils.associateLicensesWithExceptions import org.ossreviewtoolkit.model.utils.toPurl import org.ossreviewtoolkit.model.utils.toPurlExtras +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.scanner.provenance.DefaultProvenanceDownloader import org.ossreviewtoolkit.scanner.provenance.NestedProvenance -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.packZip import org.ossreviewtoolkit.utils.common.safeDeleteRecursively @@ -66,26 +66,27 @@ import org.ossreviewtoolkit.utils.ort.runBlocking * https://github.com/doubleopen-project/dos. The server runs ScanCode in the backend and stores / reuses scan results * on a per-file basis and thus uses its own scan storage. */ -class DosScanner internal constructor( - override val name: String, - private val config: DosScannerConfig, - wrapperConfig: ScannerWrapperConfig +@OrtPlugin( + id = "DOS", + displayName = "Double Open Server", + description = "The DOS scanner wrapper is a client for the scanner API implemented as part of the Double Open " + + "Server project at https://github.com/doubleopen-project/dos. The server runs ScanCode in the backend and " + + "stores / reuses scan results on a per-file basis and thus uses its own scan storage.", + factory = ScannerWrapperFactory::class +) +class DosScanner( + override val descriptor: PluginDescriptor = DosScannerFactory.descriptor, + private val config: DosScannerConfig ) : PackageScannerWrapper { - class Factory : ScannerWrapperFactory("DOS") { - override fun create(config: DosScannerConfig, wrapperConfig: ScannerWrapperConfig) = - DosScanner(type, config, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = DosScannerConfig.create(options, secrets) - } - // TODO: Introduce a DOS version and expose it through the API to use it here. override val version = "1.0.0" override val configuration = "" override val matcher: ScannerMatcher? = null - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + + override val readFromStorage = false + override val writeToStorage = config.writeToStorage private val service = DosService.create(config.url, config.token, config.timeout?.let { Duration.ofSeconds(it) }) internal val client = DosClient(service) @@ -115,9 +116,9 @@ class DosScanner internal constructor( val existingScanResults = runCatching { client.getScanResults(packages, config.fetchConcluded) }.onFailure { - issues += createAndLogIssue(name, it.collectMessages()) + issues += createAndLogIssue(descriptor.id, it.collectMessages()) }.onSuccess { - if (it == null) issues += createAndLogIssue(name, "Missing scan results response body.") + if (it == null) issues += createAndLogIssue(descriptor.id, "Missing scan results response body.") }.getOrNull() when (existingScanResults?.state?.status) { @@ -129,7 +130,7 @@ class DosScanner internal constructor( }.mapCatching { sourceDir -> runBackendScan(packages, sourceDir, startTime, issues) }.onFailure { - issues += createAndLogIssue(name, it.collectMessages()) + issues += createAndLogIssue(descriptor.id, it.collectMessages()) }.getOrNull() } @@ -183,14 +184,14 @@ class DosScanner internal constructor( val uploadUrl = client.getUploadUrl(zipName) if (uploadUrl == null) { - issues += createAndLogIssue(name, "Unable to get an upload URL for '$zipName'.") + issues += createAndLogIssue(descriptor.id, "Unable to get an upload URL for '$zipName'.") zipFile.delete() return null } val uploadSuccessful = client.uploadFile(zipFile, uploadUrl).also { zipFile.delete() } if (!uploadSuccessful) { - issues += createAndLogIssue(name, "Uploading '$zipFile' to $uploadUrl failed.") + issues += createAndLogIssue(descriptor.id, "Uploading '$zipFile' to $uploadUrl failed.") return null } @@ -199,7 +200,7 @@ class DosScanner internal constructor( if (id == null) { issues += createAndLogIssue( - name, + descriptor.id, "Failed to add scan job for the following packages:\n${packages.joinToString("\n") { it.purl }}" ) return null @@ -236,7 +237,7 @@ class DosScanner internal constructor( "failed" -> { issues += createAndLogIssue( - name, + descriptor.id, "Scan failed for job with ID '$jobId': ${jobState.state.message}" ) return null diff --git a/plugins/scanners/dos/src/main/kotlin/DosScannerConfig.kt b/plugins/scanners/dos/src/main/kotlin/DosScannerConfig.kt index 560e3df454a94..81e7ca4973a27 100644 --- a/plugins/scanners/dos/src/main/kotlin/DosScannerConfig.kt +++ b/plugins/scanners/dos/src/main/kotlin/DosScannerConfig.kt @@ -19,7 +19,7 @@ package org.ossreviewtoolkit.plugins.scanners.dos -import org.ossreviewtoolkit.utils.common.Options +import org.ossreviewtoolkit.plugins.api.OrtPluginOption /** * This is the configuration class for DOS Scanner. @@ -35,27 +35,20 @@ data class DosScannerConfig( val timeout: Long?, /** Interval (in seconds) to use for polling scanjob status from DOS API. **/ + @OrtPluginOption(defaultValue = "5") val pollInterval: Long, /** Use license conclusions as detected licenses when they exist? **/ + @OrtPluginOption(defaultValue = "false") val fetchConcluded: Boolean, /** The URL where the DOS / package curation front-end is running. **/ - val frontendUrl: String -) { - companion object { - private const val DEFAULT_FRONT_END_URL = "http://localhost:3000" - private const val DEFAULT_POLLING_INTERVAL = 5L - - fun create(options: Options, secrets: Options): DosScannerConfig { - return DosScannerConfig( - url = options.getValue("url"), - token = secrets.getValue("token"), - timeout = options["timeout"]?.toLongOrNull(), - pollInterval = options["pollInterval"]?.toLongOrNull() ?: DEFAULT_POLLING_INTERVAL, - fetchConcluded = options["fetchConcluded"].toBoolean(), - frontendUrl = options["frontendUrl"] ?: DEFAULT_FRONT_END_URL - ) - } - } -} + @OrtPluginOption(defaultValue = "http://localhost:3000") + val frontendUrl: String, + + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) diff --git a/plugins/scanners/dos/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/dos/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 49708eb12a655..0000000000000 --- a/plugins/scanners/dos/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.dos.DosScanner$Factory diff --git a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt index 504f5e07afd12..0b30a45f1ed60 100644 --- a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt +++ b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt @@ -50,7 +50,6 @@ import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.scanner.provenance.NestedProvenance class DosScannerTest : StringSpec({ @@ -65,7 +64,7 @@ class DosScannerTest : StringSpec({ beforeTest { server.start() - val config = DosScannerConfig( + scanner = DosScannerFactory.create( url = "http://localhost:${server.port()}/api/", token = "", timeout = 60L, @@ -73,8 +72,6 @@ class DosScannerTest : StringSpec({ fetchConcluded = false, frontendUrl = "http://localhost:3000" ) - - scanner = DosScanner.Factory().create(config, ScannerWrapperConfig.EMPTY) } afterTest { diff --git a/plugins/scanners/fossid/build.gradle.kts b/plugins/scanners/fossid/build.gradle.kts index 7942dd3042379..2ee3867ab7a7a 100644 --- a/plugins/scanners/fossid/build.gradle.kts +++ b/plugins/scanners/fossid/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") } dependencies { @@ -34,6 +34,8 @@ dependencies { implementation(libs.kotlinx.coroutines) + ksp(projects.scanner) + testImplementation(libs.mockk) testImplementation(libs.wiremock) } diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index db135e3b885ed..540d4e5ebb337 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -86,14 +86,14 @@ import org.ossreviewtoolkit.model.config.snippet.SnippetChoice import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason import org.ossreviewtoolkit.model.createAndLogIssue import org.ossreviewtoolkit.model.jsonMapper +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.scanner.provenance.NestedProvenance -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.enumSetOf import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri import org.ossreviewtoolkit.utils.ort.runBlocking @@ -110,11 +110,16 @@ import org.semver4j.Semver * Therefore, it implements the [PackageScannerWrapper] interface for backward compatibility, even though FossID itself * gets a Git repository URL as input and would be a good match for [ProvenanceScannerWrapper]. */ +@OrtPlugin( + id = "fossid", + displayName = "FossID", + description = "The FossID scanner plugin.", + factory = ScannerWrapperFactory::class +) @Suppress("LargeClass", "TooManyFunctions") class FossId internal constructor( - override val name: String, - private val config: FossIdConfig, - wrapperConfig: ScannerWrapperConfig + override val descriptor: PluginDescriptor = FossIdFactory.descriptor, + internal val config: FossIdConfig ) : PackageScannerWrapper { companion object { @JvmStatic @@ -195,13 +200,6 @@ class FossId internal constructor( ) } - class Factory : ScannerWrapperFactory("FossId") { - override fun create(config: FossIdConfig, wrapperConfig: ScannerWrapperConfig) = - FossId(type, config, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = FossIdConfig.create(options, secrets) - } - /** * The qualifier of a scan when delta scans are enabled. */ @@ -226,19 +224,19 @@ class FossId internal constructor( // package is wanted. private val createdScans = mutableSetOf() - private val service = runBlocking { FossIdRestService.create(config.serverUrl) } + private val service by lazy { runBlocking { FossIdRestService.create(config.serverUrl) } } - override val version = service.version + override val version by lazy { service.version } override val configuration = "" override val matcher: ScannerMatcher? = null - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } + override val readFromStorage = false - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val writeToStorage = config.writeToStorage private suspend fun getProject(projectCode: String): Project? = - service.getProject(config.user, config.apiKey, projectCode).run { + service.getProject(config.user.value, config.apiKey.value, projectCode).run { when { error == null && data != null -> { logger.info { "Project '$projectCode' exists." } @@ -281,7 +279,7 @@ class FossId internal constructor( } if (issueMessage != null) { - val issue = createAndLogIssue(name, issueMessage, Severity.WARNING) + val issue = createAndLogIssue(descriptor.id, issueMessage, Severity.WARNING) val summary = createSingleIssueSummary(startTime, issue = issue) return ScanResult(provenance, details, summary) } @@ -297,11 +295,11 @@ class FossId internal constructor( if (getProject(projectCode) == null) { logger.info { "Creating project '$projectCode'..." } - service.createProject(config.user, config.apiKey, projectCode, projectCode) + service.createProject(config.user.value, config.apiKey.value, projectCode, projectCode) .checkResponse("create project") } - val scans = service.listScansForProject(config.user, config.apiKey, projectCode) + val scans = service.listScansForProject(config.user.value, config.apiKey.value, projectCode) .checkResponse("list scans for project").data checkNotNull(scans) @@ -344,7 +342,7 @@ class FossId internal constructor( ) } else { val issue = createAndLogIssue( - source = name, + source = descriptor.id, message = "Package '${pkg.id.toCoordinates()}' has been scanned in asynchronous mode. " + "Scan results need to be inspected on the server instance.", severity = Severity.HINT @@ -366,7 +364,7 @@ class FossId internal constructor( e.showStackTrace() val issue = createAndLogIssue( - source = name, + source = descriptor.id, message = "Failed to scan package '${pkg.id.toCoordinates()}' from $url." ) val summary = createSingleIssueSummary(startTime, issue = issue) @@ -401,7 +399,7 @@ class FossId internal constructor( "The code for an existing scan must not be null." } - val response = service.checkScanStatus(config.user, config.apiKey, scanCode) + val response = service.checkScanStatus(config.user.value, config.apiKey.value, scanCode) .checkResponse("check scan status", false) when (response.data?.status) { ScanStatus.FINISHED -> true @@ -478,7 +476,7 @@ class FossId internal constructor( val scanId = createScan(projectCode, scanCode, newUrl, revision) logger.info { "Initiating the download..." } - service.downloadFromGit(config.user, config.apiKey, scanCode) + service.downloadFromGit(config.user.value, config.apiKey.value, scanCode) .checkResponse("download data from Git", false) val issues = createIgnoreRules(scanCode, context.excludes) @@ -562,7 +560,7 @@ class FossId internal constructor( val scanId = createScan(projectCode, scanCode, mappedUrl, revision, projectRevision.orEmpty()) logger.info { "Initiating the download..." } - service.downloadFromGit(config.user, config.apiKey, scanCode) + service.downloadFromGit(config.user.value, config.apiKey.value, scanCode) .checkResponse("download data from Git", false) val issues = mutableListOf() @@ -580,7 +578,7 @@ class FossId internal constructor( // TODO: This is the old way of carrying the rules to the new delta scan, by querying the previous scan. // With the introduction of support for the ORT excludes, this old behavior can be dropped. - val ignoreRules = service.listIgnoreRules(config.user, config.apiKey, existingScanCode) + val ignoreRules = service.listIgnoreRules(config.user.value, config.apiKey.value, existingScanCode) .checkResponse("list ignore rules") ignoreRules.data?.let { rules -> logger.info { "${rules.size} ignore rule(s) have been found." } @@ -653,8 +651,15 @@ class FossId internal constructor( val allRules = excludesRules + legacyRules allRules.forEach { - service.createIgnoreRule(config.user, config.apiKey, scanCode, it.type, it.value, RuleScope.SCAN) - .checkResponse("create ignore rules", false) + service.createIgnoreRule( + config.user.value, + config.apiKey.value, + scanCode, + it.type, + it.value, + RuleScope.SCAN + ).checkResponse("create ignore rules", false) + logger.info { "Ignore rule of type '${it.type}' and value '${it.value}' has been created for the new scan." } @@ -675,15 +680,9 @@ class FossId internal constructor( ): String { logger.info { "Creating scan '$scanCode'..." } - val response = service.createScan( - config.user, - config.apiKey, - projectCode, - scanCode, - url, - revision, - reference - ).checkResponse("create scan") + val response = service + .createScan(config.user.value, config.apiKey.value, projectCode, scanCode, url, revision, reference) + .checkResponse("create scan") val scanId = response.data?.get("scan_id") @@ -701,7 +700,7 @@ class FossId internal constructor( private suspend fun checkScan(scanCode: String, vararg runOptions: Pair) { waitDownloadComplete(scanCode) - val response = service.checkScanStatus(config.user, config.apiKey, scanCode) + val response = service.checkScanStatus(config.user.value, config.apiKey.value, scanCode) .checkResponse("check scan status", false) check(response.data?.status != ScanStatus.FAILED) { "Triggered scan has failed." } @@ -716,7 +715,7 @@ class FossId internal constructor( ) val scanResult = service.runScan( - config.user, config.apiKey, scanCode, mapOf(*runOptions, *optionsFromConfig) + config.user.value, config.apiKey.value, scanCode, mapOf(*runOptions, *optionsFromConfig) ) // Scans that were added to the queue are interpreted as an error by FossID before version 2021.2. @@ -750,7 +749,7 @@ class FossId internal constructor( val result = wait(config.timeout.minutes, WAIT_DELAY) { logger.info { "Checking download status for scan '$scanCode'." } - val response = service.checkDownloadStatus(config.user, config.apiKey, scanCode) + val response = service.checkDownloadStatus(config.user.value, config.apiKey.value, scanCode) .checkResponse("check download status") when (response.data) { @@ -790,7 +789,7 @@ class FossId internal constructor( val result = wait(config.timeout.minutes, WAIT_DELAY) { logger.info { "Waiting for scan '$scanCode' to complete." } - val response = service.checkScanStatus(config.user, config.apiKey, scanCode) + val response = service.checkScanStatus(config.user.value, config.apiKey.value, scanCode) .checkResponse("check scan status", false) when (response.data?.status) { @@ -816,7 +815,7 @@ class FossId internal constructor( * Delete a scan with [scanCode]. */ private suspend fun deleteScan(scanCode: String) { - val response = service.deleteScan(config.user, config.apiKey, scanCode) + val response = service.deleteScan(config.user.value, config.apiKey.value, scanCode) response.error?.let { logger.error { "Cannot delete scan '$scanCode': $it." } } @@ -827,12 +826,13 @@ class FossId internal constructor( */ @Suppress("UnsafeCallOnNullableType") private suspend fun getRawResults(scanCode: String, snippetChoices: List): RawResults { - val identifiedFiles = service.listIdentifiedFiles(config.user, config.apiKey, scanCode) + val identifiedFiles = service.listIdentifiedFiles(config.user.value, config.apiKey.value, scanCode) .checkResponse("list identified files") .data!! logger.info { "${identifiedFiles.size} identified files have been returned for scan '$scanCode'." } - val markedAsIdentifiedFiles = service.listMarkedAsIdentifiedFiles(config.user, config.apiKey, scanCode) + val markedAsIdentifiedFiles = service + .listMarkedAsIdentifiedFiles(config.user.value, config.apiKey.value, scanCode) .checkResponse("list marked as identified files") .data!! logger.info { @@ -840,11 +840,11 @@ class FossId internal constructor( } // The "match_type=ignore" info is already in the ScanResult, but here we also get the ignore reason. - val listIgnoredFiles = service.listIgnoredFiles(config.user, config.apiKey, scanCode) + val listIgnoredFiles = service.listIgnoredFiles(config.user.value, config.apiKey.value, scanCode) .checkResponse("list ignored files") .data!! - val pendingFiles = service.listPendingFiles(config.user, config.apiKey, scanCode) + val pendingFiles = service.listPendingFiles(config.user.value, config.apiKey.value, scanCode) .checkResponse("list pending files") .data!!.toMutableList() logger.info { @@ -858,7 +858,7 @@ class FossId internal constructor( "altered: putting it again as 'pending'." } - service.unmarkAsIdentified(config.user, config.apiKey, scanCode, it, false) + service.unmarkAsIdentified(config.user.value, config.apiKey.value, scanCode, it, false) } } @@ -869,7 +869,7 @@ class FossId internal constructor( val file = pendingFilesIterator.next() logger.info { "Listing snippet for $file..." } - val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, file) + val snippetResponse = service.listSnippets(config.user.value, config.apiKey.value, scanCode, file) .checkResponse("list snippets") val snippets = checkNotNull(snippetResponse.data) { "Snippet could not be listed. Response was ${snippetResponse.message}." @@ -885,9 +885,14 @@ class FossId internal constructor( coroutineScope { filteredSnippets.filter { it.matchType == MatchType.PARTIAL }.map { snippet -> async { - val matchedLinesResponse = - service.listMatchedLines(config.user, config.apiKey, scanCode, file, snippet.id) - .checkResponse("list snippets matched lines") + val matchedLinesResponse = service.listMatchedLines( + config.user.value, + config.apiKey.value, + scanCode, + file, + snippet.id + ).checkResponse("list snippets matched lines") + val lines = checkNotNull(matchedLinesResponse.data) { "Matched lines could not be listed. Response was " + "${matchedLinesResponse.message}." @@ -951,7 +956,7 @@ class FossId internal constructor( issues.add( 0, Issue( - source = name, + source = descriptor.id, message = "This scan has $pendingFilesCount file(s) pending identification in FossID.", severity = Severity.HINT ) @@ -1017,7 +1022,7 @@ class FossId internal constructor( } requests += async { - service.markAsIdentified(config.user, config.apiKey, scanCode, path, false) + service.markAsIdentified(config.user.value, config.apiKey.value, scanCode, path, false) result += path } @@ -1043,8 +1048,8 @@ class FossId internal constructor( } service.addComponentIdentification( - config.user, - config.apiKey, + config.user.value, + config.apiKey.value, scanCode, path, artifact, @@ -1079,7 +1084,7 @@ class FossId internal constructor( "relevant count $notRelevantChoicesCount." } - service.addFileComment(config.user, config.apiKey, scanCode, path, jsonComment) + service.addFileComment(config.user.value, config.apiKey.value, scanCode, path, jsonComment) } } } diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt index 4d130efdc430c..653500696ab55 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt @@ -19,66 +19,19 @@ package org.ossreviewtoolkit.plugins.scanners.fossid -import org.apache.logging.log4j.kotlin.logger +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.plugins.api.Secret -import org.ossreviewtoolkit.model.config.ScannerConfiguration -import org.ossreviewtoolkit.utils.common.Options - -/** - * A data class that holds the configuration options supported by the [FossId] scanner. An instance of this class is - * created from the [Options] contained in a [ScannerConfiguration] object under the key _FossId_. It offers the - * following configuration options: - * - * * **"options.serverUrl":** The URL of the FossID server. - * * **"secrets.user":** The user to connect to the FossID server. - * * **"secrets.apiKey":** The API key of the user which connects to the FossID server. - * * **"options.waitForResult":** When set to false, ORT does not wait for repositories to be downloaded nor scans to be - * completed. As a consequence, scan results won't be available in ORT result. - * * **"options.deltaScans":** If set, ORT will create delta scans. When only changes in a repository need to be - * scanned, delta scans reuse the identifications of the latest scan on this repository to reduce the amount of - * findings. If *deltaScans* is set and no scan exist yet, an initial scan called "origin" scan will be created. - * * **"options.deltaScanLimit":** This setting can be used to limit the number of delta scans to keep for a given - * repository. So if another delta scan is created, older delta scans are deleted until this number is reached. If - * unspecified, no limit is enforced on the number of delta scans to keep. This property is evaluated only if - * *deltaScans* is enabled. - * * **"options.detectLicenseDeclaration":** When set, the FossID scan is configured to automatically detect file - * license declarations. - * * **"options.detectCopyrightStatements":** When set, the FossID scan is configured to automatically detect copyright - * statements. - * * **"options.namingScanPattern":** A pattern for scan names when scans are created on the FossID instance. If not - * set, a default pattern is used. - * - * URL mapping options. These options allow transforming the URLs of specific repositories before they are passed to - * the FossID service. This may be necessary if FossID uses a different mechanism to clone a repository, e.g. via SSH - * instead of HTTP. Options of this form start with the prefix [FossIdUrlProvider.PREFIX_URL_MAPPING] followed by an - * arbitrary name. Their values define the mapping to be applied consisting of two parts separated by the string - * " -> ": - * * A regular expression to match the repository URL. - * * The replacement to be used for this repository URL. It can access the capture groups defined by the regular - * expression, so that rather flexible transformations can be achieved. In addition, it can contain the variables - * "#user" and "#password" that are replaced by the credentials known for the target host. - * - * The example - * - * `mapExampleRepo = https://my-repo.example.org(?.*) -> ssh://my-mapped-repo.example.org${repoPath}` - * - * would change the scheme from "https" to "ssh" and the host name for all repositories hosted on - * "my-repo.example.org". With - * - * `mapAddCredentials = - * (?)://(?)(?:\\d+)?(?.*) -> ${scheme}://#user:#password@${host}${port}${repoPath}` - * - * every repository URL would be added credentials. Mappings are applied in the order they are defined. - */ +/** A data class that holds the configuration options supported by the [FossId] scanner. */ data class FossIdConfig( /** The URL where the FossID service is running. */ val serverUrl: String, /** The user to authenticate against the server. */ - val user: String, + val user: Secret, /** The API key to access the FossID server. */ - val apiKey: String, + val apiKey: Secret, /** The name of the FossID project. If `null`, the name will be determined from the repository URL. */ val projectName: String?, @@ -86,165 +39,93 @@ data class FossIdConfig( /** The pattern for scan names when scans are created on the FossID instance. If null, a default pattern is used. */ val namingScanPattern: String?, - /** Flag whether the scanner should wait for the completion of FossID scans. */ + /** + * When set to false, ORT does not wait for repositories to be downloaded nor scans to be completed. As a + * consequence, scan results won't be available in the ORT result. + */ + @OrtPluginOption(defaultValue = "true") val waitForResult: Boolean, /** Flag whether failed scans should be kept. */ + @OrtPluginOption(defaultValue = "false") val keepFailedScans: Boolean, - /** Flag whether delta scans should be triggered. */ + /** If set, ORT will create delta scans. When only changes in a repository need to be scanned, delta scans reuse the + * identifications of the latest scan on this repository to reduce the number of findings. If *deltaScans* is set + * and no scan exists yet, an initial scan called "origin" scan will be created. + */ + @OrtPluginOption(defaultValue = "false") val deltaScans: Boolean, - /** A maximum number of delta scans to keep for a single repository. */ + /** + * This setting can be used to limit the number of delta scans to keep for a given repository. So if another delta + * scan is created, older delta scans are deleted until this number is reached. If unspecified, no limit is enforced + * on the number of delta scans to keep. This property is evaluated only if delta scans are enabled. + */ + @OrtPluginOption(defaultValue = Int.MAX_VALUE.toString()) val deltaScanLimit: Int, /** * Configure to automatically detect license declarations. Uses the `auto_identification_detect_copyright` setting. */ + @OrtPluginOption(defaultValue = "false") val detectLicenseDeclarations: Boolean, /** Configure to detect copyright statements. Uses the `auto_identification_detect_copyright` setting. */ + @OrtPluginOption(defaultValue = "false") val detectCopyrightStatements: Boolean, /** Timeout in minutes for communication with FossID. */ + @OrtPluginOption(defaultValue = "60") val timeout: Int, /** Whether matched lines of snippets are to be fetched. */ + @OrtPluginOption(defaultValue = "false") val fetchSnippetMatchedLines: Boolean, /** A limit on the amount of snippets to fetch. **/ + @OrtPluginOption(defaultValue = "500") val snippetsLimit: Int, /** The sensitivity of the scan. */ + @OrtPluginOption(defaultValue = "10") val sensitivity: Int, - /** A comma-separated list of URL mappings. */ - val urlMappings: String? -) { - companion object { - /** Name of the configuration property for the server URL. */ - private const val PROP_SERVER_URL = "serverUrl" - - /** Name of the configuration property for the username. */ - private const val PROP_USER = "user" - - /** Name of the configuration property for the API key. */ - private const val PROP_API_KEY = "apiKey" - - /** Name of the configuration property to set the project name. */ - private const val PROP_PROJECT_NAME = "projectName" - - /** Name of the configuration property controlling whether ORT should wait for FossID results. */ - private const val PROP_WAIT_FOR_RESULT = "waitForResult" - - /** Name of the configuration property defining the naming convention for scans. */ - private const val PROP_NAMING_SCAN_PATTERN = "namingScanPattern" - - /** Name of the configuration property defining whether to keep failed scans. */ - private const val PROP_KEEP_FAILED_SCANS = "keepFailedScans" - - /** Name of the configuration property controlling whether delta scans are to be created. */ - private const val PROP_DELTA_SCAN = "deltaScans" - - /** Name of the configuration property that limits the number of delta scans. */ - private const val PROP_DELTA_SCAN_LIMIT = "deltaScanLimit" - - private const val PROP_DETECT_LICENSE_DECLARATIONS = "detectLicenseDeclarations" - - private const val PROP_DETECT_COPYRIGHT_STATEMENTS = "detectCopyrightStatements" - - /** Name of the configuration property defining the timeout in minutes for communication with FossID. */ - private const val PROP_TIMEOUT = "timeout" - - /** Name of the configuration property controlling whether matched lines of snippets are to be fetched. */ - private const val PROP_FETCH_SNIPPET_MATCHED_LINES = "fetchSnippetMatchedLines" - - /** Name of the configuration property defining the limit on the amount of snippets to fetch. */ - private const val PROP_SNIPPETS_LIMIT = "snippetsLimit" - - /** Name of the configuration property defining the sensitivity of the scan. */ - private const val PROP_SENSITIVITY = "sensitivity" - - /** Name of the configuration property defining the URL mappings. */ - private const val PROP_URL_MAPPINGS = "urlMappings" - - /** - * Default timeout in minutes for communication with FossID. - */ - @JvmStatic - private val DEFAULT_TIMEOUT = 60 - - /** - * Default limit on the amount of snippets to fetch. - */ - @JvmStatic - private val DEFAULT_SNIPPETS_LIMIT = 500 - - /** - * Default scan sensitivity. - */ - @JvmStatic - private val DEFAULT_SENSITIVITY = 10 - - fun create(options: Options, secrets: Options): FossIdConfig { - require(options.isNotEmpty()) { "No FossID Scanner configuration found." } - - val serverUrl = options[PROP_SERVER_URL] - ?: throw IllegalArgumentException("No FossID server URL configuration found.") - val user = secrets[PROP_USER] - ?: throw IllegalArgumentException("No FossID User configuration found.") - val apiKey = secrets[PROP_API_KEY] - ?: throw IllegalArgumentException("No FossID API Key configuration found.") - - val projectName = options[PROP_PROJECT_NAME] - val namingScanPattern = options[PROP_NAMING_SCAN_PATTERN] - - val waitForResult = options[PROP_WAIT_FOR_RESULT]?.toBooleanStrict() ?: true - - val keepFailedScans = options[PROP_KEEP_FAILED_SCANS]?.toBooleanStrict() ?: false - val deltaScans = options[PROP_DELTA_SCAN]?.toBooleanStrict() ?: false - val deltaScanLimit = options[PROP_DELTA_SCAN_LIMIT]?.toInt() ?: Int.MAX_VALUE - - val detectLicenseDeclarations = options[PROP_DETECT_LICENSE_DECLARATIONS]?.toBooleanStrict() ?: false - val detectCopyrightStatements = options[PROP_DETECT_COPYRIGHT_STATEMENTS]?.toBooleanStrict() ?: false - - val timeout = options[PROP_TIMEOUT]?.toInt() ?: DEFAULT_TIMEOUT - - val fetchSnippetMatchedLines = options[PROP_FETCH_SNIPPET_MATCHED_LINES]?.toBooleanStrict() ?: false - val snippetsLimit = options[PROP_SNIPPETS_LIMIT]?.toInt() ?: DEFAULT_SNIPPETS_LIMIT - - val sensitivity = options[PROP_SENSITIVITY]?.toInt() ?: DEFAULT_SENSITIVITY - - val urlMappings = options[PROP_URL_MAPPINGS] - - require(deltaScanLimit > 0) { - "deltaScanLimit must be > 0, current value is $deltaScanLimit." - } - - require(sensitivity in 0..20) { - "Sensitivity must be between 0 and 20, current value is $sensitivity." - } + /** + * A comma-separated list of URL mappings that allow transforming the VCS URLs of repositories before they are + * passed to the FossID service. This may be necessary if FossID uses a different mechanism to clone a repository, + * e.g., via SSH instead of HTTP. Their values define the mapping to be applied consisting of two parts separated by + * the string " -> ": + * * A regular expression to match the repository URL. + * * The replacement to be used for this repository URL. It can access the capture groups defined by the regular + * expression, so that rather flexible transformations can be achieved. In addition, it can contain the variables + * "#user" and "#password" that are replaced by the credentials known for the target host. + * + * The example + * + * `mapExampleRepo = https://my-repo.example.org(?.*) -> ssh://my-mapped-repo.example.org${repoPath}` + * + * would change the scheme from "https" to "ssh" and the host name for all repositories hosted on + * "my-repo.example.org". With + * + * `mapAddCredentials = + * (?)://(?)(?:\\d+)?(?.*) -> ${scheme}://#user:#password@${host}${port}${repoPath}` + * + * every repository URL would be added credentials. Mappings are applied in the order they are defined. + */ + val urlMappings: String?, - logger.info { "waitForResult parameter is set to '$waitForResult'" } + /** Whether to write scan results to the storage. */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) { + init { + require(deltaScanLimit > 0) { + "deltaScanLimit must be > 0, current value is $deltaScanLimit." + } - return FossIdConfig( - serverUrl = serverUrl, - user = user, - apiKey = apiKey, - projectName = projectName, - namingScanPattern = namingScanPattern, - waitForResult = waitForResult, - keepFailedScans = keepFailedScans, - deltaScans = deltaScans, - deltaScanLimit = deltaScanLimit, - detectLicenseDeclarations = detectLicenseDeclarations, - detectCopyrightStatements = detectCopyrightStatements, - timeout = timeout, - fetchSnippetMatchedLines = fetchSnippetMatchedLines, - snippetsLimit = snippetsLimit, - sensitivity = sensitivity, - urlMappings = urlMappings - ) + require(sensitivity in 0..20) { + "Sensitivity must be between 0 and 20, current value is $sensitivity." } } diff --git a/plugins/scanners/fossid/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/fossid/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 0ebc97ec30825..0000000000000 --- a/plugins/scanners/fossid/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.fossid.FossId$Factory diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt index 9f91ceff2349e..c0e39d8e7ce2f 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdConfigTest.kt @@ -33,153 +33,43 @@ import io.kotest.matchers.shouldBe import kotlin.IllegalArgumentException import org.ossreviewtoolkit.clients.fossid.FossIdRestService +import org.ossreviewtoolkit.plugins.api.Secret class FossIdConfigTest : WordSpec({ "create" should { - "throw if no options for FossID are provided in the scanner configuration" { - shouldThrow { FossIdConfig.create(emptyMap(), emptyMap()) } - } - - "read all properties from the scanner configuration" { - val options = mapOf( - "serverUrl" to SERVER_URL, - "projectName" to PROJECT, - "namingScanPattern" to "#repositoryName_#deltaTag", - "waitForResult" to "false", - "keepFailedScans" to "true", - "deltaScans" to "true", - "deltaScanLimit" to "42", - "detectLicenseDeclarations" to "true", - "detectCopyrightStatements" to "true", - "timeout" to "300", - "fetchSnippetMatchedLines" to "true", - "snippetsLimit" to "1000", - "urlMappings" to "https://example.org(?.*) -> ssh://example.org\${repoPath}" - ) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - - val fossIdConfig = FossIdConfig.create(options, secrets) - - fossIdConfig shouldBe FossIdConfig( - serverUrl = SERVER_URL, - user = USER, - apiKey = API_KEY, - projectName = PROJECT, - namingScanPattern = "#repositoryName_#deltaTag", - waitForResult = false, - keepFailedScans = true, - deltaScans = true, - deltaScanLimit = 42, - detectLicenseDeclarations = true, - detectCopyrightStatements = true, - timeout = 300, - fetchSnippetMatchedLines = true, - snippetsLimit = 1000, - sensitivity = 10, - urlMappings = "https://example.org(?.*) -> ssh://example.org\${repoPath}" - ) - } - - "set default values for optional properties" { - val options = mapOf("serverUrl" to SERVER_URL) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - - val fossIdConfig = FossIdConfig.create(options, secrets) - - fossIdConfig shouldBe FossIdConfig( - serverUrl = SERVER_URL, - user = USER, - apiKey = API_KEY, - projectName = null, - namingScanPattern = null, - waitForResult = true, - keepFailedScans = false, - deltaScans = false, - deltaScanLimit = Int.MAX_VALUE, - detectLicenseDeclarations = false, - detectCopyrightStatements = false, - timeout = 60, - fetchSnippetMatchedLines = false, - snippetsLimit = 500, - sensitivity = 10, - urlMappings = null - ) - } - - "throw if the server URL is missing" { - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - - shouldThrow { FossIdConfig.create(emptyMap(), secrets) } - } - - "throw if the API key is missing" { - val options = mapOf("serverUrl" to SERVER_URL) - val secrets = mapOf("user" to USER) - - shouldThrow { FossIdConfig.create(options, secrets) } - } - - "throw if the user name is missing" { - val options = mapOf("serverUrl" to SERVER_URL) - val secrets = mapOf("apiKey" to API_KEY) - - shouldThrow { FossIdConfig.create(options, secrets) } - } - "throw if the deltaScanLimit is invalid" { - val options = mapOf( - "serverUrl" to SERVER_URL, - "deltaScanLimit" to "0" - ) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - - shouldThrow { FossIdConfig.create(options, secrets) } + shouldThrow { + FossIdFactory.create( + serverUrl = SERVER_URL, + user = Secret(USER), + apiKey = Secret(API_KEY), + deltaScanLimit = 0 + ) + } } "throw if the sensitivity is invalid" { - val options = mapOf( - "serverUrl" to SERVER_URL, - "sensitivity" to "21" - ) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - - shouldThrow { FossIdConfig.create(options, secrets) } + shouldThrow { + FossIdFactory.create( + serverUrl = SERVER_URL, + user = Secret(USER), + apiKey = Secret(API_KEY), + sensitivity = 21 + ) + } } } "createNamingProvider" should { "create a naming provider with a correct scan naming convention" { - val options = mapOf( - "serverUrl" to SERVER_URL, - "namingScanPattern" to "#repositoryName_#deltaTag" - ) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY + val fossId = FossIdFactory.create( + serverUrl = SERVER_URL, + user = Secret(USER), + apiKey = Secret(API_KEY), + namingScanPattern = "#repositoryName_#deltaTag" ) - val fossIdConfig = FossIdConfig.create(options, secrets) - val namingProvider = fossIdConfig.createNamingProvider() + val namingProvider = fossId.config.createNamingProvider() val scanCode = namingProvider.createScanCode("TestProject", FossId.DeltaTag.DELTA) @@ -190,18 +80,14 @@ class FossIdConfigTest : WordSpec({ "createUrlProvider" should { "initialize correct URL mappings" { val url = "https://changeit.example.org/foo" - val options = mapOf( - "serverUrl" to SERVER_URL, - "urlMappings" to "$url -> $SERVER_URL" - ) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY + val fossId = FossIdFactory.create( + serverUrl = SERVER_URL, + user = Secret(USER), + apiKey = Secret(API_KEY), + urlMappings = "$url -> $SERVER_URL" ) - val fossIdConfig = FossIdConfig.create(options, secrets) - val urlProvider = fossIdConfig.createUrlProvider() + val urlProvider = fossId.config.createUrlProvider() urlProvider.getUrl(url) shouldBe SERVER_URL } @@ -215,14 +101,6 @@ class FossIdConfigTest : WordSpec({ server.start() try { - val serverUrl = "http://localhost:${server.port()}" - val options = mapOf("serverUrl" to serverUrl) - - val secrets = mapOf( - "user" to USER, - "apiKey" to API_KEY - ) - server.stubFor( get(urlPathEqualTo("/index.php")) .withQueryParam("form", equalTo("login")) @@ -232,8 +110,7 @@ class FossIdConfigTest : WordSpec({ ) ) - val fossIdConfig = FossIdConfig.create(options, secrets) - val service = FossIdRestService.create(fossIdConfig.serverUrl) + val service = FossIdRestService.create("http://localhost:${server.port()}") service.getLoginPage().string() shouldBe loginPage } finally { @@ -243,4 +120,4 @@ class FossIdConfigTest : WordSpec({ } }) -private const val SERVER_URL = "https://www.example.org/fossid" +private const val SERVER_URL = "https://www.example.org/fossid/" diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt index 06638b1e2a4c0..c1554231bf861 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt @@ -321,7 +321,7 @@ class FossIdTest : WordSpec({ val expectedIssues = listOf( Issue( timestamp = Instant.EPOCH, - source = "FossId", + source = "fossid", message = "This scan has 2 file(s) pending identification in FossID.", severity = Severity.HINT ) diff --git a/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt b/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt index 3e05c81fbdcd9..18451d4644532 100644 --- a/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt +++ b/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt @@ -80,8 +80,8 @@ import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.Excludes import org.ossreviewtoolkit.model.config.SnippetChoices +import org.ossreviewtoolkit.plugins.api.Secret import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.scanner.provenance.NestedProvenance import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -112,7 +112,7 @@ internal val DEFAULT_IGNORE_RULE_SCOPE = RuleScope.SCAN /** * Create a new [FossId] instance with the specified [config]. */ -internal fun createFossId(config: FossIdConfig): FossId = FossId("FossId", config, ScannerWrapperConfig.EMPTY) +internal fun createFossId(config: FossIdConfig): FossId = FossId(config = config) /** * Create a standard [FossIdConfig] whose properties can be partly specified. @@ -127,8 +127,8 @@ internal fun createConfig( ): FossIdConfig { val config = FossIdConfig( serverUrl = "https://www.example.org/fossid", - user = USER, - apiKey = API_KEY, + user = Secret(USER), + apiKey = Secret(API_KEY), projectName = projectName, namingScanPattern = null, waitForResult = waitForResult, @@ -141,7 +141,8 @@ internal fun createConfig( fetchSnippetMatchedLines = fetchSnippetMatchedLines, snippetsLimit = snippetsLimit, sensitivity = 10, - urlMappings = null + urlMappings = null, + writeToStorage = false ) val namingProvider = createNamingProviderMock() diff --git a/plugins/scanners/licensee/build.gradle.kts b/plugins/scanners/licensee/build.gradle.kts index 8e9a5d578624f..22cfe5e247f39 100644 --- a/plugins/scanners/licensee/build.gradle.kts +++ b/plugins/scanners/licensee/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") // Apply third-party plugins. alias(libs.plugins.kotlinSerialization) @@ -32,5 +32,7 @@ dependencies { implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.json) + ksp(projects.scanner) + funTestApi(testFixtures(projects.scanner)) } diff --git a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt index 215a0b3572c06..18819319a046b 100644 --- a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt +++ b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt @@ -22,10 +22,9 @@ package org.ossreviewtoolkit.plugins.scanners.licensee import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig class LicenseeFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = Licensee("Licensee", ScannerWrapperConfig.EMPTY) + override val scanner = LicenseeFactory.create() override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 100.0f) diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index 40674581c088f..d77f11019ca8f 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -38,14 +38,16 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os private val JSON = Json { @@ -62,27 +64,73 @@ object LicenseeCommand : CommandLineTool { override fun displayName() = "Licensee" } -class Licensee internal constructor(name: String, private val wrapperConfig: ScannerWrapperConfig) : - LocalPathScannerWrapper(name) { +data class LicenseeConfig( + /** + * A regular expression to match the scanner name when looking up scan results in the storage. + */ + val regScannerName: String?, + + /** + * The minimum version of stored scan results to use. + */ + val minVersion: String?, + + /** + * The maximum version of stored scan results to use. + */ + val maxVersion: String?, + + /** + * The configuration to use for the scanner. Only scan results with the same configuration are used when looking up + * scan results in the storage. + */ + val configuration: String?, + + /** + * Whether to read scan results from the storage. + */ + @OrtPluginOption(defaultValue = "true") + val readFromStorage: Boolean, + + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) + +@OrtPlugin( + id = "licensee", + displayName = "Licensee", + description = "Licensee is a command line tool to detect licenses in a given project.", + factory = ScannerWrapperFactory::class +) +class Licensee( + override val descriptor: PluginDescriptor = LicenseeFactory.descriptor, + config: LicenseeConfig +) : LocalPathScannerWrapper() { companion object { val CONFIGURATION_OPTIONS = listOf("--json") } - class Factory : ScannerWrapperFactory("Licensee") { - override fun create(config: Unit, wrapperConfig: ScannerWrapperConfig) = Licensee(type, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = Unit - } - override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + override val matcher by lazy { + ScannerMatcher.create( + details, + ScannerMatcherConfig( + config.regScannerName, + config.minVersion, + config.maxVersion, + config.configuration + ) + ) + } override val version by lazy { LicenseeCommand.getVersion() } - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val readFromStorage = config.readFromStorage + override val writeToStorage = config.writeToStorage override fun runScanner(path: File, context: ScanContext): String { val process = LicenseeCommand.run( @@ -106,7 +154,7 @@ class Licensee internal constructor(name: String, private val wrapperConfig: Sca val parameters = details.getValue("parameters").jsonArray return ScannerDetails( - name = name, + name = descriptor.id, version = version, // TODO: Filter out parameters that have no influence on scan results. configuration = parameters.joinToString(" ") { it.jsonPrimitive.content } @@ -130,7 +178,7 @@ class Licensee internal constructor(name: String, private val wrapperConfig: Sca licenseFindings = licenseFindings, issues = listOf( Issue( - source = name, + source = descriptor.id, message = "This scanner is not capable of detecting copyright statements.", severity = Severity.HINT ) diff --git a/plugins/scanners/licensee/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/licensee/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 2d7233e4b82e6..0000000000000 --- a/plugins/scanners/licensee/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.licensee.Licensee$Factory diff --git a/plugins/scanners/licensee/src/test/kotlin/LicenseeTest.kt b/plugins/scanners/licensee/src/test/kotlin/LicenseeTest.kt index 836d8a9f25f04..23df84f455177 100644 --- a/plugins/scanners/licensee/src/test/kotlin/LicenseeTest.kt +++ b/plugins/scanners/licensee/src/test/kotlin/LicenseeTest.kt @@ -23,10 +23,9 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig class LicenseeTest : StringSpec({ - val scanner = Licensee("Licensee", ScannerWrapperConfig.EMPTY) + val scanner = LicenseeFactory.create() "Parsing details from results succeeds" { val result = """ @@ -47,7 +46,7 @@ class LicenseeTest : StringSpec({ """.trimIndent() scanner.parseDetails(result) shouldBe ScannerDetails( - name = "Licensee", + name = "licensee", version = "9.12.0", configuration = "--json --no-readme" ) diff --git a/plugins/scanners/scancode/build.gradle.kts b/plugins/scanners/scancode/build.gradle.kts index f1fe6d3a81374..96e6612746c1b 100644 --- a/plugins/scanners/scancode/build.gradle.kts +++ b/plugins/scanners/scancode/build.gradle.kts @@ -19,7 +19,7 @@ plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") // Apply third-party plugins. alias(libs.plugins.kotlinSerialization) @@ -40,6 +40,8 @@ dependencies { implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.json) + ksp(projects.scanner) + funTestApi(testFixtures(projects.scanner)) testImplementation(libs.mockk) diff --git a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt index 2f7bcacce6e29..46720ef9bea3c 100644 --- a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt +++ b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt @@ -28,11 +28,10 @@ import io.kotest.matchers.string.startWith import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.utils.spdx.getLicenseText class ScanCodeScannerFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = ScanCode("ScanCode", ScanCodeConfig.DEFAULT, ScannerWrapperConfig.EMPTY) + override val scanner = ScanCodeFactory.create() override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", 1, 201), 100.0f) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 4ee575e2ab60a..66ae54a2c0899 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -22,20 +22,22 @@ package org.ossreviewtoolkit.plugins.scanners.scancode import java.io.File import java.time.Instant +import kotlin.math.max + import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.config.PluginConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScanStorage import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.safeDeleteRecursively @@ -79,14 +81,16 @@ object ScanCodeCommand : CommandLineTool { * file, the end line is set to the maximum of all end lines for per-line findings in that file, and the score is set * to the arithmetic average of the scores of all per-line findings in that file. */ -class ScanCode internal constructor( - name: String, - private val config: ScanCodeConfig, - private val wrapperConfig: ScannerWrapperConfig -) : LocalPathScannerWrapper(name) { - // This constructor is required by the `RequirementsCommand`. - constructor(name: String, wrapperConfig: ScannerWrapperConfig) : this(name, ScanCodeConfig.DEFAULT, wrapperConfig) - +@OrtPlugin( + id = "ScanCode", + displayName = "ScanCode", + description = "A wrapper for [ScanCode](https://github.com/aboutcode-org/scancode-toolkit).", + factory = ScannerWrapperFactory::class +) +class ScanCode( + override val descriptor: PluginDescriptor = ScanCodeFactory.descriptor, + private val config: ScanCodeConfig +) : LocalPathScannerWrapper() { companion object { const val SCANNER_NAME = "ScanCode" @@ -94,19 +98,17 @@ class ScanCode internal constructor( private const val OUTPUT_FORMAT_OPTION = "--json" } - class Factory : ScannerWrapperFactory(SCANNER_NAME) { - override fun create(config: ScanCodeConfig, wrapperConfig: ScannerWrapperConfig) = - ScanCode(type, config, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = ScanCodeConfig.create(options) - } - private val commandLineOptions by lazy { getCommandLineOptions(version) } internal fun getCommandLineOptions(version: String) = buildList { addAll(config.commandLine) - addAll(config.commandLineNonConfig) + config.commandLineNonConfig?.let { addAll(it) } + + if ("--processes" !in config.commandLineNonConfig.orEmpty()) { + add("--processes") + add(max(1, Runtime.getRuntime().availableProcessors() - 1).toString()) + } if (Semver(version).isGreaterThanOrEqualTo(LICENSE_REFERENCES_OPTION_VERSION)) { // Required to be able to map ScanCode license keys to SPDX IDs. @@ -124,13 +126,22 @@ class ScanCode internal constructor( }.joinToString(" ") } - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + override val matcher by lazy { + ScannerMatcher.create( + details, + ScannerMatcherConfig( + config.regScannerName, + config.minVersion, + config.maxVersion, + config.configuration + ) + ) + } override val version by lazy { ScanCodeCommand.getVersion() } - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val readFromStorage = config.readFromStorage + override val writeToStorage = config.writeToStorage override fun runScanner(path: File, context: ScanContext): String { val resultFile = createOrtTempDir().resolve("result.json") @@ -153,7 +164,7 @@ class ScanCode internal constructor( val options = header.getPrimitiveOptions() return ScannerDetails( - name = name, + name = descriptor.id, version = header.toolVersion, // TODO: Filter out options that have no influence on scan results. configuration = options.joinToString(" ") { "${it.first} ${it.second}" } diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt index 0a7e5c216d5c7..826f82f07162e 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt @@ -19,49 +19,67 @@ package org.ossreviewtoolkit.plugins.scanners.scancode -import kotlin.math.max -import kotlin.time.Duration.Companion.minutes - -import org.ossreviewtoolkit.utils.common.Options -import org.ossreviewtoolkit.utils.common.splitOnWhitespace +import org.ossreviewtoolkit.model.ScannerDetails +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.scanner.ScanStorage data class ScanCodeConfig( + /** + * Command line options that modify the result. These are added to the [ScannerDetails] when looking up results from + * a [ScanStorage]. Defaults to [ScanCodeConfig.DEFAULT_COMMAND_LINE_OPTIONS]. + */ + @OrtPluginOption(defaultValue = "--copyright,--license,--info,--strip-root,--timeout,300") val commandLine: List, - val commandLineNonConfig: List, - val preferFileLicense: Boolean -) { - companion object { - /** - * The default time after which scanning a file is aborted. - */ - private val DEFAULT_TIMEOUT = 5.minutes - /** - * The default list of command line options that might have an impact on the scan results. - */ - private val DEFAULT_COMMAND_LINE_OPTIONS = listOf( - "--copyright", - "--license", - "--info", - "--strip-root", - "--timeout", "${DEFAULT_TIMEOUT.inWholeSeconds}" - ) + /** + * Command line options that do not modify the result and should therefore not be considered in [configuration], + * like "--processes". If this does not contain "--processes", it is added with a value of one less than the number + * of available processors. + */ + val commandLineNonConfig: List?, + + /** + * A flag to indicate whether the "high-level" per-file license reported by ScanCode starting with version 32 should + * be used instead of the individual "low-level" per-line license findings. The per-file license may be different + * from the conjunction of per-line licenses and is supposed to contain fewer false-positives. However, no exact + * line numbers can be associated to the per-file license anymore. If enabled, the start line of the per-file + * license finding is set to the minimum of all start lines for per-line findings in that file, the end line is set + * to the maximum of all end lines for per-line findings in that file, and the score is set to the arithmetic + * average of the scores of all per-line findings in that file. + */ + @OrtPluginOption(defaultValue = "false") + val preferFileLicense: Boolean, + + /** + * A regular expression to match the scanner name when looking up scan results in the storage. + */ + val regScannerName: String?, + + /** + * The minimum version of stored scan results to use. + */ + val minVersion: String?, + + /** + * The maximum version of stored scan results to use. + */ + val maxVersion: String?, - /** - * The default list of command line options that cannot have an impact on the scan results. - */ - private val DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS = listOf( - "--processes", max(1, Runtime.getRuntime().availableProcessors() - 1).toString() - ) + /** + * The configuration to use for the scanner. Only scan results with the same configuration are used when looking up + * scan results in the storage. + */ + val configuration: String?, - val DEFAULT = create(emptyMap()) + /** + * Whether to read scan results from the storage. + */ + @OrtPluginOption(defaultValue = "true") + val readFromStorage: Boolean, - fun create(options: Options) = - ScanCodeConfig( - options["commandLine"]?.splitOnWhitespace() ?: DEFAULT_COMMAND_LINE_OPTIONS, - options["commandLineNonConfig"]?.splitOnWhitespace() - ?: DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS, - options["preferFileLicense"].toBoolean() - ) - } -} + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) diff --git a/plugins/scanners/scancode/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/scancode/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index b55b723d10c7f..0000000000000 --- a/plugins/scanners/scancode/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.scancode.ScanCode$Factory diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt index 048ada58684f2..a41ba7e841c43 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt @@ -36,11 +36,10 @@ import java.io.File import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.utils.common.ProcessCapture class ScanCodeTest : WordSpec({ - val scanner = ScanCode("ScanCode", ScanCodeConfig.DEFAULT, ScannerWrapperConfig.EMPTY) + val scanner = ScanCodeFactory.create() "configuration" should { "return the default values if the scanner configuration is empty" { @@ -48,15 +47,11 @@ class ScanCodeTest : WordSpec({ } "return the non-config values from the scanner configuration" { - val config = ScanCodeConfig.create( - mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) + val scannerWithConfig = ScanCodeFactory.create( + commandLine = listOf("--command", "--line"), + commandLineNonConfig = listOf("--commandLineNonConfig") ) - val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) - scannerWithConfig.configuration shouldBe "--command --line --json" } } @@ -70,30 +65,13 @@ class ScanCodeTest : WordSpec({ } "contain the values from the scanner configuration" { - val config = ScanCodeConfig.create( - mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) + val scannerWithConfig = ScanCodeFactory.create( + commandLine = listOf("--command", "--line"), + commandLineNonConfig = listOf("--commandLineNonConfig") ) - val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) - - scannerWithConfig.getCommandLineOptions("31.2.4").joinToString(" ") shouldBe - "--command --line --commandLineNonConfig" - } - - "be handled correctly when containing multiple spaces" { - val config = ScanCodeConfig.create( - mapOf( - "commandLine" to " --command --line ", - "commandLineNonConfig" to " -n -c " - ) - ) - - val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) - - scannerWithConfig.getCommandLineOptions("31.2.4") shouldBe listOf("--command", "--line", "-n", "-c") + scannerWithConfig.getCommandLineOptions("31.2.4").joinToString(" ") shouldMatch + "--command --line --commandLineNonConfig --processes \\d+" } } @@ -126,7 +104,7 @@ class ScanCodeTest : WordSpec({ "transformVersion()" should { "work with a version output without a colon" { - scanner.transformVersion( + ScanCodeCommand.transformVersion( """ ScanCode version 30.0.1 ScanCode Output Format version 1.0.0 @@ -136,7 +114,7 @@ class ScanCodeTest : WordSpec({ } "work with a version output with a colon" { - scanner.transformVersion( + ScanCodeCommand.transformVersion( """ ScanCode version: 31.0.0b4 ScanCode Output Format version: 2.0.0 diff --git a/plugins/scanners/scanoss/build.gradle.kts b/plugins/scanners/scanoss/build.gradle.kts index 6a6b46797d209..505ebdd938041 100644 --- a/plugins/scanners/scanoss/build.gradle.kts +++ b/plugins/scanners/scanoss/build.gradle.kts @@ -21,7 +21,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { // Apply precompiled plugins. - id("ort-library-conventions") + id("ort-plugin-conventions") } dependencies { @@ -38,6 +38,8 @@ dependencies { .because("the logging provider conflicts with ORT's") } + ksp(projects.scanner) + funTestApi(testFixtures(projects.scanner)) testImplementation(libs.kotlinx.serialization.core) diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 91d6df147ba34..6efac9c52ce43 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -32,27 +32,25 @@ import java.util.UUID import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory -import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES -class ScanOss internal constructor( - override val name: String, - config: ScanOssConfig, - private val wrapperConfig: ScannerWrapperConfig +@OrtPlugin( + id = "SCANOSS", + displayName = "SCANOSS", + description = "A wrapper for the SCANOSS snippet scanner.", + factory = ScannerWrapperFactory::class +) +class ScanOss( + override val descriptor: PluginDescriptor = ScanOssFactory.descriptor, + config: ScanOssConfig ) : PathScannerWrapper { - class Factory : ScannerWrapperFactory("SCANOSS") { - override fun create(config: ScanOssConfig, wrapperConfig: ScannerWrapperConfig) = - ScanOss(type, config, wrapperConfig) - - override fun parseConfig(options: Options, secrets: Options) = - ScanOssConfig.create(options, secrets).also { logger.info { "The $type API URL is ${it.apiUrl}." } } - } - private val service = ScanApi.builder() // As there is only a single endpoint, the SCANOSS API client expects the path to be part of the API URL. .url(config.apiUrl.removeSuffix("/") + "/scan/direct") @@ -66,11 +64,21 @@ class ScanOss internal constructor( override val configuration = "" - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + override val matcher by lazy { + ScannerMatcher.create( + details, + ScannerMatcherConfig( + config.regScannerName, + config.minVersion, + config.maxVersion, + configuration + ) + ) + } - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } + override val readFromStorage = config.readFromStorage - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } + override val writeToStorage = config.writeToStorage /** * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. @@ -106,7 +114,7 @@ class ScanOss internal constructor( val uuid = UUID.fromString(it.filePath) val fileName = fileNamesAnonymizationMapping[uuid] ?: throw IllegalArgumentException( - "The $name server returned UUID '$uuid' which is not present in the mapping." + "The ${descriptor.id} server returned UUID '$uuid' which is not present in the mapping." ) ScanFileResult(fileName, it.fileDetails) diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt index 06fbe01a69eda..b5840bb0f789b 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt @@ -19,38 +19,40 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss -import org.ossreviewtoolkit.model.config.ScannerConfiguration -import org.ossreviewtoolkit.utils.common.Options +import org.ossreviewtoolkit.plugins.api.OrtPluginOption -/** - * A data class that holds the configuration options supported by the [ScanOss] scanner. An instance of this class is - * created from the [scanner config][ScannerConfiguration.config] object under the key _ScanOss_. It offers the - * following configuration options: - * - * **"options.apiUrl":** The URL of the ScanOSS server. - * **"secrets.apiKey":** The API key to authenticate with the ScanOSS server. - */ data class ScanOssConfig( - /** URL of the ScanOSS server. */ + /** The URL of the ScanOSS server. */ + @OrtPluginOption(defaultValue = "https://api.osskb.org/") val apiUrl: String, - /** API Key required to authenticate with the ScanOSS server. */ - val apiKey: String -) { - companion object { - /** Name of the configuration property for the API URL. */ - const val API_URL_PROPERTY = "apiUrl" - - /** Name of the configuration property for the API key. */ - const val API_KEY_PROPERTY = "apiKey" - - fun create(options: Options, secrets: Options): ScanOssConfig { - // TODO: Remove the hard-coded default URL once https://github.com/scanoss/scanoss.java/issues/11 is - // resolved. - val apiUrl = options[API_URL_PROPERTY] ?: "https://api.osskb.org/" - val apiKey = secrets[API_KEY_PROPERTY].orEmpty() - - return ScanOssConfig(apiUrl, apiKey) - } - } -} + /** The API key required to authenticate with the ScanOSS server. */ + val apiKey: String, + + /** + * A regular expression to match the scanner name when looking up scan results in the storage. + */ + val regScannerName: String?, + + /** + * The minimum version of stored scan results to use. + */ + val minVersion: String?, + + /** + * The maximum version of stored scan results to use. + */ + val maxVersion: String?, + + /** + * Whether to read scan results from the storage. + */ + @OrtPluginOption(defaultValue = "true") + val readFromStorage: Boolean, + + /** + * Whether to write scan results to the storage. + */ + @OrtPluginOption(defaultValue = "true") + val writeToStorage: Boolean +) diff --git a/plugins/scanners/scanoss/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory b/plugins/scanners/scanoss/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory deleted file mode 100644 index 4bcf9ad82cabe..0000000000000 --- a/plugins/scanners/scanoss/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory +++ /dev/null @@ -1 +0,0 @@ -org.ossreviewtoolkit.plugins.scanners.scanoss.ScanOss$Factory diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt deleted file mode 100644 index 774c51af61771..0000000000000 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.plugins.scanners.scanoss - -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.should -import io.kotest.matchers.shouldBe -import io.kotest.matchers.string.beEmpty - -class ScanOssConfigTest : StringSpec({ - "Default values are used" { - with(ScanOssConfig.create(emptyMap(), emptyMap())) { - apiUrl shouldBe "https://api.osskb.org/" - apiKey should beEmpty() - } - } - - "Default values can be overridden" { - val options = mapOf(ScanOssConfig.API_URL_PROPERTY to "url") - val secrets = mapOf(ScanOssConfig.API_KEY_PROPERTY to "key") - - with(ScanOssConfig.create(options, secrets)) { - apiUrl shouldBe "url" - apiKey shouldBe "key" - } - } -}) diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index 55f2c456a59ff..1296c2ed61af8 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -43,7 +43,6 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.utils.spdx.SpdxExpression private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/filesToScan") @@ -62,8 +61,7 @@ class ScanOssScannerDirectoryTest : StringSpec({ beforeSpec { server.start() - val config = ScanOssConfig(apiUrl = "http://localhost:${server.port()}", apiKey = "") - scanner = spyk(ScanOss.Factory().create(config, ScannerWrapperConfig.EMPTY)) + scanner = spyk(ScanOssFactory.create(apiUrl = "http://localhost:${server.port()}", apiKey = "")) } afterSpec { diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt index 7e8b6867d3351..000f34f45aeb4 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt @@ -37,7 +37,6 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScannerWrapperConfig private val TEST_FILE_TO_SCAN = File("src/test/assets/filesToScan/ScannerFactory.kt") @@ -55,8 +54,7 @@ class ScanOssScannerFileTest : StringSpec({ beforeSpec { server.start() - val config = ScanOssConfig(apiUrl = "http://localhost:${server.port()}", apiKey = "") - scanner = spyk(ScanOss.Factory().create(config, ScannerWrapperConfig.EMPTY)) + scanner = spyk(ScanOssFactory.create(apiUrl = "http://localhost:${server.port()}", apiKey = "")) } afterSpec { diff --git a/scanner/build.gradle.kts b/scanner/build.gradle.kts index bc81b7479adaf..21d142dafe065 100644 --- a/scanner/build.gradle.kts +++ b/scanner/build.gradle.kts @@ -27,6 +27,7 @@ plugins { dependencies { api(projects.model) + api(projects.plugins.api) implementation(projects.clients.clearlyDefinedClient) implementation(projects.downloader) diff --git a/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt b/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt index 9040b43b36a6f..da149e1d259c6 100644 --- a/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt +++ b/scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt @@ -42,6 +42,7 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.toYaml +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.Scanner @@ -210,7 +211,8 @@ private val pkg4 = createPackage( ) ) -internal class DummyScanner(override val name: String = "Dummy") : PathScannerWrapper { +internal class DummyScanner(id: String = "Dummy") : PathScannerWrapper { + override val descriptor = PluginDescriptor(id = id, displayName = id, description = "") override val version = "1.0.0" override val configuration = "" diff --git a/scanner/src/main/kotlin/LocalPathScannerWrapper.kt b/scanner/src/main/kotlin/LocalPathScannerWrapper.kt index a7d0e5ffe4085..cfcd4d4a46da6 100644 --- a/scanner/src/main/kotlin/LocalPathScannerWrapper.kt +++ b/scanner/src/main/kotlin/LocalPathScannerWrapper.kt @@ -28,7 +28,7 @@ import org.ossreviewtoolkit.model.ScannerDetails /** * A [PathScannerWrapper] that is executed on the local machine. */ -abstract class LocalPathScannerWrapper(override val name: String) : PathScannerWrapper { +abstract class LocalPathScannerWrapper : PathScannerWrapper { final override fun scanPath(path: File, context: ScanContext): ScanSummary { val startTime = Instant.now() val result = runScanner(path, context) diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt index 615d4a76c9996..cc12589a4aca1 100644 --- a/scanner/src/main/kotlin/Scanner.kt +++ b/scanner/src/main/kotlin/Scanner.kt @@ -139,7 +139,7 @@ class Scanner( scannerWrappers.values.flatten().forEach { scanner -> if (scanner is CommandLineTool) { - toolVersions[scanner.name] = scanner.getVersion() + toolVersions[scanner.descriptor.id] = scanner.getVersion() } } @@ -232,8 +232,8 @@ class Scanner( filteredScanResults } - val scannerNames = scannerWrappers.mapTo(mutableSetOf()) { it.name } - val scanners = packages.associateBy({ it.id }) { scannerNames } + val scannerIds = scannerWrappers.mapTo(mutableSetOf()) { it.descriptor.id } + val scanners = packages.associateBy({ it.id }) { scannerIds } return ScannerRun.EMPTY.copy( config = scannerConfig, @@ -318,16 +318,16 @@ class Scanner( val hasNestedProvenance = controller.getNestedProvenance(pkg.id) != null if (!hasNestedProvenance) { logger.debug { - "Skipping scan of '${pkg.id.toCoordinates()}' with package scanner '${scanner.name}' as " + - "no nested provenance for the package could be resolved." + "Skipping scan of '${pkg.id.toCoordinates()}' with package scanner " + + "'${scanner.descriptor.id}' as no nested provenance for the package could be resolved." } } val hasCompleteScanResult = controller.hasCompleteScanResult(scanner, pkg) if (hasCompleteScanResult) { logger.debug { - "Skipping scan of '${pkg.id.toCoordinates()}' with package scanner '${scanner.name}' as " + - "stored results are available." + "Skipping scan of '${pkg.id.toCoordinates()}' with package scanner " + + "'${scanner.descriptor.id}' as stored results are available." } } @@ -336,7 +336,7 @@ class Scanner( if (packagesWithIncompleteScanResult.isEmpty()) { logger.info { - "Skipping scan with package scanner '${scanner.name}' as all packages have results." + "Skipping scan with package scanner '${scanner.descriptor.id}' as all packages have results." } return@scanner @@ -354,14 +354,14 @@ class Scanner( logger.info { val coveredCoordinates = adjustedContext.coveredPackages.joinToString { it.id.toCoordinates() } - "Starting scan of ${nestedProvenance.root} with package scanner '${scanner.name}' which covers " + - "the following packages: $coveredCoordinates" + "Starting scan of ${nestedProvenance.root} with package scanner '${scanner.descriptor.id}' which " + + "covers the following packages: $coveredCoordinates" } val scanResult = scanner.scanPackage(nestedProvenance, adjustedContext) logger.info { - "Finished scan of ${nestedProvenance.root} with package scanner '${scanner.name}'." + "Finished scan of ${nestedProvenance.root} with package scanner '${scanner.descriptor.id}'." } val provenanceScanResultsToStore = mutableSetOf>() @@ -401,7 +401,7 @@ class Scanner( if (controller.hasScanResult(scanner, provenance)) { logger.debug { "Skipping $provenance scan (${index + 1} of ${provenances.size}) with provenance scanner " + - "'${scanner.name}' as a result is already available." + "'${scanner.descriptor.id}' as a result is already available." } return@scanner @@ -409,7 +409,7 @@ class Scanner( logger.info { "Scanning $provenance (${index + 1} of ${provenances.size}) with provenance scanner " + - "'${scanner.name}'." + "'${scanner.descriptor.id}'." } // Filter the scan context to hide the excludes from scanner with scan matcher. @@ -506,7 +506,8 @@ class Scanner( controller.scanners.forEach { scanner -> val results = controller.getScanResults(scanner) logger.info { - "\t${scanner.name}: Result(s) for ${results.size} of ${allKnownProvenances.size} provenance(s)." + "\t${scanner.descriptor.id}: Result(s) for ${results.size} of ${allKnownProvenances.size} " + + "provenance(s)." } } } @@ -596,7 +597,7 @@ class Scanner( } val results = scanners.associateWith { scanner -> - logger.info { "Scan of $provenance with path scanner '${scanner.name}' started." } + logger.info { "Scan of $provenance with path scanner '${scanner.descriptor.id}' started." } // Filter the scan context to hide the excludes from scanner with scan matcher. val filteredContext = if (scanner.matcher == null) context else context.copy(excludes = null) @@ -605,8 +606,8 @@ class Scanner( scanner.scanPath(downloadDir, filteredContext) }.getOrElse { e -> val issue = createAndLogIssue( - scanner.name, - "Failed to scan $provenance with path scanner '${scanner.name}': ${e.collectMessages()}" + scanner.descriptor.id, + "Failed to scan $provenance with path scanner '${scanner.descriptor.id}': ${e.collectMessages()}" ) val time = Instant.now() @@ -617,7 +618,7 @@ class Scanner( ) } - logger.info { "Scan of $provenance with path scanner '${scanner.name}' finished." } + logger.info { "Scan of $provenance with path scanner '${scanner.descriptor.id}' finished." } val summaryWithMappedLicenses = summary.copy( licenseFindings = summary.licenseFindings.mapTo(mutableSetOf()) { diff --git a/scanner/src/main/kotlin/ScannerWrapper.kt b/scanner/src/main/kotlin/ScannerWrapper.kt index 3023d0abad19a..10b0517432a4d 100644 --- a/scanner/src/main/kotlin/ScannerWrapper.kt +++ b/scanner/src/main/kotlin/ScannerWrapper.kt @@ -29,17 +29,13 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.config.PluginConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.plugins.api.Plugin import org.ossreviewtoolkit.scanner.provenance.NestedProvenance /** * The base interface for all types of wrappers for scanners. */ -sealed interface ScannerWrapper { - /** - * The name of the scanner. - */ - val name: String - +sealed interface ScannerWrapper : Plugin { /** * The version of the scanner. */ @@ -54,7 +50,7 @@ sealed interface ScannerWrapper { * The details of the scanner. */ val details: ScannerDetails - get() = ScannerDetails(name, version, configuration) + get() = ScannerDetails(descriptor.id, version, configuration) /** * The [ScannerMatcher] object to be used when looking up existing scan results from a scan storage. By default, diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index cca5cf96de7a5..2089bd9ebb1ec 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -21,39 +21,16 @@ package org.ossreviewtoolkit.scanner import java.util.ServiceLoader -import org.ossreviewtoolkit.utils.common.Options -import org.ossreviewtoolkit.utils.common.Plugin -import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory +import org.ossreviewtoolkit.plugins.api.PluginFactory /** * A common abstract class for use with [ServiceLoader] that all [ScannerWrapperFactory] classes need to implement. */ -abstract class ScannerWrapperFactory(override val type: String) : - TypedConfigurablePluginFactory { +interface ScannerWrapperFactory : PluginFactory { companion object { /** * All [scanner wrapper factories][ScannerWrapperFactory] available in the classpath, associated by their names. */ - val ALL by lazy { Plugin.getAll>() } + val ALL by lazy { PluginFactory.getAll() } } - - override fun create(options: Options, secrets: Options): ScannerWrapper { - val (wrapperConfig, filteredOptions) = ScannerWrapperConfig.create(options) - return create(parseConfig(filteredOptions, secrets), wrapperConfig) - } - - final override fun create(config: CONFIG): ScannerWrapper { - throw UnsupportedOperationException("Use 'create(CONFIG, ScannerMatcherConfig)' instead.") - } - - /** - * Create a [ScannerWrapper] from the provided [config] and [wrapperConfig]. - */ - abstract fun create(config: CONFIG, wrapperConfig: ScannerWrapperConfig): ScannerWrapper - - /** - * Return the scanner wrapper's name here to allow Clikt to display something meaningful when listing the scanner - * wrapper factories which are enabled by default. - */ - override fun toString() = type } diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index 84c681f6443ad..4ca153826a3d0 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -47,6 +47,7 @@ import org.ossreviewtoolkit.model.config.ClearlyDefinedStorageConfiguration import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates import org.ossreviewtoolkit.model.utils.toClearlyDefinedSourceLocation +import org.ossreviewtoolkit.plugins.api.PluginConfig import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper import org.ossreviewtoolkit.scanner.ScanStorageException import org.ossreviewtoolkit.scanner.ScannerMatcher @@ -109,7 +110,7 @@ class ClearlyDefinedStorage( val supportedScanners = toolVersionsByName.mapNotNull { (name, versions) -> // For the ClearlyDefined tool names see https://github.com/clearlydefined/service#tool-name-registry. ScannerWrapperFactory.ALL[name]?.let { factory -> - val scanner = factory.create(emptyMap(), emptyMap()) + val scanner = factory.create(PluginConfig()) (scanner as? LocalPathScannerWrapper)?.let { cliScanner -> cliScanner to versions.last() } }.also { factory -> factory ?: logger.debug { "Unsupported tool '$name' for coordinates '$coordinates'." } @@ -118,12 +119,12 @@ class ClearlyDefinedStorage( supportedScanners.mapNotNull { (cliScanner, version) -> val startTime = Instant.now() - val name = cliScanner.name.lowercase() + val name = cliScanner.descriptor.id.lowercase() val data = loadToolData(coordinates, name, version) val provenance = getProvenance(coordinates) val endTime = Instant.now() - when (cliScanner.name) { + when (cliScanner.descriptor.id) { "ScanCode" -> { data["content"]?.let { result -> val resultString = result.toString() diff --git a/scanner/src/test/kotlin/ScannerTest.kt b/scanner/src/test/kotlin/ScannerTest.kt index 03868c42acc3e..0e437fc0b539a 100644 --- a/scanner/src/test/kotlin/ScannerTest.kt +++ b/scanner/src/test/kotlin/ScannerTest.kt @@ -62,6 +62,7 @@ import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.FileArchiverConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.toYaml +import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.provenance.NestedProvenance import org.ossreviewtoolkit.scanner.provenance.NestedProvenanceResolver import org.ossreviewtoolkit.scanner.provenance.NestedProvenanceScanResult @@ -82,8 +83,8 @@ class ScannerTest : WordSpec({ "Scanning with different scanners for projects and packages" should { "Use the correct scanners for each data entity" { val pkgWithArtifact = Package.new(name = "artifact").withValidSourceArtifact() - val packageScannerWrapper = FakePackageScannerWrapper(name = "package scanner") - val projectScannerWrapper = FakePackageScannerWrapper(name = "project scanner") + val packageScannerWrapper = FakePackageScannerWrapper(id = "package scanner") + val projectScannerWrapper = FakePackageScannerWrapper(id = "project scanner") val scanner = createScanner( packageScannerWrappers = listOf(packageScannerWrapper), projectScannerWrappers = listOf(projectScannerWrapper) @@ -916,7 +917,8 @@ class ScannerTest : WordSpec({ * An implementation of [PackageScannerWrapper] that creates empty scan results. */ @Suppress("RedundantNullableReturnType") -private class FakePackageScannerWrapper(override val name: String = "fake") : PackageScannerWrapper { +private class FakePackageScannerWrapper(id: String = "fake") : PackageScannerWrapper { + override val descriptor = PluginDescriptor(id = id, displayName = id, description = "") override val version = "1.0.0" override val configuration = "config" @@ -933,7 +935,7 @@ private class FakePackageScannerWrapper(override val name: String = "fake") : Pa * An implementation of [ProvenanceScannerWrapper] that creates empty scan results. */ private class FakeProvenanceScannerWrapper : ProvenanceScannerWrapper { - override val name = "fake" + override val descriptor = PluginDescriptor(id = "fake", displayName = "fake", description = "") override val version = "1.0.0" override val configuration = "config" @@ -949,7 +951,7 @@ private class FakeProvenanceScannerWrapper : ProvenanceScannerWrapper { * An implementation of [PathScannerWrapper] that creates scan results with one license finding for each file. */ private class FakePathScannerWrapper : PathScannerWrapper { - override val name = "fake" + override val descriptor = PluginDescriptor(id = "fake", displayName = "fake", description = "") override val version = "1.0.0" override val configuration = "config"