feat(advisor): Add BlackDuck as advisor

Signed-off-by: Frank Viernau <[email protected]>

buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts

@@ -44,6 +44,15 @@ repositories {
             includeGroup("org.gradle")
         }
     }
+
+    maven {
+        // com.blackducksoftware.bdio:bdio2
+        url = uri("https://sig-repo.synopsys.com/bds-bdio-release")
+    }
+
+    maven { // com.blackducksoftware.magpie:magpie
+        url = uri("https://repo.blackduck.com/bds-integrations-release")
+    }
 }
 
 tasks.withType<Jar>().configureEach {

gradle/libs.versions.toml

@@ -15,6 +15,8 @@ versionsPlugin = "0.51.0"
 aeSecurity = "0.132.0"
 asciidoctorj = "3.0.0"
 asciidoctorjPdf = "2.3.19"
+blackduckCommon = "67.0.3"
+blackduckCommonApi = "2023.10.0.6"
 clikt = "5.0.2"
 commonsCompress = "1.27.1"
 cyclonedx = "10.0.0"
@@ -25,6 +27,7 @@ exposed = "0.57.0"
 flexmark = "0.64.8"
 freemarker = "2.3.33"
 greenmail = "2.1.2"
+gson = "2.11.0"
 hikari = "6.2.1"
 hoplite = "2.9.0"
 jackson = "2.18.2"
@@ -90,6 +93,8 @@ aeSecurity = { module = "org.metaeffekt.core:ae-security", version.ref = "aeSecu
 asciidoctorj = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" }
 asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorjPdf" }
 awsS3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" }
+blackduck-common = { module = "com.blackduck.integration:blackduck-common", version.ref = "blackduckCommon" }
+blackduck-common-api = { module = "com.blackduck.integration:blackduck-common-api", version.ref = "blackduckCommonApi" }
 clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" }
 commonsCompress = { module = "org.apache.commons:commons-compress", version.ref = "commonsCompress" }
 cyclonedx = { module = "org.cyclonedx:cyclonedx-core-java", version.ref = "cyclonedx" }
@@ -105,6 +110,7 @@ exposed-jdbc = { module = "org.jetbrains.exposed:exposed-jdbc", version.ref = "e
 exposed-json = { module = "org.jetbrains.exposed:exposed-json", version.ref = "exposed" }
 flexmark = { module = "com.vladsch.flexmark:flexmark", version.ref = "flexmark" }
 freemarker = { module = "org.freemarker:freemarker", version.ref = "freemarker" }
+gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
 greenmail = { module = "com.icegreen:greenmail", version.ref = "greenmail" }
 hikari = { module = "com.zaxxer:HikariCP", version.ref = "hikari" }
 hoplite-core = { module = "com.sksamuel.hoplite:hoplite-core", version.ref = "hoplite" }

plugins/advisors/black-duck/build.gradle.kts

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2020 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+plugins {
+    // Apply precompiled plugins.
+    id("ort-plugin-conventions")
+
+    // Apply third-party plugins.
+    alias(libs.plugins.kotlinSerialization)
+}
+
+dependencies {
+    api(projects.advisor)
+    api(projects.model)
+
+    implementation(libs.blackduck.common)
+    implementation(libs.blackduck.common.api)
+    implementation(libs.bundles.ks3)
+    implementation(libs.kotlinx.serialization.core)
+    implementation(libs.kotlinx.serialization.json)
+
+    implementation(projects.utils.commonUtils)
+    implementation(projects.utils.ortUtils)
+
+    funTestImplementation(libs.gson)
+
+    ksp(projects.advisor)
+}

plugins/advisors/black-duck/src/funTest/assets/retrieve-package-findings-expected-result.yml

@@ -0,0 +1,29 @@
+---
+Crate::sys-info:0.7.0:
+  advisor:
+    name: "BlackDuck"
+    capabilities:
+    - "VULNERABILITIES"
+  summary:
+    start_time: "1970-01-01T00:00:00Z"
+    end_time: "1970-01-01T00:00:00Z"
+  vulnerabilities:
+  - id: "CVE-2020-36434"
+    description: "An issue was discovered in the sys-info crate before 0.8.0 for Rust.\
+      \ sys_info::disk_info calls can trigger a double free."
+    references:
+    - url: "https://BLACK_DUCK_SERVER_HOST/api/vulnerabilities/CVE-2020-36434"
+      scoring_system: "CVSS:3.1"
+      severity: "CRITICAL"
+      score: 9.8
+      vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
+    - url: "https://BLACK_DUCK_SERVER_HOST/api/cwes/CWE-415"
+      scoring_system: "CVSS:3.1"
+      severity: "CRITICAL"
+      score: 9.8
+      vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
+    - url: "https://BLACK_DUCK_SERVER_HOST/api/vulnerabilities/BDSA-2020-4804"
+      scoring_system: "CVSS:3.1"
+      severity: "CRITICAL"
+      score: 9.8
+      vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"

plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt

@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.advisors.blackduck
+
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.inspectors.forAll
+import io.kotest.matchers.collections.beEmpty
+import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
+import io.kotest.matchers.shouldBe
+import io.kotest.matchers.shouldNot
+
+import java.time.Instant
+
+import org.ossreviewtoolkit.advisor.normalizeVulnerabilityData
+import org.ossreviewtoolkit.model.AdvisorResult
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.readValue
+import org.ossreviewtoolkit.model.toYaml
+import org.ossreviewtoolkit.utils.common.Os
+import org.ossreviewtoolkit.utils.test.getAssetFile
+import org.ossreviewtoolkit.utils.test.identifierToPackage
+
+class BlackDuckFunTest : WordSpec({
+    /**
+     * To run the test against a real instance, and / or to re-record the responses:
+     *
+     * 1. Define the below environment variables: BLACK_DUCK_SERVER_URL and BLACK_DUCK_API_TOKEN
+     * 2. Delete 'recorded-responses.json'
+     * 3. Run the functional test.
+     */
+    val serverUrl = Os.env["BLACK_DUCK_SERVER_URL"]
+    val apiToken = Os.env["BLACK_DUCK_API_TOKEN"]
+    val componentServiceClient = ResponseCachingComponentServiceClient(
+        overrideFile = getAssetFile("recorded-responses.json"),
+        serverUrl = serverUrl,
+        apiToken = apiToken
+    )
+
+    val blackDuck = BlackDuck(BlackDuckFactory.descriptor, componentServiceClient)
+
+    afterEach { componentServiceClient.flush() }
+
+    "retrievePackageFindings()" should {
+        "return the vulnerabilities for the supported ecosystems" {
+            val packages = setOf(
+                // TODO: Add hackage / pod
+                "Crate::sys-info:0.7.0",
+                "Gem::rack:2.0.4",
+                "Maven:com.jfinal:jfinal:1.4",
+                "NPM::rebber:1.0.0",
+                "NuGet::Bunkum:4.0.0",
+                "Pub::http:0.13.1",
+                "PyPI::django:3.2"
+            ).mapTo(mutableSetOf()) {
+                identifierToPackage(it)
+            }
+
+            val packageFindings = blackDuck.retrievePackageFindings(packages).mapKeys { it.key.id.toCoordinates() }
+
+            packageFindings.keys shouldContainExactlyInAnyOrder packages.map { it.id.toCoordinates() }
+            packageFindings.keys.forAll { id ->
+                packageFindings.getValue(id).vulnerabilities shouldNot beEmpty()
+            }
+        }
+
+        "return the expected result for the given package(s)" {
+            val expectedResult = getAssetFile("retrieve-package-findings-expected-result.yml")
+                .readValue<Map<Identifier, AdvisorResult>>()
+            val packages = setOf(
+                // Package using CVSS 3.1 vector:
+                "Crate::sys-info:0.7.0"
+                // Todo: Add a package using CVSS 2 vector:
+            ).mapTo(mutableSetOf()) {
+                identifierToPackage(it)
+            }
+
+            val packageFindings = blackDuck.retrievePackageFindings(packages).mapKeys { it.key.id }
+
+            packageFindings.patchTimes().toYaml().patchServerUrl(serverUrl) shouldBe
+                expectedResult.patchTimes().toYaml()
+        }
+    }
+})
+
+internal fun String.patchServerUrl(serverUrl: String?) =
+    serverUrl?.let { replace(it, "https://BLACK_DUCK_SERVER_HOST") } ?: this
+
+private fun Map<Identifier, AdvisorResult>.patchTimes(): Map<Identifier, AdvisorResult> =
+    mapValues { (_, advisorResult) ->
+        advisorResult.normalizeVulnerabilityData().copy(
+            summary = advisorResult.summary.copy(
+                startTime = Instant.EPOCH,
+                endTime = Instant.EPOCH
+            )
+        )
+    }

plugins/advisors/black-duck/src/funTest/kotlin/ResponseCachingComponentServiceClient.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.advisors.blackduck
+
+import com.blackduck.integration.blackduck.api.generated.response.ComponentsView
+import com.blackduck.integration.blackduck.api.generated.view.OriginView
+import com.blackduck.integration.blackduck.api.generated.view.VulnerabilityView
+
+import com.google.gson.GsonBuilder
+
+import java.io.File
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * This ComponentServiceClient uses a cache for responses. The cache is initialized with the content from a preceding
+ * run in the given overrideFile.
+ *
+ * Note: In case the cache initially contains all responses for a particular test, an instance of this class can be used
+ * as a fake ComponentServiceClient.
+ */
+internal class ResponseCachingComponentServiceClient(
+    private val overrideFile: File,
+    private val serverUrl: String?,
+    apiToken: String?
+) : ComponentServiceClient {
+    // The BlackDuck library uses GSON to serialize its POJOs. So use GSON too because this is the simplest option.
+    private val gson = GsonBuilder().setPrettyPrinting().create()
+
+    private val cache = if (overrideFile.isFile) {
+        gson.fromJson(overrideFile.readText(), ResponseCache::class.java)
+    } else {
+        ResponseCache()
+    }
+
+    private val delegate = if (serverUrl != null && apiToken != null) {
+        ExtendedComponentService.create(serverUrl, apiToken)
+    } else {
+        null
+    }
+
+    override fun searchKbComponentsByPurl(purl: String): List<ComponentsView> =
+        cache.componentsViewsForPurl.getOrPut(purl) {
+            delegate?.searchKbComponentsByPurl(purl).orEmpty()
+        }
+
+    override fun getOriginView(searchResult: ComponentsView): OriginView? =
+        cache.originViewForOriginUuid.getOrPut(searchResult.originId) {
+            delegate?.getOriginView(searchResult)
+        }
+
+    override fun getVulnerabilities(originView: OriginView): List<VulnerabilityView> =
+        cache.vulnerabilitiesForOriginUuid.getOrPut(originView.originId) {
+            delegate?.getVulnerabilities(originView).orEmpty()
+        }
+
+    fun flush() {
+        if (delegate != null) {
+            val json = gson.toJson(cache).patchServerUrl(serverUrl)
+            overrideFile.writeText(json)
+        }
+    }
+}
+
+private class ResponseCache {
+    val componentsViewsForPurl = ConcurrentHashMap<String, List<ComponentsView>>()
+    val originViewForOriginUuid = ConcurrentHashMap<String, OriginView?>()
+    val vulnerabilitiesForOriginUuid = ConcurrentHashMap<String, List<VulnerabilityView>>()
+}