feat(scanner): Add flag to scanner to detect unlicensed files

Add flag `includeUnlicensed` to the scanner configuration. Its default is `false`. When set to `true`, the scanner add to a `ScanResult` files without license as LicenseFindings with license set to `NONE`. This contribution makes possible to the scanner to display all files as license findings. The ultimate goal is that any file without license is catched by the scanner, so that curation mechanism can override files without licenses in cases where a license applies to a whole folder. Signed-off-by: Kiko Fernandez-Reyes <[email protected]>
oss-review-toolkit · Jan 21, 2025 · c737701 · c737701
1 parent d26dd73
commit c737701
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 1 deletion.
diff --git a/model/src/main/kotlin/config/ScannerConfiguration.kt b/model/src/main/kotlin/config/ScannerConfiguration.kt
@@ -44,6 +44,11 @@ data class ScannerConfiguration(
      */
     val skipExcluded: Boolean = false,
 
+    /**
+     * A flag to indicate whether the scanner should add files without license to the scanner results.
+     */
+    val includeFilesWithoutFindings: Boolean = false,
+
     /**
      * Configuration of a [FileArchiver] that archives certain scanned files in an external [FileStorage].
      */

diff --git a/scanner/src/funTest/assets/scanner-integration-all-pkgs-expected-ort-result.yml b/scanner/src/funTest/assets/scanner-integration-all-pkgs-expected-ort-result.yml
@@ -197,6 +197,7 @@ scanner:
   config:
     skip_concluded: false
     skip_excluded: false
+    include_files_without_findings: false
     detected_license_mapping:
       LicenseRef-scancode-agpl-generic-additional-terms: "NOASSERTION"
       LicenseRef-scancode-free-unknown: "NOASSERTION"

diff --git a/scanner/src/funTest/assets/scanner-integration-subset-pkgs-expected-ort-result.yml b/scanner/src/funTest/assets/scanner-integration-subset-pkgs-expected-ort-result.yml
@@ -116,6 +116,7 @@ scanner:
   config:
     skip_concluded: false
     skip_excluded: false
+    include_files_without_findings: false
     detected_license_mapping:
       LicenseRef-scancode-agpl-generic-additional-terms: "NOASSERTION"
       LicenseRef-scancode-free-unknown: "NOASSERTION"

diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt
@@ -36,13 +36,16 @@ import org.ossreviewtoolkit.model.FileList
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.KnownProvenance
+import org.ossreviewtoolkit.model.LicenseFinding
 import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.ProvenanceResolutionResult
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerRun
+import org.ossreviewtoolkit.model.TextLocation
+import org.ossreviewtoolkit.model.TextLocation.Companion.UNKNOWN_LINE
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
@@ -67,6 +70,7 @@ import org.ossreviewtoolkit.utils.common.collectMessages
 import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
 import org.ossreviewtoolkit.utils.ort.Environment
 import org.ossreviewtoolkit.utils.ort.showStackTrace
+import org.ossreviewtoolkit.utils.spdx.SpdxConstants
 import org.ossreviewtoolkit.utils.spdx.toSpdx
 
 class Scanner(
@@ -205,13 +209,36 @@ class Scanner(
             }
         }
 
+        val scanResults = if (scannerConfig.includeFilesWithoutFindings) {
+            filteredScanResults.mapTo(mutableSetOf()) { scanResult ->
+                val allPaths = controller.getAllFileLists()[scanResult.provenance]?.files?.mapTo(mutableSetOf()) {
+                    it.path
+                }.orEmpty()
+
+                val pathsWithFindings = scanResult.summary.licenseFindings.mapTo(mutableSetOf()) { it.location.path }
+                val pathsWithoutFindings = allPaths - pathsWithFindings
+
+                val findingsThatAreNone = pathsWithoutFindings.mapTo(mutableSetOf()) {
+                    LicenseFinding(SpdxConstants.NONE, TextLocation(it, UNKNOWN_LINE))
+                }
+
+                scanResult.copy(
+                    summary = scanResult.summary.copy(
+                        licenseFindings = scanResult.summary.licenseFindings + findingsThatAreNone
+                    )
+                )
+            }
+        } else {
+            filteredScanResults
+        }
+
         val scannerNames = scannerWrappers.mapTo(mutableSetOf()) { it.name }
         val scanners = packages.associateBy({ it.id }) { scannerNames }
 
         return ScannerRun.EMPTY.copy(
             config = scannerConfig,
             provenances = provenances,
-            scanResults = filteredScanResults,
+            scanResults = scanResults,
             files = files,
             scanners = scanners
         )