Skip to content

Commit

Permalink
fix(pnpm): Fix parsing of JSON output for nested projects
Browse files Browse the repository at this point in the history
If PNPM encounters nested projects, the output of the `list` command
is not well-formed JSON, but consists of multiple arrays. Change the
`parsePnpmList()` function to handle this format correctly.

In `Pnpm`, only analyze the top-level project, since nested projects
will be handled by the `pnpm` command transparently.

Fixes #9784.

Signed-off-by: Oliver Heger <[email protected]>
  • Loading branch information
oheger-bosch committed Jan 31, 2025
1 parent c15996b commit 6a6170b
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 4 deletions.
17 changes: 16 additions & 1 deletion plugins/package-managers/node/src/main/kotlin/pnpm/ModuleInfo.kt
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,27 @@

package org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm

import java.io.ByteArrayInputStream

import kotlinx.serialization.Serializable
import kotlinx.serialization.json.DecodeSequenceMode
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeToSequence

private val JSON = Json { ignoreUnknownKeys = true }

internal fun parsePnpmList(json: String): List<ModuleInfo> = JSON.decodeFromString(json)
/**
* Parse the given [json] output of a PNPM list command. Normally, the resulting [Sequence] contains only a single
* [List] with the [ModuleInfo] objects of the project. If there are nested projects, PNPM outputs multiple arrays,
* which leads to syntactically invalid JSON. This is handled by this function by returning a [Sequence] with a
* corresponding number of elements. In this case, callers are responsible for correctly mapping the elements to
* projects.
*/
internal fun parsePnpmList(json: String): Sequence<List<ModuleInfo>> =
JSON.decodeToSequence<List<ModuleInfo>>(
ByteArrayInputStream(json.toByteArray()),
DecodeSequenceMode.WHITESPACE_SEPARATED
)

@Serializable
data class ModuleInfo(
Expand Down
26 changes: 23 additions & 3 deletions plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
import org.ossreviewtoolkit.utils.common.CommandLineTool
import org.ossreviewtoolkit.utils.common.DirectoryStash
import org.ossreviewtoolkit.utils.common.Os
import org.ossreviewtoolkit.utils.common.nextOrNull

import org.semver4j.RangesList
import org.semver4j.RangesListFactory
Expand Down Expand Up @@ -117,10 +118,11 @@ class Pnpm(
val json = PnpmCommand.run(workingDir, "list", "--json", "--only-projects", "--recursive").requireSuccess()
.stdout

return parsePnpmList(json).mapTo(mutableSetOf()) { File(it.path) }
val listResult = parsePnpmList(json)
return listResult.findModulesFor(workingDir).mapTo(mutableSetOf()) { File(it.path) }
}

private fun listModules(workingDir: File, scope: Scope): List<ModuleInfo> {
private fun listModules(workingDir: File, scope: Scope): Sequence<ModuleInfo> {
val scopeOption = when (scope) {
Scope.DEPENDENCIES -> "--prod"
Scope.DEV_DEPENDENCIES -> "--dev"
Expand All @@ -129,7 +131,7 @@ class Pnpm(
val json = PnpmCommand.run(workingDir, "list", "--json", "--recursive", "--depth", "Infinity", scopeOption)
.requireSuccess().stdout

return parsePnpmList(json)
return parsePnpmList(json).flatten()
}

private fun installDependencies(workingDir: File) =
Expand Down Expand Up @@ -170,3 +172,21 @@ private fun ModuleInfo.getScopeDependencies(scope: Scope) =

Scope.DEV_DEPENDENCIES -> devDependencies.values.toList()
}

/**
* Find the [List] of [ModuleInfo] objects for the project in the given [workingDir]. If there are nested projects,
* the `pnpm list` command yields multiple arrays with modules. In this case, only the top-level project should be
* analyzed. This function tries to detect the corresponding [ModuleInfo]s based on the [workingDir]. If this is not
* possible, as a fallback the first list of [ModuleInfo] objects is returned.
*/
private fun Sequence<List<ModuleInfo>>.findModulesFor(workingDir: File): List<ModuleInfo> {
val moduleInfoIterator = iterator()
val first = moduleInfoIterator.nextOrNull() ?: return emptyList()

fun List<ModuleInfo>.matchesWorkingDir() = any { File(it.path).absoluteFile == workingDir }

fun findMatchingModules(): List<ModuleInfo>? =
moduleInfoIterator.nextOrNull()?.takeIf { it.matchesWorkingDir() } ?: findMatchingModules()

return first.takeIf { it.matchesWorkingDir() } ?: findMatchingModules() ?: first
}
28 changes: 28 additions & 0 deletions plugins/package-managers/node/src/test/assets/pnpm-list.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[
{
"name": "some-project",
"version": "1.0.0",
"path": "/tmp/work/root",
"private": false,
"dependencies": {
"eslint-scope": {
"from": "eslint-scope",
"version": "link:node_modules/.pnpm/[email protected]/node_modules/eslint-scope",
"path": "/tmp/work/root/node_modules/.pnpm/[email protected]/node_modules/eslint-scope"
}
}
},
{
"name": "other-project",
"version": "1.0.1",
"path": "/tmp/work/other_root",
"private": false,
"dependencies": {
"@types/eslint": {
"from": "@types/eslint",
"version": "link:node_modules/.pnpm/@[email protected]/node_modules/@types/eslint",
"path": "/tmp/work/other_root/node_modules/.pnpm/@[email protected]/node_modules/@types/eslint"
}
}
}
]
24 changes: 24 additions & 0 deletions plugins/package-managers/node/src/test/assets/pnpm-multi-list.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[
{
"name": "outer-project",
"version": "1.0.0",
"path": "/tmp/work/top",
"private": false,
"dependencies": {
"eslint-scope": {
"from": "eslint-scope",
"version": "link:node_modules/.pnpm/[email protected]/node_modules/eslint-scope",
"path": "/tmp/work/top/node_modules/.pnpm/[email protected]/node_modules/eslint-scope"
}
}
}
]

[
{
"name": "nested-project",
"version": "1.0.0",
"path": "/tmp/work/top/nested",
"private": false
}
]
103 changes: 103 additions & 0 deletions plugins/package-managers/node/src/test/kotlin/pnpm/ModuleInfoTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* Copyright (C) 2025 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm

import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.sequences.shouldContainExactly

import java.io.File

class ModuleInfoTest : WordSpec({
"parsePnpmList()" should {
"handle normal PNPM output" {
val input = File("src/test/assets/pnpm-list.json").readText()

val expectedResults = sequenceOf(
listOf(
ModuleInfo(
name = "some-project",
version = "1.0.0",
path = "/tmp/work/root",
private = false,
dependencies = mapOf(
"eslint-scope" to ModuleInfo.Dependency(
from = "eslint-scope",
version = "link:node_modules/.pnpm/[email protected]/node_modules/eslint-scope",
path = "/tmp/work/root/node_modules/.pnpm/[email protected]/node_modules/eslint-scope"
)
)
),
ModuleInfo(
name = "other-project",
version = "1.0.1",
path = "/tmp/work/other_root",
private = false,
dependencies = mapOf(
"@types/eslint" to ModuleInfo.Dependency(
from = "@types/eslint",
version = "link:node_modules/.pnpm/@[email protected]/node_modules/@types/eslint",
path = "/tmp/work/other_root/node_modules/.pnpm/@[email protected]" +
"/node_modules/@types/eslint"
)
)
)
)
)

val moduleInfos = parsePnpmList(input)

moduleInfos shouldContainExactly expectedResults
}

"handle multiple JSON arrays" {
val input = File("src/test/assets/pnpm-multi-list.json").readText()

val expectedResults = sequenceOf(
listOf(
ModuleInfo(
name = "outer-project",
version = "1.0.0",
path = "/tmp/work/top",
private = false,
dependencies = mapOf(
"eslint-scope" to ModuleInfo.Dependency(
from = "eslint-scope",
version = "link:node_modules/.pnpm/[email protected]/node_modules/eslint-scope",
path = "/tmp/work/top/node_modules/.pnpm/[email protected]/node_modules/eslint-scope"
)
)
)
),
listOf(
ModuleInfo(
name = "nested-project",
version = "1.0.0",
path = "/tmp/work/top/nested",
private = false
)
)
)

val moduleInfos = parsePnpmList(input)

moduleInfos shouldContainExactly expectedResults
}
}
})

0 comments on commit 6a6170b

Please sign in to comment.