Skip to content

Commit

Permalink
fix: electron-builder fails when list of node_modules files is too bi…
Browse files Browse the repository at this point in the history 
…g to pass in a glob (#8725)

fix #8705

**root cause**

When listing all files, if there are too many files, it will trigger
glob's length limit.

**how to fix**

If all files in a folder are unpacked, use the folder name directly
instead of listing all files.
  • Loading branch information
@beyondkmp
beyondkmp authored Jan 14, 2025
1 parent 6a9597b commit ccbf0a5
Show file tree
Hide file tree
Showing 6 changed files with 126 additions and 4 deletions.
5 changes: 5 additions & 0 deletions .changeset/khaki-buckets-jog.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"app-builder-lib": patch
---

fix: electron-builder fails when list of node_modules files is too big to pass in a glob
17 changes: 14 additions & 3 deletions packages/app-builder-lib/src/asar/asarUtil.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,10 @@ export class AsarPackager {
const links: Array<Link> = []
const symlinkType = platform() === "win32" ? "junction" : "file"

const matchUnpacker = (file: string, dest: string, stat: fs.Stats) => {
const matchUnpacker = (file: string, dest: string, stat: fs.Stats, tmpUnpackedPaths: Set<string>) => {
if (this.config.unpackPattern?.(file, stat)) {
log.debug({ file }, "unpacking")
unpackedPaths.add(dest)
tmpUnpackedPaths.add(dest)
return
}
}
Expand Down Expand Up @@ -145,6 +145,7 @@ export class AsarPackager {
}

// Don't use BluebirdPromise, we need to retain order of execution/iteration through the ordered fileset
const tmpUnpackedPaths = new Set<string>()
for (let i = 0; i < fileSet.files.length; i++) {
const file = fileSet.files[i]
const transformedData = fileSet.transformedFiles?.get(i)
Expand All @@ -153,13 +154,23 @@ export class AsarPackager {
const relative = path.relative(this.config.defaultDestination, getDestinationPath(file, fileSet))
const destination = path.resolve(this.rootForAppFilesWithoutAsar, relative)

matchUnpacker(file, destination, stat)
matchUnpacker(file, destination, stat, tmpUnpackedPaths)
taskManager.addTask(writeFileOrProcessSymlink({ transformedData, file, destination, stat, fileSet }))

if (taskManager.tasks.length > MAX_FILE_REQUESTS) {
await taskManager.awaitTasks()
}
}

if (tmpUnpackedPaths.size === fileSet.files.length) {
const relative = path.relative(this.config.defaultDestination, fileSet.destination)
unpackedPaths.add(relative)
} else {
// add all tmpUnpackedPaths to unpackedPaths
for (const it of tmpUnpackedPaths) {
unpackedPaths.add(it)
}
}
}
// finish copy then set up all symlinks
await taskManager.awaitTasks()
Expand Down
2 changes: 1 addition & 1 deletion packages/app-builder-lib/src/targets/nsis/NsisTarget.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export class NsisTarget extends Target {
const setupText = this.isPortable ? "" : "Setup "
const archSuffix = !this.shouldBuildUniversalInstaller && primaryArch != null ? getArchSuffix(primaryArch, defaultArch) : ""

return "${productName} " + setupText + "${version}" + archSuffix + ".${ext}";
return "${productName} " + setupText + "${version}" + archSuffix + ".${ext}"
}

private get isPortable(): boolean {
Expand Down
1 change: 1 addition & 0 deletions test/snapshots/HoistedNodeModuleTest.js.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ exports[`yarn several workspaces and asarUnpack 2`] = `
"unpacked": true,
},
},
"unpacked": true,
},
},
},
Expand Down
13 changes: 13 additions & 0 deletions test/snapshots/globTest.js.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,24 @@ exports[`asarUnpack node_modules 2`] = `
"unpacked": true,
},
},
"unpacked": true,
},
},
}
`;

exports[`asarUnpack node_modules which has many modules 1`] = `
{
"linux": [],
}
`;

exports[`exclude some modules when asarUnpack node_modules which has many modules 1`] = `
{
"linux": [],
}
`;

exports[`failed peer dep 1`] = `
{
"linux": [],
Expand Down
92 changes: 92 additions & 0 deletions test/src/globTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,3 +265,95 @@ test.ifAll.ifDevOrLinuxCi("asarUnpack node_modules", () => {
}
)
})

test.ifAll.ifDevOrLinuxCi("asarUnpack node_modules which has many modules", () => {
return assertPack(
"test-app-one",
{
targets: Platform.LINUX.createTarget(DIR_TARGET),
config: {
asarUnpack: "node_modules",
},
},
{
isInstallDepsBefore: true,
projectDirCreated: projectDir =>
modifyPackageJson(projectDir, data => {
data.dependencies = {
"@react-navigation/stack": "^6.3.7",
"@sentry/electron": "^4.4.0",
"axios": "^1.1.3",
"deep-equal": "^2.1.0",
"dotenv": "^16.4.5",
"electron-log": "^4.4.8",
"electron-updater": "^6.0.4",
"electron-window-state": "^5.0.3",
"jwt-decode": "^3.1.2",
"keytar": "^7.9.0",
"webpack": "^5.74.0",
"pubsub-js": "^1.9.4",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-native-web": "^0.18.10",
"react-router-dom": "^6.4.0",
"source-map-support": "^0.5.16",
"yargs": "^16.2.0",
"ci-info": "2.0.0",
}
}),
packed: async context => {
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/jwt-decode")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/keytar")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/yargs")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/@sentry/electron")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/ci-info")).isDirectory()
},
}
)
})

test.ifAll.ifDevOrLinuxCi("exclude some modules when asarUnpack node_modules which has many modules", () => {
return assertPack(
"test-app-one",
{
targets: Platform.LINUX.createTarget(DIR_TARGET),
config: {
asarUnpack: ["node_modules","!**/node_modules/ci-info/**/*"]
},
},
{
isInstallDepsBefore: true,
projectDirCreated: projectDir =>
modifyPackageJson(projectDir, data => {
data.dependencies = {
"@react-navigation/stack": "^6.3.7",
"@sentry/electron": "^4.4.0",
"axios": "^1.1.3",
"deep-equal": "^2.1.0",
"dotenv": "^16.4.5",
"electron-log": "^4.4.8",
"electron-updater": "^6.0.4",
"electron-window-state": "^5.0.3",
"jwt-decode": "^3.1.2",
"keytar": "^7.9.0",
"webpack": "^5.74.0",
"pubsub-js": "^1.9.4",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-native-web": "^0.18.10",
"react-router-dom": "^6.4.0",
"source-map-support": "^0.5.16",
"yargs": "^16.2.0",
"ci-info": "2.0.0",
}
}),
packed: async context => {
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/jwt-decode")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/keytar")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/yargs")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/@sentry/electron")).isDirectory()
await assertThat(path.join(context.getResources(Platform.LINUX), "app.asar.unpacked/node_modules/ci-info")).doesNotExist()
},
}
)
})

0 comments on commit ccbf0a5

Please sign in to comment.