Skip to content

Commit

Permalink
Rewrite userdev setup pipeline
Browse files Browse the repository at this point in the history
Uses a graph for ordering steps instead of manual ordering, improved input/output hashing and up-to-date checking.

Except for the final outputs which are keyed to the dev bundle, steps are independent of the dev bundle and can be reused between bundles that have the same inputs for the step. For example when using a Paper and Folia 1.21.4 dev bundle in two projects, decompile will only run once as only the final apply dev bundle patches step differs.

Final outputs are copied into a project local location to avoid projects sharing a bundle from having a task with a shared output location.
  • Loading branch information
jpenilla committed Dec 26, 2024
1 parent fa56235 commit 43f2c06
Show file tree
Hide file tree
Showing 48 changed files with 2,440 additions and 1,811 deletions.
4 changes: 0 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,3 @@ ehthumbs_vista.db
!gradle-wrapper.ja

/.kotlin/

# todo remove again
/test/
/testfork/
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import io.papermc.paperweight.util.constants.*
import java.nio.file.Path
import java.util.concurrent.ConcurrentHashMap
import kotlin.io.path.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
Expand Down Expand Up @@ -63,21 +63,24 @@ abstract class IndexLibraryFiles : BaseTask() {

@TaskAction
fun run() {
val possible = findPossibleLibraryImports(libraries.sourcesJars())
.groupBy { it.libraryFileName }
.mapValues {
it.value.map { v -> v.importFilePath }
}
val possible = ioDispatcher("IndexLibraryFiles").use { dispatcher ->
findPossibleLibraryImports(libraries.sourcesJars(), dispatcher)
.groupBy { it.libraryFileName }
.mapValues {
it.value.map { v -> v.importFilePath }
}
}

outputFile.path.cleanFile().outputStream().gzip().bufferedWriter().use { writer ->
gson.toJson(possible, writer)
}
}

private fun findPossibleLibraryImports(libFiles: List<Path>): Collection<LibraryImport> = runBlocking {
private fun findPossibleLibraryImports(libFiles: List<Path>, dispatcher: CoroutineDispatcher): Collection<LibraryImport> = runBlocking {
val found = ConcurrentHashMap.newKeySet<LibraryImport>()
val suffix = ".java"
libFiles.forEach { libFile ->
launch(Dispatchers.IO) {
launch(dispatcher) {
libFile.openZipSafe().use { zipFile ->
zipFile.walkSequence()
.filter { it.isRegularFile() && it.name.endsWith(suffix) }
Expand Down Expand Up @@ -125,14 +128,17 @@ abstract class ImportLibraryFiles : BaseTask() {
gson.fromJson<Map<String, List<String>>>(reader, typeToken<Map<String, List<String>>>())
}.flatMap { entry -> entry.value.map { LibraryImport(entry.key, it) } }.toSet()
val patchFiles = patches.files.flatMap { it.toPath().filesMatchingRecursive("*.patch") }
importLibraryFiles(
patchFiles,
devImports.pathOrNull,
outputDir.path,
libraries.sourcesJars(),
index,
true
)
ioDispatcher("ImportLibraryFiles").use { dispatcher ->
importLibraryFiles(
patchFiles,
devImports.pathOrNull,
outputDir.path,
libraries.sourcesJars(),
index,
true,
dispatcher,
)
}
}
}

Expand All @@ -143,16 +149,17 @@ abstract class ImportLibraryFiles : BaseTask() {
libFiles: List<Path>,
index: Set<LibraryImport>,
printOutput: Boolean,
dispatcher: CoroutineDispatcher,
) = runBlocking {
// Import library classes
val allImports = findLibraryImports(importsFile, libFiles, index, patches)
val allImports = findLibraryImports(importsFile, libFiles, index, patches, dispatcher)
val importsByLib = allImports.groupBy { it.libraryFileName }
logger.log(if (printOutput) LogLevel.LIFECYCLE else LogLevel.DEBUG, "Importing {} classes from library sources...", allImports.size)

for ((libraryFileName, imports) in importsByLib) {
val libFile = libFiles.firstOrNull { it.name == libraryFileName }
?: throw PaperweightException("Failed to find library: $libraryFileName for classes ${imports.map { it.importFilePath }}")
launch(Dispatchers.IO) {
launch(dispatcher) {
libFile.openZipSafe().use { zipFile ->
for (import in imports) {
val outputFile = targetDir.resolve(import.importFilePath)
Expand All @@ -169,9 +176,9 @@ abstract class ImportLibraryFiles : BaseTask() {
}
}

private suspend fun usePatchLines(patches: Iterable<Path>, consumer: (String) -> Unit) = coroutineScope {
private suspend fun usePatchLines(patches: Iterable<Path>, dispatcher: CoroutineDispatcher, consumer: (String) -> Unit) = coroutineScope {
for (patch in patches) {
launch(Dispatchers.IO) {
launch(dispatcher) {
patch.useLines { lines ->
lines.forEach { consumer(it) }
}
Expand All @@ -183,7 +190,8 @@ abstract class ImportLibraryFiles : BaseTask() {
libraryImports: Path?,
libFiles: List<Path>,
index: Set<LibraryImport>,
patchFiles: Iterable<Path>
patchFiles: Iterable<Path>,
dispatcher: CoroutineDispatcher,
): Set<LibraryImport> {
val result = hashSetOf<LibraryImport>()

Expand All @@ -200,19 +208,20 @@ abstract class ImportLibraryFiles : BaseTask() {
}

// Scan patches for necessary imports
result += findNeededLibraryImports(patchFiles, index)
result += findNeededLibraryImports(patchFiles, index, dispatcher)

return result
}

private suspend fun findNeededLibraryImports(
patchFiles: Iterable<Path>,
index: Set<LibraryImport>,
dispatcher: CoroutineDispatcher,
): Set<LibraryImport> {
val knownImportMap = index.associateBy { it.importFilePath }
val prefix = "+++ b/"
val needed = ConcurrentHashMap.newKeySet<LibraryImport>()
usePatchLines(patchFiles) { line ->
usePatchLines(patchFiles, dispatcher) { line ->
if (!line.startsWith(prefix)) {
return@usePatchLines
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ import java.time.format.DateTimeFormatter
import java.util.Locale
import java.util.concurrent.TimeUnit
import kotlin.io.path.*
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.apache.http.HttpHost
import org.apache.http.HttpStatus
import org.apache.http.client.config.CookieSpecs
Expand Down Expand Up @@ -71,12 +69,6 @@ abstract class DownloadService : BuildService<DownloadService.Params>, AutoClose
download(url, file, hash)
}

suspend fun downloadAsync(source: Any, target: Any, hash: Hash? = null) = coroutineScope {
async {
download(source.convertToUrl(), target.convertToPath(), hash, false)
}
}

private fun download(source: URL, target: Path, hash: Hash?, retry: Boolean = false) {
download(source, target)
if (hash == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ fun macheDecompileJar(
) {
val out = outputJar.cleanFile()

val cfgFile = out.resolveSibling("${out.name}.cfg")
val cfgFile = workDir.resolve("${out.name}.cfg")
val cfgText = buildString {
for (file in minecraftClasspath) {
append("-e=")
Expand All @@ -98,7 +98,7 @@ fun macheDecompileJar(
}
cfgFile.writeText(cfgText)

val logs = out.resolveSibling("${out.name}.log")
val logs = workDir.resolve("${out.name}.log")

val args = mutableListOf<String>()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ fun macheRemapJar(
) {
val out = outputJar.cleanFile()

val logFile = out.resolveSibling("${out.name}.log")
val logFile = tempDir.resolve("${out.name}.log")

val args = mutableListOf<String>()

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*
* paperweight is a Gradle plugin for the PaperMC project.
*
* Copyright (c) 2023 Kyle Wood (DenWav)
* Contributors
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 only, no later versions.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
* USA
*/

package io.papermc.paperweight.util

import io.papermc.paperweight.PaperweightException
import java.nio.file.Path
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
import kotlin.io.path.*
import org.gradle.api.logging.Logger
import org.gradle.api.logging.Logging

private val openCurrentJvm: MutableMap<Path, ReentrantLock> = mutableMapOf()

fun <R> withLock(
lockFile: Path,
printInfoAfter: Long = 1000 * 60 * 5, // 5 minutes
timeoutMs: Long = 1000L * 60 * 60, // one hour
action: () -> R,
): R {
val logger = Logging.getLogger("paperweight lock file")

var waitedMs = 0L
var firstFailedAcquire = true
while (true) {
val normalized = lockFile.normalize().absolute()

val lock = synchronized(openCurrentJvm) {
openCurrentJvm.computeIfAbsent(normalized) { ReentrantLock() }
}
if (!lock.tryLock()) {
if (firstFailedAcquire) {
logger.lifecycle("Lock for '$lockFile' is currently held by another thread.")
logger.lifecycle("Waiting for lock to be released...")
firstFailedAcquire = false
}
val startWait = System.nanoTime()
val acquired = lock.tryLock(printInfoAfter, TimeUnit.MILLISECONDS)
if (!acquired) {
waitedMs += (System.nanoTime() - startWait) / 1_000_000
if (waitedMs >= timeoutMs) {
throw PaperweightException("Have been waiting on lock for '$lockFile' for $waitedMs ms. Giving up as timeout is $timeoutMs ms.")
}
logger.lifecycle(
"Have been waiting on lock for '$lockFile' for ${waitedMs / 1000 / 60} minute(s).\n" +
"If this persists for an unreasonable length of time, kill this process, run './gradlew --stop', and then try again."
)
}
}
val cont = synchronized(openCurrentJvm) {
if (openCurrentJvm[normalized] !== lock) {
lock.unlock()
true
} else {
false
}
}
if (cont) {
continue
}

try {
acquireProcessLockWaiting(lockFile, logger, waitedMs, printInfoAfter, timeoutMs)
try {
return action()
} finally {
lockFile.deleteForcefully()
}
} finally {
synchronized(openCurrentJvm) {
lock.unlock()
openCurrentJvm.remove(normalized)
}
}
}
}

// TODO: Open an actual exclusive lock using FileChannel
private fun acquireProcessLockWaiting(
lockFile: Path,
logger: Logger,
alreadyWaited: Long = 0,
printInfoAfter: Long,
timeoutMs: Long,
) {
val currentPid = ProcessHandle.current().pid()

if (lockFile.exists()) {
val lockingProcessId = lockFile.readText().toLong()
if (lockingProcessId == currentPid) {
throw IllegalStateException("Lock file '$lockFile' is currently held by this process.")
} else {
logger.lifecycle("Lock file '$lockFile' is currently held by pid '$lockingProcessId'.")
}

if (ProcessHandle.of(lockingProcessId).isEmpty) {
logger.lifecycle("Locking process does not exist, assuming abrupt termination and deleting lock file.")
lockFile.deleteIfExists()
} else {
logger.lifecycle("Waiting for lock to be released...")
var sleptMs: Long = alreadyWaited
while (lockFile.exists()) {
Thread.sleep(100)
sleptMs += 100
if (sleptMs >= printInfoAfter && sleptMs % printInfoAfter == 0L) {
logger.lifecycle(
"Have been waiting on lock file '$lockFile' held by pid '$lockingProcessId' for ${sleptMs / 1000 / 60} minute(s).\n" +
"If this persists for an unreasonable length of time, kill this process, run './gradlew --stop' and then try again.\n" +
"If the problem persists, the lock file may need to be deleted manually."
)
}
if (sleptMs >= timeoutMs) {
throw PaperweightException("Have been waiting on lock file '$lockFile' for $sleptMs ms. Giving up as timeout is $timeoutMs ms.")
}
}
}
}

if (!lockFile.parent.exists()) {
lockFile.parent.createDirectories()
}
lockFile.writeText(currentPid.toString())
}
Loading

0 comments on commit 43f2c06

Please sign in to comment.