Rewrite userdev setup pipeline

.gitignore

@@ -45,7 +45,3 @@ ehthumbs_vista.db
 !gradle-wrapper.ja
 
 /.kotlin/
-
-# todo remove again
-/test/
-/testfork/

paperweight-core/src/main/kotlin/io/papermc/paperweight/core/tasks/ImportLibraryFiles.kt

@@ -30,7 +30,7 @@ import io.papermc.paperweight.util.constants.*
 import java.nio.file.Path
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.io.path.*
-import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
@@ -63,21 +63,24 @@ abstract class IndexLibraryFiles : BaseTask() {
 
     @TaskAction
     fun run() {
-        val possible = findPossibleLibraryImports(libraries.sourcesJars())
-            .groupBy { it.libraryFileName }
-            .mapValues {
-                it.value.map { v -> v.importFilePath }
-            }
+        val possible = ioDispatcher("IndexLibraryFiles").use { dispatcher ->
+            findPossibleLibraryImports(libraries.sourcesJars(), dispatcher)
+                .groupBy { it.libraryFileName }
+                .mapValues {
+                    it.value.map { v -> v.importFilePath }
+                }
+        }
+
         outputFile.path.cleanFile().outputStream().gzip().bufferedWriter().use { writer ->
             gson.toJson(possible, writer)
         }
     }
 
-    private fun findPossibleLibraryImports(libFiles: List<Path>): Collection<LibraryImport> = runBlocking {
+    private fun findPossibleLibraryImports(libFiles: List<Path>, dispatcher: CoroutineDispatcher): Collection<LibraryImport> = runBlocking {
         val found = ConcurrentHashMap.newKeySet<LibraryImport>()
         val suffix = ".java"
         libFiles.forEach { libFile ->
-            launch(Dispatchers.IO) {
+            launch(dispatcher) {
                 libFile.openZipSafe().use { zipFile ->
                     zipFile.walkSequence()
                         .filter { it.isRegularFile() && it.name.endsWith(suffix) }
@@ -125,14 +128,17 @@ abstract class ImportLibraryFiles : BaseTask() {
                 gson.fromJson<Map<String, List<String>>>(reader, typeToken<Map<String, List<String>>>())
             }.flatMap { entry -> entry.value.map { LibraryImport(entry.key, it) } }.toSet()
             val patchFiles = patches.files.flatMap { it.toPath().filesMatchingRecursive("*.patch") }
-            importLibraryFiles(
-                patchFiles,
-                devImports.pathOrNull,
-                outputDir.path,
-                libraries.sourcesJars(),
-                index,
-                true
-            )
+            ioDispatcher("ImportLibraryFiles").use { dispatcher ->
+                importLibraryFiles(
+                    patchFiles,
+                    devImports.pathOrNull,
+                    outputDir.path,
+                    libraries.sourcesJars(),
+                    index,
+                    true,
+                    dispatcher,
+                )
+            }
         }
     }
 
@@ -143,16 +149,17 @@ abstract class ImportLibraryFiles : BaseTask() {
         libFiles: List<Path>,
         index: Set<LibraryImport>,
         printOutput: Boolean,
+        dispatcher: CoroutineDispatcher,
     ) = runBlocking {
         // Import library classes
-        val allImports = findLibraryImports(importsFile, libFiles, index, patches)
+        val allImports = findLibraryImports(importsFile, libFiles, index, patches, dispatcher)
         val importsByLib = allImports.groupBy { it.libraryFileName }
         logger.log(if (printOutput) LogLevel.LIFECYCLE else LogLevel.DEBUG, "Importing {} classes from library sources...", allImports.size)
 
         for ((libraryFileName, imports) in importsByLib) {
             val libFile = libFiles.firstOrNull { it.name == libraryFileName }
                 ?: throw PaperweightException("Failed to find library: $libraryFileName for classes ${imports.map { it.importFilePath }}")
-            launch(Dispatchers.IO) {
+            launch(dispatcher) {
                 libFile.openZipSafe().use { zipFile ->
                     for (import in imports) {
                         val outputFile = targetDir.resolve(import.importFilePath)
@@ -169,9 +176,9 @@ abstract class ImportLibraryFiles : BaseTask() {
         }
     }
 
-    private suspend fun usePatchLines(patches: Iterable<Path>, consumer: (String) -> Unit) = coroutineScope {
+    private suspend fun usePatchLines(patches: Iterable<Path>, dispatcher: CoroutineDispatcher, consumer: (String) -> Unit) = coroutineScope {
         for (patch in patches) {
-            launch(Dispatchers.IO) {
+            launch(dispatcher) {
                 patch.useLines { lines ->
                     lines.forEach { consumer(it) }
                 }
@@ -183,7 +190,8 @@ abstract class ImportLibraryFiles : BaseTask() {
         libraryImports: Path?,
         libFiles: List<Path>,
         index: Set<LibraryImport>,
-        patchFiles: Iterable<Path>
+        patchFiles: Iterable<Path>,
+        dispatcher: CoroutineDispatcher,
     ): Set<LibraryImport> {
         val result = hashSetOf<LibraryImport>()
 
@@ -200,19 +208,20 @@ abstract class ImportLibraryFiles : BaseTask() {
         }
 
         // Scan patches for necessary imports
-        result += findNeededLibraryImports(patchFiles, index)
+        result += findNeededLibraryImports(patchFiles, index, dispatcher)
 
         return result
     }
 
     private suspend fun findNeededLibraryImports(
         patchFiles: Iterable<Path>,
         index: Set<LibraryImport>,
+        dispatcher: CoroutineDispatcher,
     ): Set<LibraryImport> {
         val knownImportMap = index.associateBy { it.importFilePath }
         val prefix = "+++ b/"
         val needed = ConcurrentHashMap.newKeySet<LibraryImport>()
-        usePatchLines(patchFiles) { line ->
+        usePatchLines(patchFiles, dispatcher) { line ->
             if (!line.startsWith(prefix)) {
                 return@usePatchLines
             }

paperweight-lib/src/main/kotlin/io/papermc/paperweight/DownloadService.kt

@@ -32,8 +32,6 @@ import java.time.format.DateTimeFormatter
 import java.util.Locale
 import java.util.concurrent.TimeUnit
 import kotlin.io.path.*
-import kotlinx.coroutines.async
-import kotlinx.coroutines.coroutineScope
 import org.apache.http.HttpHost
 import org.apache.http.HttpStatus
 import org.apache.http.client.config.CookieSpecs
@@ -71,12 +69,6 @@ abstract class DownloadService : BuildService<DownloadService.Params>, AutoClose
         download(url, file, hash)
     }
 
-    suspend fun downloadAsync(source: Any, target: Any, hash: Hash? = null) = coroutineScope {
-        async {
-            download(source.convertToUrl(), target.convertToPath(), hash, false)
-        }
-    }
-
     private fun download(source: URL, target: Path, hash: Hash?, retry: Boolean = false) {
         download(source, target)
         if (hash == null) {

paperweight-lib/src/main/kotlin/io/papermc/paperweight/tasks/mache/DecompileJar.kt

@@ -88,7 +88,7 @@ fun macheDecompileJar(
 ) {
     val out = outputJar.cleanFile()
 
-    val cfgFile = out.resolveSibling("${out.name}.cfg")
+    val cfgFile = workDir.resolve("${out.name}.cfg")
     val cfgText = buildString {
         for (file in minecraftClasspath) {
             append("-e=")
@@ -98,7 +98,7 @@ fun macheDecompileJar(
     }
     cfgFile.writeText(cfgText)
 
-    val logs = out.resolveSibling("${out.name}.log")
+    val logs = workDir.resolve("${out.name}.log")
 
     val args = mutableListOf<String>()
 

paperweight-lib/src/main/kotlin/io/papermc/paperweight/tasks/mache/RemapJar.kt

@@ -99,7 +99,7 @@ fun macheRemapJar(
 ) {
     val out = outputJar.cleanFile()
 
-    val logFile = out.resolveSibling("${out.name}.log")
+    val logFile = tempDir.resolve("${out.name}.log")
 
     val args = mutableListOf<String>()
 

paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/file-lock.kt

@@ -0,0 +1,147 @@
+/*
+ * paperweight is a Gradle plugin for the PaperMC project.
+ *
+ * Copyright (c) 2023 Kyle Wood (DenWav)
+ *                    Contributors
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 only, no later versions.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+package io.papermc.paperweight.util
+
+import io.papermc.paperweight.PaperweightException
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.io.path.*
+import org.gradle.api.logging.Logger
+import org.gradle.api.logging.Logging
+
+private val openCurrentJvm: MutableMap<Path, ReentrantLock> = ConcurrentHashMap()
+
+fun <R> withLock(
+    lockFile: Path,
+    printInfoAfter: Long = 1000 * 60 * 5, // 5 minutes
+    timeoutMs: Long = 1000L * 60 * 60, // one hour
+    action: () -> R,
+): R {
+    val logger = Logging.getLogger("paperweight lock file")
+
+    var waitedMs = 0L
+    var firstFailedAcquire = true
+    while (true) {
+        val normalized = lockFile.normalize().absolute()
+
+        val lock = openCurrentJvm.computeIfAbsent(normalized) { ReentrantLock() }
+        if (!lock.tryLock()) {
+            if (firstFailedAcquire) {
+                logger.lifecycle("Lock for '$lockFile' is currently held by another thread.")
+                logger.lifecycle("Waiting for lock to be released...")
+                firstFailedAcquire = false
+            }
+            val startWait = System.nanoTime()
+            val acquired = lock.tryLock(printInfoAfter, TimeUnit.MILLISECONDS)
+            if (!acquired) {
+                waitedMs += (System.nanoTime() - startWait) / 1_000_000
+                if (waitedMs >= timeoutMs) {
+                    throw PaperweightException("Have been waiting on lock for '$lockFile' for $waitedMs ms. Giving up as timeout is $timeoutMs ms.")
+                }
+                logger.lifecycle(
+                    "Have been waiting on lock for '$lockFile' for ${waitedMs / 1000 / 60} minute(s).\n" +
+                        "If this persists for an unreasonable length of time, kill this process, run './gradlew --stop', and then try again."
+                )
+            }
+        }
+        if (openCurrentJvm[normalized] !== lock) {
+            lock.unlock()
+            continue
+        }
+
+        try {
+            acquireProcessLockWaiting(lockFile, logger, waitedMs, printInfoAfter, timeoutMs)
+            try {
+                return action()
+            } finally {
+                lockFile.deleteForcefully()
+            }
+        } finally {
+            openCurrentJvm.remove(normalized)
+            lock.unlock()
+        }
+    }
+}
+
+private fun acquireProcessLockWaiting(
+    lockFile: Path,
+    logger: Logger,
+    alreadyWaited: Long,
+    printInfoAfter: Long,
+    timeoutMs: Long,
+) {
+    if (!lockFile.parent.exists()) {
+        lockFile.parent.createDirectories()
+    }
+
+    val currentPid = ProcessHandle.current().pid()
+    var sleptMs: Long = alreadyWaited
+
+    while (true) {
+        if (lockFile.exists()) {
+            val lockingProcessId = lockFile.readText().toLong()
+            if (lockingProcessId == currentPid) {
+                throw IllegalStateException("Lock file '$lockFile' is currently held by this process.")
+            } else {
+                logger.lifecycle("Lock file '$lockFile' is currently held by pid '$lockingProcessId'.")
+            }
+
+            if (ProcessHandle.of(lockingProcessId).isEmpty) {
+                logger.lifecycle("Locking process does not exist, assuming abrupt termination and deleting lock file.")
+                lockFile.deleteIfExists()
+            } else {
+                logger.lifecycle("Waiting for lock to be released...")
+                while (lockFile.exists()) {
+                    Thread.sleep(100)
+                    sleptMs += 100
+                    if (sleptMs >= printInfoAfter && sleptMs % printInfoAfter == 0L) {
+                        logger.lifecycle(
+                            "Have been waiting on lock file '$lockFile' held by pid '$lockingProcessId' for ${sleptMs / 1000 / 60} minute(s).\n" +
+                                "If this persists for an unreasonable length of time, kill this process, run './gradlew --stop', and then" +
+                                " try again.\nIf the problem persists, the lock file may need to be deleted manually."
+                        )
+                    }
+                    if (sleptMs >= timeoutMs) {
+                        throw PaperweightException(
+                            "Have been waiting on lock file '$lockFile' for $sleptMs ms. Giving up as timeout is $timeoutMs ms."
+                        )
+                    }
+                }
+            }
+        }
+
+        try {
+            lockFile.writeText(
+                currentPid.toString(),
+                options = arrayOf(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW, StandardOpenOption.SYNC)
+            )
+        } catch (e: FileAlreadyExistsException) {
+            continue
+        }
+
+        break
+    }
+}