Skip to content

Commit

Permalink
Persist 'expired' userdev cache entries when a non-expired entry uses it
Browse files Browse the repository at this point in the history
Useful because we don't update the last used time for intermediate outputs when the final output is up to date.
  • Loading branch information
jpenilla committed Dec 28, 2024
1 parent 21b1e82 commit 2c814fb
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 32 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

package io.papermc.paperweight.userdev.internal.action

import io.papermc.paperweight.userdev.internal.util.formatNs
import io.papermc.paperweight.util.*
import java.nio.file.Files
import java.nio.file.Path
Expand All @@ -40,20 +41,38 @@ class CacheCleaner(private val work: Path) {
}

val start = System.nanoTime()
var deleted = 0
var deletedSize = 0L

val delete = mutableListOf<Path>()
val keep = mutableListOf<Path>()
work.listDirectoryEntries().forEach {
val lockFile = it.resolve("lock")
if (lockFile.exists()) {
return@forEach
if (it.resolve("lock").exists()) {
val took = System.nanoTime() - start
logger.info("paperweight-userdev: Aborted cache cleanup in ${formatNs(took)} due to locked cache entry (${it.name})")
return
}
val metadataFile = it.resolve("metadata.json")
if (!metadataFile.isRegularFile()) {
return@forEach
}
val since = System.currentTimeMillis() - metadataFile.getLastModifiedTime().toMillis()
if (since > deleteUnusedAfter) {
delete.add(it)
} else {
keep.add(it)
}
}

var deleted = 0
var deletedSize = 0L
if (delete.isNotEmpty()) {
keep.forEach { k ->
val metadataFile = k.resolve("metadata.json")
gson.fromJson<WorkGraph.Metadata>(metadataFile).skippedWhenUpToDate?.let {
delete.removeIf { o -> o.name in it }
}
}

delete.forEach {
deleted++
it.deleteRecursive { toDelete ->
if (toDelete.isRegularFile()) {
Expand All @@ -65,6 +84,6 @@ class CacheCleaner(private val work: Path) {

val took = System.nanoTime() - start
val level = if (deleted > 0) LogLevel.LIFECYCLE else LogLevel.INFO
logger.log(level, "paperweight-userdev: Deleted $deleted expired cache entries totaling ${deletedSize / 1024}KB in ${took / 1_000_000}ms")
logger.log(level, "paperweight-userdev: Deleted $deleted expired cache entries totaling ${deletedSize / 1024}KB in ${formatNs(took)}")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class WorkGraph(
class Node(
val registration: WorkDispatcherImpl.Registration,
val dependencies: List<Node>,
var inputHash: String? = null,
)

private val roots: List<Node> = buildGraph(requested.toList())
Expand Down Expand Up @@ -101,6 +102,7 @@ class WorkGraph(

data class Metadata(
val outputHashes: List<String>,
val skippedWhenUpToDate: Set<String>?,
val lastUsed: Long = System.currentTimeMillis(),
) {
fun updateLastUsed() = copy(lastUsed = System.currentTimeMillis())
Expand Down Expand Up @@ -143,18 +145,19 @@ class WorkGraph(
.hash(HashingAlgorithm.SHA256)
.asHexString()
}
node.inputHash = inputHash

realizeOutputPaths(node, work, inputHash)

val lockFile = work.resolve("${node.registration.name}_$inputHash/lock")

val hashFile = work.resolve("${node.registration.name}_$inputHash/metadata.json")
val metadataFile = work.resolve("${node.registration.name}_$inputHash/metadata.json")
val upToDate = withLock(lockFile) {
if (hashFile.exists()) {
val metadata = hashFile.bufferedReader().use { gson.fromJson(it, Metadata::class.java) }
if (metadataFile.exists()) {
val metadata = metadataFile.bufferedReader().use { gson.fromJson(it, Metadata::class.java) }
if (node.registration.outputs.hash(hashCache) == metadata.outputHashes) {
logger.lifecycle("Skipping ${node.registration.name} (up-to-date)")
metadata.updateLastUsed().writeTo(hashFile)
metadata.updateLastUsed().writeTo(metadataFile)
val took = System.nanoTime() - start
logger.info("Up-to-date check for ${node.registration.name} took ${formatNs(took)} (up-to-date)")
return@withLock true
Expand All @@ -175,8 +178,9 @@ class WorkGraph(
throw PaperweightException("Exception executing ${node.registration.name}", e)
}

val metadata = Metadata(node.registration.outputs.hash(hashCache))
metadata.writeTo(hashFile)
val deps = if (root && terminalInputHash != null) collectDependencies(node) else null
val metadata = Metadata(node.registration.outputs.hash(hashCache), deps?.takeIf { it.isNotEmpty() })
metadata.writeTo(metadataFile)

val tookExec = System.nanoTime() - startExec
logger.lifecycle("Finished ${node.registration.name} in ${formatNs(tookExec)}")
Expand All @@ -193,6 +197,22 @@ class WorkGraph(
}
}

private fun collectDependencies(node: Node): Set<String> {
val deps = mutableSetOf<String>()
val nodes = mutableListOf<Node>()
nodes.add(node)
while (nodes.isNotEmpty()) {
val n = nodes.removeAt(0)
for (dep in n.dependencies) {
nodes.add(dep)
if (dep != node) {
deps += "${dep.registration.name}_${dep.inputHash}"
}
}
}
return deps
}

private fun List<Value<*>>.hash(cache: MutableMap<Value<*>, String>): List<String> {
return map {
cache.computeIfAbsent(it) { v ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,28 +63,28 @@ class VanillaServerDownloads(
)
val versionManifest: MinecraftVersionManifest = gson.fromJson(versionManifestPath)

val dispatcher = ioDispatcher("VanillaServerDownloads")
runBlocking {
launch(dispatcher) {
val serverTmp = tmp.resolve("server.jar")
downloadService.downloadFile(
versionManifest.serverDownload().url,
serverTmp,
expectedHash = versionManifest.serverDownload().hash()
)
serverTmp.copyTo(serverJar.get(), overwrite = true)
}
launch(dispatcher) {
val mappingsTmp = tmp.resolve("mappings.txt")
downloadService.downloadFile(
versionManifest.serverMappingsDownload().url,
mappingsTmp,
expectedHash = versionManifest.serverMappingsDownload().hash()
)
mappingsTmp.copyTo(serverMappings.get(), overwrite = true)
ioDispatcher("VanillaServerDownloads").use { dispatcher ->
runBlocking {
launch(dispatcher) {
val serverTmp = tmp.resolve("server.jar")
downloadService.downloadFile(
versionManifest.serverDownload().url,
serverTmp,
expectedHash = versionManifest.serverDownload().hash()
)
serverTmp.copyTo(serverJar.get(), overwrite = true)
}
launch(dispatcher) {
val mappingsTmp = tmp.resolve("mappings.txt")
downloadService.downloadFile(
versionManifest.serverMappingsDownload().url,
mappingsTmp,
expectedHash = versionManifest.serverMappingsDownload().hash()
)
mappingsTmp.copyTo(serverMappings.get(), overwrite = true)
}
}
}
dispatcher.close()

tmp.deleteRecursive()
}
Expand Down

0 comments on commit 2c814fb

Please sign in to comment.