Dynamically scrape search key

Dockerfile

@@ -1,9 +1,9 @@
-FROM gradle:8.0.1-jdk17 AS build
+FROM gradle:8.10.1-jdk21 AS build
 COPY --chown=gradle:gradle . /home/gradle/src
 WORKDIR /home/gradle/src
 RUN gradle shadowJar --no-daemon
 
-FROM eclipse-temurin:17-jre-alpine
+FROM eclipse-temurin:21-jre-alpine
 
 COPY --from=build /home/gradle/src/build/libs/ /app/
 ENTRYPOINT ["java","-jar","/app/HowLongToBeat-Proxy-API-1.0-SNAPSHOT-all.jar"]

build.gradle.kts

@@ -1,11 +1,12 @@
 val coroutinesVersion = "1.6.4"
-val http4kVersion = "4.37.0.0"
+val http4kVersion = "5.26.0.0"
 
 plugins {
-    kotlin("jvm") version "1.8.0"
-    kotlin("plugin.serialization") version "1.8.0"
+    kotlin("jvm") version "2.0.20"
+    kotlin("plugin.serialization") version "2.0.0"
     id("com.github.johnrengelman.shadow") version "7.1.2"
     application
+    idea
 }
 
 group = "io.github.darefox"
@@ -15,6 +16,12 @@ repositories {
     mavenCentral()
 }
 
+idea {
+    module {
+        isDownloadJavadoc = true
+        isDownloadSources = true
+    }
+}
 
 dependencies {
     testImplementation(kotlin("test"))
@@ -26,30 +33,26 @@ dependencies {
     implementation("org.http4k:http4k-format-kotlinx-serialization:$http4kVersion")
 
     // loggers
-    implementation("org.slf4j:slf4j-simple:2.0.6")
-    implementation("io.github.microutils:kotlin-logging-jvm:3.0.4")
+    implementation("org.slf4j:slf4j-simple:2.0.13")
+    implementation("io.github.microutils:kotlin-logging-jvm:3.0.5")
 
     // serialization
-    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1")
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.1")
 
     // dependency for mutex
-    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4")
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0")
 
     // html parser
-    implementation("org.jsoup:jsoup:1.15.4")
+    implementation("org.jsoup:jsoup:1.18.1")
 
-    implementation("org.apache.commons:commons-collections4:4.1")
+    implementation("org.apache.commons:commons-collections4:4.4")
 }
 
 
 tasks.test {
     useJUnitPlatform()
 }
 
-kotlin {
-    jvmToolchain(17)
-}
-
 tasks.jar {
     manifest.attributes["Main-Class"] = "io.github.darefox.hltbproxy.MainKt"
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/kotlin/io/github/darefox/hltbproxy/hltb/HLTB.kt

@@ -1,28 +1,59 @@
 package io.github.darefox.hltbproxy.hltb
 
 import io.github.darefox.hltbproxy.http4k.client
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import mu.KotlinLogging
 import org.http4k.core.Body
 import org.http4k.core.Method.GET
 import org.http4k.core.Method.POST
 import org.http4k.core.Request
+import org.http4k.core.Response
+import org.http4k.core.Status
 import org.http4k.format.KotlinxSerialization.auto
 import org.jsoup.Jsoup
+import org.jsoup.select.Elements
 import java.net.URLEncoder
 
 object HLTB {
-    fun queryGames(title: String, page: Int): HltbQueryResponse {
+    private val log = KotlinLogging.logger { }
+    lateinit var searchKey: String
+    private val keyMutex = Mutex()
+    private val updateMutex = Mutex()
+    private val initMutex = Mutex()
+
+    suspend fun queryGames(title: String, page: Int): HltbQueryResponse {
         val encoded = URLEncoder.encode(title, "utf-8")
         val url = "https://howlongtobeat.com/?q=$encoded"
 
         val queryObj = createQueryObj(title, page)
-        val response = client(
-            Request(POST, "https://howlongtobeat.com/api/search/43ff72690aa2496d").hltbJsonRequest(url, queryObj)
-        )
+
+        var keyIsNotUpdated = true
+        lateinit var response: Response
+        while (true) {
+            val call = client(
+                Request(POST, "https://howlongtobeat.com/api/search/${getSearchKey()}")
+                    .hltbJsonRequest(url, queryObj)
+            )
+
+            if (call.bodyString().startsWith("<!DOCTYPE html>") && keyIsNotUpdated) {
+                log.info("Search key is no longer valid, calling update")
+                updateSearchKey()
+                keyIsNotUpdated = false
+            } else {
+                response = call
+                break
+            }
+        }
+
+        if (response.bodyString().startsWith("<!DOCTYPE html>")) {
+            updateSearchKey()
+        }
 
         return Body.auto<HltbQueryResponse>().toLens().invoke(response)
     }
 
-    fun getOverviewInfoAboutGame(id: Long): HltbOverviewParser? {
+    suspend fun getOverviewInfoAboutGame(id: Long): HltbOverviewParser? {
         val url = "https://howlongtobeat.com/game/$id"
         val response = client(Request(GET, url).hltbDefaultHeaders(url, false))
 
@@ -33,4 +64,84 @@ object HLTB {
         val html = Jsoup.parse(response.bodyString())
         return HltbOverviewParser(html)
     }
+
+    private suspend fun getSearchKey(): String {
+        if (!this::searchKey.isInitialized) {
+            initMutex.withLock {
+                if (!this::searchKey.isInitialized) {
+                    return updateSearchKey()
+                } else {
+                    return keyMutex.withLock { searchKey }
+                }
+            }
+        } else {
+            keyMutex.withLock { return searchKey }
+        }
+    }
+
+    private suspend fun updateSearchKey(): String {
+        // Prevent creating multiple calls to updateSearchKey
+        // First call updates key, remaining calls wait for new key
+        return if (updateMutex.isLocked) getSearchKey() else {
+            return updateMutex.withLock {
+                log.info("Updating key...")
+                keyMutex.withLock { // keyMutex needed for API calls to wait new key
+                    val url = "https://howlongtobeat.com"
+                    val response = client(Request(GET, url).hltbDefaultHeaders(url, false))
+
+                    if (response.status != Status.OK) {
+                        error("Response is not 199 from HLTB server, can't get search key")
+                    }
+
+                    val html = Jsoup.parse(response.bodyString())
+                    html.setBaseUri(url)
+
+                    val scripts = html.select("script")
+
+                    val codeOfScripts = getCodeOfInlineScripts(scripts) + getCodeOfExternalScripts(scripts)
+                    val key = findKeyInScripts(codeOfScripts)
+
+                    searchKey = key
+                    key
+                }
+            }
+        }
+    }
+
+    private fun getCodeOfInlineScripts(scripts: Elements): Sequence<String> {
+        val inlinedScripts = scripts.filter { element ->
+            val isJS = element.attr("type") == "text/javascript"
+            val isExternal = element.attr("src").isNotEmpty()
+            isJS && !isExternal
+        }
+        return inlinedScripts.asSequence().map { it.text() }
+    }
+
+    private fun getCodeOfExternalScripts(scripts: Elements): Sequence<String> {
+        val externalScripts = scripts.filter { element ->
+            val isExternal = element.attr("src").isNotEmpty()
+            isExternal
+        }.map {
+            it.attr("abs:src")
+        }
+
+        return externalScripts.asSequence().map { scriptUrl ->
+            log.info("Downloading script: $scriptUrl")
+            val response = client(Request(GET, scriptUrl).hltbDefaultHeaders(scriptUrl, false))
+            if (response.status != Status.OK) {
+                error("On downloading script ($scriptUrl), server returned ${response.status}")
+            }
+            response.bodyString()
+        }
+    }
+
+    private fun findKeyInScripts(codes: Sequence<String>): String {
+        val regex = "(?<=api/search/\"\\.concat\\(\")[a-zA-Z0-9]+".toRegex()
+        for (code in codes) {
+            val result = regex.find(code)?.value ?: continue
+            return result
+        }
+
+        error("Can't find search key")
+    }
 }

src/main/kotlin/io/github/darefox/hltbproxy/hltb/HltbQueryResponse.kt

@@ -36,8 +36,6 @@ data class HltbGameData(
     val compLvlCo: Long,
     @JsonNames("comp_lvl_mp")
     val compLvlMp: Long,
-    @JsonNames("comp_lvl_spd")
-    val compLvlSpd: Long,
     @JsonNames("comp_main")
     val compMain: Long,
     @JsonNames("comp_plus")
@@ -76,14 +74,8 @@ data class HltbGameData(
     val countPlaying: Long,
     @JsonNames("count_retired")
     val countRetired: Long,
-    @JsonNames("profile_dev")
-    val profileDev: String,
     @JsonNames("profile_popular")
     val profilePopular: Long,
-    @JsonNames("profile_steam")
-    val profileSteam: Long,
-    @JsonNames("profile_platform")
-    val profilePlatform: String,
     @JsonNames("release_world")
     val releaseWorld: Long,
 )

src/main/kotlin/io/github/darefox/hltbproxy/proxy/OverviewInfo.kt

@@ -3,6 +3,7 @@ package io.github.darefox.hltbproxy.proxy
 import io.github.darefox.hltbproxy.cache.getOrGenerateBlocking
 import io.github.darefox.hltbproxy.hltb.*
 import io.github.darefox.hltbproxy.http4k.ErrorResponse
+import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.Serializable
 import org.http4k.core.*
 import org.http4k.format.KotlinxSerialization.auto
@@ -15,7 +16,7 @@ val getOverviewInfo: HttpHandler = { req ->
 
     val key = "overviewInfo;$id"
     cache.getOrGenerateBlocking(mutexMap, key) {
-        val obj = HLTB.getOverviewInfoAboutGame(id)?.toProxy()
+        val obj = runBlocking { HLTB.getOverviewInfoAboutGame(id)?.toProxy() }
             ?: return@getOrGenerateBlocking ErrorResponse(Status.NOT_FOUND, "Not Found")
         Response(Status.OK).with(Body.auto<OverviewInfo>().toLens() of obj)
     }

src/main/kotlin/io/github/darefox/hltbproxy/proxy/QueryGames.kt

@@ -2,6 +2,7 @@ package io.github.darefox.hltbproxy.proxy
 
 import io.github.darefox.hltbproxy.cache.getOrGenerateBlocking
 import io.github.darefox.hltbproxy.hltb.HLTB
+import kotlinx.coroutines.runBlocking
 import org.http4k.core.*
 import org.http4k.format.KotlinxSerialization.auto
 import org.http4k.lens.Query
@@ -16,7 +17,7 @@ val queryGames: HttpHandler = { req ->
     val key = "queryGames;$name;$page"
 
     cache.getOrGenerateBlocking(mutexMap, key) {
-        val body = HLTB.queryGames(name, page).data.map {
+        val body = runBlocking { HLTB.queryGames(name, page) }.data.map {
             it.toProxyObj()
         }
 

src/main/kotlin/io/github/darefox/hltbproxy/proxy/QueryResponse.kt

@@ -46,8 +46,8 @@ fun HltbGameData.toProxyObj(): QueryGamesResponse {
         type = this.gameType,
         gameId = this.gameId,
         gameImage = this.gameImage,
-        dev = this.profileDev,
-        platforms = this.profilePlatform.trim().split(", "),
+        dev = "API Deprecated",
+        platforms = listOf(),
         releaseYear = this.releaseWorld.toInt(),
         beatTime = QueryGamesBeatTime(
             main = QueryGamesBeatTimeEntry(compMain, compMainCount),
@@ -62,6 +62,6 @@ fun HltbGameData.toProxyObj(): QueryGamesResponse {
             retired = countRetired,
             reviews = countReview,
         ),
-        steamId = if (profileSteam == 0L) null else profileSteam
+        steamId = null
     )
 }