Feature/convert r2dbc

build.gradle.kts

@@ -6,7 +6,6 @@ plugins {
     id("org.jlleitschuh.gradle.ktlint") version "11.3.2"
     kotlin("jvm") version "1.9.21"
     kotlin("plugin.spring") version "1.9.21"
-    kotlin("plugin.jpa") version "1.9.24"
 }
 
 group = "com.yourssu"
@@ -29,8 +28,10 @@ dependencies {
     implementation("org.jetbrains.kotlin:kotlin-reflect")
     implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor")
-    implementation("org.springframework.boot:spring-boot-starter-data-jpa")
-    runtimeOnly("com.mysql:mysql-connector-j")
+    // implementation("org.springframework.boot:spring-boot-starter-data-jpa")
+    implementation("org.springframework.boot:spring-boot-starter-data-r2dbc")
+    // implementation("io.r2dbc:r2dbc-pool")
+    implementation("org.mariadb:r2dbc-mariadb:1.1.2")
 
     implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.1.0")
 

src/main/kotlin/com/yourssu/search/SearchApplication.kt

@@ -2,7 +2,13 @@ package com.yourssu.search
 
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.runApplication
+import org.springframework.data.r2dbc.config.EnableR2dbcAuditing
+import org.springframework.data.r2dbc.repository.config.EnableR2dbcRepositories
+import org.springframework.scheduling.annotation.EnableScheduling
 
+@EnableScheduling
+@EnableR2dbcRepositories
+@EnableR2dbcAuditing
 @SpringBootApplication
 class SearchApplication
 

src/main/kotlin/com/yourssu/search/crawling/config/R2dbcConfig.kt

@@ -0,0 +1,15 @@
+package com.yourssu.search.crawling.config
+
+import io.r2dbc.spi.ConnectionFactory
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.data.r2dbc.core.R2dbcEntityTemplate
+
+@Configuration
+class R2dbcConfig(private val connectionFactory: ConnectionFactory) {
+
+    @Bean
+    fun r2dbcEntityTemplate(): R2dbcEntityTemplate {
+        return R2dbcEntityTemplate(connectionFactory)
+    }
+}

src/main/kotlin/com/yourssu/search/crawling/domain/InformationUrl.kt

@@ -1,25 +1,17 @@
 package com.yourssu.search.crawling.domain
 
-import jakarta.persistence.Column
-import jakarta.persistence.Entity
-import jakarta.persistence.EnumType
-import jakarta.persistence.Enumerated
-import jakarta.persistence.GeneratedValue
-import jakarta.persistence.GenerationType
-import jakarta.persistence.Id
-import jakarta.persistence.Table
+import org.springframework.data.annotation.Id
+import org.springframework.data.relational.core.mapping.Column
+import org.springframework.data.relational.core.mapping.Table
 
-@Entity
-@Table(name = "information_url")
+@Table("information_url")
 class InformationUrl(
-    @field:Id
-    @field:GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Id
     val id: Long? = null,
 
-    @field:Column(name = "content_url", nullable = false, unique = true, length = 500)
+    @Column("content_url")
     val contentUrl: String,
 
-    @field:Column(name = "source_type", nullable = false)
-    @field:Enumerated(EnumType.STRING)
+    @Column("source_type")
     val sourceType: SourceType
 )

src/main/kotlin/com/yourssu/search/crawling/repository/CoroutineInformationRepository.kt

@@ -0,0 +1,6 @@
+package com.yourssu.search.crawling.repository
+
+import com.yourssu.search.crawling.domain.Information
+import org.springframework.data.elasticsearch.repository.CoroutineElasticsearchRepository
+
+interface CoroutineInformationRepository : CoroutineElasticsearchRepository<Information, String>

src/main/kotlin/com/yourssu/search/crawling/repository/InformationUrlRepository.kt

@@ -2,10 +2,11 @@ package com.yourssu.search.crawling.repository
 
 import com.yourssu.search.crawling.domain.InformationUrl
 import com.yourssu.search.crawling.domain.SourceType
-import org.springframework.data.jpa.repository.JpaRepository
+import kotlinx.coroutines.flow.Flow
+import org.springframework.data.repository.kotlin.CoroutineCrudRepository
 import org.springframework.stereotype.Repository
 
 @Repository
-interface InformationUrlRepository : JpaRepository<InformationUrl, String> {
-    fun findAllBySourceType(sourceType: SourceType): List<InformationUrl>
+interface InformationUrlRepository : CoroutineCrudRepository<InformationUrl, Long> {
+    fun findAllBySourceType(sourceType: SourceType): Flow<InformationUrl>
 }

src/main/kotlin/com/yourssu/search/crawling/scheduler/CrawlingScheduler.kt

@@ -0,0 +1,18 @@
+package com.yourssu.search.crawling.scheduler
+
+import com.yourssu.search.crawling.service.CrawlingService
+import kotlinx.coroutines.runBlocking
+import org.springframework.scheduling.annotation.Scheduled
+import org.springframework.stereotype.Component
+
+@Component
+class CrawlingScheduler(
+    private val crawlingService: CrawlingService
+) {
+    @Scheduled(cron = "0 0 0 * * ?") // 매일 자정 12시 실행
+    fun scheduleCrawling() = runBlocking {
+        crawlingService.strategies.keys.forEach { key -> // strategies의 의존성 주입 된 객체 순회
+            crawlingService.executeCrawling(key)
+        }
+    }
+}

src/main/kotlin/com/yourssu/search/crawling/service/CrawlingService.kt

@@ -4,11 +4,12 @@ import com.yourssu.search.crawling.repository.InformationRepository
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import org.springframework.stereotype.Service
+import org.springframework.transaction.annotation.Transactional
 
 @Service
 class CrawlingService(
-    private val strategies: Map<String, CrawlingStrategy>,
-    private val informationRepository: InformationRepository,
+    public val strategies: Map<String, CrawlingStrategy>,
+    private val informationRepository: InformationRepository
 ) {
 
     suspend fun executeCrawling(strategyKey: String) {
@@ -17,6 +18,7 @@ class CrawlingService(
         strategy.crawl()
     }
 
+    @Transactional
     suspend fun deleteData() {
         withContext(Dispatchers.IO) {
             informationRepository.deleteAll()

src/main/kotlin/com/yourssu/search/crawling/service/CrawlingStrategy.kt

@@ -2,4 +2,4 @@ package com.yourssu.search.crawling.service
 
 interface CrawlingStrategy {
     suspend fun crawl()
-}
+}

src/main/kotlin/com/yourssu/search/crawling/service/strategy/FunCrawlingStrategy.kt

@@ -11,7 +11,7 @@ import kotlin.time.measureTimedValue
 @Component("fun")
 class FunCrawlingStrategy(
     private val crawlingUtils: CrawlingUtils
-): CrawlingStrategy {
+) : CrawlingStrategy {
     private val log = LoggerFactory.getLogger(this::class.java)
 
     override suspend fun crawl() {
@@ -25,8 +25,10 @@ class FunCrawlingStrategy(
 
             val toSaveDocuments: List<Element> =
                 crawlingUtils.filteringToSaveDocuments(allDocuments, SourceType.FUN, urlSelector)
+            val flattenedDocuments: List<Element> = allDocuments.flatten()
 
             crawlingUtils.crawlingContents(
+                // toSaveDocuments = flattenedDocuments,
                 toSaveDocuments = toSaveDocuments,
                 titleSelector = ".content .title",
                 contentSelector = "div .description p",
@@ -38,4 +40,4 @@ class FunCrawlingStrategy(
         }
         log.info("all time use {}", duration.duration.inWholeSeconds)
     }
-}
+}

src/main/kotlin/com/yourssu/search/crawling/service/strategy/NoticeCrawlingStrategy.kt

@@ -11,7 +11,7 @@ import kotlin.time.measureTimedValue
 @Component("notice")
 class NoticeCrawlingStrategy(
     private val crawlingUtils: CrawlingUtils
-): CrawlingStrategy {
+) : CrawlingStrategy {
     private val log = LoggerFactory.getLogger(this::class.java)
 
     override suspend fun crawl() {
@@ -25,12 +25,12 @@ class NoticeCrawlingStrategy(
 
             val toSaveDocuments: List<Element> =
                 crawlingUtils.filteringToSaveDocuments(
-                    lists = allDocuments, 
-                    sourceType = SourceType.NOTICE, 
-                    urlSelector = urlSelector
+                    lists = allDocuments, sourceType = SourceType.NOTICE, urlSelector = urlSelector
                 )
+            val flattenedDocuments: List<Element> = allDocuments.flatten()
 
             crawlingUtils.crawlingContents(
+                // toSaveDocuments = flattenedDocuments,
                 toSaveDocuments = toSaveDocuments,
                 titleSelector = ".notice_col3 a .d-inline-blcok.m-pt-5",
                 contentSelector = "div.bg-white p",
@@ -42,4 +42,4 @@ class NoticeCrawlingStrategy(
         }
         log.info("all time use {}", duration.duration.inWholeSeconds)
     }
-}
+}

src/main/kotlin/com/yourssu/search/crawling/utils/CrawlingUtils.kt

@@ -3,13 +3,14 @@ package com.yourssu.search.crawling.utils
 import com.yourssu.search.crawling.domain.Information
 import com.yourssu.search.crawling.domain.InformationUrl
 import com.yourssu.search.crawling.domain.SourceType
-import com.yourssu.search.crawling.repository.InformationRepository
+import com.yourssu.search.crawling.repository.CoroutineInformationRepository
 import com.yourssu.search.crawling.repository.InformationUrlRepository
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.joinAll
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
@@ -18,6 +19,7 @@ import org.jsoup.nodes.Element
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
+import org.springframework.transaction.annotation.Transactional
 import java.io.FileNotFoundException
 import java.time.LocalDate
 import java.util.concurrent.atomic.AtomicBoolean
@@ -26,13 +28,14 @@ import java.util.regex.Pattern
 
 @Component
 class CrawlingUtils(
-    private val informationRepository: InformationRepository,
+    private val coroutineElasticsearchRepository: CoroutineInformationRepository,
     private val informationUrlRepository: InformationUrlRepository,
 
     @Value("\${general.user-agent}")
     private val userAgent: String,
 
     private val coroutineScope: CoroutineScope = CoroutineScope(Dispatchers.IO),
+    private val coroutineInformationRepository: CoroutineInformationRepository
 ) {
     private val log = LoggerFactory.getLogger(this::class.java)
 
@@ -43,7 +46,7 @@ class CrawlingUtils(
     ): List<Element> {
         val savedData: List<InformationUrl>
         withContext(Dispatchers.IO) {
-            savedData = informationUrlRepository.findAllBySourceType(sourceType)
+            savedData = informationUrlRepository.findAllBySourceType(sourceType).toList()
         }
         val savedUrls = savedData.map { it.contentUrl }
 
@@ -97,7 +100,6 @@ class CrawlingUtils(
         resultList
     }
 
-
     private suspend fun fetchPage(baseUrl: String, pageNumber: Int, ulSelector: String): List<Element> {
         val document = Jsoup.connect("$baseUrl/$pageNumber")
             .userAgent(userAgent)
@@ -117,6 +119,7 @@ class CrawlingUtils(
         return contents
     }
 
+    @Transactional
     suspend fun crawlingContents(
         toSaveDocuments: List<Element>,
         titleSelector: String,
@@ -126,8 +129,6 @@ class CrawlingUtils(
         favicon: String?,
         sourceType: SourceType
     ) {
-        val urlChannel = Channel<InformationUrl>(Channel.UNLIMITED)
-
         val contentJobs: List<Job> = toSaveDocuments.map { element ->
             coroutineScope.launch {
                 val rawDate = element.selectFirst(dateSelector)?.text() ?: ""
@@ -151,32 +152,37 @@ class CrawlingUtils(
                     return@launch
                 }
 
-                urlChannel.send(InformationUrl(contentUrl = contentUrl, sourceType = sourceType))
-
-                informationRepository.save(
-                    Information(
-                        title = title,
-                        content = content.toString().trim(),
-                        date = extractedDate,
-                        contentUrl = contentUrl,
-                        imgList = imgList,
-                        favicon = favicon,
-                        source = sourceType.value
+                try {
+                    // `InformationUrl`을 즉시 저장
+                    informationUrlRepository.save(
+                        InformationUrl(
+                            contentUrl = contentUrl,
+                            sourceType = sourceType
+                        )
+                    )
+                    log.info("Saved URL: $contentUrl")
+
+                    // `Information`도 저장
+                    coroutineInformationRepository.save(
+                        Information(
+                            title = title,
+                            content = content.toString().trim(),
+                            date = extractedDate,
+                            contentUrl = contentUrl,
+                            imgList = imgList,
+                            favicon = favicon,
+                            source = sourceType.value
+                        )
                     )
-                )
+                    // log.info("Saved Information for URL: $contentUrl")
+                } catch (e: Exception) {
+                    // log.error("Error saving URL or Information for $contentUrl", e)
+                }
             }
         }
 
         contentJobs.joinAll()
-        urlChannel.close()
-
-        val toSaveUrls = mutableListOf<InformationUrl>()
-        for (url in urlChannel) {
-            toSaveUrls.add(url)
-        }
-
-        val distinctUrls = toSaveUrls.distinctBy { it.contentUrl }
-        informationUrlRepository.saveAll(distinctUrls)
+        log.info("Crawling and saving completed.")
     }
 
     private fun extractDate(dateStr: String): LocalDate? {
@@ -197,4 +203,16 @@ class CrawlingUtils(
             null // 정규표현식에 맞지 않으면 null 반환
         }
     }
-}
+
+    /*@Transactional
+    suspend fun saveAllWithRollback(urls: List<InformationUrl>) {
+        urls.forEachIndexed { index, url ->
+            informationUrlRepository.save(url)
+
+            // 인위적으로 예외 발생
+            if (index == 90) {
+                throw RuntimeException("Simulated exception for rollback")
+            }
+        }
+    }*/
+}