From cb6b72d8b5694eb9deda128f7c7fa1eddd42a8b9 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Fri, 15 Dec 2023 11:59:27 +0100 Subject: [PATCH] Update cache to include records to recompile --- .../scala/com/github/sbt/avro/SbtAvro.scala | 71 ++++++++++++------- src/sbt-test/sbt-avro/recompile/record.avsc | 11 +++ src/sbt-test/sbt-avro/recompile/test | 29 +++++++- 3 files changed, 82 insertions(+), 29 deletions(-) create mode 100644 src/sbt-test/sbt-avro/recompile/record.avsc diff --git a/src/main/scala/com/github/sbt/avro/SbtAvro.scala b/src/main/scala/com/github/sbt/avro/SbtAvro.scala index 227dc9e..cc54e53 100644 --- a/src/main/scala/com/github/sbt/avro/SbtAvro.scala +++ b/src/main/scala/com/github/sbt/avro/SbtAvro.scala @@ -279,6 +279,7 @@ object SbtAvro extends AutoPlugin { } private[this] def compileAvroSchema( + records: Seq[Class[_ <: SpecificRecord]], srcDirs: Seq[File], target: File, log: Logger, @@ -294,6 +295,7 @@ object SbtAvro extends AutoPlugin { val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))) val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get) + recompile(records, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters, builder) compileIdls(avdls, target, log, stringType, fieldVisibility, enableDecimalLogicalType, optionalGetters, createSetters) compileAvscs(avscs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters, builder) compileAvprs(avprs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, optionalGetters, createSetters) @@ -303,7 +305,7 @@ object SbtAvro extends AutoPlugin { private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task { val out = (key / streams).value - val records = avroSpecificRecords.value + val srcDir = avroSource.value val externalSrcDir = (avroUnpackDependencies / target).value val includes = avroIncludes.value @@ -320,34 +322,51 @@ object SbtAvro extends AutoPlugin { } val builder = avroSchemaParserBuilder.value val cachedCompile = { - FileFunction.cached(out.cacheDirectory / "avro", FilesInfo.lastModified, FilesInfo.exists) { _ => - out.log.info(s"Avro compiler $avroCompilerVersion using stringType=$strType") - compileAvroSchema( - srcDirs, - outDir, - out.log, - strType, - fieldVis, - enbDecimal, - useNs, - optionalGetters, - createSetters, - builder - ) - } - } + import sbt.util.CacheStoreFactory + import sbt.util.CacheImplicits._ - val recordOutputFiles = if(records.isEmpty) { - Set.empty[File] - } else { - recompile( - records, outDir, out.log, strType, fieldVis, enbDecimal, useNs, - optionalGetters, createSetters, builder - ) - (outDir ** JavaFileFilter).get.toSet + out.log.info(s"Avro compiler $avroCompilerVersion using stringType=$strType") + + val cacheStoreFactory = CacheStoreFactory(out.cacheDirectory / "avro") + val lastCache = { (action: Option[Set[File]] => Set[File]) => + Tracked.lastOutput[Unit, Set[File]](cacheStoreFactory.make("last-cache")) { + case (_, l) => action(l) + }.apply(()) + } + val inCache = Difference.inputs(cacheStoreFactory.make("in-cache"), FileInfo.lastModified) + val outCache = Difference.outputs(cacheStoreFactory.make("out-cache"), FileInfo.exists) + + (inputs: Set[File], records: Seq[Class[_ <: SpecificRecord]]) => + lastCache { lastCache => + inCache(inputs) { inReport => + outCache { outReport => + if ((lastCache.isEmpty && records.nonEmpty) || inReport.modified.nonEmpty || outReport.modified.nonEmpty) { + // compile if + // - no previous cache and we have records to recompile + // - input files have changed + // - output files are missing + compileAvroSchema( + records, + srcDirs, + outDir, + out.log, + strType, + fieldVis, + enbDecimal, + useNs, + optionalGetters, + createSetters, + builder + ) + } else { + outReport.checked + } + } + } + } } - (cachedCompile((srcDirs ** AvroFilter).get.toSet) ++ recordOutputFiles).toSeq + cachedCompile((srcDirs ** AvroFilter).get.toSet, avroSpecificRecords.value).toSeq } } diff --git a/src/sbt-test/sbt-avro/recompile/record.avsc b/src/sbt-test/sbt-avro/recompile/record.avsc new file mode 100644 index 0000000..7b689a9 --- /dev/null +++ b/src/sbt-test/sbt-avro/recompile/record.avsc @@ -0,0 +1,11 @@ +{ + "name": "Record", + "namespace": "com.github.sbt.avro.test", + "type": "record", + "fields": [ + { + "name": "value", + "type": "string" + } + ] +} diff --git a/src/sbt-test/sbt-avro/recompile/test b/src/sbt-test/sbt-avro/recompile/test index ec172a1..ba58487 100644 --- a/src/sbt-test/sbt-avro/recompile/test +++ b/src/sbt-test/sbt-avro/recompile/test @@ -1,5 +1,28 @@ -> compile +> avroGenerate +$ exists target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java + +# reference file created after 1st generation +$ touch target/reference.txt + +# cache should not recompile +> avroGenerate +$ newer target/reference.txt target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java +# regenerate after clean +> clean +> avroGenerate +$exists target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java + +# add new source file should re-compile +$ copy-file record.avsc src/main/avro/record.avsc +> avroGenerate +$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/Record.java $ exists target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java --$ exec grep BigDecimal target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java --$ exec grep CharSequence target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java + +# update reference file +$ touch target/reference.txt + +# cache should not recompile +> avroGenerate +$ newer target/reference.txt target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/Record.java +$ newer target/reference.txt target/scala-2.13/src_managed/compiled_avro/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java