From a7683c57055beb6740d85d75b7705afc76f8f23c Mon Sep 17 00:00:00 2001 From: Benjamin Wollmer Date: Tue, 26 Nov 2024 18:24:41 +0100 Subject: [PATCH 1/3] Avoid unintend garbage collection of raw data in PreparedDictionaryImpl --- .../main/java/com/aayushatharva/brotli4j/encoder/EncoderJNI.java | 1 + 1 file changed, 1 insertion(+) diff --git a/brotli4j/src/main/java/com/aayushatharva/brotli4j/encoder/EncoderJNI.java b/brotli4j/src/main/java/com/aayushatharva/brotli4j/encoder/EncoderJNI.java index 0ccc98ed..370b031f 100644 --- a/brotli4j/src/main/java/com/aayushatharva/brotli4j/encoder/EncoderJNI.java +++ b/brotli4j/src/main/java/com/aayushatharva/brotli4j/encoder/EncoderJNI.java @@ -36,6 +36,7 @@ private static class PreparedDictionaryImpl implements PreparedDictionary { private PreparedDictionaryImpl(ByteBuffer data, ByteBuffer rawData) { this.data = data; + this.rawData = rawData; } @Override From 3ec114d8d3941cbff21b13fc9ecb7a20d39f2a2a Mon Sep 17 00:00:00 2001 From: Benjamin Wollmer Date: Wed, 27 Nov 2024 14:56:07 +0100 Subject: [PATCH 2/3] Add test to check potential gc of dictionary data --- .../brotli4j/encoder/EncoderTest.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java b/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java index cc7effd4..4f732ecb 100644 --- a/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java +++ b/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java @@ -17,13 +17,16 @@ package com.aayushatharva.brotli4j.encoder; import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.common.BrotliCommon; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,4 +80,32 @@ void encodeModeEnumValues() { assertEquals(Encoder.Mode.TEXT, Encoder.Mode.of(Encoder.Mode.TEXT.ordinal())); assertEquals(Encoder.Mode.GENERIC, Encoder.Mode.of(Encoder.Mode.GENERIC.ordinal())); } + + + @Test + void ensureDictionaryDataRemainsAfterGC() throws IOException, InterruptedException { + // We hard code the compressed data, since the dictionary could also be collected just before our first compression + final byte[] expectedCompression = new byte[]{27, 43, 0, -8, 37, 0, -62, -104, -40, -63, 0}; + final String dictionaryData = "This is some data to be used as a dictionary"; + final byte[] rawBytes = dictionaryData.getBytes(); // Use dictionary also as data to keep it small + final PreparedDictionary dic = Encoder.prepareDictionary(BrotliCommon.makeNative(dictionaryData.getBytes()), 0); + + // Create gc pressure to trigger potential collection of dictionary data + ArrayList hashes = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + String obj = String.valueOf(Math.random()); + hashes.add(obj.hashCode()); + } + hashes = null; + System.gc(); + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + BrotliOutputStream brotliOutputStream = new BrotliOutputStream(byteArrayOutputStream)) { + brotliOutputStream.attachDictionary(dic); + brotliOutputStream.write(rawBytes); + brotliOutputStream.close(); + byteArrayOutputStream.close(); + assertArrayEquals(expectedCompression, byteArrayOutputStream.toByteArray()); // Otherwise the GC already cleared the data + } + } } From fa618a8fae8fdde2afbac892e9763edbdbd5235d Mon Sep 17 00:00:00 2001 From: Aayush Atharva Date: Wed, 27 Nov 2024 23:24:03 +0530 Subject: [PATCH 3/3] Increase loop iteration count in EncoderTest --- .../java/com/aayushatharva/brotli4j/encoder/EncoderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java b/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java index 4f732ecb..34792f99 100644 --- a/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java +++ b/brotli4j/src/test/java/com/aayushatharva/brotli4j/encoder/EncoderTest.java @@ -92,7 +92,7 @@ void ensureDictionaryDataRemainsAfterGC() throws IOException, InterruptedExcepti // Create gc pressure to trigger potential collection of dictionary data ArrayList hashes = new ArrayList<>(); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 1_000_000; i++) { String obj = String.valueOf(Math.random()); hashes.add(obj.hashCode()); }