From ed22f2d423c16fe2fdd56affe850fc620904e786 Mon Sep 17 00:00:00 2001 From: Jane Wang Date: Sat, 16 May 2026 10:00:16 -0700 Subject: [PATCH 01/10] docs: add compression algorithm enum design spec Co-Authored-By: Claude Sonnet 4.6 --- ...05-16-compression-algorithm-enum-design.md | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md diff --git a/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md new file mode 100644 index 00000000..08b51696 --- /dev/null +++ b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md @@ -0,0 +1,118 @@ +# Compression Algorithm Enum — Design Spec + +**Date:** 2026-05-16 +**Scope:** `evcache-core` — `EVCacheSerializingTranscoder`, `EVCacheTranscoder`, `build.gradle` + +--- + +## Goal + +Add first-class support for pluggable compression algorithms (gzip, zstd) in EVCache, while preserving full backward compatibility with existing gzip-compressed data already in cache. + +--- + +## Changes + +### 1. `EVCacheSerializingTranscoder` — enum, constant, field, constructors, compress/decompress + +**Nested enum:** +```java +public enum CompressionAlgorithm { GZIP, ZSTD } +``` + +**Constant:** +```java +public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; +``` +Level 3 is zstd's own default — good speed/ratio balance for a cache write path. Level 9 (VHS's choice) is 10x slower to compress for only ~10% better ratio, appropriate for at-rest storage but not a latency-sensitive cache. + +**New field:** +```java +private final CompressionAlgorithm compressionAlgorithm; +``` +Defaults to `GZIP` in existing constructors — no behavioral change for current callers. + +**New constructors:** +```java +// existing — unchanged, default to GZIP +public EVCacheSerializingTranscoder() // → this(MAX_SIZE) +public EVCacheSerializingTranscoder(int max) // → this(max, GZIP) + +// new +public EVCacheSerializingTranscoder(int max, CompressionAlgorithm algo) +public EVCacheSerializingTranscoder(int max, CompressionAlgorithm algo, int zstdLevel) +``` + +**`compress()` override — dispatches on `compressionAlgorithm`:** +- `GZIP`: `GZIPOutputStream` at Java default level (6) — identical to current `BaseSerializingTranscoder` behavior +- `ZSTD`: `Zstd.compress(data, zstdLevel)` + +**`decompress()` override — magic-byte auto-detection (modeled on VHS `CompressionUtils`):** + +| Magic bytes | Algorithm | +|---|---| +| `0x1F 0x8B` | gzip | +| `0x28 0xB5 0x2F 0xFD` (little-endian int) | zstd | +| neither | return data as-is (backward compat for unrecognized data) | + +Zstd decompression uses a fast path when the frame carries a content-size header (`Zstd.decompressedSize() > 0`), falling back to `ZstdInputStream` stream-decode otherwise. + +The `compressionAlgorithm` field is **not consulted during decode** — detection is always by magic bytes. This ensures any transcoder instance can decode data written by any other instance regardless of its configured algorithm. + +**Metrics:** `updateTimerWithCompressionRatio` replaces hardcoded `"gzip"` tag with `compressionAlgorithm.name().toLowerCase()`. + +--- + +### 2. `EVCacheTranscoder` — two new FPs, new constructors + +**New FP — algorithm:** +``` +Property: default.evcache.compression.algorithm +Type: String +Default: "GZIP" +``` +Cast to enum via `CompressionAlgorithm.valueOf(value.toUpperCase())`. Throws `IllegalArgumentException` for unsupported values — fast-fail at startup rather than silent misconfiguration. + +**New FP — zstd level:** +``` +Property: default.evcache.compression.zstd.level +Type: Integer +Default: EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL (3) +``` +Only used when algorithm is `ZSTD`. Gzip level is not exposed — Java default (6) is the established behavior in this repo. + +**New constructors threading both values through to super:** +```java +public EVCacheTranscoder(int max) +public EVCacheTranscoder(int max, int compressionThreshold) +public EVCacheTranscoder(int max, int compressionThreshold, CompressionAlgorithm algo, int zstdLevel) +``` + +The no-arg and `(int max)` constructors read both FPs and delegate to the full constructor. + +--- + +### 3. `evcache-core/build.gradle` — new dependency + +```groovy +api group: 'com.github.luben', name: 'zstd-jni', version: 'latest.release' +``` + +Same library used by `viewing_history_service/CompressionUtils.java`. + +--- + +## Backward Compatibility + +- All existing constructors default to `GZIP` — no behavior change for current callers. +- `decompress()` auto-detects by magic bytes, so existing gzip-compressed cache entries decode correctly even if the transcoder is reconfigured to write zstd. +- The `COMPRESSED` flag bit is unchanged — no wire format changes. + +--- + +## Out of Scope + +- zlib detection (no existing zlib-compressed data in EVCache) +- Gzip level FP (Java default level 6 is the established behavior; easy to add later) +- Updating `ChunkTranscoder` (separate concern) + From 742014544121a320bcf0f857047293c08ec440b0 Mon Sep 17 00:00:00 2001 From: Jane Wang Date: Sat, 16 May 2026 10:04:35 -0700 Subject: [PATCH 02/10] docs: remove zlib from compression algorithm spec Co-Authored-By: Claude Sonnet 4.6 --- .../specs/2026-05-16-compression-algorithm-enum-design.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md index 08b51696..0adc3393 100644 --- a/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md +++ b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md @@ -53,7 +53,7 @@ public EVCacheSerializingTranscoder(int max, CompressionAlgorithm algo, int zstd |---|---| | `0x1F 0x8B` | gzip | | `0x28 0xB5 0x2F 0xFD` (little-endian int) | zstd | -| neither | return data as-is (backward compat for unrecognized data) | +| neither | return data as-is (backward compat) | Zstd decompression uses a fast path when the frame carries a content-size header (`Zstd.decompressedSize() > 0`), falling back to `ZstdInputStream` stream-decode otherwise. @@ -112,7 +112,6 @@ Same library used by `viewing_history_service/CompressionUtils.java`. ## Out of Scope -- zlib detection (no existing zlib-compressed data in EVCache) - Gzip level FP (Java default level 6 is the established behavior; easy to add later) - Updating `ChunkTranscoder` (separate concern) From 87022acd1e169833499de5ac9ef34d5698385149 Mon Sep 17 00:00:00 2001 From: Jane Wang Date: Sat, 16 May 2026 10:05:13 -0700 Subject: [PATCH 03/10] docs: add zstdLevel field to EVCacheSerializingTranscoder spec Co-Authored-By: Claude Sonnet 4.6 --- .../specs/2026-05-16-compression-algorithm-enum-design.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md index 0adc3393..31e8c8ff 100644 --- a/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md +++ b/docs/superpowers/specs/2026-05-16-compression-algorithm-enum-design.md @@ -26,11 +26,12 @@ public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; ``` Level 3 is zstd's own default — good speed/ratio balance for a cache write path. Level 9 (VHS's choice) is 10x slower to compress for only ~10% better ratio, appropriate for at-rest storage but not a latency-sensitive cache. -**New field:** +**New fields:** ```java private final CompressionAlgorithm compressionAlgorithm; +private final int zstdLevel; ``` -Defaults to `GZIP` in existing constructors — no behavioral change for current callers. +Both default in existing constructors (`GZIP`, `DEFAULT_ZSTD_COMPRESSION_LEVEL`) — no behavioral change for current callers. **New constructors:** ```java From 7b55bc9b8148b0da70d46844bba2da852f36ecf4 Mon Sep 17 00:00:00 2001 From: Jane Wang Date: Sat, 16 May 2026 10:10:59 -0700 Subject: [PATCH 04/10] docs: add compression algorithm enum implementation plan Co-Authored-By: Claude Sonnet 4.6 --- .../2026-05-16-compression-algorithm-enum.md | 649 ++++++++++++++++++ 1 file changed, 649 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-16-compression-algorithm-enum.md diff --git a/docs/superpowers/plans/2026-05-16-compression-algorithm-enum.md b/docs/superpowers/plans/2026-05-16-compression-algorithm-enum.md new file mode 100644 index 00000000..4bee1fa9 --- /dev/null +++ b/docs/superpowers/plans/2026-05-16-compression-algorithm-enum.md @@ -0,0 +1,649 @@ +# Compression Algorithm Enum Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `CompressionAlgorithm` enum (GZIP, ZSTD) to `EVCacheSerializingTranscoder` with magic-byte auto-detection on decode and FP-driven algorithm/level selection in `EVCacheTranscoder`. + +**Architecture:** The enum and all compress/decompress logic live in `EVCacheSerializingTranscoder`. Existing constructors default to GZIP — no behavioral change for current callers. Decode always auto-detects by magic bytes regardless of the configured algorithm, ensuring backward compatibility. `EVCacheTranscoder` reads two new FPs (`default.evcache.compression.algorithm`, `default.evcache.compression.zstd.level`) and threads them into the parent constructor. + +**Tech Stack:** Java, TestNG, `com.github.luben:zstd-jni`, `java.util.zip.GZIPOutputStream/GZIPInputStream`, `java.nio.ByteBuffer` + +--- + +## Files + +| Action | Path | +|---|---| +| Modify | `evcache-core/build.gradle` | +| Modify | `evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java` | +| Modify | `evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java` | +| Create | `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` | +| Modify | `evcache-core/src/test/java/test-suite.xml` | + +--- + +## Task 1: Add zstd-jni dependency + +**Files:** +- Modify: `evcache-core/build.gradle` + +- [ ] **Step 1: Add dependency** + +In `evcache-core/build.gradle`, add after the last `api` line in the `dependencies` block: + +```groovy +api group: 'com.github.luben', name: 'zstd-jni', version: 'latest.release' +``` + +- [ ] **Step 2: Verify it resolves** + +```bash +./gradlew :evcache-core:dependencies --configuration compileClasspath | grep zstd +``` + +Expected output includes a line like: +``` +\--- com.github.luben:zstd-jni:... +``` + +- [ ] **Step 3: Commit** + +```bash +git add evcache-core/build.gradle +git commit -m "build: add zstd-jni dependency to evcache-core" +``` + +--- + +## Task 2: Add enum, constants, fields, and constructors + +**Files:** +- Modify: `evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java` +- Create: `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` +- Modify: `evcache-core/src/test/java/test-suite.xml` + +- [ ] **Step 1: Create the test file with failing tests** + +Create `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java`: + +```java +package com.netflix.evcache; + +import net.spy.memcached.CachedData; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class EVCacheSerializingTranscoderTest { + + @Test + public void testEnumValues() { + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("GZIP"), + EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("ZSTD"), + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + } + + @Test + public void testDefaultZstdLevelConstant() { + assertEquals(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL, 3); + } + + @Test + public void testDefaultConstructorUsesGzip() { + // Default constructor must not throw; algorithm defaults to GZIP + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(); + assertNotNull(t); + } + + @Test + public void testConstructorWithAlgorithm() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + assertNotNull(t); + } + + @Test + public void testConstructorWithAlgorithmAndLevel() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD, + 5); + assertNotNull(t); + } +} +``` + +- [ ] **Step 2: Register the test class in test-suite.xml** + +In `evcache-core/src/test/java/test-suite.xml`, add inside ``: + +```xml + + + + + +``` + +- [ ] **Step 3: Run tests to confirm they fail** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: compilation errors — `CompressionAlgorithm` and `DEFAULT_ZSTD_COMPRESSION_LEVEL` don't exist yet. + +- [ ] **Step 4: Add enum, constant, fields, and constructors to `EVCacheSerializingTranscoder`** + +Replace the existing two constructors and add new fields. The complete updated top section of the class (after the existing `static final` flag constants and before `asyncDecode`) should look like this: + +```java + public enum CompressionAlgorithm { GZIP, ZSTD } + + public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; + + private static final int ZSTD_MAGIC = 0xFD2FB528; + private static final byte GZIP_MAGIC_0 = (byte) 0x1f; + private static final byte GZIP_MAGIC_1 = (byte) 0x8b; + + private final TranscoderUtils tu = new TranscoderUtils(true); + private Timer timer; + private final CompressionAlgorithm compressionAlgorithm; + private final int zstdLevel; + + public EVCacheSerializingTranscoder() { + this(CachedData.MAX_SIZE); + } + + public EVCacheSerializingTranscoder(int max) { + this(max, CompressionAlgorithm.GZIP); + } + + public EVCacheSerializingTranscoder(int max, CompressionAlgorithm algo) { + this(max, algo, DEFAULT_ZSTD_COMPRESSION_LEVEL); + } + + public EVCacheSerializingTranscoder(int max, CompressionAlgorithm algo, int zstdLevel) { + super(max); + this.compressionAlgorithm = algo; + this.zstdLevel = zstdLevel; + } +``` + +Also add these imports at the top of the file: + +```java +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +``` + +Remove the old two constructors: +```java +// DELETE these: +public EVCacheSerializingTranscoder() { + this(CachedData.MAX_SIZE); +} +public EVCacheSerializingTranscoder(int max) { + super(max); +} +``` + +- [ ] **Step 5: Run tests to confirm they pass** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `BUILD SUCCESSFUL` and `TranscoderTests` pass. + +- [ ] **Step 6: Commit** + +```bash +git add evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java \ + evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java \ + evcache-core/src/test/java/test-suite.xml +git commit -m "feat: add CompressionAlgorithm enum, constants, fields, and constructors to EVCacheSerializingTranscoder" +``` + +--- + +## Task 3: Override compress() + +**Files:** +- Modify: `evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java` +- Modify: `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` + +- [ ] **Step 1: Add failing tests** + +Add these test methods to `EVCacheSerializingTranscoderTest`: + +```java + @Test + public void testGzipEncodeSetsGzipMagicBytes() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + t.setCompressionThreshold(0); // compress everything + // Use a String — encode() compresses when above threshold + CachedData encoded = t.encode("hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "Expected gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "Expected gzip magic byte 1"); + } + + @Test + public void testZstdEncodeSetsZstdMagicBytes() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + // Zstd magic is 0xFD2FB528 in little-endian: bytes 0x28 0xB5 0x2F 0xFD + assertEquals(data[0], (byte) 0x28, "Expected zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "Expected zstd magic byte 1"); + assertEquals(data[2], (byte) 0x2F, "Expected zstd magic byte 2"); + assertEquals(data[3], (byte) 0xFD, "Expected zstd magic byte 3"); + } +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: both new tests fail — `EVCacheSerializingTranscoder` still delegates to `BaseSerializingTranscoder.compress()` which always writes gzip, so the zstd test fails. + +- [ ] **Step 3: Add compress() override and helpers** + +Add these methods to `EVCacheSerializingTranscoder`, before `updateTimerWithCompressionRatio`: + +```java + @Override + protected byte[] compress(byte[] in) { + if (in == null) throw new NullPointerException("Can't compress null"); + switch (compressionAlgorithm) { + case ZSTD: + return Zstd.compress(in, zstdLevel); + case GZIP: + default: + return compressGzip(in); + } + } + + private byte[] compressGzip(byte[] in) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(in.length / 10); + GZIPOutputStream gz = null; + try { + gz = new GZIPOutputStream(bos); + gz.write(in); + } catch (IOException e) { + throw new RuntimeException("IO exception compressing data", e); + } finally { + closeQuietly(gz); + closeQuietly(bos); + } + return bos.toByteArray(); + } + + private static void closeQuietly(java.io.Closeable c) { + if (c != null) try { c.close(); } catch (IOException ignored) {} + } +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 5: Commit** + +```bash +git add evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java \ + evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java +git commit -m "feat: override compress() in EVCacheSerializingTranscoder to dispatch on algorithm" +``` + +--- + +## Task 4: Override decompress() with magic-byte auto-detection + +**Files:** +- Modify: `evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java` +- Modify: `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` + +- [ ] **Step 1: Add failing tests** + +Add these test methods to `EVCacheSerializingTranscoderTest`: + +```java + @Test + public void testGzipRoundTrip() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + t.setCompressionThreshold(0); + String original = "round trip gzip round trip gzip round trip gzip"; + CachedData encoded = t.encode(original); + assertEquals(t.decode(encoded), original); + } + + @Test + public void testZstdRoundTrip() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + t.setCompressionThreshold(0); + String original = "round trip zstd round trip zstd round trip zstd"; + CachedData encoded = t.encode(original); + assertEquals(t.decode(encoded), original); + } + + @Test + public void testGzipTranscoderDecodesZstdData() { + // Encode with ZSTD + EVCacheSerializingTranscoder writer = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + writer.setCompressionThreshold(0); + CachedData encoded = writer.encode("cross decode test cross decode test cross decode test"); + + // Decode with a GZIP-configured transcoder — must auto-detect zstd + EVCacheSerializingTranscoder reader = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + assertEquals(reader.decode(encoded), "cross decode test cross decode test cross decode test"); + } + + @Test + public void testZstdTranscoderDecodesGzipData() { + // Encode with GZIP (legacy data already in cache) + EVCacheSerializingTranscoder writer = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + writer.setCompressionThreshold(0); + CachedData encoded = writer.encode("legacy gzip data legacy gzip data legacy gzip data"); + + // Decode with a ZSTD-configured transcoder — must auto-detect gzip + EVCacheSerializingTranscoder reader = new EVCacheSerializingTranscoder( + CachedData.MAX_SIZE, EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + assertEquals(reader.decode(encoded), "legacy gzip data legacy gzip data legacy gzip data"); + } +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `testZstdRoundTrip` and `testGzipTranscoderDecodesZstdData` fail — zstd-compressed bytes are fed to the gzip decompressor in the parent class, which throws. + +- [ ] **Step 3: Add decompress() override and helpers** + +Add these methods to `EVCacheSerializingTranscoder`, after `compressGzip` and before `updateTimerWithCompressionRatio`: + +```java + @Override + protected byte[] decompress(byte[] in) { + if (in == null) return null; + if (isGzipCompressed(in)) return decompressGzip(in); + if (isZstdCompressed(in)) return decompressZstd(in); + return in; + } + + private boolean isGzipCompressed(byte[] data) { + return data.length >= 2 && data[0] == GZIP_MAGIC_0 && data[1] == GZIP_MAGIC_1; + } + + private boolean isZstdCompressed(byte[] data) { + if (data.length < 4) return false; + int magic = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); + return magic == ZSTD_MAGIC; + } + + private byte[] decompressGzip(byte[] in) { + ByteArrayInputStream bis = new ByteArrayInputStream(in); + ByteArrayOutputStream bos = new ByteArrayOutputStream(in.length * 2); + GZIPInputStream gis = null; + try { + gis = new GZIPInputStream(bis); + byte[] buf = new byte[8192]; + int r; + while ((r = gis.read(buf)) > 0) { + bos.write(buf, 0, r); + } + } catch (IOException e) { + getLogger().warn("Failed to decompress gzip data", e); + return null; + } finally { + closeQuietly(gis); + closeQuietly(bis); + closeQuietly(bos); + } + return bos.toByteArray(); + } + + private byte[] decompressZstd(byte[] in) { + try { + long originalSize = Zstd.decompressedSize(in); + if (originalSize > Integer.MAX_VALUE) { + getLogger().warn("Zstd frame declares size > Integer.MAX_VALUE: {}", originalSize); + return null; + } + if (originalSize > 0) { + return Zstd.decompress(in, (int) originalSize); + } + // Slow path: frame has no content-size header — stream decode + ZstdInputStream zis = new ZstdInputStream(new ByteArrayInputStream(in)); + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = zis.read(buf)) != -1) { + bos.write(buf, 0, n); + } + return bos.toByteArray(); + } finally { + zis.close(); + } + } catch (IOException e) { + getLogger().warn("Failed to decompress zstd data", e); + return null; + } + } +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 5: Commit** + +```bash +git add evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java \ + evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java +git commit -m "feat: override decompress() with magic-byte auto-detection for gzip and zstd" +``` + +--- + +## Task 5: Update compression metrics tag + +**Files:** +- Modify: `evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java` +- Modify: `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` + +Note: `EVCacheMetricsFactory` requires a running registry so we verify this change by inspection only — no unit test for the metric tag itself. + +- [ ] **Step 1: Update the hardcoded `"gzip"` tag** + +In `updateTimerWithCompressionRatio`, replace: + +```java +tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, "gzip")); +``` + +with: + +```java +tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, compressionAlgorithm.name().toLowerCase())); +``` + +- [ ] **Step 2: Run all tests to confirm nothing broke** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 3: Commit** + +```bash +git add evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java +git commit -m "feat: use algorithm name in compression metrics tag instead of hardcoded gzip" +``` + +--- + +## Task 6: Update EVCacheTranscoder with new FPs and constructors + +**Files:** +- Modify: `evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java` +- Modify: `evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java` + +- [ ] **Step 1: Add failing tests** + +Add these test methods to `EVCacheSerializingTranscoderTest`: + +```java + @Test + public void testEVCacheTranscoderExplicitZstd() { + EVCacheTranscoder t = new EVCacheTranscoder( + CachedData.MAX_SIZE, 0, + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD, + EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); + String original = "evcachetranscoder zstd test evcachetranscoder zstd test"; + CachedData encoded = t.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0); + // Verify zstd magic bytes + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28); + assertEquals(data[1], (byte) 0xB5); + assertEquals(data[2], (byte) 0x2F); + assertEquals(data[3], (byte) 0xFD); + assertEquals(t.decode(encoded), original); + } + + @Test + public void testEVCacheTranscoderExplicitGzip() { + EVCacheTranscoder t = new EVCacheTranscoder( + CachedData.MAX_SIZE, 0, + EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP, + EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); + String original = "evcachetranscoder gzip test evcachetranscoder gzip test"; + CachedData encoded = t.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f); + assertEquals(data[1], (byte) 0x8b); + assertEquals(t.decode(encoded), original); + } +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: compilation errors — the four-arg constructor on `EVCacheTranscoder` doesn't exist yet. + +- [ ] **Step 3: Rewrite EVCacheTranscoder** + +Replace the full contents of `EVCacheTranscoder.java` with: + +```java +package com.netflix.evcache; + +import com.netflix.evcache.util.EVCacheConfig; +import net.spy.memcached.CachedData; + +public class EVCacheTranscoder extends EVCacheSerializingTranscoder { + + public EVCacheTranscoder() { + this(EVCacheConfig.getInstance().getPropertyRepository() + .get("default.evcache.max.data.size", Integer.class) + .orElse(20 * 1024 * 1024).get()); + } + + public EVCacheTranscoder(int max) { + this(max, EVCacheConfig.getInstance().getPropertyRepository() + .get("default.evcache.compression.threshold", Integer.class) + .orElse(120).get()); + } + + public EVCacheTranscoder(int max, int compressionThreshold) { + this(max, compressionThreshold, + CompressionAlgorithm.valueOf( + EVCacheConfig.getInstance().getPropertyRepository() + .get("default.evcache.compression.algorithm", String.class) + .orElse("GZIP").get().toUpperCase()), + EVCacheConfig.getInstance().getPropertyRepository() + .get("default.evcache.compression.zstd.level", Integer.class) + .orElse(DEFAULT_ZSTD_COMPRESSION_LEVEL).get()); + } + + public EVCacheTranscoder(int max, int compressionThreshold, CompressionAlgorithm algo, int zstdLevel) { + super(max, algo, zstdLevel); + setCompressionThreshold(compressionThreshold); + } + + @Override + public boolean asyncDecode(CachedData d) { + return super.asyncDecode(d); + } + + @Override + public Object decode(CachedData d) { + return super.decode(d); + } + + @Override + public CachedData encode(Object o) { + if (o != null && o instanceof CachedData) return (CachedData) o; + return super.encode(o); + } +} +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +./gradlew :evcache-core:test 2>&1 | tail -30 +``` + +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 5: Commit** + +```bash +git add evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java \ + evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java +git commit -m "feat: add compression algorithm and zstd level FPs to EVCacheTranscoder" +``` From 4d97319d7af4cdd7405abc4e24ee5b52b9f37ec6 Mon Sep 17 00:00:00 2001 From: Jane Wang Date: Sat, 16 May 2026 13:33:00 -0700 Subject: [PATCH 05/10] feat: add pluggable compression algorithm support (gzip, zstd) Adds CompressionAlgorithm enum (GZIP, ZSTD) to EVCacheSerializingTranscoder with magic-byte auto-detection on decode for full backward compatibility. EVCacheTranscoder wires algorithm and zstd level from Fast Properties. Fixes compression ratio metric which was always reporting 100%. Co-Authored-By: Claude Sonnet 4.6 --- evcache-core/build.gradle | 1 + .../evcache/EVCacheSerializingTranscoder.java | 108 ++++++++++- .../netflix/evcache/EVCacheTranscoder.java | 20 +- .../EVCacheSerializingTranscoderTest.java | 176 ++++++++++++++++++ evcache-core/src/test/java/test-suite.xml | 5 + 5 files changed, 290 insertions(+), 20 deletions(-) create mode 100644 evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java diff --git a/evcache-core/build.gradle b/evcache-core/build.gradle index 63874384..b67317a4 100644 --- a/evcache-core/build.gradle +++ b/evcache-core/build.gradle @@ -43,6 +43,7 @@ dependencies { api group:"joda-time", name:"joda-time", version:"latest.release" api group:"javax.annotation", name:"javax.annotation-api", version:"latest.release" api group:"com.github.fzakaria", name:"ascii85", version:"latest.release" + api group:"com.github.luben", name:"zstd-jni", version:"latest.release" testImplementation group:"org.testng", name:"testng", version:"7.5" testImplementation group:"com.beust", name:"jcommander", version:"1.72" diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java index 95c7a86e..4a481b74 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java @@ -22,8 +22,9 @@ package com.netflix.evcache; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.netflix.evcache.metrics.EVCacheMetricsFactory; -import com.netflix.evcache.pool.ServerGroup; import com.netflix.spectator.api.BasicTag; import com.netflix.spectator.api.Tag; import com.netflix.spectator.api.Timer; @@ -31,14 +32,17 @@ import net.spy.memcached.transcoders.BaseSerializingTranscoder; import net.spy.memcached.transcoders.Transcoder; import net.spy.memcached.transcoders.TranscoderUtils; -import net.spy.memcached.util.StringUtils; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.time.Duration; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -65,8 +69,18 @@ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder impl static final String COMPRESSION = "COMPRESSION_METRIC"; + public enum CompressionAlgorithm { GZIP, ZSTD } + + public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; + + private static final int ZSTD_MAGIC = 0xFD2FB528; + private static final byte GZIP_MAGIC_0 = (byte) 0x1f; + private static final byte GZIP_MAGIC_1 = (byte) 0x8b; + private final TranscoderUtils tu = new TranscoderUtils(true); private Timer timer; + private CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.GZIP; + private int zstdLevel = DEFAULT_ZSTD_COMPRESSION_LEVEL; /** * Get a serializing transcoder with the default max data size. @@ -82,6 +96,14 @@ public EVCacheSerializingTranscoder(int max) { super(max); } + public void setCompressionAlgorithm(CompressionAlgorithm algo) { + this.compressionAlgorithm = algo; + } + + public void setCompressionLevel(int level) { + this.zstdLevel = level; + } + @Override public boolean asyncDecode(CachedData d) { if ((d.getFlags() & COMPRESSED) != 0 || (d.getFlags() & SERIALIZED) != 0) { @@ -179,29 +201,95 @@ public CachedData encode(Object o) { } assert b != null; if (b.length > compressionThreshold) { + int originalLength = b.length; byte[] compressed = compress(b); - if (compressed.length < b.length) { + if (compressed.length < originalLength) { getLogger().trace("Compressed %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + o.getClass().getName(), originalLength, compressed.length); b = compressed; flags |= COMPRESSED; } else { getLogger().debug("Compression increased the size of %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + o.getClass().getName(), originalLength, compressed.length); } - long compression_ratio = Math.round((double) compressed.length / b.length * 100); + long compression_ratio = Math.round((double) compressed.length / originalLength * 100); updateTimerWithCompressionRatio(compression_ratio); } return new CachedData(flags, b, getMaxSize()); } + @Override + protected byte[] compress(byte[] in) { + if (in == null) throw new NullPointerException("Can't compress null"); + switch (compressionAlgorithm) { + case ZSTD: + return Zstd.compress(in, zstdLevel); + case GZIP: + return super.compress(in); + default: + throw new IllegalArgumentException("Unsupported compression algorithm: " + compressionAlgorithm); + } + } + + @Override + protected byte[] decompress(byte[] in) { + if (in == null || in.length == 0) return in; + if (isZstdCompressed(in)) return decompressZstd(in); + if (isGzipCompressed(in)) return super.decompress(in); + return in; + } + + private boolean isGzipCompressed(byte[] data) { + return data.length >= 2 && data[0] == GZIP_MAGIC_0 && data[1] == GZIP_MAGIC_1; + } + + private boolean isZstdCompressed(byte[] data) { + if (data.length < 4) return false; + int magic = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); + return magic == ZSTD_MAGIC; + } + + private byte[] decompressZstd(byte[] in) { + long originalSize = Zstd.decompressedSize(in); + if (originalSize > Integer.MAX_VALUE) { + getLogger().warn("Zstd decompressed size exceeds int range: " + originalSize); + return null; + } + if (originalSize > 0) { + // Fast path: frame carries a content-size header (compress() above always does). + return Zstd.decompress(in, (int) originalSize); + } + // Slow path: declared size is 0, unknown (-1), or invalid (-2) — stream-decode and let + // ZstdInputStream surface any frame errors. + ZstdInputStream zis = null; + try { + zis = new ZstdInputStream(new ByteArrayInputStream(in)); + return readAll(zis); + } catch (IOException e) { + getLogger().error("Error reading Zstd input stream", e); + return null; + } finally { + try { if (zis != null) zis.close(); } catch (IOException ignored) {} + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } + private void updateTimerWithCompressionRatio(long ratio_percentage) { if(timer == null) { final List tagList = new ArrayList(1); - tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, "gzip")); + tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, compressionAlgorithm.name().toLowerCase())); timer = EVCacheMetricsFactory.getInstance().getPercentileTimer(EVCacheMetricsFactory.COMPRESSION_RATIO, tagList, Duration.ofMillis(100)); - }; + } timer.record(ratio_percentage, TimeUnit.MILLISECONDS); } diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java index 97be808b..2fabc0ff 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java @@ -1,5 +1,6 @@ package com.netflix.evcache; +import com.netflix.archaius.api.PropertyRepository; import com.netflix.evcache.util.EVCacheConfig; import net.spy.memcached.CachedData; @@ -17,16 +18,15 @@ public EVCacheTranscoder(int max) { public EVCacheTranscoder(int max, int compressionThreshold) { super(max); setCompressionThreshold(compressionThreshold); - } - - @Override - public boolean asyncDecode(CachedData d) { - return super.asyncDecode(d); - } - - @Override - public Object decode(CachedData d) { - return super.decode(d); + PropertyRepository config = EVCacheConfig.getInstance().getPropertyRepository(); + CompressionAlgorithm algo = CompressionAlgorithm.valueOf( + config.get("default.evcache.compression.algorithm", String.class) + .orElse("GZIP").get().toUpperCase()); + setCompressionAlgorithm(algo); + if (algo == CompressionAlgorithm.ZSTD) { + setCompressionLevel(config.get("default.evcache.compression.zstd.level", Integer.class) + .orElse(DEFAULT_ZSTD_COMPRESSION_LEVEL).get()); + } } @Override diff --git a/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java new file mode 100644 index 00000000..d73fb6e4 --- /dev/null +++ b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java @@ -0,0 +1,176 @@ +package com.netflix.evcache; + +import net.spy.memcached.CachedData; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class EVCacheSerializingTranscoderTest { + + @Test + public void testEnumValues() { + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("GZIP"), + EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("ZSTD"), + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + } + + @Test + public void testDefaultZstdLevelConstant() { + assertEquals(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL, 3); + } + + @Test + public void testDefaultConstructorUsesGzip() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "Default constructor must use gzip"); + assertEquals(data[1], (byte) 0x8b, "Default constructor must use gzip"); + } + + @Test + public void testSetCompressionAlgorithmProducesZstd() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + t.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "setCompressionAlgorithm(ZSTD) must produce zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "setCompressionAlgorithm(ZSTD) must produce zstd magic byte 1"); + } + + @Test + public void testSetCompressionLevelRoundTrip() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + t.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + t.setCompressionLevel(5); + t.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = t.encode(original); + String decoded = (String) t.decode(encoded); + assertEquals(decoded, original, "Round-trip must succeed with custom zstd level 5"); + } + + @Test + public void testGzipEncodeSetsGzipMagicBytes() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "Expected gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "Expected gzip magic byte 1"); + } + + @Test + public void testZstdEncodeSetsZstdMagicBytes() { + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + t.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + // Zstd magic is 0xFD2FB528 in little-endian: bytes 0x28 0xB5 0x2F 0xFD + assertEquals(data[0], (byte) 0x28, "Expected zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "Expected zstd magic byte 1"); + assertEquals(data[2], (byte) 0x2F, "Expected zstd magic byte 2"); + assertEquals(data[3], (byte) 0xFD, "Expected zstd magic byte 3"); + } + + @Test + public void testGzipRoundTrip() { + EVCacheSerializingTranscoder transcoder = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdRoundTrip() { + EVCacheSerializingTranscoder transcoder = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + transcoder.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testGzipTranscoderDecodesZstdData() { + // zstd transcoder writes, gzip transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + writer.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdTranscoderDecodesGzipData() { + // gzip transcoder writes, zstd transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + reader.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderDefaultsToGzip() { + EVCacheTranscoder transcoder = new EVCacheTranscoder(); + transcoder.setCompressionThreshold(0); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "EVCacheTranscoder must default to gzip"); + assertEquals(data[1], (byte) 0x8b, "EVCacheTranscoder must default to gzip"); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderExplicitAlgorithm() { + EVCacheTranscoder transcoder = new EVCacheTranscoder(CachedData.MAX_SIZE, 1); + transcoder.setCompressionAlgorithm(EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + transcoder.setCompressionLevel(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testUncompressedDataPassesThroughDecompress() { + // backward compat: data with no known magic bytes is returned as-is (not an error) + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + byte[] raw = new byte[]{0x01, 0x02, 0x03, 0x04, 0x05}; + byte[] result = t.decompress(raw); + assertEquals(result, raw, "Unrecognized data must be returned unchanged"); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidAlgorithmEnumThrows() { + EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("INVALID"); + } +} diff --git a/evcache-core/src/test/java/test-suite.xml b/evcache-core/src/test/java/test-suite.xml index f031a615..194ea07e 100644 --- a/evcache-core/src/test/java/test-suite.xml +++ b/evcache-core/src/test/java/test-suite.xml @@ -10,6 +10,11 @@ + + + + +