diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index d94729673..58ee89c1d 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -7,10 +7,14 @@ package com.linkedin.avroutil1.builder; import com.linkedin.avroutil1.compatibility.AvroCodecUtil; +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroRecordUtil; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.RandomRecordGenerator; import com.linkedin.avroutil1.compatibility.RecordGenerationConfig; import com.linkedin.avroutil1.compatibility.StringConverterUtil; + +import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; @@ -29,9 +33,13 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; + +import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt; import noutf8.TestCollections; import org.apache.avro.AvroRuntimeException; import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.Encoder; import org.apache.avro.util.Utf8; import org.testng.Assert; import org.testng.annotations.BeforeClass; @@ -2040,6 +2048,36 @@ public void testNoUtf8Encoding() throws IOException { Assert.assertTrue(instance.arOfMap.get(0).containsValue(strValue)); } + @DataProvider + private Object[][] customDecodeDataProvider() { + return new Object[][]{ + {vs19.MoneyRange.class}, + {vs110.MoneyRange.class}, + {vs111.MoneyRange.class} + }; + } + + @Test(dataProvider = "customDecodeDataProvider") + public void testCustomDecode(Class specificRecordClass) throws Exception { + RandomRecordGenerator generator = new RandomRecordGenerator(); + SpecificRecordBaseExt instance = generator.randomSpecific(specificRecordClass); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder encoder = AvroCompatibilityHelper.newBinaryEncoder(outputStream); + Method customEncodeMethod = instance.getClass().getMethod("customEncode", Encoder.class); + customEncodeMethod.invoke(instance, encoder); + encoder.flush(); + + byte[] data = outputStream.toByteArray(); + Decoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(data); + CustomDecoder decoder = + (CustomDecoder) AvroCompatibilityHelper.newCachedResolvingDecoder( + instance.getSchema(), instance.getSchema(), binaryDecoder); + SpecificRecordBaseExt decodedInstance = specificRecordClass.getDeclaredConstructor().newInstance(); + decodedInstance.customDecode(decoder); + Assert.assertEquals(instance, decodedInstance); + } + @BeforeClass public void setup() { System.setProperty("org.apache.avro.specific.use_custom_coders", "true"); diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java index 5e5ca3b17..879c10bc0 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java @@ -356,7 +356,7 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi classBuilder.addSuperinterface(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD); // extends - classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE); + classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE_EXT); //add class-level doc from schema doc //file-level (top of file) comment is added to the file object later @@ -519,9 +519,27 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in") .addException(IOException.class) .addModifiers(Modifier.PUBLIC); - addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter); + addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter, false); classBuilder.addMethod(customDecodeBuilder.build()); + //customDecode with CustomDecoder + MethodSpec.Builder methodBuilder = MethodSpec + .methodBuilder("customDecode") + .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER, "in") + .addException(IOException.class) + .addModifiers(Modifier.PUBLIC) + .addAnnotation(Override.class); + addCustomDecodeMethod(methodBuilder, recordSchema, config, classBuilder, sizeValCounter, true); + classBuilder.addMethod(methodBuilder.build()); + + MethodSpec.Builder isCustomDecodingEnabledMethod = MethodSpec + .methodBuilder("isCustomDecodingEnabled") + .addModifiers(Modifier.PUBLIC) + .returns(TypeName.BOOLEAN) + .addAnnotation(Override.class) + .addCode("return hasCustomCoders();"); + classBuilder.addMethod(isCustomDecodingEnabledMethod.build()); + // Builder TypeSpec.Builder recordBuilder = TypeSpec.classBuilder("Builder"); recordBuilder.addModifiers(Modifier.PUBLIC, Modifier.STATIC); @@ -945,8 +963,13 @@ private String getMethodNameForFieldWithPrefix(String prefix, String fieldName) } private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroRecordSchema recordSchema, - SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter) { + SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter, boolean isCustomDecoder) { int blockSize = 25, fieldCounter = 0, chunkCounter = 0; + + // Decoder class name. + ClassName decoderClassName = isCustomDecoder ? SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER : + SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER; + // reset var counter sizeValCounter.reset(); customDecodeBuilder.addStatement( @@ -959,7 +982,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR customDecodeBuilder.addStatement(chunkMethodName + "(in)"); // create new method MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName) - .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in") + .addParameter(decoderClassName, "in") .addException(IOException.class) .addModifiers(Modifier.PUBLIC); for (; fieldCounter < Math.min(blockSize * chunkCounter + blockSize, recordSchema.getFields().size()); @@ -989,7 +1012,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR customDecodeBuilder.addStatement(chunkMethodName + "(in, fieldOrder)"); // create new method MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName) - .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in") + .addParameter(decoderClassName, "in") .addParameter(ArrayTypeName.of(SpecificRecordGeneratorUtil.CLASSNAME_SCHEMA_FIELD), "fieldOrder") .addException(IOException.class) .addModifiers(Modifier.PUBLIC); diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java index e5eb32e57..6110aa4ba 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java @@ -51,9 +51,11 @@ public class SpecificRecordGeneratorUtil { public static final ClassName CLASSNAME_SPECIFIC_DATA = ClassName.get("org.apache.avro.specific", "SpecificData"); public static final ClassName CLASSNAME_SPECIFIC_RECORD = ClassName.get("org.apache.avro.specific", "SpecificRecord"); public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE = ClassName.get("org.apache.avro.specific", "SpecificRecordBase"); + public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE_EXT = ClassName.get("com.linkedin.avroutil1.compatibility.backports", "SpecificRecordBaseExt"); public static final ClassName CLASSNAME_SPECIFIC_DATUM_READER = ClassName.get("org.apache.avro.specific", "SpecificDatumReader"); public static final ClassName CLASSNAME_SPECIFIC_DATUM_WRITER = ClassName.get("org.apache.avro.specific", "SpecificDatumWriter"); public static final ClassName CLASSNAME_ENCODER = ClassName.get("org.apache.avro.io", "Encoder"); + public static final ClassName CLASSNAME_CUSTOM_DECODER = ClassName.get("com.linkedin.avroutil1.compatibility", "CustomDecoder"); public static final ClassName CLASSNAME_RESOLVING_DECODER = ClassName.get("org.apache.avro.io", "ResolvingDecoder"); public static final ClassName CLASSNAME_DATUM_READER = ClassName.get("org.apache.avro.io", "DatumReader"); public static final ClassName CLASSNAME_DATUM_WRITER = ClassName.get("org.apache.avro.io", "DatumWriter"); diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/CustomDecoder.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/CustomDecoder.java new file mode 100644 index 000000000..18a0be710 --- /dev/null +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/CustomDecoder.java @@ -0,0 +1,286 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility; + +import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; +import org.apache.avro.io.Encoder; +import org.apache.avro.util.Utf8; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Interface that enables custom decoding of + * {@link com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt} instances. + */ +public interface CustomDecoder { + + /** Returns the actual order in which the reader's fields will be + * returned to the reader. + * Throws a runtime exception if we're not just about to read the + * field of a record. Also, this method will consume the field + * information, and thus may only be called once before reading + * the field value. (However, if the client knows the order of + * incoming fields, then the client does not need to call this + * method but rather can just start reading the field values.) + * + * @throws AvroTypeException If we're not starting a new record + * + */ + Schema.Field[] readFieldOrder() throws IOException; + + /** + * Skips the Symbol.String and returns the size of the string to copy. + */ + int readStringSize() throws IOException; + + /** + * Skips the Symbol.BYTES and returns the size of the bytes to copy + */ + int readBytesSize() throws IOException; + + /** + * Reads fixed sized String object + */ + void readStringData(byte[] bytes, int start, int len) throws IOException; + + /** + * Reads fixed sized Byte object + */ + void readBytesData(byte[] bytes, int start, int len) throws IOException; + + /** + * "Reads" a null value. (Doesn't actually read anything, but + * advances the state of the parser if the implementation is + * stateful.) + * @throws AvroTypeException If this is a stateful reader and + * null is not the type of the next value to be read + */ + void readNull() throws IOException; + + /** + * Reads a boolean value written by {@link Encoder#writeBoolean}. + * @throws AvroTypeException If this is a stateful reader and + * boolean is not the type of the next value to be read + */ + + boolean readBoolean() throws IOException; + + /** + * Reads an integer written by {@link Encoder#writeInt}. + * @throws AvroTypeException If encoded value is larger than + * 32-bits + * @throws AvroTypeException If this is a stateful reader and + * int is not the type of the next value to be read + */ + int readInt() throws IOException; + + /** + * Reads a long written by {@link Encoder#writeLong}. + * @throws AvroTypeException If this is a stateful reader and + * long is not the type of the next value to be read + */ + long readLong() throws IOException; + + /** + * Reads a float written by {@link Encoder#writeFloat}. + * @throws AvroTypeException If this is a stateful reader and + * is not the type of the next value to be read + */ + float readFloat() throws IOException; + + /** + * Reads a double written by {@link Encoder#writeDouble}. + * @throws AvroTypeException If this is a stateful reader and + * is not the type of the next value to be read + */ + double readDouble() throws IOException; + + /** + * Reads a char-string written by {@link Encoder#writeString}. + * @throws AvroTypeException If this is a stateful reader and + * char-string is not the type of the next value to be read + */ + Utf8 readString(Utf8 old) throws IOException; + + /** + * Discards a char-string written by {@link Encoder#writeString}. + * @throws AvroTypeException If this is a stateful reader and + * char-string is not the type of the next value to be read + */ + void skipString() throws IOException; + + /** + * Reads a byte-string written by {@link Encoder#writeBytes}. + * if old is not null and has sufficient capacity to take in + * the bytes being read, the bytes are returned in old. + * @throws AvroTypeException If this is a stateful reader and + * byte-string is not the type of the next value to be read + */ + ByteBuffer readBytes(ByteBuffer old) throws IOException; + + /** + * Discards a byte-string written by {@link Encoder#writeBytes}. + * @throws AvroTypeException If this is a stateful reader and + * byte-string is not the type of the next value to be read + */ + void skipBytes() throws IOException; + + /** + * Reads fixed sized binary object. + * @param bytes The buffer to store the contents being read. + * @param start The position where the data needs to be written. + * @param length The size of the binary object. + * @throws AvroTypeException If this is a stateful reader and + * fixed sized binary object is not the type of the next + * value to be read or the length is incorrect. + * @throws IOException + */ + void readFixed(byte[] bytes, int start, int length) + throws IOException; + + /** + * A shorthand for readFixed(bytes, 0, bytes.length). + * @throws AvroTypeException If this is a stateful reader and + * fixed sized binary object is not the type of the next + * value to be read or the length is incorrect. + * @throws IOException + */ + void readFixed(byte[] bytes) throws IOException; + + /** + * Discards fixed sized binary object. + * @param length The size of the binary object to be skipped. + * @throws AvroTypeException If this is a stateful reader and + * fixed sized binary object is not the type of the next + * value to be read or the length is incorrect. + * @throws IOException + */ + void skipFixed(int length) throws IOException; + + /** + * Reads an enumeration. + * @return The enumeration's value. + * @throws AvroTypeException If this is a stateful reader and + * enumeration is not the type of the next value to be read. + * @throws IOException + */ + int readEnum() throws IOException; + + /** + * Reads and returns the size of the first block of an array. If + * this method returns non-zero, then the caller should read the + * indicated number of items, and then call {@link + * #arrayNext} to find out the number of items in the next + * block. The typical pattern for consuming an array looks like: + * 
{@code
+     *   for(long i = in.readArrayStart(); i != 0; i = in.arrayNext()) {
+     *     for (long j = 0; j < i; j++) {
+     *       read next element of the array;
+     *     }
+     *   }
+     * }
+ * @throws AvroTypeException If this is a stateful reader and + * array is not the type of the next value to be read */ + long readArrayStart() throws IOException; + + /** + * Processes the next block of an array andreturns the number of items in + * the block and let's the caller + * read those items. + * @throws AvroTypeException When called outside of an + * array context + */ + long arrayNext() throws IOException; + + /** + * Used for quickly skipping through an array. Note you can + * either skip the entire array, or read the entire array (with + * {@link #readArrayStart}), but you can't mix the two on the + * same array. + * + * This method will skip through as many items as it can, all of + * them if possible. It will return zero if there are no more + * items to skip through, or an item count if it needs the client's + * help in skipping. The typical usage pattern is: + * 
{@code
+     *   for(long i = in.skipArray(); i != 0; i = i.skipArray()) {
+     *     for (long j = 0; j < i; j++) {
+     *       read and discard the next element of the array;
+     *     }
+     *   }
+     * }
+ * Note that this method can automatically skip through items if a + * byte-count is found in the underlying data, or if a schema has + * been provided to the implementation, but + * otherwise the client will have to skip through items itself. + * + * @throws AvroTypeException If this is a stateful reader and + * array is not the type of the next value to be read + */ + long skipArray() throws IOException; + + /** + * Reads and returns the size of the next block of map-entries. + * Similar to {@link #readArrayStart}. + * + * As an example, let's say you want to read a map of records, + * the record consisting of an Long field and a Boolean field. + * Your code would look something like this: + * 
{@code
+     *   Map m = new HashMap();
+     *   Record reuse = new Record();
+     *   for(long i = in.readMapStart(); i != 0; i = in.readMapNext()) {
+     *     for (long j = 0; j < i; j++) {
+     *       String key = in.readString();
+     *       reuse.intField = in.readInt();
+     *       reuse.boolField = in.readBoolean();
+     *       m.put(key, reuse);
+     *     }
+     *   }
+     * }
+ * @throws AvroTypeException If this is a stateful reader and + * map is not the type of the next value to be read + */ + long readMapStart() throws IOException; + + /** + * Processes the next block of map entries and returns the count of them. + * Similar to {@link #arrayNext}. See {@link #readMapStart} for details. + * @throws AvroTypeException When called outside of a + * map context + */ + long mapNext() throws IOException; + + /** + * Support for quickly skipping through a map similar to {@link #skipArray}. + * + * As an example, let's say you want to skip a map of records, + * the record consisting of an Long field and a Boolean field. + * Your code would look something like this: + * 
{@code
+     *   for(long i = in.skipMap(); i != 0; i = in.skipMap()) {
+     *     for (long j = 0; j < i; j++) {
+     *       in.skipString();  // Discard key
+     *       in.readInt(); // Discard int-field of value
+     *       in.readBoolean(); // Discard boolean-field of value
+     *     }
+     *   }
+     * }
+ * @throws AvroTypeException If this is a stateful reader and + * array is not the type of the next value to be read */ + + long skipMap() throws IOException; + + /** + * Reads the tag of a union written by {@link Encoder#writeIndex}. + * @throws AvroTypeException If this is a stateful reader and + * union is not the type of the next value to be read + */ + int readIndex() throws IOException; +} diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/backports/SpecificRecordBaseExt.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/backports/SpecificRecordBaseExt.java new file mode 100644 index 000000000..05295864d --- /dev/null +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/backports/SpecificRecordBaseExt.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.backports; + +import com.linkedin.avroutil1.compatibility.CustomDecoder; +import org.apache.avro.specific.SpecificRecordBase; + +import java.io.IOException; + +/** + * Extension of {@link SpecificRecordBase} that allows for custom decoding using a custom decoder. + */ +public abstract class SpecificRecordBaseExt extends SpecificRecordBase { + + /** + * Indicates whether this record has custom decoding support enabled. + * + * @return true if custom decoding is enabled, false otherwise + */ + public boolean isCustomDecodingEnabled() { + return false; + } + + /** + * Custom decode method to be implemented by subclasses for custom decoding logic. + * + * @param in the custom decoder to use for decoding + * @throws IOException if an I/O error occurs during decoding + */ + public void customDecode(CustomDecoder in) throws IOException { + throw new UnsupportedOperationException("customDecode not implemented"); + } +} diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java index b0e1d95e5..a3df729b3 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java @@ -24,6 +24,8 @@ import com.linkedin.avroutil1.compatibility.SkipDecoder; import com.linkedin.avroutil1.compatibility.StringRepresentation; import com.linkedin.avroutil1.compatibility.avro110.backports.Avro110DefaultValuesCache; +import com.linkedin.avroutil1.compatibility.avro110.backports.GenericDatumReaderExt; +import com.linkedin.avroutil1.compatibility.avro110.backports.SpecificDatumReaderExt; import com.linkedin.avroutil1.compatibility.avro110.codec.AliasAwareSpecificDatumReader; import com.linkedin.avroutil1.compatibility.avro110.codec.BoundedMemoryDecoder; import com.linkedin.avroutil1.compatibility.avro110.codec.CachedResolvingDecoder; @@ -38,7 +40,6 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.Avro110BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; @@ -239,7 +240,7 @@ public DatumWriter newGenericDatumWriter(Schema writer, GenericData genericDa @Override public DatumReader newGenericDatumReader(Schema writer, Schema reader, GenericData genericData) { - return new GenericDatumReader<>(writer, reader, genericData); + return new GenericDatumReaderExt<>(writer, reader, genericData); } @Override @@ -249,7 +250,7 @@ public DatumWriter newSpecificDatumWriter(Schema writer, SpecificData specifi @Override public DatumReader newSpecificDatumReader(Schema writer, Schema reader, SpecificData specificData) { - return new SpecificDatumReader<>(writer, reader, specificData); + return new SpecificDatumReaderExt<>(writer, reader, specificData); } @Override diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/GenericDatumReaderExt.java new file mode 100644 index 000000000..d4de9f5ec --- /dev/null +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/GenericDatumReaderExt.java @@ -0,0 +1,177 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro110.backports; + +import com.linkedin.avroutil1.compatibility.avro110.codec.CachedResolvingDecoder; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro110GenericDataAccessUtil; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.io.Decoder; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link GenericDatumReader} with + * a specified {@link GenericData} instance under avro 1.10 + * + * @param + */ +public class GenericDatumReaderExt extends GenericDatumReader { + + public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericData) { + super(writer, reader, genericData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro110GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro110GenericDataAccessUtil.getField(data, r, name, pos, state); + } + Avro110GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), + read(oldDatum, f.schema(), in), state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + LogicalType logicalType = expectedType.getLogicalType(); + Conversion conversion = getData().getConversionFor(logicalType); + Object array = newArray(old, (int) l, expected); + do { + if (logicalType != null && conversion != null) { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, + readWithConversion(peekArray(array), expectedType, logicalType, conversion, in)); + } + } else { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, readWithoutConversion(peekArray(array), expectedType, in)); + } + } + base += l; + } while ((l = in.arrayNext()) > 0); + return pruneArray(array); + } else { + return pruneArray(newArray(old, 0, expected)); + } + } + + private Object pruneArray(Object object) { + if (object instanceof GenericArray) { + ((GenericArray) object).prune(); + } + return object; + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } + + private Object readWithConversion(Object old, Schema expected, + LogicalType logicalType, + Conversion conversion, + CachedResolvingDecoder in) throws IOException { + return convert(readWithoutConversion(old, expected, in), + expected, logicalType, conversion); + } +} diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/SpecificDatumReaderExt.java new file mode 100644 index 000000000..9fe212158 --- /dev/null +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/backports/SpecificDatumReaderExt.java @@ -0,0 +1,213 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro110.backports; + +import com.linkedin.avroutil1.compatibility.avro110.codec.CachedResolvingDecoder; +import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro110GenericDataAccessUtil; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Decoder; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link SpecificDatumReader} with + * a specified {@link SpecificData} instance under avro 1.9. + * + * @param + */ +public class SpecificDatumReaderExt extends SpecificDatumReader { + + public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specificData) { + super(writer, reader, specificData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + SpecificData specificData = getSpecificData(); + if (specificData.useCustomCoders()) { + old = specificData.newRecord(old, expected); + if (old instanceof SpecificRecordBaseExt) { + SpecificRecordBaseExt d = (SpecificRecordBaseExt) old; + if (d.isCustomDecodingEnabled()) { + d.customDecode(in); + return d; + } + } + } + + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro110GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro110GenericDataAccessUtil.getField(data, r, name, pos, state); + } + readField(r, f, oldDatum, in, state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + LogicalType logicalType = expectedType.getLogicalType(); + Conversion conversion = getData().getConversionFor(logicalType); + Object array = newArray(old, (int) l, expected); + do { + if (logicalType != null && conversion != null) { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, + readWithConversion(peekArray(array), expectedType, logicalType, conversion, in)); + } + } else { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, readWithoutConversion(peekArray(array), expectedType, in)); + } + } + base += l; + } while ((l = in.arrayNext()) > 0); + return pruneArray(array); + } else { + return pruneArray(newArray(old, 0, expected)); + } + } + + private Object pruneArray(Object object) { + if (object instanceof GenericArray) { + ((GenericArray) object).prune(); + } + return object; + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } + + private void readField(Object r, Schema.Field f, Object oldDatum, + CachedResolvingDecoder in, Object state) + throws IOException { + if (r instanceof SpecificRecordBase) { + Conversion conversion = ((SpecificRecordBase) r).getConversion(f.pos()); + + Object datum; + if (conversion != null) { + datum = readWithConversion( + oldDatum, f.schema(), f.schema().getLogicalType(), conversion, in); + } else { + datum = readWithoutConversion(oldDatum, f.schema(), in); + } + + getData().setField(r, f.name(), f.pos(), datum); + + } else { + Avro110GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), + read(oldDatum, f.schema(), in), state); + } + } + + private Object readWithConversion(Object old, Schema expected, + LogicalType logicalType, + Conversion conversion, + CachedResolvingDecoder in) throws IOException { + return convert(readWithoutConversion(old, expected, in), + expected, logicalType, conversion); + } +} diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/AliasAwareSpecificDatumReader.java index cceb502ec..320d805a5 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro110.codec; +import com.linkedin.avroutil1.compatibility.avro110.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -20,7 +21,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { public AliasAwareSpecificDatumReader() { this(null, null); @@ -40,10 +41,12 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } protected AliasAwareSpecificDatumReader(SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } } diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/ResolvingDecoder.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/ResolvingDecoder.java index 43bdde577..230a349d8 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro110.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro110.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro110.parsing.Symbol; import java.io.IOException; @@ -38,7 +39,7 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.util.Utf8; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; ResolvingDecoder(Schema writer, Schema reader, Decoder in) throws IOException { @@ -69,6 +70,24 @@ public final void drain() throws IOException { this.parser.processImplicitActions(); } + @Override + public int readInt() throws IOException { + Symbol actual = parser.advance(Symbol.INT); + + if (actual == Symbol.INT) { + return in.readInt(); + } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { + long value = in.readLong(); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new AvroTypeException(value + " cannot be represented as int"); + } + + return (int) value; + } + + throw new AvroTypeException("Expected int but found " + actual); + } + public long readLong() throws IOException { Symbol actual = this.parser.advance(Symbol.LONG); if (actual == Symbol.INT) { @@ -225,6 +244,10 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { this.backup = this.in; this.in = DecoderFactory.get().binaryDecoder(dsa.contents, (BinaryDecoder)null); } else { + if (top == Symbol.IntLongAdjustAction.INSTANCE) { + return top; + } + if (top != Symbol.DEFAULT_END_ACTION) { throw new AvroTypeException("Unknown action: " + top); } diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/ResolvingGrammarGenerator.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/ResolvingGrammarGenerator.java index 09aa81439..e7424366a 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/ResolvingGrammarGenerator.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/ResolvingGrammarGenerator.java @@ -82,6 +82,11 @@ private Symbol generate(Resolver.Action action, Map seen, boolea return simpleGen(action.writer, seen, useFqcns); } else if (action instanceof Resolver.ErrorAction) { + // We should be able to selectively demote long to int if it fits within the range of int. + if (isLongToIntDemotion((Resolver.ErrorAction) action, false)) { + return Symbol.IntLongAdjustAction.INSTANCE; + } + return Symbol.error(action.toString()); } else if (action instanceof Resolver.Skip) { @@ -92,6 +97,16 @@ private Symbol generate(Resolver.Action action, Map seen, boolea } else if (action instanceof Resolver.ReaderUnion) { Resolver.ReaderUnion ru = (Resolver.ReaderUnion) action; + + // Check if we need to handle selective long-to-int demotion within unions. + if (ru.actualAction instanceof Resolver.ErrorAction) { + if (isLongToIntDemotion((Resolver.ErrorAction) ru.actualAction, true)) { + return Symbol.seq( + Symbol.unionAdjustAction(ru.firstMatch, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } + } + Symbol s = generate(ru.actualAction, seen, useFqcns); return Symbol.seq(Symbol.unionAdjustAction(ru.firstMatch, s), Symbol.UNION); @@ -107,18 +122,28 @@ private Symbol generate(Resolver.Action action, Map seen, boolea if (((Resolver.WriterUnion) action).unionEquiv) { return simpleGen(action.writer, seen, useFqcns); } - Resolver.Action[] branches = ((Resolver.WriterUnion) action).actions; + + Resolver.WriterUnion wu = (Resolver.WriterUnion) action; + Resolver.Action[] branches = wu.actions; Symbol[] symbols = new Symbol[branches.length]; String[] oldLabels = new String[branches.length]; String[] newLabels = new String[branches.length]; - int i = 0; - for (Resolver.Action branch : branches) { - symbols[i] = generate(branch, seen, useFqcns); + + for (int i = 0; i < branches.length; i++) { + // Check if this branch needs long-to-int conversion + if (branches[i] instanceof Resolver.ErrorAction && isLongToIntDemotion((Resolver.ErrorAction) branches[i], true)) { + symbols[i] = Symbol.seq( + Symbol.unionAdjustAction(i, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } else { + symbols[i] = generate(branches[i], seen, useFqcns); + } + Schema schema = action.writer.getTypes().get(i); oldLabels[i] = schema.getName(); newLabels[i] = schema.getFullName(); - i++; } + return Symbol.seq(Symbol.alt(symbols, oldLabels, newLabels, useFqcns), Symbol.WRITER_UNION_ACTION); } else if (action instanceof Resolver.EnumAdjust) { Resolver.EnumAdjust e = (Resolver.EnumAdjust) action; @@ -352,6 +377,39 @@ public static void encode(Encoder e, Schema s, JsonNode n) throws IOException { } } + private static boolean isLongToIntDemotion(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + return isInt(errorAction.reader, includeIntUnions) + && errorAction.writer.getType() == Schema.Type.LONG + && isLongToIntDemotionError(errorAction, includeIntUnions); + } + + private static boolean isLongToIntDemotionError(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + if (errorAction.error == Resolver.ErrorAction.ErrorType.INCOMPATIBLE_SCHEMA_TYPES) { + return true; + } + + return includeIntUnions && errorAction.error == Resolver.ErrorAction.ErrorType.NO_MATCHING_BRANCH; + } + + private static boolean isInt(Schema schema, boolean includeIntUnions) { + if (schema.getType() == Schema.Type.INT) { + return true; + } + + if (!includeIntUnions) { + return false; + } + + if (schema.getType() != Schema.Type.UNION) { + return false; + } + + List types = schema.getTypes(); + return (types.size() == 2 + && types.get(0).getType() == Schema.Type.NULL + && types.get(1).getType() == Schema.Type.INT); + } + /** * Clever trick which differentiates items put into * seen by {@link ValidatingGrammarGenerator validating()} diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/Symbol.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/Symbol.java index 07ae5918c..9ba84b171 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/Symbol.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/parsing/Symbol.java @@ -615,6 +615,10 @@ public FieldAdjustAction(int rindex, String fname, Set aliases) { } } + public static class IntLongAdjustAction extends ImplicitAction { + public static final IntLongAdjustAction INSTANCE = new IntLongAdjustAction(); + } + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } diff --git a/helper/impls/helper-impl-110/src/main/java/org/apache/avro/generic/Avro110GenericDataAccessUtil.java b/helper/impls/helper-impl-110/src/main/java/org/apache/avro/generic/Avro110GenericDataAccessUtil.java new file mode 100644 index 000000000..1f2c62f21 --- /dev/null +++ b/helper/impls/helper-impl-110/src/main/java/org/apache/avro/generic/Avro110GenericDataAccessUtil.java @@ -0,0 +1,29 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.generic; + +import org.apache.avro.Schema; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link GenericData} + */ +public class Avro110GenericDataAccessUtil { + private Avro110GenericDataAccessUtil() { + } + + public static Object getRecordState(GenericData data, Object record, Schema schema) { + return data.getRecordState(record, schema); + } + + public static Object getField(GenericData data, Object record, String name, int pos, Object state) { + return data.getField(record, name, pos, state); + } + + public static void setField(GenericData data, Object record, String name, int pos, Object value, Object state) { + data.setField(record, name, pos, value, state); + } +} diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java index c471aa6ce..ebed681ab 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java @@ -24,6 +24,8 @@ import com.linkedin.avroutil1.compatibility.SkipDecoder; import com.linkedin.avroutil1.compatibility.StringRepresentation; import com.linkedin.avroutil1.compatibility.avro111.backports.Avro111DefaultValuesCache; +import com.linkedin.avroutil1.compatibility.avro111.backports.GenericDatumReaderExt; +import com.linkedin.avroutil1.compatibility.avro111.backports.SpecificDatumReaderExt; import com.linkedin.avroutil1.compatibility.avro111.codec.AliasAwareSpecificDatumReader; import com.linkedin.avroutil1.compatibility.avro111.codec.BoundedMemoryDecoder; import com.linkedin.avroutil1.compatibility.avro111.codec.CachedResolvingDecoder; @@ -38,7 +40,6 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.Avro111BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; @@ -239,7 +240,7 @@ public DatumWriter newGenericDatumWriter(Schema writer, GenericData genericDa @Override public DatumReader newGenericDatumReader(Schema writer, Schema reader, GenericData genericData) { - return new GenericDatumReader<>(writer, reader, genericData); + return new GenericDatumReaderExt<>(writer, reader, genericData); } @Override @@ -249,7 +250,7 @@ public DatumWriter newSpecificDatumWriter(Schema writer, SpecificData specifi @Override public DatumReader newSpecificDatumReader(Schema writer, Schema reader, SpecificData specificData) { - return new SpecificDatumReader<>(writer, reader, specificData); + return new SpecificDatumReaderExt<>(writer, reader, specificData); } @Override diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/GenericDatumReaderExt.java new file mode 100644 index 000000000..79ca35844 --- /dev/null +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/GenericDatumReaderExt.java @@ -0,0 +1,152 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro111.backports; + +import com.linkedin.avroutil1.compatibility.avro111.codec.CachedResolvingDecoder; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro111GenericDataAccessUtil; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.io.Decoder; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link GenericDatumReader} with + * a specified {@link GenericData} instance under avro 1.10 + * + * @param + */ +public class GenericDatumReaderExt extends GenericDatumReader { + + public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericData) { + super(writer, reader, genericData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro111GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro111GenericDataAccessUtil.getField(data, r, name, pos, state); + } + Avro111GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), + read(oldDatum, f.schema(), in), state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + Object array = newArray(old, (int) l, expected); + do { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, read(peekArray(array), expectedType, in)); + } + base += l; + } while ((l = in.arrayNext()) > 0); + return array; + } else { + return newArray(old, 0, expected); + } + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } +} diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/SpecificDatumReaderExt.java new file mode 100644 index 000000000..4515001f1 --- /dev/null +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/backports/SpecificDatumReaderExt.java @@ -0,0 +1,196 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro111.backports; + +import com.linkedin.avroutil1.compatibility.avro111.codec.CachedResolvingDecoder; +import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro111GenericDataAccessUtil; +import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Decoder; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link SpecificDatumReader} with + * a specified {@link SpecificData} instance under avro 1.9. + * + * @param + */ +public class SpecificDatumReaderExt extends SpecificDatumReader { + + public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specificData) { + super(writer, reader, specificData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + SpecificData specificData = getSpecificData(); + if (specificData.useCustomCoders()) { + old = specificData.newRecord(old, expected); + if (old instanceof SpecificRecordBaseExt) { + SpecificRecordBaseExt d = (SpecificRecordBaseExt) old; + if (d.isCustomDecodingEnabled()) { + d.customDecode(in); + return d; + } + } + } + + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro111GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro111GenericDataAccessUtil.getField(data, r, name, pos, state); + } + readField(r, f, oldDatum, in, state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + Object array = newArray(old, (int) l, expected); + do { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, read(peekArray(array), expectedType, in)); + } + base += l; + } while ((l = in.arrayNext()) > 0); + return array; + } else { + return newArray(old, 0, expected); + } + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } + + private void readField(Object r, Schema.Field f, Object oldDatum, + CachedResolvingDecoder in, Object state) + throws IOException { + if (r instanceof SpecificRecordBase) { + Conversion conversion = ((SpecificRecordBase) r).getConversion(f.pos()); + + Object datum; + if (conversion != null) { + datum = readWithConversion( + oldDatum, f.schema(), f.schema().getLogicalType(), conversion, in); + } else { + datum = readWithoutConversion(oldDatum, f.schema(), in); + } + + getData().setField(r, f.name(), f.pos(), datum); + + } else { + Avro111GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), + read(oldDatum, f.schema(), in), state); + } + } + + private Object readWithConversion(Object old, Schema expected, + LogicalType logicalType, + Conversion conversion, + CachedResolvingDecoder in) throws IOException { + return convert(readWithoutConversion(old, expected, in), + expected, logicalType, conversion); + } +} diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/AliasAwareSpecificDatumReader.java index 854cb939a..83aa085f0 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro111.codec; +import com.linkedin.avroutil1.compatibility.avro111.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -20,7 +21,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { public AliasAwareSpecificDatumReader() { this(null, null); @@ -40,10 +41,12 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } protected AliasAwareSpecificDatumReader(SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } } diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/ResolvingDecoder.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/ResolvingDecoder.java index 56e031754..8e060fca9 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro111.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro111.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro111.parsing.Symbol; import org.apache.avro.AvroTypeException; @@ -39,7 +40,7 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; ResolvingDecoder(Schema writer, Schema reader, Decoder in) throws IOException { @@ -70,6 +71,23 @@ public final void drain() throws IOException { this.parser.processImplicitActions(); } + @Override + public int readInt() throws IOException { + Symbol actual = parser.advance(Symbol.INT); + if (actual == Symbol.INT) { + return in.readInt(); + } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { + long value = in.readLong(); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new AvroTypeException(value + " cannot be represented as int"); + } + + return (int) value; + } + + throw new AvroTypeException("Expected int but found " + actual); + } + public long readLong() throws IOException { Symbol actual = this.parser.advance(Symbol.LONG); if (actual == Symbol.INT) { @@ -226,6 +244,10 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { this.backup = this.in; this.in = DecoderFactory.get().binaryDecoder(dsa.contents, (BinaryDecoder)null); } else { + if (top == Symbol.IntLongAdjustAction.INSTANCE) { + return top; + } + if (top != Symbol.DEFAULT_END_ACTION) { throw new AvroTypeException("Unknown action: " + top); } diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/ResolvingGrammarGenerator.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/ResolvingGrammarGenerator.java index c2bfb6f5b..84a3db788 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/ResolvingGrammarGenerator.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/ResolvingGrammarGenerator.java @@ -82,6 +82,11 @@ private Symbol generate(Resolver.Action action, Map seen, boolea return simpleGen(action.writer, seen, useFqcns); } else if (action instanceof Resolver.ErrorAction) { + // We should be able to selectively demote long to int if it fits within the range of int. + if (isLongToIntDemotion((Resolver.ErrorAction) action, false)) { + return Symbol.IntLongAdjustAction.INSTANCE; + } + return Symbol.error(action.toString()); } else if (action instanceof Resolver.Skip) { @@ -92,6 +97,16 @@ private Symbol generate(Resolver.Action action, Map seen, boolea } else if (action instanceof Resolver.ReaderUnion) { Resolver.ReaderUnion ru = (Resolver.ReaderUnion) action; + + // Check if we need to handle selective long-to-int demotion within unions. + if (ru.actualAction instanceof Resolver.ErrorAction) { + if (isLongToIntDemotion((Resolver.ErrorAction) ru.actualAction, true)) { + return Symbol.seq( + Symbol.unionAdjustAction(ru.firstMatch, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } + } + Symbol s = generate(ru.actualAction, seen, useFqcns); return Symbol.seq(Symbol.unionAdjustAction(ru.firstMatch, s), Symbol.UNION); @@ -107,18 +122,28 @@ private Symbol generate(Resolver.Action action, Map seen, boolea if (((Resolver.WriterUnion) action).unionEquiv) { return simpleGen(action.writer, seen, useFqcns); } - Resolver.Action[] branches = ((Resolver.WriterUnion) action).actions; + + Resolver.WriterUnion wu = (Resolver.WriterUnion) action; + Resolver.Action[] branches = wu.actions; Symbol[] symbols = new Symbol[branches.length]; String[] oldLabels = new String[branches.length]; String[] newLabels = new String[branches.length]; - int i = 0; - for (Resolver.Action branch : branches) { - symbols[i] = generate(branch, seen, useFqcns); + + for (int i = 0; i < branches.length; i++) { + // Check if this branch needs long-to-int conversion + if (branches[i] instanceof Resolver.ErrorAction && isLongToIntDemotion((Resolver.ErrorAction) branches[i], true)) { + symbols[i] = Symbol.seq( + Symbol.unionAdjustAction(i, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } else { + symbols[i] = generate(branches[i], seen, useFqcns); + } + Schema schema = action.writer.getTypes().get(i); oldLabels[i] = schema.getName(); newLabels[i] = schema.getFullName(); - i++; } + return Symbol.seq(Symbol.alt(symbols, oldLabels, newLabels, useFqcns), Symbol.WRITER_UNION_ACTION); } else if (action instanceof Resolver.EnumAdjust) { Resolver.EnumAdjust e = (Resolver.EnumAdjust) action; @@ -352,6 +377,39 @@ public static void encode(Encoder e, Schema s, JsonNode n) throws IOException { } } + private static boolean isLongToIntDemotion(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + return isInt(errorAction.reader, includeIntUnions) + && errorAction.writer.getType() == Schema.Type.LONG + && isLongToIntDemotionError(errorAction, includeIntUnions); + } + + private static boolean isLongToIntDemotionError(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + if (errorAction.error == Resolver.ErrorAction.ErrorType.INCOMPATIBLE_SCHEMA_TYPES) { + return true; + } + + return includeIntUnions && errorAction.error == Resolver.ErrorAction.ErrorType.NO_MATCHING_BRANCH; + } + + private static boolean isInt(Schema schema, boolean includeIntUnions) { + if (schema.getType() == Schema.Type.INT) { + return true; + } + + if (!includeIntUnions) { + return false; + } + + if (schema.getType() != Schema.Type.UNION) { + return false; + } + + List types = schema.getTypes(); + return (types.size() == 2 + && types.get(0).getType() == Schema.Type.NULL + && types.get(1).getType() == Schema.Type.INT); + } + /** * Clever trick which differentiates items put into * seen by {@link ValidatingGrammarGenerator validating()} diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/Symbol.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/Symbol.java index 25d467bae..62b0d6766 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/Symbol.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/parsing/Symbol.java @@ -615,6 +615,10 @@ public FieldAdjustAction(int rindex, String fname, Set aliases) { } } + public static class IntLongAdjustAction extends ImplicitAction { + public static final IntLongAdjustAction INSTANCE = new IntLongAdjustAction(); + } + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } diff --git a/helper/impls/helper-impl-111/src/main/java/org/apache/avro/generic/Avro111GenericDataAccessUtil.java b/helper/impls/helper-impl-111/src/main/java/org/apache/avro/generic/Avro111GenericDataAccessUtil.java new file mode 100644 index 000000000..45e1ae846 --- /dev/null +++ b/helper/impls/helper-impl-111/src/main/java/org/apache/avro/generic/Avro111GenericDataAccessUtil.java @@ -0,0 +1,29 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.generic; + +import org.apache.avro.Schema; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link GenericData} + */ +public class Avro111GenericDataAccessUtil { + private Avro111GenericDataAccessUtil() { + } + + public static Object getRecordState(GenericData data, Object record, Schema schema) { + return data.getRecordState(record, schema); + } + + public static Object getField(GenericData data, Object record, String name, int pos, Object state) { + return data.getField(record, name, pos, state); + } + + public static void setField(GenericData data, Object record, String name, int pos, Object value, Object state) { + data.setField(record, name, pos, value, state); + } +} diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/AliasAwareSpecificDatumReader.java index 38b3cda6a..31fe3aa11 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro14.codec; +import com.linkedin.avroutil1.compatibility.avro14.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -23,20 +24,21 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { //same idea as the one in SpecificData protected final static Map> CLASS_CACHE = new ConcurrentHashMap<>(); public AliasAwareSpecificDatumReader() { + this(null, null); } public AliasAwareSpecificDatumReader(Class c) { - super(c); + this(SpecificData.get().getSchema(c)); } public AliasAwareSpecificDatumReader(Schema schema) { - super(schema); + this(schema, schema); } public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/ResolvingDecoder.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/ResolvingDecoder.java index 8b5b61a9a..e18e67ca7 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro14.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro14.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro14.parsing.Symbol; import java.io.IOException; @@ -45,7 +46,7 @@ *

See the parser documentation for * information on how this works. */ -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; @@ -153,7 +154,7 @@ public final void drain() throws IOException { @Override public int readInt() throws IOException { - Symbol actual = parser.popSymbol(); + Symbol actual = parser.advance(Symbol.INT); if (actual == Symbol.INT) { return in.readInt(); } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { @@ -245,8 +246,7 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { } else if (top == Symbol.DEFAULT_END_ACTION) { in = backup; } else if (top == Symbol.IntLongAdjustAction.INSTANCE) { - parser.pushSymbol(Symbol.INT); - return Symbol.INT; + return top; } else { throw new AvroTypeException("Unknown action: " + top); } diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/GenericDatumReaderExt.java index bfb89434f..9b373996e 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/GenericDatumReaderExt.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/GenericDatumReaderExt.java @@ -59,7 +59,7 @@ public void setSchema(Schema writer) { @SuppressWarnings("unchecked") @Override public T read(T reuse, Decoder in) throws IOException { - CachedResolvingDecoder resolver = new CachedResolvingDecoder(writer, reader, in); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.init(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/SpecificDatumReaderExt.java index 1f487e4dc..5b01aba17 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/SpecificDatumReaderExt.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/backports/SpecificDatumReaderExt.java @@ -59,7 +59,7 @@ public void setSchema(Schema writer) { @SuppressWarnings("unchecked") @Override public T read(T reuse, Decoder in) throws IOException { - CachedResolvingDecoder resolver = new CachedResolvingDecoder(writer, reader, in); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.init(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/AliasAwareSpecificDatumReader.java index 4e4b6ddf2..fc5586a02 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro15.codec; +import com.linkedin.avroutil1.compatibility.avro15.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -23,24 +24,25 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { //same idea as the one in SpecificData protected final static Map> CLASS_CACHE = new ConcurrentHashMap<>(); public AliasAwareSpecificDatumReader() { + this(null, null); } public AliasAwareSpecificDatumReader(Class c) { - super(c); + this(SpecificData.get().getSchema(c)); } public AliasAwareSpecificDatumReader(Schema schema) { - super(schema); + this(schema, schema); } public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { - super(writer, reader); + super(writer, reader, SpecificData.get()); } @Override diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/ResolvingDecoder.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/ResolvingDecoder.java index 0243b0ced..dde1ac8e4 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro15.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro15.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro15.parsing.Symbol; import java.io.IOException; @@ -45,7 +46,7 @@ *

See the parser documentation for * information on how this works. */ -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; @@ -158,7 +159,7 @@ public final void drain() throws IOException { @Override public int readInt() throws IOException { - Symbol actual = parser.popSymbol(); + Symbol actual = parser.advance(Symbol.INT); if (actual == Symbol.INT) { return in.readInt(); } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { @@ -263,8 +264,7 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { } else if (top == Symbol.DEFAULT_END_ACTION) { in = backup; } else if (top == Symbol.IntLongAdjustAction.INSTANCE) { - parser.pushSymbol(Symbol.INT); - return Symbol.INT; + return top; } else { throw new AvroTypeException("Unknown action: " + top); } diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/GenericDatumReaderExt.java index 4f286e8cf..479d1080f 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/GenericDatumReaderExt.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/GenericDatumReaderExt.java @@ -34,7 +34,8 @@ public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericDa @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/SpecificDatumReaderExt.java index 0f9be5e86..4db31af34 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/SpecificDatumReaderExt.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/backports/SpecificDatumReaderExt.java @@ -35,7 +35,8 @@ public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specifi @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/AliasAwareSpecificDatumReader.java index 9258fe819..7f52b63fd 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro16.codec; +import com.linkedin.avroutil1.compatibility.avro16.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -23,7 +24,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { //same idea as the one in SpecificData protected final static Map> CLASS_CACHE = new ConcurrentHashMap<>(); @@ -45,6 +46,7 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/ResolvingDecoder.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/ResolvingDecoder.java index bd44d71a0..ee8b6871f 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro16.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro16.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro16.parsing.Symbol; import java.io.IOException; @@ -34,7 +35,7 @@ import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; ResolvingDecoder(Schema writer, Schema reader, Decoder in) throws IOException { @@ -66,7 +67,7 @@ public final void drain() throws IOException { @Override public int readInt() throws IOException { - Symbol actual = parser.popSymbol(); + Symbol actual = parser.advance(Symbol.INT); if (actual == Symbol.INT) { return in.readInt(); } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { @@ -164,8 +165,7 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { } if (top == Symbol.IntLongAdjustAction.INSTANCE) { - parser.pushSymbol(Symbol.INT); - return Symbol.INT; + return top; } if (top instanceof Symbol.DefaultStartAction) { diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/GenericDatumReaderExt.java index a05ea957c..82025d73c 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/GenericDatumReaderExt.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/GenericDatumReaderExt.java @@ -34,7 +34,8 @@ public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericDa @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/SpecificDatumReaderExt.java index 72a01910b..c8445451d 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/SpecificDatumReaderExt.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/backports/SpecificDatumReaderExt.java @@ -35,7 +35,8 @@ public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specifi @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/AliasAwareSpecificDatumReader.java index 5fbe2ba05..989b91001 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro17.codec; +import com.linkedin.avroutil1.compatibility.avro17.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -24,7 +25,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { //same idea as the one in SpecificData protected final static Map> CLASS_CACHE = new ConcurrentHashMap<>(); @@ -47,10 +48,12 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } protected AliasAwareSpecificDatumReader(SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/ResolvingDecoder.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/ResolvingDecoder.java index ff656b811..5c9690061 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro17.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro17.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro17.parsing.Symbol; import java.io.IOException; @@ -37,7 +38,7 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.util.Utf8; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; private static final Charset UTF8 = Charset.forName("UTF-8"); @@ -70,7 +71,7 @@ public final void drain() throws IOException { @Override public int readInt() throws IOException { - Symbol actual = parser.popSymbol(); + Symbol actual = parser.advance(Symbol.INT); if (actual == Symbol.INT) { return in.readInt(); } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { @@ -226,8 +227,7 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { } if (top == Symbol.IntLongAdjustAction.INSTANCE) { - parser.pushSymbol(Symbol.INT); - return Symbol.INT; + return top; } if (top instanceof Symbol.DefaultStartAction) { diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/GenericDatumReaderExt.java index df1c80b52..3a09719c3 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/GenericDatumReaderExt.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/GenericDatumReaderExt.java @@ -38,7 +38,8 @@ public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericDa @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/SpecificDatumReaderExt.java index 67e3395b5..4b02fbd06 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/SpecificDatumReaderExt.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/backports/SpecificDatumReaderExt.java @@ -40,7 +40,8 @@ public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specifi @Override public T read(T reuse, Decoder in) throws IOException { final Schema reader = getExpected(); - CachedResolvingDecoder resolver = new CachedResolvingDecoder(getSchema(), reader, in); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); resolver.configure(in); T result = (T) read(reuse, reader, resolver); resolver.drain(); diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/AliasAwareSpecificDatumReader.java index bae57cf7c..04956f057 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro18.codec; +import com.linkedin.avroutil1.compatibility.avro18.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -20,7 +21,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { public AliasAwareSpecificDatumReader() { this(null, null); @@ -40,10 +41,12 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } protected AliasAwareSpecificDatumReader(SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } } diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/ResolvingDecoder.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/ResolvingDecoder.java index 765447e94..bf60440ce 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/codec/ResolvingDecoder.java @@ -24,6 +24,7 @@ package com.linkedin.avroutil1.compatibility.avro18.codec; +import com.linkedin.avroutil1.compatibility.CustomDecoder; import com.linkedin.avroutil1.compatibility.avro18.parsing.ResolvingGrammarGenerator; import com.linkedin.avroutil1.compatibility.avro18.parsing.Symbol; import java.io.IOException; @@ -37,7 +38,7 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.util.Utf8; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; private static final Charset UTF8 = Charset.forName("UTF-8"); @@ -70,7 +71,7 @@ public final void drain() throws IOException { @Override public int readInt() throws IOException { - Symbol actual = parser.popSymbol(); + Symbol actual = parser.advance(Symbol.INT); if (actual == Symbol.INT) { return in.readInt(); } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { @@ -226,8 +227,7 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { } if (top == Symbol.IntLongAdjustAction.INSTANCE) { - parser.pushSymbol(Symbol.INT); - return Symbol.INT; + return top; } if (top instanceof Symbol.DefaultStartAction) { diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java index 0af84ff78..45b8bebca 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java @@ -24,6 +24,8 @@ import com.linkedin.avroutil1.compatibility.SkipDecoder; import com.linkedin.avroutil1.compatibility.StringRepresentation; import com.linkedin.avroutil1.compatibility.avro19.backports.Avro19DefaultValuesCache; +import com.linkedin.avroutil1.compatibility.avro19.backports.GenericDatumReaderExt; +import com.linkedin.avroutil1.compatibility.avro19.backports.SpecificDatumReaderExt; import com.linkedin.avroutil1.compatibility.avro19.codec.AliasAwareSpecificDatumReader; import com.linkedin.avroutil1.compatibility.avro19.codec.BoundedMemoryDecoder; import com.linkedin.avroutil1.compatibility.avro19.codec.CachedResolvingDecoder; @@ -38,7 +40,6 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.Avro19BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; @@ -242,7 +243,7 @@ public DatumWriter newGenericDatumWriter(Schema writer, GenericData genericDa @Override public DatumReader newGenericDatumReader(Schema writer, Schema reader, GenericData genericData) { - return new GenericDatumReader<>(writer, reader, genericData); + return new GenericDatumReaderExt<>(writer, reader, genericData); } @Override @@ -252,7 +253,7 @@ public DatumWriter newSpecificDatumWriter(Schema writer, SpecificData specifi @Override public DatumReader newSpecificDatumReader(Schema writer, Schema reader, SpecificData specificData) { - return new SpecificDatumReader<>(writer, reader, specificData); + return new SpecificDatumReaderExt<>(writer, reader, specificData); } @Override diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/GenericDatumReaderExt.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/GenericDatumReaderExt.java new file mode 100644 index 000000000..06d714f1c --- /dev/null +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/GenericDatumReaderExt.java @@ -0,0 +1,151 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro19.backports; + +import com.linkedin.avroutil1.compatibility.avro19.codec.CachedResolvingDecoder; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro19GenericDataAccessUtil; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.io.Decoder; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link GenericDatumReader} with + * a specified {@link GenericData} instance under avro 1.9 + * + * @param + */ +public class GenericDatumReaderExt extends GenericDatumReader { + + public GenericDatumReaderExt(Schema writer, Schema reader, GenericData genericData) { + super(writer, reader, genericData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro19GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro19GenericDataAccessUtil.getField(data, r, name, pos, state); + } + Avro19GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), read(oldDatum, f.schema(), in), state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + Object array = newArray(old, (int) l, expected); + do { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, read(peekArray(array), expectedType, in)); + } + base += l; + } while ((l = in.arrayNext()) > 0); + return array; + } else { + return newArray(old, 0, expected); + } + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } +} diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/SpecificDatumReaderExt.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/SpecificDatumReaderExt.java new file mode 100644 index 000000000..eee2f73f7 --- /dev/null +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/backports/SpecificDatumReaderExt.java @@ -0,0 +1,196 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro19.backports; + +import com.linkedin.avroutil1.compatibility.avro19.codec.CachedResolvingDecoder; +import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.generic.Avro19GenericDataAccessUtil; +import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Decoder; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; + +import java.io.IOException; + + +/** + * this class allows constructing a {@link SpecificDatumReader} with + * a specified {@link SpecificData} instance under avro 1.9. + * + * @param + */ +public class SpecificDatumReaderExt extends SpecificDatumReader { + + public SpecificDatumReaderExt(Schema writer, Schema reader, SpecificData specificData) { + super(writer, reader, specificData); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings("unchecked") + @Override + public T read(T reuse, Decoder in) throws IOException { + final Schema reader = getExpected(); + final Schema writer = getSchema(); + CachedResolvingDecoder resolver = new CachedResolvingDecoder(Schema.applyAliases(writer, reader), reader, in); + resolver.configure(in); + T result = (T) read(reuse, reader, resolver); + resolver.drain(); + return result; + } + + private Object read(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Object datum = readWithoutConversion(old, expected, in); + LogicalType logicalType = expected.getLogicalType(); + if (logicalType != null) { + Conversion conversion = getData().getConversionFor(logicalType); + if (conversion != null) { + return convert(datum, expected, logicalType, conversion); + } + } + return datum; + } + + private Object readWithoutConversion(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + switch (expected.getType()) { + case RECORD: + return readRecord(old, expected, in); + case ENUM: + return readEnum(expected, in); + case ARRAY: + return readArray(old, expected, in); + case MAP: + return readMap(old, expected, in); + case UNION: + return read(old, expected.getTypes().get(in.readIndex()), in); + case FIXED: + return readFixed(old, expected, in); + case STRING: + return readString(old, expected, in); + case BYTES: + return readBytes(old, expected, in); + case INT: + return readInt(old, expected, in); + case LONG: + return in.readLong(); + case FLOAT: + return in.readFloat(); + case DOUBLE: + return in.readDouble(); + case BOOLEAN: + return in.readBoolean(); + case NULL: + in.readNull(); + return null; + default: + throw new AvroRuntimeException("Unknown type: " + expected); + } + } + + private Object readRecord(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + SpecificData specificData = getSpecificData(); + if (specificData.useCustomCoders()) { + old = specificData.newRecord(old, expected); + if (old instanceof SpecificRecordBaseExt) { + SpecificRecordBaseExt d = (SpecificRecordBaseExt) old; + if (d.isCustomDecodingEnabled()) { + d.customDecode(in); + return d; + } + } + } + + final GenericData data = getData(); + Object r = data.newRecord(old, expected); + Object state = Avro19GenericDataAccessUtil.getRecordState(data, r, expected); + + for (Schema.Field f : in.readFieldOrder()) { + int pos = f.pos(); + String name = f.name(); + Object oldDatum = null; + if (old != null) { + oldDatum = Avro19GenericDataAccessUtil.getField(data, r, name, pos, state); + } + readField(r, f, oldDatum, in, state); + } + + return r; + } + + private Object readArray(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema expectedType = expected.getElementType(); + long l = in.readArrayStart(); + long base = 0; + if (l > 0) { + Object array = newArray(old, (int) l, expected); + do { + for (long i = 0; i < l; i++) { + addToArray(array, base + i, read(peekArray(array), expectedType, in)); + } + base += l; + } while ((l = in.arrayNext()) > 0); + return array; + } else { + return newArray(old, 0, expected); + } + } + + private Object readMap(Object old, Schema expected, + CachedResolvingDecoder in) throws IOException { + Schema eValue = expected.getValueType(); + long l = in.readMapStart(); + Object map = newMap(old, (int) l); + if (l > 0) { + do { + for (int i = 0; i < l; i++) { + addToMap(map, readString(null, in), read(null, eValue, in)); + } + } while ((l = in.mapNext()) > 0); + } + return map; + } + + private void readField(Object r, Schema.Field f, Object oldDatum, + CachedResolvingDecoder in, Object state) + throws IOException { + if (r instanceof SpecificRecordBase) { + Conversion conversion = ((SpecificRecordBase) r).getConversion(f.pos()); + + Object datum; + if (conversion != null) { + datum = readWithConversion( + oldDatum, f.schema(), f.schema().getLogicalType(), conversion, in); + } else { + datum = readWithoutConversion(oldDatum, f.schema(), in); + } + + getData().setField(r, f.name(), f.pos(), datum); + + } else { + Avro19GenericDataAccessUtil.setField(getData(), r, f.name(), f.pos(), + read(oldDatum, f.schema(), in), state); + } + } + + private Object readWithConversion(Object old, Schema expected, + LogicalType logicalType, + Conversion conversion, + CachedResolvingDecoder in) throws IOException { + return convert(readWithoutConversion(old, expected, in), + expected, logicalType, conversion); + } +} diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/AliasAwareSpecificDatumReader.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/AliasAwareSpecificDatumReader.java index 35539178d..d7a7ff9a0 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/AliasAwareSpecificDatumReader.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/AliasAwareSpecificDatumReader.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.compatibility.avro19.codec; +import com.linkedin.avroutil1.compatibility.avro19.backports.SpecificDatumReaderExt; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; @@ -20,7 +21,7 @@ * * @param */ -public class AliasAwareSpecificDatumReader extends SpecificDatumReader { +public class AliasAwareSpecificDatumReader extends SpecificDatumReaderExt { public AliasAwareSpecificDatumReader() { this(null, null); @@ -40,10 +41,12 @@ public AliasAwareSpecificDatumReader(Schema writer, Schema reader) { } public AliasAwareSpecificDatumReader(Schema writer, Schema reader, SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } protected AliasAwareSpecificDatumReader(SpecificData data) { + super(null, null, data); throw new UnsupportedOperationException("providing custom SpecificData not supported (yet?)"); } } diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/CachedResolvingDecoder.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/CachedResolvingDecoder.java index 7564fd3fc..2665a56af 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/CachedResolvingDecoder.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/CachedResolvingDecoder.java @@ -10,6 +10,7 @@ import com.linkedin.avroutil1.compatibility.avro19.parsing.Symbol; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; + import org.apache.avro.Schema; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.Decoder; diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/ResolvingDecoder.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/ResolvingDecoder.java index 851cb2f48..0276a5bfa 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/ResolvingDecoder.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/codec/ResolvingDecoder.java @@ -29,13 +29,15 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; + +import com.linkedin.avroutil1.compatibility.CustomDecoder; import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; import org.apache.avro.util.Utf8; -public class ResolvingDecoder extends ValidatingDecoder { +public class ResolvingDecoder extends ValidatingDecoder implements CustomDecoder { private Decoder backup; @@ -152,6 +154,23 @@ public final void drain() throws IOException { parser.processImplicitActions(); } + @Override + public int readInt() throws IOException { + Symbol actual = parser.advance(Symbol.INT); + if (actual == Symbol.INT) { + return in.readInt(); + } else if (actual == Symbol.IntLongAdjustAction.INSTANCE) { + long value = in.readLong(); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new AvroTypeException(value + " cannot be represented as int"); + } + + return (int) value; + } + + throw new AvroTypeException("Expected int but found " + actual); + } + @Override public long readLong() throws IOException { Symbol actual = parser.advance(Symbol.LONG); @@ -307,6 +326,8 @@ public Symbol doAction(Symbol input, Symbol top) throws IOException { in = DecoderFactory.get().binaryDecoder(dsa.contents, null); } else if (top == Symbol.DEFAULT_END_ACTION) { in = backup; + } else if (top == Symbol.IntLongAdjustAction.INSTANCE) { + return top; } else { throw new AvroTypeException("Unknown action: " + top); } diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/ResolvingGrammarGenerator.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/ResolvingGrammarGenerator.java index 3d13aa075..efb37e6ef 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/ResolvingGrammarGenerator.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/ResolvingGrammarGenerator.java @@ -82,6 +82,11 @@ private Symbol generate(Resolver.Action action, Map seen, boolea return simpleGen(action.writer, seen, useFqcns); } else if (action instanceof Resolver.ErrorAction) { + // We should be able to selectively demote long to int if it fits within the range of int. + if (isLongToIntDemotion((Resolver.ErrorAction) action, false)) { + return Symbol.IntLongAdjustAction.INSTANCE; + } + return Symbol.error(action.toString()); } else if (action instanceof Resolver.Skip) { @@ -92,6 +97,16 @@ private Symbol generate(Resolver.Action action, Map seen, boolea } else if (action instanceof Resolver.ReaderUnion) { Resolver.ReaderUnion ru = (Resolver.ReaderUnion) action; + + // Check if we need to handle selective long-to-int demotion within unions. + if (ru.actualAction instanceof Resolver.ErrorAction) { + if (isLongToIntDemotion((Resolver.ErrorAction) ru.actualAction, true)) { + return Symbol.seq( + Symbol.unionAdjustAction(ru.firstMatch, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } + } + Symbol s = generate(ru.actualAction, seen, useFqcns); return Symbol.seq(Symbol.unionAdjustAction(ru.firstMatch, s), Symbol.UNION); @@ -107,17 +122,25 @@ private Symbol generate(Resolver.Action action, Map seen, boolea if (((Resolver.WriterUnion) action).unionEquiv) { return simpleGen(action.writer, seen, useFqcns); } - Resolver.Action[] branches = ((Resolver.WriterUnion) action).actions; + Resolver.WriterUnion wu = (Resolver.WriterUnion) action; + Resolver.Action[] branches = wu.actions; Symbol[] symbols = new Symbol[branches.length]; String[] oldLabels = new String[branches.length]; String[] newLabels = new String[branches.length]; - int i = 0; - for (Resolver.Action branch : branches) { - symbols[i] = generate(branch, seen, useFqcns); + + for (int i = 0; i < branches.length; i++) { + // Check if this branch needs long-to-int conversion + if (branches[i] instanceof Resolver.ErrorAction && isLongToIntDemotion((Resolver.ErrorAction) branches[i], true)) { + symbols[i] = Symbol.seq( + Symbol.unionAdjustAction(i, Symbol.IntLongAdjustAction.INSTANCE), + Symbol.UNION); + } else { + symbols[i] = generate(branches[i], seen, useFqcns); + } + Schema schema = action.writer.getTypes().get(i); oldLabels[i] = schema.getName(); newLabels[i] = schema.getFullName(); - i++; } return Symbol.seq(Symbol.alt(symbols, oldLabels, newLabels, useFqcns), Symbol.WRITER_UNION_ACTION); } else if (action instanceof Resolver.EnumAdjust) { @@ -352,6 +375,39 @@ public static void encode(Encoder e, Schema s, JsonNode n) throws IOException { } } + private static boolean isLongToIntDemotion(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + return isInt(errorAction.reader, includeIntUnions) + && errorAction.writer.getType() == Schema.Type.LONG + && isLongToIntDemotionError(errorAction, includeIntUnions); + } + + private static boolean isLongToIntDemotionError(Resolver.ErrorAction errorAction, boolean includeIntUnions) { + if (errorAction.error == Resolver.ErrorAction.ErrorType.INCOMPATIBLE_SCHEMA_TYPES) { + return true; + } + + return includeIntUnions && errorAction.error == Resolver.ErrorAction.ErrorType.NO_MATCHING_BRANCH; + } + + private static boolean isInt(Schema schema, boolean includeIntUnions) { + if (schema.getType() == Schema.Type.INT) { + return true; + } + + if (!includeIntUnions) { + return false; + } + + if (schema.getType() != Schema.Type.UNION) { + return false; + } + + List types = schema.getTypes(); + return (types.size() == 2 + && types.get(0).getType() == Schema.Type.NULL + && types.get(1).getType() == Schema.Type.INT); + } + /** * Clever trick which differentiates items put into * seen by {@link ValidatingGrammarGenerator validating()} diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/Symbol.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/Symbol.java index ff50d07d8..4b8bbd4c2 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/Symbol.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/parsing/Symbol.java @@ -633,6 +633,10 @@ public FieldOrderAction(Schema.Field[] fields) { } } + public static class IntLongAdjustAction extends ImplicitAction { + public static final IntLongAdjustAction INSTANCE = new IntLongAdjustAction(); + } + public static DefaultStartAction defaultStartAction(byte[] contents) { return new DefaultStartAction(contents); } diff --git a/helper/impls/helper-impl-19/src/main/java/org/apache/avro/generic/Avro19GenericDataAccessUtil.java b/helper/impls/helper-impl-19/src/main/java/org/apache/avro/generic/Avro19GenericDataAccessUtil.java new file mode 100644 index 000000000..9b486ecf8 --- /dev/null +++ b/helper/impls/helper-impl-19/src/main/java/org/apache/avro/generic/Avro19GenericDataAccessUtil.java @@ -0,0 +1,29 @@ +/* + * Copyright 2025 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.generic; + +import org.apache.avro.Schema; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link GenericData} + */ +public class Avro19GenericDataAccessUtil { + private Avro19GenericDataAccessUtil() { + } + + public static Object getRecordState(GenericData data, Object record, Schema schema) { + return data.getRecordState(record, schema); + } + + public static Object getField(GenericData data, Object record, String name, int pos, Object state) { + return data.getField(record, name, pos, state); + } + + public static void setField(GenericData data, Object record, String name, int pos, Object value, Object state) { + data.setField(record, name, pos, value, state); + } +} diff --git a/helper/tests/codegen-110/src/main/raw-avro/by110/IntRecord.avsc b/helper/tests/codegen-110/src/main/raw-avro/by110/IntRecord.avsc new file mode 100644 index 000000000..5bb9f233f --- /dev/null +++ b/helper/tests/codegen-110/src/main/raw-avro/by110/IntRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by110", + "name": "IntRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "int" + ], + "default": null + }, + { + "name": "field", + "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/codegen-110/src/main/raw-avro/by110/LongRecord.avsc b/helper/tests/codegen-110/src/main/raw-avro/by110/LongRecord.avsc new file mode 100644 index 000000000..4a1681abf --- /dev/null +++ b/helper/tests/codegen-110/src/main/raw-avro/by110/LongRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by110", + "name": "LongRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "long" + ], + "default": null + }, + { + "name": "field", + "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/codegen-111/src/main/raw-avro/by111/IntRecord.avsc b/helper/tests/codegen-111/src/main/raw-avro/by111/IntRecord.avsc new file mode 100644 index 000000000..245991a62 --- /dev/null +++ b/helper/tests/codegen-111/src/main/raw-avro/by111/IntRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by111", + "name": "IntRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "int" + ], + "default": null + }, + { + "name": "field", + "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/codegen-111/src/main/raw-avro/by111/LongRecord.avsc b/helper/tests/codegen-111/src/main/raw-avro/by111/LongRecord.avsc new file mode 100644 index 000000000..6e0d8c085 --- /dev/null +++ b/helper/tests/codegen-111/src/main/raw-avro/by111/LongRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by111", + "name": "LongRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "long" + ], + "default": null + }, + { + "name": "field", + "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/codegen-14/src/main/raw-avro/by14/IntRecord.avsc b/helper/tests/codegen-14/src/main/raw-avro/by14/IntRecord.avsc index ab2641547..fba71f0de 100644 --- a/helper/tests/codegen-14/src/main/raw-avro/by14/IntRecord.avsc +++ b/helper/tests/codegen-14/src/main/raw-avro/by14/IntRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-14/src/main/raw-avro/by14/LongRecord.avsc b/helper/tests/codegen-14/src/main/raw-avro/by14/LongRecord.avsc index c357c9856..1c7cb5dd0 100644 --- a/helper/tests/codegen-14/src/main/raw-avro/by14/LongRecord.avsc +++ b/helper/tests/codegen-14/src/main/raw-avro/by14/LongRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-15/src/main/raw-avro/by15/IntRecord.avsc b/helper/tests/codegen-15/src/main/raw-avro/by15/IntRecord.avsc index eafebd345..09a7cb0a8 100644 --- a/helper/tests/codegen-15/src/main/raw-avro/by15/IntRecord.avsc +++ b/helper/tests/codegen-15/src/main/raw-avro/by15/IntRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-15/src/main/raw-avro/by15/LongRecord.avsc b/helper/tests/codegen-15/src/main/raw-avro/by15/LongRecord.avsc index c39ba7335..08ffe4057 100644 --- a/helper/tests/codegen-15/src/main/raw-avro/by15/LongRecord.avsc +++ b/helper/tests/codegen-15/src/main/raw-avro/by15/LongRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-16/src/main/raw-avro/by16/IntRecord.avsc b/helper/tests/codegen-16/src/main/raw-avro/by16/IntRecord.avsc index f37656601..953aa6527 100644 --- a/helper/tests/codegen-16/src/main/raw-avro/by16/IntRecord.avsc +++ b/helper/tests/codegen-16/src/main/raw-avro/by16/IntRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-16/src/main/raw-avro/by16/LongRecord.avsc b/helper/tests/codegen-16/src/main/raw-avro/by16/LongRecord.avsc index 818b83c90..040a93ee2 100644 --- a/helper/tests/codegen-16/src/main/raw-avro/by16/LongRecord.avsc +++ b/helper/tests/codegen-16/src/main/raw-avro/by16/LongRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-17/src/main/raw-avro/by17/IntRecord.avsc b/helper/tests/codegen-17/src/main/raw-avro/by17/IntRecord.avsc index d5b5c80a3..7e2d16166 100644 --- a/helper/tests/codegen-17/src/main/raw-avro/by17/IntRecord.avsc +++ b/helper/tests/codegen-17/src/main/raw-avro/by17/IntRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-17/src/main/raw-avro/by17/LongRecord.avsc b/helper/tests/codegen-17/src/main/raw-avro/by17/LongRecord.avsc index 12e8b8623..a3e518484 100644 --- a/helper/tests/codegen-17/src/main/raw-avro/by17/LongRecord.avsc +++ b/helper/tests/codegen-17/src/main/raw-avro/by17/LongRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-18/src/main/raw-avro/by18/IntRecord.avsc b/helper/tests/codegen-18/src/main/raw-avro/by18/IntRecord.avsc index 40397758b..2f3da2788 100644 --- a/helper/tests/codegen-18/src/main/raw-avro/by18/IntRecord.avsc +++ b/helper/tests/codegen-18/src/main/raw-avro/by18/IntRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-18/src/main/raw-avro/by18/LongRecord.avsc b/helper/tests/codegen-18/src/main/raw-avro/by18/LongRecord.avsc index 8f78a8b0d..c832d6eae 100644 --- a/helper/tests/codegen-18/src/main/raw-avro/by18/LongRecord.avsc +++ b/helper/tests/codegen-18/src/main/raw-avro/by18/LongRecord.avsc @@ -14,6 +14,44 @@ { "name": "field", "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} } ] } \ No newline at end of file diff --git a/helper/tests/codegen-19/src/main/raw-avro/by19/IntRecord.avsc b/helper/tests/codegen-19/src/main/raw-avro/by19/IntRecord.avsc new file mode 100644 index 000000000..26eed2528 --- /dev/null +++ b/helper/tests/codegen-19/src/main/raw-avro/by19/IntRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by19", + "name": "IntRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "int" + ], + "default": null + }, + { + "name": "field", + "type": "int" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "int" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "int" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "int" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/codegen-19/src/main/raw-avro/by19/LongRecord.avsc b/helper/tests/codegen-19/src/main/raw-avro/by19/LongRecord.avsc new file mode 100644 index 000000000..d8c3c6f18 --- /dev/null +++ b/helper/tests/codegen-19/src/main/raw-avro/by19/LongRecord.avsc @@ -0,0 +1,57 @@ +{ + "type": "record", + "namespace": "by19", + "name": "LongRecord", + "fields": [ + { + "name": "unionField", + "type": [ + "null", + "long" + ], + "default": null + }, + { + "name": "field", + "type": "long" + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "mapField", + "type": { + "type": "map", + "values": "long" + }, + "default": {} + }, + { + "name": "unionArrayField", + "type": { + "type": "array", + "items": [ + "null", + "long" + ] + }, + "default": [] + }, + { + "name": "unionMapField", + "type": { + "type": "map", + "values": [ + "null", + "long" + ] + }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/AvroCompatibilityHelperAvro110Test.java b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/AvroCompatibilityHelperAvro110Test.java index b5226fb3a..7d4150a20 100644 --- a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/AvroCompatibilityHelperAvro110Test.java +++ b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/AvroCompatibilityHelperAvro110Test.java @@ -6,15 +6,33 @@ package com.linkedin.avroutil1.compatibility.avro110; +import by110.IntRecord; +import by110.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.testcommon.TestUtil; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroVersion; + +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; import java.util.Map; + +import org.apache.avro.AvroTypeException; import org.apache.avro.JsonProperties; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -74,4 +92,180 @@ public void testCreateSchemaFieldWithProvidedDefaultValue() throws IOException { (List>) AvroCompatibilityHelper.createSchemaField("arrayOfArrayWithDefault", field.schema(), "", field.defaultVal()).defaultVal(); Assert.assertEquals(actualListValue.get(0).get(0), "dummyElement"); } + + @Test + public void testIntRoundtrip() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongRoundtrip() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testIntToLongPromotion() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testLongToIntDemotion() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongToIntDemotionOutOfRange() throws IOException { + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); + byte[] binary = toBinary(longRecord); + + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); + byte[] binary2 = toBinary(longRecord2); + + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private byte[] toBinary(IndexedRecord record) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos); + GenericDatumWriter writer = new GenericDatumWriter<>(record.getSchema()); + writer.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private T toSpecificRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newSpecificDatumReader(writerSchema, readerSchema, SpecificData.get()); + return reader.read(null, decoder); + } + + private GenericRecord toGenericRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newGenericDatumReader(writerSchema, readerSchema, GenericData.get()); + return reader.read(null, decoder); + } } diff --git a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/AvroCompatibilityHelperAvro111Test.java b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/AvroCompatibilityHelperAvro111Test.java index e539da7ab..42b925547 100644 --- a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/AvroCompatibilityHelperAvro111Test.java +++ b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/AvroCompatibilityHelperAvro111Test.java @@ -6,11 +6,30 @@ package com.linkedin.avroutil1.compatibility.avro111; +import by111.IntRecord; +import by111.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroVersion; +import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.testng.Assert; import org.testng.annotations.Test; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + public class AvroCompatibilityHelperAvro111Test { @@ -27,4 +46,180 @@ public void testAvroCompilerVersionDetection() { AvroVersion detected = AvroCompatibilityHelper.getRuntimeAvroCompilerVersion(); Assert.assertEquals(detected, expected, "expected " + expected + ", got " + detected); } + + @Test + public void testIntRoundtrip() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongRoundtrip() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testIntToLongPromotion() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testLongToIntDemotion() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongToIntDemotionOutOfRange() throws IOException { + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); + byte[] binary = toBinary(longRecord); + + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); + byte[] binary2 = toBinary(longRecord2); + + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private byte[] toBinary(IndexedRecord record) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos); + GenericDatumWriter writer = new GenericDatumWriter<>(record.getSchema()); + writer.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private T toSpecificRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newSpecificDatumReader(writerSchema, readerSchema, SpecificData.get()); + return reader.read(null, decoder); + } + + private GenericRecord toGenericRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newGenericDatumReader(writerSchema, readerSchema, GenericData.get()); + return reader.read(null, decoder); + } } diff --git a/helper/tests/helper-tests-111_0/src/test/java/com.linkedin.avroutil1.compatibility.avro111/AvroCompatibilityHelperAvro1110Test.java b/helper/tests/helper-tests-111_0/src/test/java/com.linkedin.avroutil1.compatibility.avro111/AvroCompatibilityHelperAvro1110Test.java index 23addc6d6..4866a59f7 100644 --- a/helper/tests/helper-tests-111_0/src/test/java/com.linkedin.avroutil1.compatibility.avro111/AvroCompatibilityHelperAvro1110Test.java +++ b/helper/tests/helper-tests-111_0/src/test/java/com.linkedin.avroutil1.compatibility.avro111/AvroCompatibilityHelperAvro1110Test.java @@ -6,11 +6,30 @@ package com.linkedin.avroutil1.compatibility.avro111; +import by111.IntRecord; +import by111.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroVersion; +import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.testng.Assert; import org.testng.annotations.Test; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + // Tests version detection against Avro 1.11.0 public class AvroCompatibilityHelperAvro1110Test { @@ -28,4 +47,180 @@ public void testAvroCompilerVersionDetection() { AvroVersion detected = AvroCompatibilityHelper.getRuntimeAvroCompilerVersion(); Assert.assertEquals(detected, expected, "expected " + expected + ", got " + detected); } + + @Test + public void testIntRoundtrip() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongRoundtrip() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testIntToLongPromotion() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testLongToIntDemotion() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongToIntDemotionOutOfRange() throws IOException { + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); + byte[] binary = toBinary(longRecord); + + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); + byte[] binary2 = toBinary(longRecord2); + + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private byte[] toBinary(IndexedRecord record) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos); + GenericDatumWriter writer = new GenericDatumWriter<>(record.getSchema()); + writer.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private T toSpecificRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newSpecificDatumReader(writerSchema, readerSchema, SpecificData.get()); + return reader.read(null, decoder); + } + + private GenericRecord toGenericRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newGenericDatumReader(writerSchema, readerSchema, GenericData.get()); + return reader.read(null, decoder); + } } \ No newline at end of file diff --git a/helper/tests/helper-tests-14/src/test/java/com/linkedin/avroutil1/compatibility/avro14/AvroCompatibilityHelperAvro14Test.java b/helper/tests/helper-tests-14/src/test/java/com/linkedin/avroutil1/compatibility/avro14/AvroCompatibilityHelperAvro14Test.java index c2a5e7134..41a6ffe2e 100644 --- a/helper/tests/helper-tests-14/src/test/java/com/linkedin/avroutil1/compatibility/avro14/AvroCompatibilityHelperAvro14Test.java +++ b/helper/tests/helper-tests-14/src/test/java/com/linkedin/avroutil1/compatibility/avro14/AvroCompatibilityHelperAvro14Test.java @@ -8,6 +8,8 @@ import by14.IntRecord; import by14.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroVersion; @@ -15,12 +17,14 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -28,6 +32,7 @@ import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DatumReader; import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -98,15 +103,27 @@ public void testIntRoundtrip() throws IOException { IntRecord intRecord = new IntRecord(); intRecord.field = 42; intRecord.unionField = 55; + intRecord.arrayField = ImmutableList.of(100, -200); + intRecord.mapField = ImmutableMap.of("key1", 300, "key2", -400); + intRecord.unionArrayField = ImmutableList.of(99, -199); + intRecord.unionMapField = ImmutableMap.of("key1", 298, "key2", 355); byte[] binary = toBinary(intRecord); IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(roundtrip.field, 42); Assert.assertEquals(roundtrip.unionField.intValue(), 55); + Assert.assertEquals(roundtrip.arrayField, ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.mapField, ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.unionArrayField, ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.unionMapField, ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test @@ -114,15 +131,27 @@ public void testLongRoundtrip() throws IOException { LongRecord longRecord = new LongRecord(); longRecord.field = 42L; longRecord.unionField = 55L; + longRecord.arrayField = ImmutableList.of(100L, -200L); + longRecord.mapField = ImmutableMap.of("key1", 300L, "key2", -400L); + longRecord.unionArrayField = ImmutableList.of(99L, -199L); + longRecord.unionMapField = ImmutableMap.of("key1", 298L, "key2", 355L); byte[] binary = toBinary(longRecord); LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(roundtrip.field, 42L); Assert.assertEquals(roundtrip.unionField.longValue(), 55L); + Assert.assertEquals(roundtrip.arrayField, ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.mapField, ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.unionArrayField, ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.unionMapField, ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test @@ -130,15 +159,27 @@ public void testIntToLongPromotion() throws IOException { IntRecord intRecord = new IntRecord(); intRecord.field = 42; intRecord.unionField = 55; + intRecord.arrayField = ImmutableList.of(100, -200); + intRecord.mapField = ImmutableMap.of("key1", 300, "key2", -400); + intRecord.unionArrayField = ImmutableList.of(99, -199); + intRecord.unionMapField = ImmutableMap.of("key1", 298, "key2", 355); byte[] binary = toBinary(intRecord); LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(longRecord.field, 42L); Assert.assertEquals(longRecord.unionField.longValue(), 55L); + Assert.assertEquals(longRecord.arrayField, ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.mapField, ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.unionArrayField, ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.unionMapField, ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test @@ -146,32 +187,83 @@ public void testLongToIntDemotion() throws IOException { LongRecord longRecord = new LongRecord(); longRecord.field = 42L; longRecord.unionField = 55L; + longRecord.arrayField = ImmutableList.of(100L, -200L); + longRecord.mapField = ImmutableMap.of("key1", 300L, "key2", -400L); + longRecord.unionArrayField = ImmutableList.of(99L, -199L); + longRecord.unionMapField = ImmutableMap.of("key1", 298L, "key2", 355L); byte[] binary = toBinary(longRecord); IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(intRecord.field, 42); Assert.assertEquals(intRecord.unionField.intValue(), 55); + Assert.assertEquals(intRecord.arrayField, ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.mapField, ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.unionArrayField, ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.unionMapField, ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongToIntDemotionOutOfRange() throws IOException { - LongRecord longRecord = new LongRecord(); + LongRecord longRecord = newLongRecord(); longRecord.field = (long) Integer.MAX_VALUE + 1L; byte[] binary = toBinary(longRecord); - LongRecord longRecord2 = new LongRecord(); + LongRecord longRecord2 = newLongRecord(); longRecord2.unionField = (long) Integer.MIN_VALUE - 1L; byte[] binary2 = toBinary(longRecord2); + LongRecord longRecord3 = newLongRecord(); + longRecord3.arrayField = ImmutableList.of((long) Integer.MAX_VALUE + 1L); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = newLongRecord(); + longRecord4.mapField = ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = newLongRecord(); + longRecord5.unionArrayField = ImmutableList.of((long) Integer.MAX_VALUE + 1L); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = newLongRecord(); + longRecord6.unionMapField = ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L); + byte[] binary6 = toBinary(longRecord6); + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private LongRecord newLongRecord() { + LongRecord longRecord = new LongRecord(); + longRecord.field = 0L; + longRecord.unionField = 0L; + longRecord.arrayField = ImmutableList.of(); + longRecord.mapField = ImmutableMap.of(); + longRecord.unionArrayField = ImmutableList.of(); + longRecord.unionMapField = ImmutableMap.of(); + return longRecord; } private byte[] toBinary(IndexedRecord record) throws IOException { diff --git a/helper/tests/helper-tests-15/src/test/java/com/linkedin/avroutil1/compatibility/avro15/AvroCompatibilityHelperAvro15Test.java b/helper/tests/helper-tests-15/src/test/java/com/linkedin/avroutil1/compatibility/avro15/AvroCompatibilityHelperAvro15Test.java index f7e9e1e39..be3f0b395 100644 --- a/helper/tests/helper-tests-15/src/test/java/com/linkedin/avroutil1/compatibility/avro15/AvroCompatibilityHelperAvro15Test.java +++ b/helper/tests/helper-tests-15/src/test/java/com/linkedin/avroutil1/compatibility/avro15/AvroCompatibilityHelperAvro15Test.java @@ -8,6 +8,8 @@ import by15.IntRecord; import by15.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.avro.AvroTypeException; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumWriter; @@ -25,10 +27,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.avro.Schema; +import org.apache.avro.util.Utf8; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -100,15 +104,27 @@ public void testIntRoundtrip() throws IOException { IntRecord intRecord = new IntRecord(); intRecord.field = 42; intRecord.unionField = 55; + intRecord.arrayField = ImmutableList.of(100, -200); + intRecord.mapField = ImmutableMap.of("key1", 300, "key2", -400); + intRecord.unionArrayField = ImmutableList.of(99, -199); + intRecord.unionMapField = ImmutableMap.of("key1", 298, "key2", 355); byte[] binary = toBinary(intRecord); IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(roundtrip.field, 42); Assert.assertEquals(roundtrip.unionField.intValue(), 55); + Assert.assertEquals(roundtrip.arrayField, ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.mapField, ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.unionArrayField, ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.unionMapField, ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test @@ -116,15 +132,27 @@ public void testLongRoundtrip() throws IOException { LongRecord longRecord = new LongRecord(); longRecord.field = 42L; longRecord.unionField = 55L; + longRecord.arrayField = ImmutableList.of(100L, -200L); + longRecord.mapField = ImmutableMap.of("key1", 300L, "key2", -400L); + longRecord.unionArrayField = ImmutableList.of(99L, -199L); + longRecord.unionMapField = ImmutableMap.of("key1", 298L, "key2", 355L); byte[] binary = toBinary(longRecord); LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(roundtrip.field, 42L); Assert.assertEquals(roundtrip.unionField.longValue(), 55L); + Assert.assertEquals(roundtrip.arrayField, ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.mapField, ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.unionArrayField, ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.unionMapField, ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test @@ -132,15 +160,27 @@ public void testIntToLongPromotion() throws IOException { IntRecord intRecord = new IntRecord(); intRecord.field = 42; intRecord.unionField = 55; + intRecord.arrayField = ImmutableList.of(100, -200); + intRecord.mapField = ImmutableMap.of("key1", 300, "key2", -400); + intRecord.unionArrayField = ImmutableList.of(99, -199); + intRecord.unionMapField = ImmutableMap.of("key1", 298, "key2", 355); byte[] binary = toBinary(intRecord); LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(longRecord.field, 42L); Assert.assertEquals(longRecord.unionField.longValue(), 55L); + Assert.assertEquals(longRecord.arrayField, ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.mapField, ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.unionArrayField, ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.unionMapField, ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test @@ -148,32 +188,83 @@ public void testLongToIntDemotion() throws IOException { LongRecord longRecord = new LongRecord(); longRecord.field = 42L; longRecord.unionField = 55L; + longRecord.arrayField = ImmutableList.of(100L, -200L); + longRecord.mapField = ImmutableMap.of("key1", 300L, "key2", -400L); + longRecord.unionArrayField = ImmutableList.of(99L, -199L); + longRecord.unionMapField = ImmutableMap.of("key1", 298L, "key2", 355L); byte[] binary = toBinary(longRecord); IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(intRecord.field, 42); Assert.assertEquals(intRecord.unionField.intValue(), 55); + Assert.assertEquals(intRecord.arrayField, ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.mapField, ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.unionArrayField, ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.unionMapField, ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("arrayField")), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(new ArrayList<>((GenericData.Array) genericRecord.get("unionArrayField")), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongToIntDemotionOutOfRange() throws IOException { - LongRecord longRecord = new LongRecord(); + LongRecord longRecord = newLongRecord(); longRecord.field = (long) Integer.MAX_VALUE + 1L; byte[] binary = toBinary(longRecord); - LongRecord longRecord2 = new LongRecord(); + LongRecord longRecord2 = newLongRecord(); longRecord2.unionField = (long) Integer.MIN_VALUE - 1L; byte[] binary2 = toBinary(longRecord2); + LongRecord longRecord3 = newLongRecord(); + longRecord3.arrayField = ImmutableList.of((long) Integer.MAX_VALUE + 1L); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = newLongRecord(); + longRecord4.mapField = ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = newLongRecord(); + longRecord5.unionArrayField = ImmutableList.of((long) Integer.MAX_VALUE + 1L); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = newLongRecord(); + longRecord6.unionMapField = ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L); + byte[] binary6 = toBinary(longRecord6); + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private LongRecord newLongRecord() { + LongRecord longRecord = new LongRecord(); + longRecord.field = 0L; + longRecord.unionField = 0L; + longRecord.arrayField = ImmutableList.of(); + longRecord.mapField = ImmutableMap.of(); + longRecord.unionArrayField = ImmutableList.of(); + longRecord.unionMapField = ImmutableMap.of(); + return longRecord; } private byte[] toBinary(IndexedRecord record) throws IOException { diff --git a/helper/tests/helper-tests-16/src/test/java/com/linkedin/avroutil1/compatibility/avro16/AvroCompatibilityHelperAvro16Test.java b/helper/tests/helper-tests-16/src/test/java/com/linkedin/avroutil1/compatibility/avro16/AvroCompatibilityHelperAvro16Test.java index be0f49ae9..0925c70a8 100644 --- a/helper/tests/helper-tests-16/src/test/java/com/linkedin/avroutil1/compatibility/avro16/AvroCompatibilityHelperAvro16Test.java +++ b/helper/tests/helper-tests-16/src/test/java/com/linkedin/avroutil1/compatibility/avro16/AvroCompatibilityHelperAvro16Test.java @@ -8,6 +8,8 @@ import by16.IntRecord; import by16.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.testcommon.TestUtil; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; @@ -30,6 +32,7 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -99,84 +102,154 @@ public void testCreateSchemaFieldWithProvidedDefaultValue() throws IOException { @Test public void testIntRoundtrip() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42); - Assert.assertEquals(roundtrip.unionField.intValue(), 55); + Assert.assertEquals((int) roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongRoundtrip() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42L); - Assert.assertEquals(roundtrip.unionField.longValue(), 55L); + Assert.assertEquals((long) roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testIntToLongPromotion() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(longRecord.field, 42L); - Assert.assertEquals(longRecord.unionField.longValue(), 55L); + Assert.assertEquals((long) longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testLongToIntDemotion() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(intRecord.field, 42); - Assert.assertEquals(intRecord.unionField.intValue(), 55); + Assert.assertEquals((int) intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongToIntDemotionOutOfRange() throws IOException { - LongRecord longRecord = new LongRecord(); - longRecord.field = (long) Integer.MAX_VALUE + 1L; + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); byte[] binary = toBinary(longRecord); - LongRecord longRecord2 = new LongRecord(); - longRecord2.unionField = (long) Integer.MIN_VALUE - 1L; + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); byte[] binary2 = toBinary(longRecord2); + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); - } + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + private byte[] toBinary(IndexedRecord record) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos); diff --git a/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroCompatibilityHelperAvro17Test.java b/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroCompatibilityHelperAvro17Test.java index 66082f5b8..b8421155e 100644 --- a/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroCompatibilityHelperAvro17Test.java +++ b/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroCompatibilityHelperAvro17Test.java @@ -8,6 +8,8 @@ import by17.IntRecord; import by17.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.testcommon.TestUtil; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; @@ -30,6 +32,7 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -116,82 +119,152 @@ public void testGetGenericDefaultValueCloningForEnums() throws IOException { @Test public void testIntRoundtrip() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42); - Assert.assertEquals(roundtrip.unionField.intValue(), 55); + Assert.assertEquals((int) roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongRoundtrip() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42L); - Assert.assertEquals(roundtrip.unionField.longValue(), 55L); + Assert.assertEquals((long) roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testIntToLongPromotion() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(longRecord.field, 42L); - Assert.assertEquals(longRecord.unionField.longValue(), 55L); + Assert.assertEquals((long) longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testLongToIntDemotion() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(intRecord.field, 42); - Assert.assertEquals(intRecord.unionField.intValue(), 55); + Assert.assertEquals((int) intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongToIntDemotionOutOfRange() throws IOException { - LongRecord longRecord = new LongRecord(); - longRecord.field = (long) Integer.MAX_VALUE + 1L; + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); byte[] binary = toBinary(longRecord); - LongRecord longRecord2 = new LongRecord(); - longRecord2.unionField = (long) Integer.MIN_VALUE - 1L; + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); byte[] binary2 = toBinary(longRecord2); + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); } private byte[] toBinary(IndexedRecord record) throws IOException { diff --git a/helper/tests/helper-tests-18/src/test/java/com/linkedin/avroutil1/compatibility/avro18/AvroCompatibilityHelperAvro18Test.java b/helper/tests/helper-tests-18/src/test/java/com/linkedin/avroutil1/compatibility/avro18/AvroCompatibilityHelperAvro18Test.java index 0eda22b6a..b3b94ddf7 100644 --- a/helper/tests/helper-tests-18/src/test/java/com/linkedin/avroutil1/compatibility/avro18/AvroCompatibilityHelperAvro18Test.java +++ b/helper/tests/helper-tests-18/src/test/java/com/linkedin/avroutil1/compatibility/avro18/AvroCompatibilityHelperAvro18Test.java @@ -8,6 +8,8 @@ import by18.IntRecord; import by18.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.testcommon.TestUtil; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; @@ -30,6 +32,7 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -99,82 +102,152 @@ public void testCreateSchemaFieldWithProvidedDefaultValue() throws IOException { @Test public void testIntRoundtrip() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42); - Assert.assertEquals(roundtrip.unionField.intValue(), 55); + Assert.assertEquals((int) roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongRoundtrip() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(roundtrip.field, 42L); - Assert.assertEquals(roundtrip.unionField.longValue(), 55L); + Assert.assertEquals((long) roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testIntToLongPromotion() throws IOException { IntRecord intRecord = new IntRecord(); - intRecord.field = 42; - intRecord.unionField = 55; + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); byte[] binary = toBinary(intRecord); LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); - Assert.assertEquals(longRecord.field, 42L); - Assert.assertEquals(longRecord.unionField.longValue(), 55L); + Assert.assertEquals((long) longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42L); Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); } @Test public void testLongToIntDemotion() throws IOException { LongRecord longRecord = new LongRecord(); - longRecord.field = 42L; - longRecord.unionField = 55L; + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); byte[] binary = toBinary(longRecord); IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); - Assert.assertEquals(intRecord.field, 42); - Assert.assertEquals(intRecord.unionField.intValue(), 55); + Assert.assertEquals((int) intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); Assert.assertEquals(genericRecord.get("field"), 42); Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); } @Test public void testLongToIntDemotionOutOfRange() throws IOException { - LongRecord longRecord = new LongRecord(); - longRecord.field = (long) Integer.MAX_VALUE + 1L; + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); byte[] binary = toBinary(longRecord); - LongRecord longRecord2 = new LongRecord(); - longRecord2.unionField = (long) Integer.MIN_VALUE - 1L; + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); byte[] binary2 = toBinary(longRecord2); + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); } private byte[] toBinary(IndexedRecord record) throws IOException { diff --git a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroCompatibilityHelperAvro19Test.java b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroCompatibilityHelperAvro19Test.java index 10c9c70fd..67f9d3fd3 100644 --- a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroCompatibilityHelperAvro19Test.java +++ b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroCompatibilityHelperAvro19Test.java @@ -6,15 +6,33 @@ package com.linkedin.avroutil1.compatibility.avro19; +import by19.IntRecord; +import by19.LongRecord; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.avroutil1.Pojo; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroVersion; import com.linkedin.avroutil1.testcommon.TestUtil; + +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; import java.util.Map; + +import org.apache.avro.AvroTypeException; import org.apache.avro.JsonProperties; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -75,4 +93,179 @@ public void testCreateSchemaFieldWithProvidedDefaultValue() throws IOException { Assert.assertEquals(actualListValue.get(0).get(0), "dummyElement"); } + @Test + public void testIntRoundtrip() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + IntRecord roundtrip = toSpecificRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42); + Assert.assertEquals(roundtrip.getUnionField().intValue(), 55); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongRoundtrip() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + LongRecord roundtrip = toSpecificRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(roundtrip.getField(), 42L); + Assert.assertEquals(roundtrip.getUnionField().longValue(), 55L); + Assert.assertEquals(roundtrip.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(roundtrip.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(roundtrip.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(roundtrip.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testIntToLongPromotion() throws IOException { + IntRecord intRecord = new IntRecord(); + intRecord.setField(42); + intRecord.setUnionField(55); + intRecord.setArrayField(ImmutableList.of(100, -200)); + intRecord.setMapField(ImmutableMap.of("key1", 300, "key2", -400)); + intRecord.setUnionArrayField(ImmutableList.of(99, -199)); + intRecord.setUnionMapField(ImmutableMap.of("key1", 298, "key2", 355)); + byte[] binary = toBinary(intRecord); + + LongRecord longRecord = toSpecificRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(longRecord.getField(), 42L); + Assert.assertEquals(longRecord.getUnionField().longValue(), 55L); + Assert.assertEquals(longRecord.getArrayField(), ImmutableList.of(100L, -200L)); + Assert.assertEquals(longRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(longRecord.getUnionArrayField(), ImmutableList.of(99L, -199L)); + Assert.assertEquals(longRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + + GenericRecord genericRecord = toGenericRecord(binary, IntRecord.SCHEMA$, LongRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42L); + Assert.assertEquals(genericRecord.get("unionField"), 55L); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100L, -200L)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300L, new Utf8("key2"), -400L)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99L, -199L)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298L, new Utf8("key2"), 355L)); + } + + @Test + public void testLongToIntDemotion() throws IOException { + LongRecord longRecord = new LongRecord(); + longRecord.setField(42L); + longRecord.setUnionField(55L); + longRecord.setArrayField(ImmutableList.of(100L, -200L)); + longRecord.setMapField(ImmutableMap.of("key1", 300L, "key2", -400L)); + longRecord.setUnionArrayField(ImmutableList.of(99L, -199L)); + longRecord.setUnionMapField(ImmutableMap.of("key1", 298L, "key2", 355L)); + byte[] binary = toBinary(longRecord); + + IntRecord intRecord = toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(intRecord.getField(), 42); + Assert.assertEquals(intRecord.getUnionField().intValue(), 55); + Assert.assertEquals(intRecord.getArrayField(), ImmutableList.of(100, -200)); + Assert.assertEquals(intRecord.getMapField(), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(intRecord.getUnionArrayField(), ImmutableList.of(99, -199)); + Assert.assertEquals(intRecord.getUnionMapField(), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + + GenericRecord genericRecord = toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$); + Assert.assertEquals(genericRecord.get("field"), 42); + Assert.assertEquals(genericRecord.get("unionField"), 55); + Assert.assertEquals(genericRecord.get("arrayField"), ImmutableList.of(100, -200)); + Assert.assertEquals(genericRecord.get("mapField"), ImmutableMap.of(new Utf8("key1"), 300, new Utf8("key2"), -400)); + Assert.assertEquals(genericRecord.get("unionArrayField"), ImmutableList.of(99, -199)); + Assert.assertEquals(genericRecord.get("unionMapField"), ImmutableMap.of(new Utf8("key1"), 298, new Utf8("key2"), 355)); + } + + @Test + public void testLongToIntDemotionOutOfRange() throws IOException { + LongRecord longRecord = LongRecord.newBuilder().setField((long) Integer.MAX_VALUE + 1L).build(); + byte[] binary = toBinary(longRecord); + + LongRecord longRecord2 = LongRecord.newBuilder().setField(0L).setUnionField((long) Integer.MIN_VALUE - 1L).build(); + byte[] binary2 = toBinary(longRecord2); + + LongRecord longRecord3 = LongRecord.newBuilder().setField(0L).setArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary3 = toBinary(longRecord3); + + LongRecord longRecord4 = LongRecord.newBuilder().setField(0L).setMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary4 = toBinary(longRecord4); + + LongRecord longRecord5 = LongRecord.newBuilder().setField(0L).setUnionArrayField(ImmutableList.of((long) Integer.MAX_VALUE + 1L)).build(); + byte[] binary5 = toBinary(longRecord5); + + LongRecord longRecord6 = LongRecord.newBuilder().setField(0L).setUnionMapField(ImmutableMap.of("haha", (long) Integer.MIN_VALUE - 1L)).build(); + byte[] binary6 = toBinary(longRecord6); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary2, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary3, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary4, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary5, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + + Assert.assertThrows(AvroTypeException.class, () -> toSpecificRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + Assert.assertThrows(AvroTypeException.class, () -> toGenericRecord(binary6, LongRecord.SCHEMA$, IntRecord.SCHEMA$)); + } + + private byte[] toBinary(IndexedRecord record) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos); + GenericDatumWriter writer = new GenericDatumWriter<>(record.getSchema()); + writer.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private T toSpecificRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newSpecificDatumReader(writerSchema, readerSchema, SpecificData.get()); + return reader.read(null, decoder); + } + + private GenericRecord toGenericRecord(byte[] binary, + Schema writerSchema, + Schema readerSchema) throws IOException { + BinaryDecoder decoder = AvroCompatibilityHelper.newBinaryDecoder(binary); + DatumReader reader = AvroCompatibilityHelper.newGenericDatumReader(writerSchema, readerSchema, GenericData.get()); + return reader.read(null, decoder); + } }