Skip to content

Support for selectively demoting long to int in avro 1.9 to 1.11 #593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
li-ukumar merged 8 commits into from
Oct 1, 2025
Merged

Support for selectively demoting long to int in avro 1.9 to 1.11 #593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0057dde
Support for selectively demoting long to int in avro 1.9 to 1.11
Sep 22, 2025
8ae8dfd
fix javadoc
Sep 29, 2025
2b61a32
fix javadoc
Sep 29, 2025
cd08199
Fix array/map
Sep 30, 2025
9dce7e6
Fix avro 1.9
Sep 30, 2025
60676c1
Fix 1.4 to 1.8
Sep 30, 2025
2a1056c
Fix checkstyle
Sep 30, 2025
4963671
java8
Sep 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .../tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@
package com.linkedin.avroutil1.builder;

import com.linkedin.avroutil1.compatibility.AvroCodecUtil;
import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
import com.linkedin.avroutil1.compatibility.AvroRecordUtil;
import com.linkedin.avroutil1.compatibility.CustomDecoder;
import com.linkedin.avroutil1.compatibility.RandomRecordGenerator;
import com.linkedin.avroutil1.compatibility.RecordGenerationConfig;
import com.linkedin.avroutil1.compatibility.StringConverterUtil;

import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
Expand All @@ -29,9 +33,13 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt;
import noutf8.TestCollections;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.Encoder;
import org.apache.avro.util.Utf8;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
Expand Down Expand Up @@ -2040,6 +2048,36 @@ public void testNoUtf8Encoding() throws IOException {
Assert.assertTrue(instance.arOfMap.get(0).containsValue(strValue));
}

@DataProvider
private Object[][] customDecodeDataProvider() {
return new Object[][]{
{vs19.MoneyRange.class},
{vs110.MoneyRange.class},
{vs111.MoneyRange.class}
};
}

@Test(dataProvider = "customDecodeDataProvider")
public void testCustomDecode(Class<? extends SpecificRecordBaseExt> specificRecordClass) throws Exception {
RandomRecordGenerator generator = new RandomRecordGenerator();
SpecificRecordBaseExt instance = generator.randomSpecific(specificRecordClass);

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Encoder encoder = AvroCompatibilityHelper.newBinaryEncoder(outputStream);
Method customEncodeMethod = instance.getClass().getMethod("customEncode", Encoder.class);
customEncodeMethod.invoke(instance, encoder);
encoder.flush();

byte[] data = outputStream.toByteArray();
Decoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(data);
CustomDecoder decoder =
(CustomDecoder) AvroCompatibilityHelper.newCachedResolvingDecoder(
instance.getSchema(), instance.getSchema(), binaryDecoder);
SpecificRecordBaseExt decodedInstance = specificRecordClass.getDeclaredConstructor().newInstance();
decodedInstance.customDecode(decoder);
Assert.assertEquals(instance, decodedInstance);
}

@BeforeClass
public void setup() {
System.setProperty("org.apache.avro.specific.use_custom_coders", "true");
Expand Down
33 changes: 28 additions & 5 deletions avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi
classBuilder.addSuperinterface(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD);

// extends
classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE);
classBuilder.superclass(SpecificRecordGeneratorUtil.CLASSNAME_SPECIFIC_RECORD_BASE_EXT);

//add class-level doc from schema doc
//file-level (top of file) comment is added to the file object later
Expand Down Expand Up @@ -519,9 +519,27 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter);
addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter, false);
classBuilder.addMethod(customDecodeBuilder.build());

//customDecode with CustomDecoder
MethodSpec.Builder methodBuilder = MethodSpec
.methodBuilder("customDecode")
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC)
.addAnnotation(Override.class);
addCustomDecodeMethod(methodBuilder, recordSchema, config, classBuilder, sizeValCounter, true);
classBuilder.addMethod(methodBuilder.build());

MethodSpec.Builder isCustomDecodingEnabledMethod = MethodSpec
.methodBuilder("isCustomDecodingEnabled")
.addModifiers(Modifier.PUBLIC)
.returns(TypeName.BOOLEAN)
.addAnnotation(Override.class)
.addCode("return hasCustomCoders();");
classBuilder.addMethod(isCustomDecodingEnabledMethod.build());

// Builder
TypeSpec.Builder recordBuilder = TypeSpec.classBuilder("Builder");
recordBuilder.addModifiers(Modifier.PUBLIC, Modifier.STATIC);
Expand Down Expand Up @@ -945,8 +963,13 @@ private String getMethodNameForFieldWithPrefix(String prefix, String fieldName)
}

private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroRecordSchema recordSchema,
SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter) {
SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder, Counter sizeValCounter, boolean isCustomDecoder) {
int blockSize = 25, fieldCounter = 0, chunkCounter = 0;

// Decoder class name.
ClassName decoderClassName = isCustomDecoder ? SpecificRecordGeneratorUtil.CLASSNAME_CUSTOM_DECODER :
SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER;

// reset var counter
sizeValCounter.reset();
customDecodeBuilder.addStatement(
Expand All @@ -959,7 +982,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR
customDecodeBuilder.addStatement(chunkMethodName + "(in)");
// create new method
MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName)
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addParameter(decoderClassName, "in")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
for (; fieldCounter < Math.min(blockSize * chunkCounter + blockSize, recordSchema.getFields().size());
Expand Down Expand Up @@ -989,7 +1012,7 @@ private void addCustomDecodeMethod(MethodSpec.Builder customDecodeBuilder, AvroR
customDecodeBuilder.addStatement(chunkMethodName + "(in, fieldOrder)");
// create new method
MethodSpec.Builder customDecodeChunkMethod = MethodSpec.methodBuilder(chunkMethodName)
.addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in")
.addParameter(decoderClassName, "in")
.addParameter(ArrayTypeName.of(SpecificRecordGeneratorUtil.CLASSNAME_SCHEMA_FIELD), "fieldOrder")
.addException(IOException.class)
.addModifiers(Modifier.PUBLIC);
Expand Down
2 changes: 2 additions & 0 deletions avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ public class SpecificRecordGeneratorUtil {
public static final ClassName CLASSNAME_SPECIFIC_DATA = ClassName.get("org.apache.avro.specific", "SpecificData");
public static final ClassName CLASSNAME_SPECIFIC_RECORD = ClassName.get("org.apache.avro.specific", "SpecificRecord");
public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE = ClassName.get("org.apache.avro.specific", "SpecificRecordBase");
public static final ClassName CLASSNAME_SPECIFIC_RECORD_BASE_EXT = ClassName.get("com.linkedin.avroutil1.compatibility.backports", "SpecificRecordBaseExt");
public static final ClassName CLASSNAME_SPECIFIC_DATUM_READER = ClassName.get("org.apache.avro.specific", "SpecificDatumReader");
public static final ClassName CLASSNAME_SPECIFIC_DATUM_WRITER = ClassName.get("org.apache.avro.specific", "SpecificDatumWriter");
public static final ClassName CLASSNAME_ENCODER = ClassName.get("org.apache.avro.io", "Encoder");
public static final ClassName CLASSNAME_CUSTOM_DECODER = ClassName.get("com.linkedin.avroutil1.compatibility", "CustomDecoder");
public static final ClassName CLASSNAME_RESOLVING_DECODER = ClassName.get("org.apache.avro.io", "ResolvingDecoder");
public static final ClassName CLASSNAME_DATUM_READER = ClassName.get("org.apache.avro.io", "DatumReader");
public static final ClassName CLASSNAME_DATUM_WRITER = ClassName.get("org.apache.avro.io", "DatumWriter");
Expand Down
Loading