Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,17 @@ allprojects {
tasks.configureEach<Test> {
val javaToolchains = project.extensions.getByType<JavaToolchainService>()
useJUnitPlatform()
javaLauncher.set(javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(11)) })
// run tests on Java 8
// toolchain resolver plugin will download Java 8 if not installed
javaLauncher.set(javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(8)) })
testLogging { exceptionFormat = TestExceptionFormat.FULL }
}
tasks.withType<JavaCompile> {
sourceCompatibility = "17"
if (project.name != "core") {
options.release.set(11)
} else {
options.release.set(8)
}
// compile for Java 8 release level
options.release = 8
// use a Java 17 to compile code, will download Java 17 if not installed
javaCompiler = javaToolchains.compilerFor { languageVersion = JavaLanguageVersion.of(17) }
dependsOn(submodulesUpdate)
}

Expand Down
6 changes: 6 additions & 0 deletions examples/substrait-spark/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,9 @@ tasks.named<Test>("test") {
}

java { toolchain { languageVersion.set(JavaLanguageVersion.of(17)) } }

// dependency io.substrait:spark:0.36.0 above has been compiled for Java 11, setting release to 11
tasks.withType<JavaCompile> {
sourceCompatibility = "17"
options.release = 11
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import io.substrait.proto.ExtendedExpression;
import io.substrait.proto.Plan;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import org.apache.calcite.avatica.util.Casing;
Expand All @@ -42,7 +43,7 @@ public class IsthmusEntryPoint implements Callable<Integer> {
names = {"-c", "--create"},
description =
"One or multiple create table statements e.g. CREATE TABLE T1(foo int, bar bigint)")
private List<String> createStatements = List.of();
private List<String> createStatements = Collections.emptyList();

@Option(
names = {"--outputformat"},
Expand Down Expand Up @@ -94,7 +95,7 @@ public Integer call() throws Exception {
SqlToSubstrait converter = new SqlToSubstrait(featureBoard);
Prepare.CatalogReader catalog =
SubstraitCreateStatementParser.processCreateStatementsToCatalog(
createStatements.toArray(String[]::new));
createStatements.toArray(new String[0]));
Plan plan = converter.execute(sql, catalog);
printMessage(plan);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import io.substrait.type.NamedStruct;
import io.substrait.type.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
Expand Down Expand Up @@ -141,7 +142,7 @@ private Result registerCreateTablesForExtendedExpression(List<String> tables)
Map<String, RexNode> nameToNodeMap = new HashMap<>();
CalciteSchema rootSchema = CalciteSchema.createRootSchema(false);
CalciteCatalogReader catalogReader =
new CalciteCatalogReader(rootSchema, List.of(), factory, config);
new CalciteCatalogReader(rootSchema, Collections.emptyList(), factory, config);
if (tables != null) {
for (String tableDef : tables) {
List<SubstraitTable> tList =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,10 @@ public Rel visit(org.apache.calcite.rel.core.Values values) {
var fields =
list.stream()
.map(l -> literalConverter.convert(l))
.collect(Collectors.toUnmodifiableList());
.collect(Collectors.toList());
return ExpressionCreator.struct(false, fields);
})
.collect(Collectors.toUnmodifiableList());
.collect(Collectors.toList());
return VirtualTableScan.builder().initialSchema(type).addAllRows(structs).build();
}

Expand Down Expand Up @@ -266,7 +266,7 @@ Aggregate.Measure fromAggCall(RelNode input, Type.Struct inputType, AggregateCal
var invocation =
aggregateFunctionConverter.convert(
input, inputType, call, t -> t.accept(rexExpressionConverter));
if (invocation.isEmpty()) {
if (!invocation.isPresent()) {
throw new UnsupportedOperationException("Unable to find binding for call " + call);
}
var builder = Aggregate.Measure.builder().function(invocation.get());
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.substrait.isthmus.calcite;

import io.substrait.isthmus.AggregateFunctions;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
Expand Down Expand Up @@ -29,7 +30,7 @@ private SubstraitOperatorTable() {}

private static final SqlOperatorTable SUBSTRAIT_OPERATOR_TABLE =
SqlOperatorTables.of(
List.of(
Arrays.asList(
AggregateFunctions.MAX,
AggregateFunctions.MIN,
AggregateFunctions.AVG,
Expand All @@ -51,7 +52,7 @@ private SubstraitOperatorTable() {}
// filter out the kinds that have been overriden from the standard operator table
STANDARD_OPERATOR_TABLE.getOperatorList().stream()
.filter(op -> !OVERRIDE_KINDS.contains(op.kind)))
.collect(Collectors.toUnmodifiableList());
.collect(Collectors.toList());

@Override
public void lookupOperatorOverloads(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.substrait.isthmus.expression;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import io.substrait.expression.AbstractExpressionVisitor;
import io.substrait.expression.EnumArg;
import io.substrait.expression.Expression;
Expand Down Expand Up @@ -426,10 +427,12 @@ public RexNode visit(Expression.WindowFunctionInvocation expr, Context context)
sf -> {
Set<SqlKind> direction =
switch (sf.direction()) {
case ASC_NULLS_FIRST -> Set.of(SqlKind.NULLS_FIRST);
case ASC_NULLS_LAST -> Set.of(SqlKind.NULLS_LAST);
case DESC_NULLS_FIRST -> Set.of(SqlKind.DESCENDING, SqlKind.NULLS_FIRST);
case DESC_NULLS_LAST -> Set.of(SqlKind.DESCENDING, SqlKind.NULLS_LAST);
case ASC_NULLS_FIRST -> ImmutableSet.of(SqlKind.NULLS_FIRST);
case ASC_NULLS_LAST -> ImmutableSet.of(SqlKind.NULLS_LAST);
case DESC_NULLS_FIRST -> ImmutableSet.of(
SqlKind.DESCENDING, SqlKind.NULLS_FIRST);
case DESC_NULLS_LAST -> ImmutableSet.of(
SqlKind.DESCENDING, SqlKind.NULLS_LAST);
case CLUSTERED -> throw new IllegalArgumentException(
"SORT_DIRECTION_CLUSTERED is not supported");
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public Optional<Expression> convert(
case ROW:
{
var index = toInt(literal);
if (index.isEmpty()) {
if (!index.isPresent()) {
return Optional.empty();
}
if (input instanceof FieldReference) {
Expand All @@ -61,7 +61,7 @@ public Optional<Expression> convert(
case ARRAY:
{
var index = toInt(literal);
if (index.isEmpty()) {
if (!index.isPresent()) {
return Optional.empty();
}

Expand All @@ -75,7 +75,7 @@ public Optional<Expression> convert(
case MAP:
{
var mapKey = toString(literal);
if (mapKey.isEmpty()) {
if (!mapKey.isPresent()) {
return Optional.empty();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@
import io.substrait.type.Type;
import io.substrait.util.Util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -325,7 +327,7 @@ private Stream<String> matchKeys(List<RexNode> rexOperands, List<String> opTypes
if (rexArg instanceof RexLiteral) {
isOption = ((RexLiteral) rexArg).getValue() instanceof Enum;
}
return isOption ? List.of("req", "opt") : List.of(opType);
return isOption ? Arrays.asList("req", "opt") : Arrays.asList(opType);
})
.collect(Collectors.toList());

Expand Down Expand Up @@ -376,7 +378,7 @@ public Optional<T> attemptMatch(C call, Function<RexNode, Expression> topLevelCo
}
})
.collect(Collectors.toList());
boolean allArgsMapped = funcArgs.stream().filter(Objects::isNull).findFirst().isEmpty();
boolean allArgsMapped = !funcArgs.stream().filter(Objects::isNull).findFirst().isPresent();
if (allArgsMapped) {
return Optional.of(generateBinding(call, variant, funcArgs, outputType));
} else {
Expand Down Expand Up @@ -406,7 +408,10 @@ private Optional<T> matchByLeastRestrictive(
return Optional.empty();
}
Type type = typeConverter.toSubstrait(leastRestrictive);
var out = singularInputType.orElseThrow().tryMatch(type, outputType);
if (!singularInputType.isPresent()) {
throw new NoSuchElementException("No value present");
}
var out = singularInputType.get().tryMatch(type, outputType);

return out.map(
declaration -> {
Expand All @@ -426,7 +431,7 @@ private Optional<T> matchCoerced(C call, Type outputType, List<Expression> expre

// See if all the input types can be made to match the function
Optional<F> matchFunction = signatureMatch(operandTypes, outputType);
if (matchFunction.isEmpty()) {
if (!matchFunction.isPresent()) {
return Optional.empty();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package io.substrait.isthmus.expression;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.substrait.isthmus.AggregateFunctions;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -98,17 +100,21 @@ public class FunctionMappings {

// contains return-type based resolver for both scalar and aggregator operator
public static final Map<SqlOperator, TypeBasedResolver> OPERATOR_RESOLVER =
Map.of(
ImmutableMap.of(
SqlStdOperatorTable.PLUS,
resolver(
SqlStdOperatorTable.PLUS, Set.of("i8", "i16", "i32", "i64", "fp32", "fp64", "dec")),
SqlStdOperatorTable.PLUS,
ImmutableSet.of("i8", "i16", "i32", "i64", "fp32", "fp64", "dec")),
SqlStdOperatorTable.DATETIME_PLUS,
resolver(SqlStdOperatorTable.PLUS, Set.of("date", "time", "timestamp")),
resolver(SqlStdOperatorTable.PLUS, ImmutableSet.of("date", "time", "timestamp")),
SqlStdOperatorTable.MINUS,
resolver(
SqlStdOperatorTable.MINUS, Set.of("i8", "i16", "i32", "i64", "fp32", "fp64", "dec")),
SqlStdOperatorTable.MINUS,
ImmutableSet.of("i8", "i16", "i32", "i64", "fp32", "fp64", "dec")),
SqlStdOperatorTable.MINUS_DATE,
resolver(SqlStdOperatorTable.MINUS_DATE, Set.of("date", "timestamp_tz", "timestamp")));
resolver(
SqlStdOperatorTable.MINUS_DATE,
ImmutableSet.of("date", "timestamp_tz", "timestamp")));

public static void main(String[] args) {
SCALAR_SIGS.forEach(System.out::println);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ public Expression.Literal convert(RexLiteral literal) {
var interval = Duration.ofMillis(totalMillis);

var days = interval.toDays();
var seconds = interval.minusDays(days).toSeconds();
var micros = interval.toMillisPart() * 1000;
var seconds = interval.minusDays(days).getSeconds();
var micros = interval.getNano() / 1000;

yield intervalDay(n, (int) days, (int) seconds, micros, 6);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.substrait.isthmus.CallConverter;
import io.substrait.isthmus.TypeConverter;
import io.substrait.type.Type;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
Expand Down Expand Up @@ -43,7 +44,7 @@ public ScalarFunctionConverter(
TypeConverter typeConverter) {
super(functions, additionalSignatures, typeFactory, typeConverter);

mappers = List.of(new TrimFunctionMapper(functions));
mappers = Arrays.asList(new TrimFunctionMapper(functions));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,7 @@ public TrimFunctionMapper(List<ScalarFunctionVariant> functions) {

private List<ScalarFunctionVariant> findFunction(
String name, Collection<ScalarFunctionVariant> functions) {
return functions.stream()
.filter(f -> name.equals(f.name()))
.collect(Collectors.toUnmodifiableList());
return functions.stream().filter(f -> name.equals(f.name())).collect(Collectors.toList());
}

@Override
Expand All @@ -92,14 +90,13 @@ public Optional<SubstraitFunctionMapping> toSubstrait(final RexCall call) {

return trimType.map(
trim -> {
var functions = trimFunctions.getOrDefault(trim, List.of());
var functions = trimFunctions.getOrDefault(trim, Collections.emptyList());
if (functions.isEmpty()) {
return null;
}

var name = trim.substraitName();
var operands =
call.getOperands().stream().skip(1).collect(Collectors.toUnmodifiableList());
var operands = call.getOperands().stream().skip(1).collect(Collectors.toList());
return new SubstraitFunctionMapping(name, operands, functions);
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ public class SubstraitCreateStatementParser {

private static final CalciteCatalogReader EMPTY_CATALOG =
new CalciteCatalogReader(
CalciteSchema.createRootSchema(false), List.of(), TYPE_FACTORY, CONNECTION_CONFIG);
CalciteSchema.createRootSchema(false),
Collections.emptyList(),
TYPE_FACTORY,
CONNECTION_CONFIG);

// A validator is needed to convert the types in column declarations to Calcite types
private static final SqlValidator VALIDATOR =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.substrait.relation.Rel;
import io.substrait.type.Type;
import io.substrait.type.TypeCreator;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
Expand All @@ -22,8 +23,8 @@ public class AggregationFunctionsTest extends PlanTestBase {
static final TypeCreator N = TypeCreator.of(true);

// Create a table with that has a column of every numeric type, both NOT NULL and NULL
private List<Type> numericTypesR = List.of(R.I8, R.I16, R.I32, R.I64, R.FP32, R.FP64);
private List<Type> numericTypesN = List.of(N.I8, N.I16, N.I32, N.I64, N.FP32, N.FP64);
private List<Type> numericTypesR = Arrays.asList(R.I8, R.I16, R.I32, R.I64, R.FP32, R.FP64);
private List<Type> numericTypesN = Arrays.asList(N.I8, N.I16, N.I32, N.I64, N.FP32, N.FP64);
private List<Type> numericTypes =
Stream.concat(numericTypesR.stream(), numericTypesN.stream()).collect(Collectors.toList());

Expand All @@ -37,7 +38,8 @@ public class AggregationFunctionsTest extends PlanTestBase {
private List<String> columnNames =
Streams.mapWithIndex(tableTypes.stream(), (t, index) -> String.valueOf(index))
.collect(Collectors.toList());
private NamedScan numericTypesTable = b.namedScan(List.of("example"), columnNames, tableTypes);
private NamedScan numericTypesTable =
b.namedScan(Arrays.asList("example"), columnNames, tableTypes);

// Create the given function call on the given field of the input
private Aggregate.Measure functionPicker(Rel input, int field, String fname) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ void tVarChar() {
@Test
void tDecimalLiteral() {
List<BigDecimal> decimalList =
List.of(
Arrays.asList(
new BigDecimal("-123.457890"),
new BigDecimal("123.457890"),
new BigDecimal("123.450000"),
Expand All @@ -320,7 +320,7 @@ void tDecimalLiteral() {
@Test
void tDecimalLiteral2() {
List<BigDecimal> decimalList =
List.of(
Arrays.asList(
new BigDecimal("-99.123456789123456789123456789123456789"), // scale = 36, precision =38
new BigDecimal("99.123456789123456789123456789123456789") // scale = 36, precision = 38
);
Expand Down
Loading
Loading