result = new LinkedHashSet<>();
+ for (MetaColumn c : cfg.metaColumns.values()) {
+ if (c.asRangeBoundOf != null) {
+ result.add(c.asRangeBoundOf);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns a mapping from data column names to their associated range bound metadata columns.
+ *
+ * Each entry maps a data column name to another map with keys {@code "lower"} and/or
+ * {@code "upper"}, representing the metadata column names that define those bounds.
+ *
+ * @param name the {@link SchemaTableName} of the target table
+ * @return a nested mapping from data column name → ("lower"/"upper" → metadata column name)
+ */
+ public Map> getDataColumnRangeMapping(SchemaTableName name)
+ {
+ TableConfig cfg = getTableConfig(name);
+ Map> mapping = new HashMap<>();
+ for (MetaColumn c : cfg.metaColumns.values()) {
+ if (c.asRangeBoundOf != null && c.boundType != null) {
+ mapping.computeIfAbsent(c.asRangeBoundOf, k -> new HashMap<>())
+ .put(c.boundType, c.name);
+ }
+ }
+ return mapping;
+ }
+
+ /**
+ * Merges and returns the effective {@link TableConfig} for the given table, taking into account
+ * the hierarchical configuration structure: global → schema → table.
+ *
+ * @param name the {@link SchemaTableName} of the target table
+ * @return the merged table configuration
+ */
+ private TableConfig getTableConfig(SchemaTableName name)
+ {
+ TableConfig merged = new TableConfig();
+
+ List namespaces = new ArrayList<>();
+ namespaces.add("");
+ namespaces.add(name.getSchemaName());
+ namespaces.add(name.getSchemaName() + "." + name.getTableName());
+
+ for (String ns : namespaces) {
+ TableConfig cfg = tableConfigs.get(ns);
+ if (cfg != null) {
+ merged.metaColumns.putAll(cfg.metaColumns);
+
+ for (FilterRule rule : cfg.filterRules) {
+ boolean exists = merged.filterRules.stream()
+ .anyMatch(r -> r.column.equals(rule.column));
+ if (!exists) {
+ merged.filterRules.add(rule);
+ }
+ }
+ }
+ }
+
+ return merged;
+ }
+}
diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpMySqlSplitFilterProvider.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpMySqlSplitFilterProvider.java
deleted file mode 100644
index 31d24fd4df71c..0000000000000
--- a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpMySqlSplitFilterProvider.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.facebook.presto.plugin.clp.ClpConfig;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.Inject;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static com.facebook.presto.plugin.clp.split.filter.ClpSplitFilterConfig.CustomSplitFilterOptions;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
-import static java.lang.String.format;
-
-/**
- * Implementation for the CLP package's MySQL metadata database.
- */
-public class ClpMySqlSplitFilterProvider
- extends ClpSplitFilterProvider
-{
- @Inject
- public ClpMySqlSplitFilterProvider(ClpConfig config)
- {
- super(config);
- }
-
- /**
- * Performs regex-based replacements to rewrite {@code pushDownExpression} according to the
- * {@code "rangeMapping"} field in {@link ClpMySqlCustomSplitFilterOptions}. For example:
- *
- * - {@code "msg.timestamp" >= 1234} → {@code end_timestamp >= 1234}
- * - {@code "msg.timestamp" <= 5678} → {@code begin_timestamp <= 5678}
- * - {@code "msg.timestamp" = 4567} →
- * {@code (begin_timestamp <= 4567 AND end_timestamp >= 4567)}
- *
- *
- * @param scope the filter's scope
- * @param pushDownExpression the expression to be rewritten
- * @return the rewritten expression
- */
- @Override
- public String remapSplitFilterPushDownExpression(String scope, String pushDownExpression)
- {
- String[] splitScope = scope.split("\\.");
-
- Map mappings = new HashMap<>(getAllMappingsFromFilters(filterMap.get(splitScope[0])));
-
- if (1 < splitScope.length) {
- mappings.putAll(getAllMappingsFromFilters(filterMap.get(splitScope[0] + "." + splitScope[1])));
- }
-
- if (3 == splitScope.length) {
- mappings.putAll(getAllMappingsFromFilters(filterMap.get(scope)));
- }
-
- String remappedSql = pushDownExpression;
- for (Map.Entry entry : mappings.entrySet()) {
- String key = entry.getKey();
- ClpMySqlCustomSplitFilterOptions.RangeMapping value = entry.getValue();
- remappedSql = remappedSql.replaceAll(
- format("\"(%s)\"\\s(>=?)\\s(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)", key),
- format("%s $2 $3", value.upperBound));
- remappedSql = remappedSql.replaceAll(
- format("\"(%s)\"\\s(<=?)\\s(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)", key),
- format("%s $2 $3", value.lowerBound));
- remappedSql = remappedSql.replaceAll(
- format("\"(%s)\"\\s(=)\\s(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)", key),
- format("(%s <= $3 AND %s >= $3)", value.lowerBound, value.upperBound));
- }
- return remappedSql;
- }
-
- @Override
- protected Class getCustomSplitFilterOptionsClass()
- {
- return ClpMySqlCustomSplitFilterOptions.class;
- }
-
- private Map getAllMappingsFromFilters(List filters)
- {
- return null != filters
- ? filters.stream()
- .filter(filter ->
- filter.customOptions instanceof ClpMySqlCustomSplitFilterOptions &&
- ((ClpMySqlCustomSplitFilterOptions) filter.customOptions).rangeMapping != null)
- .collect(toImmutableMap(
- filter -> filter.columnName,
- filter -> ((ClpMySqlCustomSplitFilterOptions) filter.customOptions).rangeMapping))
- : ImmutableMap.of();
- }
-
- /**
- * Custom options:
- *
- * - {@code rangeMapping} (optional): an object with the following properties:
- *
- * - {@code lowerBound}: The numeric metadata column that represents the lower bound
- * of values in a split for the numeric data column.
- * - {@code upperBound}: The numeric metadata column that represents the upper bound
- * of values in a split for the numeric data column.
- *
- *
- *
- */
- protected static class ClpMySqlCustomSplitFilterOptions
- implements CustomSplitFilterOptions
- {
- @JsonProperty("rangeMapping")
- public RangeMapping rangeMapping;
-
- public static class RangeMapping
- {
- @JsonProperty("lowerBound")
- public String lowerBound;
-
- @JsonProperty("upperBound")
- public String upperBound;
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (!(o instanceof RangeMapping)) {
- return false;
- }
- RangeMapping that = (RangeMapping) o;
- return Objects.equals(lowerBound, that.lowerBound) &&
- Objects.equals(upperBound, that.upperBound);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(lowerBound, upperBound);
- }
- }
- }
-}
diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfig.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfig.java
deleted file mode 100644
index 0d7e37a803515..0000000000000
--- a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfig.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Options for a how a column in a Presto query should be pushed down into a query against CLP's
- * metadata database (during split pruning):
- *
- * - {@code columnName}: The column's name in the Presto query.
- *
- * - {@code customOptions}: Options specific to the current
- * {@link ClpSplitFilterProvider}.
- *
- * - {@code required} (optional, defaults to {@code false}): Whether the filter must be
- * present in the generated metadata query. If a required filter is missing or cannot be added to
- * the metadata query, the original query will be rejected.
- *
- */
-public class ClpSplitFilterConfig
-{
- @JsonProperty("columnName")
- public String columnName;
-
- @JsonProperty("customOptions")
- public CustomSplitFilterOptions customOptions;
-
- @JsonProperty("required")
- public boolean required;
-
- public interface CustomSplitFilterOptions
- {}
-}
diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfigCustomOptionsDeserializer.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfigCustomOptionsDeserializer.java
deleted file mode 100644
index 3464c14883e26..0000000000000
--- a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterConfigCustomOptionsDeserializer.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import java.io.IOException;
-
-import static com.facebook.presto.plugin.clp.split.filter.ClpSplitFilterConfig.CustomSplitFilterOptions;
-
-/**
- * Uses the given {@link CustomSplitFilterOptions} implementation to deserialize the
- * {@code "customOptions"} field in a {@link ClpSplitFilterConfig}. The implementation is determined
- * by the implemented {@link ClpSplitFilterProvider}.
- */
-public class ClpSplitFilterConfigCustomOptionsDeserializer
- extends JsonDeserializer
-{
- private final Class actualCustomSplitFilterOptionsClass;
-
- public ClpSplitFilterConfigCustomOptionsDeserializer(Class actualCustomSplitFilterOptionsClass)
- {
- this.actualCustomSplitFilterOptionsClass = actualCustomSplitFilterOptionsClass;
- }
-
- @Override
- public CustomSplitFilterOptions deserialize(JsonParser p, DeserializationContext ctxt) throws IOException
- {
- ObjectNode node = p.getCodec().readTree(p);
- ObjectMapper mapper = (ObjectMapper) p.getCodec();
-
- return mapper.treeToValue(node, actualCustomSplitFilterOptionsClass);
- }
-}
diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterProvider.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterProvider.java
deleted file mode 100644
index 0609843aaf22f..0000000000000
--- a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/filter/ClpSplitFilterProvider.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.facebook.presto.plugin.clp.ClpConfig;
-import com.facebook.presto.spi.PrestoException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-
-import static com.facebook.presto.plugin.clp.ClpErrorCode.CLP_MANDATORY_SPLIT_FILTER_NOT_VALID;
-import static com.facebook.presto.plugin.clp.ClpErrorCode.CLP_SPLIT_FILTER_CONFIG_NOT_FOUND;
-import static com.facebook.presto.plugin.clp.split.filter.ClpSplitFilterConfig.CustomSplitFilterOptions;
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Loads and manages {@link ClpSplitFilterConfig}s from a config file.
- *
- * The config file is specified by the {@code clp.split-filter-config} property.
- *
- * Filter configs can be declared at either a catalog, schema, or table scope. Filter configs under
- * a particular scope will apply to all child scopes (e.g., schema-level filter configs will apply
- * to all tables within that schema).
- *
- * Implementations of this class can customize filter configs through the {@code "customOptions"}
- * field within each {@link ClpSplitFilterConfig}.
- */
-public abstract class ClpSplitFilterProvider
-{
- protected final Map> filterMap;
-
- public ClpSplitFilterProvider(ClpConfig config)
- {
- requireNonNull(config, "config is null");
-
- if (null == config.getSplitFilterConfig()) {
- filterMap = ImmutableMap.of();
- return;
- }
- ObjectMapper mapper = new ObjectMapper();
- SimpleModule module = new SimpleModule();
- module.addDeserializer(
- CustomSplitFilterOptions.class,
- new ClpSplitFilterConfigCustomOptionsDeserializer(getCustomSplitFilterOptionsClass()));
- mapper.registerModule(module);
- try {
- filterMap = mapper.readValue(
- new File(config.getSplitFilterConfig()),
- new TypeReference>>() {});
- }
- catch (IOException e) {
- throw new PrestoException(CLP_SPLIT_FILTER_CONFIG_NOT_FOUND, "Failed to open split filter config file", e);
- }
- }
-
- /**
- * Rewrites {@code pushDownExpression} to remap filter conditions based on the
- * {@code "customOptions"} for the given scope.
- *
- * {@code scope} follows the format {@code catalog[.schema][.table]}, and determines which
- * filter mappings to apply, since mappings from more specific scopes (e.g., table-level)
- * override or supplement those from broader scopes (e.g., catalog-level). For each scope
- * (catalog, schema, table), this method collects all mappings defined in
- * {@code "customOptions"}.
- *
- * @param scope the scope of the filter
- * @param pushDownExpression the expression to be rewritten
- * @return the rewritten expression
- */
- public abstract String remapSplitFilterPushDownExpression(String scope, String pushDownExpression);
-
- /**
- * Checks for the given table, if {@code splitFilterPushDownExpression} contains all required
- * fields.
- *
- * @param tableScopeSet the set of scopes of the tables that are being queried
- * @param splitFilterPushDownExpression the expression to be checked
- */
- public void checkContainsRequiredFilters(Set tableScopeSet, String splitFilterPushDownExpression)
- {
- boolean hasRequiredSplitFilterColumns = true;
- ImmutableList.Builder notFoundListBuilder = ImmutableList.builder();
- for (String tableScope : tableScopeSet) {
- for (String columnName : getRequiredColumnNames(tableScope)) {
- if (!splitFilterPushDownExpression.contains(columnName)) {
- hasRequiredSplitFilterColumns = false;
- notFoundListBuilder.add(columnName);
- }
- }
- }
- if (!hasRequiredSplitFilterColumns) {
- throw new PrestoException(
- CLP_MANDATORY_SPLIT_FILTER_NOT_VALID,
- notFoundListBuilder.build() + " is a mandatory split filter column but not valid");
- }
- }
-
- public Set getColumnNames(String scope)
- {
- return collectColumnNamesFromScopes(scope, this::getAllColumnNamesFromFilters);
- }
-
- /**
- * Returns the implemented {@link CustomSplitFilterOptions} class. To respect our code style, we
- * recommend implementing a {@code protected static class} as an inner class in the implemented
- * {@link ClpSplitFilterProvider} class.
- *
- * @return the implemented {@link CustomSplitFilterOptions} class
- */
- protected abstract Class getCustomSplitFilterOptionsClass();
-
- private Set getRequiredColumnNames(String scope)
- {
- return collectColumnNamesFromScopes(scope, this::getRequiredColumnNamesFromFilters);
- }
-
- private Set collectColumnNamesFromScopes(String scope, Function, Set> extractor)
- {
- String[] splitScope = scope.split("\\.");
- ImmutableSet.Builder builder = ImmutableSet.builder();
-
- builder.addAll(extractor.apply(filterMap.get(splitScope[0])));
-
- if (splitScope.length > 1) {
- builder.addAll(extractor.apply(filterMap.get(splitScope[0] + "." + splitScope[1])));
- }
-
- if (splitScope.length == 3) {
- builder.addAll(extractor.apply(filterMap.get(scope)));
- }
-
- return builder.build();
- }
-
- private Set getAllColumnNamesFromFilters(List filters)
- {
- return null != filters ? filters.stream()
- .map(filter -> filter.columnName)
- .collect(toImmutableSet()) : ImmutableSet.of();
- }
-
- private Set getRequiredColumnNamesFromFilters(List filters)
- {
- return null != filters ? filters.stream()
- .filter(filter -> filter.required)
- .map(filter -> filter.columnName)
- .collect(toImmutableSet()) : ImmutableSet.of();
- }
-}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java
deleted file mode 100644
index d1d0ee6964c8e..0000000000000
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp;
-
-import com.facebook.airlift.log.Logger;
-import com.facebook.presto.plugin.clp.metadata.ClpMetadataProvider;
-import com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider;
-import com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType;
-import com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.math3.util.Pair;
-
-import java.io.File;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
-import java.util.Map;
-
-import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.COLUMN_METADATA_TABLE_COLUMN_NAME;
-import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.COLUMN_METADATA_TABLE_COLUMN_TYPE;
-import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.DATASETS_TABLE_COLUMN_ARCHIVE_STORAGE_DIRECTORY;
-import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.DATASETS_TABLE_COLUMN_NAME;
-import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.DATASETS_TABLE_SUFFIX;
-import static com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider.ARCHIVES_TABLE_COLUMN_ID;
-import static com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider.ARCHIVES_TABLE_SUFFIX;
-import static java.lang.String.format;
-import static java.util.UUID.randomUUID;
-import static org.testng.Assert.fail;
-
-public final class ClpMetadataDbSetUp
-{
- public static final String METADATA_DB_PASSWORD = "";
- public static final String METADATA_DB_TABLE_PREFIX = "clp_";
- public static final String METADATA_DB_URL_TEMPLATE = "jdbc:h2:file:%s;MODE=MySQL;DATABASE_TO_UPPER=FALSE";
- public static final String METADATA_DB_USER = "sa";
- public static final String ARCHIVES_STORAGE_DIRECTORY_BASE = "/tmp/archives/";
-
- private static final Logger log = Logger.get(ClpMetadataDbSetUp.class);
- private static final String DATASETS_TABLE_NAME = METADATA_DB_TABLE_PREFIX + DATASETS_TABLE_SUFFIX;
- private static final String ARCHIVES_TABLE_COLUMN_BEGIN_TIMESTAMP = "begin_timestamp";
- private static final String ARCHIVES_TABLE_COLUMN_PAGINATION_ID = "pagination_id";
- private static final String ARCHIVES_TABLE_COLUMN_END_TIMESTAMP = "end_timestamp";
-
- private ClpMetadataDbSetUp()
- {
- throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
- }
-
- public static DbHandle getDbHandle(String dbName)
- {
- return new DbHandle(format("/tmp/presto-clp-test-%s/%s", randomUUID(), dbName));
- }
-
- public static ClpMetadata setupMetadata(DbHandle dbHandle, Map>> clpFields)
- {
- final String metadataDbUrl = format(METADATA_DB_URL_TEMPLATE, dbHandle.dbPath);
- final String columnMetadataTableSuffix = "_column_metadata";
-
- try (Connection conn = DriverManager.getConnection(metadataDbUrl, METADATA_DB_USER, METADATA_DB_PASSWORD); Statement stmt = conn.createStatement()) {
- createDatasetsTable(stmt);
-
- for (Map.Entry>> entry : clpFields.entrySet()) {
- String tableName = entry.getKey();
- String columnMetadataTableName = METADATA_DB_TABLE_PREFIX + tableName + columnMetadataTableSuffix;
- String createColumnMetadataSQL = format(
- "CREATE TABLE IF NOT EXISTS %s (" +
- " %s VARCHAR(512) NOT NULL," +
- " %s TINYINT NOT NULL," +
- " PRIMARY KEY (%s, %s))",
- columnMetadataTableName,
- COLUMN_METADATA_TABLE_COLUMN_NAME,
- COLUMN_METADATA_TABLE_COLUMN_TYPE,
- COLUMN_METADATA_TABLE_COLUMN_NAME,
- COLUMN_METADATA_TABLE_COLUMN_TYPE);
- String insertColumnMetadataSQL = format(
- "INSERT INTO %s (%s, %s) VALUES (?, ?)",
- columnMetadataTableName,
- COLUMN_METADATA_TABLE_COLUMN_NAME,
- COLUMN_METADATA_TABLE_COLUMN_TYPE);
-
- stmt.execute(createColumnMetadataSQL);
- updateDatasetsTable(conn, tableName);
-
- try (PreparedStatement pstmt = conn.prepareStatement(insertColumnMetadataSQL)) {
- for (Pair record : entry.getValue()) {
- pstmt.setString(1, record.getFirst());
- pstmt.setByte(2, record.getSecond().getType());
- pstmt.addBatch();
- }
- pstmt.executeBatch();
- }
- }
- }
- catch (SQLException e) {
- fail(e.getMessage());
- }
-
- ClpConfig config = new ClpConfig()
- .setPolymorphicTypeEnabled(true)
- .setMetadataDbUrl(metadataDbUrl)
- .setMetadataDbUser(METADATA_DB_USER)
- .setMetadataDbPassword(METADATA_DB_PASSWORD)
- .setMetadataTablePrefix(METADATA_DB_TABLE_PREFIX);
- ClpMetadataProvider metadataProvider = new ClpMySqlMetadataProvider(config);
- return new ClpMetadata(config, metadataProvider);
- }
-
- public static ClpMySqlSplitProvider setupSplit(DbHandle dbHandle, Map> splits)
- {
- final String metadataDbUrl = format(METADATA_DB_URL_TEMPLATE, dbHandle.dbPath);
- final String archiveTableFormat = METADATA_DB_TABLE_PREFIX + "%s" + ARCHIVES_TABLE_SUFFIX;
-
- try (Connection conn = DriverManager.getConnection(metadataDbUrl, METADATA_DB_USER, METADATA_DB_PASSWORD); Statement stmt = conn.createStatement()) {
- createDatasetsTable(stmt);
-
- // Create and populate archive tables
- for (Map.Entry> tableSplits : splits.entrySet()) {
- String tableName = tableSplits.getKey();
- updateDatasetsTable(conn, tableName);
-
- String archiveTableName = format(archiveTableFormat, tableSplits.getKey());
- String createArchiveTableSQL = format(
- "CREATE TABLE IF NOT EXISTS %s (" +
- "%s BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, " +
- "%s VARCHAR(64) NOT NULL, " +
- "%s BIGINT, " +
- "%s BIGINT)",
- archiveTableName,
- ARCHIVES_TABLE_COLUMN_PAGINATION_ID,
- ARCHIVES_TABLE_COLUMN_ID,
- ARCHIVES_TABLE_COLUMN_BEGIN_TIMESTAMP,
- ARCHIVES_TABLE_COLUMN_END_TIMESTAMP);
-
- stmt.execute(createArchiveTableSQL);
-
- String insertArchiveTableSQL = format(
- "INSERT INTO %s (%s, %s, %s) VALUES (?, ?, ?)",
- archiveTableName,
- ARCHIVES_TABLE_COLUMN_ID,
- ARCHIVES_TABLE_COLUMN_BEGIN_TIMESTAMP,
- ARCHIVES_TABLE_COLUMN_END_TIMESTAMP);
- try (PreparedStatement pstmt = conn.prepareStatement(insertArchiveTableSQL)) {
- for (ArchivesTableRow split : tableSplits.getValue()) {
- pstmt.setString(1, split.id);
- pstmt.setLong(2, split.beginTimestamp);
- pstmt.setLong(3, split.endTimestamp);
- pstmt.addBatch();
- }
- pstmt.executeBatch();
- }
- }
- }
- catch (SQLException e) {
- fail(e.getMessage());
- }
-
- return new ClpMySqlSplitProvider(
- new ClpConfig()
- .setPolymorphicTypeEnabled(true)
- .setMetadataDbUrl(metadataDbUrl)
- .setMetadataDbUser(METADATA_DB_USER)
- .setMetadataDbPassword(METADATA_DB_PASSWORD)
- .setMetadataTablePrefix(METADATA_DB_TABLE_PREFIX));
- }
-
- public static void tearDown(DbHandle dbHandle)
- {
- File dir = new File(dbHandle.dbPath).getParentFile();
- if (dir.exists()) {
- try {
- FileUtils.deleteDirectory(dir);
- log.info("Deleted database dir" + dir.getAbsolutePath());
- }
- catch (IOException e) {
- log.warn("Failed to delete directory " + dir + ": " + e.getMessage());
- }
- }
- }
-
- private static void createDatasetsTable(Statement stmt)
- throws SQLException
- {
- final String createDatasetsTableSql = format(
- "CREATE TABLE IF NOT EXISTS %s (%s VARCHAR(255) PRIMARY KEY, %s VARCHAR(4096) NOT NULL)",
- DATASETS_TABLE_NAME,
- DATASETS_TABLE_COLUMN_NAME,
- DATASETS_TABLE_COLUMN_ARCHIVE_STORAGE_DIRECTORY);
- stmt.execute(createDatasetsTableSql);
- }
-
- private static void updateDatasetsTable(Connection conn, String tableName)
- throws SQLException
- {
- final String insertDatasetsTableSql = format(
- "INSERT INTO %s (%s, %s) VALUES (?, ?)",
- DATASETS_TABLE_NAME,
- DATASETS_TABLE_COLUMN_NAME,
- DATASETS_TABLE_COLUMN_ARCHIVE_STORAGE_DIRECTORY);
- try (PreparedStatement pstmt = conn.prepareStatement(insertDatasetsTableSql)) {
- pstmt.setString(1, tableName);
- pstmt.setString(2, ARCHIVES_STORAGE_DIRECTORY_BASE + tableName);
- pstmt.executeUpdate();
- }
- }
-
- static final class DbHandle
- {
- private final String dbPath;
-
- DbHandle(String dbPath)
- {
- this.dbPath = dbPath;
- }
-
- public String getDbPath()
- {
- return dbPath;
- }
- }
-
- static final class ArchivesTableRow
- {
- private final String id;
- private final long beginTimestamp;
- private final long endTimestamp;
-
- ArchivesTableRow(String id, long beginTimestamp, long endTimestamp)
- {
- this.id = id;
- this.beginTimestamp = beginTimestamp;
- this.endTimestamp = endTimestamp;
- }
-
- public String getId()
- {
- return id;
- }
-
- public long getBeginTimestamp()
- {
- return beginTimestamp;
- }
-
- public long getEndTimestamp()
- {
- return endTimestamp;
- }
- }
-}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpFilterToKql.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpFilterToKql.java
index 413c9cd773a6d..29146e107bb2d 100644
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpFilterToKql.java
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpFilterToKql.java
@@ -13,10 +13,13 @@
*/
package com.facebook.presto.plugin.clp;
+import com.facebook.presto.plugin.clp.optimization.ClpExpression;
import com.facebook.presto.plugin.clp.optimization.ClpFilterToKqlConverter;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.relation.RowExpression;
import com.facebook.presto.spi.relation.VariableReferenceExpression;
+import com.facebook.presto.sql.planner.TypeProvider;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.testng.annotations.Test;
@@ -25,6 +28,7 @@
import java.util.Optional;
import java.util.Set;
+import static com.facebook.presto.common.type.BigintType.BIGINT;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -227,48 +231,62 @@ public void testComplexPushDown()
}
@Test
- public void testMetadataSqlGeneration()
+ public void testMetadataExpressionGeneration()
{
SessionHolder sessionHolder = new SessionHolder();
Set testMetadataFilterColumns = ImmutableSet.of("fare");
+ Set testDataColumnsWithRangeBounds = ImmutableSet.of("city.Region.Id");
// Normal case
testPushDown(
sessionHolder,
"(fare > 0 AND city.Name like 'b%')",
- "(fare > 0 AND city.Name: \"b*\")",
- "(\"fare\" > 0)",
- testMetadataFilterColumns);
+ "(city.Name: \"b*\")",
+ "fare > 0",
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
// With BETWEEN
testPushDown(
sessionHolder,
"((fare BETWEEN 0 AND 5) AND city.Name like 'b%')",
- "(fare >= 0 AND fare <= 5 AND city.Name: \"b*\")",
- "(\"fare\" >= 0 AND \"fare\" <= 5)",
- testMetadataFilterColumns);
+ "(city.Name: \"b*\")",
+ "fare >= 0 AND fare <= 5",
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
// The cases of that the metadata filter column exist but cannot be push down
testPushDown(
sessionHolder,
"(fare > 0 OR city.Name like 'b%')",
- "(fare > 0 OR city.Name: \"b*\")",
+ "(city.Name: \"b*\")",
null,
- testMetadataFilterColumns);
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
testPushDown(
sessionHolder,
"(fare > 0 AND city.Name like 'b%') OR city.Region.Id = 1",
- "((fare > 0 AND city.Name: \"b*\") OR city.Region.Id: 1)",
+ "((city.Name: \"b*\") OR city.Region.Id: 1)",
null,
- testMetadataFilterColumns);
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
// Complicated case
testPushDown(
sessionHolder,
"fare = 0 AND (city.Name like 'b%' OR city.Region.Id = 1)",
- "(fare: 0 AND (city.Name: \"b*\" OR city.Region.Id: 1))",
- "(\"fare\" = 0)",
- testMetadataFilterColumns);
+ "((city.Name: \"b*\" OR city.Region.Id: 1))",
+ "fare = 0",
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
+ testPushDown(
+ sessionHolder,
+ "city.Region.Id = 1 AND (city.Name like 'b%' OR fare = 0)",
+ "(city.Region.Id: 1 AND (city.Name: \"b*\"))",
+ "\"city.Region.Id\" = 1",
+ TypeProvider.viewOf(ImmutableMap.of("city.Region.Id", BIGINT)),
+ testMetadataFilterColumns,
+ testDataColumnsWithRangeBounds);
}
@Test
@@ -306,27 +324,55 @@ public void testClpWildcardUdf()
private void testPushDown(SessionHolder sessionHolder, String sql, String expectedKql, String expectedRemaining)
{
- ClpExpression clpExpression = tryPushDown(sql, sessionHolder, ImmutableSet.of());
+ ClpExpression clpExpression = tryPushDown(sql, sessionHolder, ImmutableSet.of(), ImmutableSet.of());
testFilter(clpExpression, expectedKql, expectedRemaining, sessionHolder);
}
- private void testPushDown(SessionHolder sessionHolder, String sql, String expectedKql, String expectedMetadataSqlQuery, Set metadataFilterColumns)
+ private void testPushDown(
+ SessionHolder sessionHolder,
+ String sql,
+ String expectedKql,
+ String expectedMetadataSqlQuery,
+ Set metadataFilterColumns,
+ Set dataColumnsWithRangeBounds)
+ {
+ testPushDown(
+ sessionHolder,
+ sql,
+ expectedKql,
+ expectedMetadataSqlQuery,
+ typeProvider,
+ metadataFilterColumns,
+ dataColumnsWithRangeBounds);
+ }
+
+ private void testPushDown(
+ SessionHolder sessionHolder,
+ String sql,
+ String expectedKql,
+ String expectedMetadataSqlQuery,
+ TypeProvider typeProviderForMetadataExpression,
+ Set metadataFilterColumns,
+ Set dataColumnsWithRangeBounds)
{
- ClpExpression clpExpression = tryPushDown(sql, sessionHolder, metadataFilterColumns);
+ ClpExpression clpExpression = tryPushDown(sql, sessionHolder, metadataFilterColumns, dataColumnsWithRangeBounds);
testFilter(clpExpression, expectedKql, null, sessionHolder);
if (expectedMetadataSqlQuery != null) {
- assertTrue(clpExpression.getMetadataSqlQuery().isPresent());
- assertEquals(clpExpression.getMetadataSqlQuery().get(), expectedMetadataSqlQuery);
+ assertTrue(clpExpression.getMetadataExpression().isPresent());
+ assertEquals(
+ clpExpression.getMetadataExpression().get(),
+ getRowExpression(expectedMetadataSqlQuery, typeProviderForMetadataExpression, sessionHolder));
}
else {
- assertFalse(clpExpression.getMetadataSqlQuery().isPresent());
+ assertFalse(clpExpression.getMetadataExpression().isPresent());
}
}
private ClpExpression tryPushDown(
String sqlExpression,
SessionHolder sessionHolder,
- Set metadataFilterColumns)
+ Set metadataFilterColumns,
+ Set dataColumnsWithRangeBounds)
{
RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder);
Map assignments = new HashMap<>(variableToColumnHandleMap);
@@ -335,7 +381,8 @@ private ClpExpression tryPushDown(
standardFunctionResolution,
functionAndTypeManager,
assignments,
- metadataFilterColumns),
+ metadataFilterColumns,
+ dataColumnsWithRangeBounds),
null);
}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java
index 9350729aec2d7..681d70ab1f1a6 100644
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java
@@ -19,6 +19,7 @@
import com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider;
import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
import com.facebook.presto.plugin.clp.mockdb.table.ColumnMetadataTableRows;
+import com.facebook.presto.plugin.clp.split.ClpSplitMetadataConfig;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ConnectorTableMetadata;
import com.facebook.presto.spi.SchemaTableName;
@@ -29,6 +30,10 @@
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import java.io.FileNotFoundException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Optional;
@@ -36,6 +41,7 @@
import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
import static com.facebook.presto.common.type.DoubleType.DOUBLE;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
+import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager;
import static com.facebook.presto.plugin.clp.ClpMetadata.DEFAULT_SCHEMA_NAME;
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Boolean;
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.ClpString;
@@ -54,7 +60,7 @@ public class TestClpMetadata
private ClpMetadata metadata;
@BeforeMethod
- public void setUp()
+ public void setUp() throws FileNotFoundException, URISyntaxException
{
mockMetadataDatabase = ClpMockMetadataDatabase
.builder()
@@ -83,7 +89,15 @@ public void setUp()
.setMetadataDbUser(mockMetadataDatabase.getUsername())
.setMetadataDbPassword(mockMetadataDatabase.getPassword())
.setMetadataTablePrefix(mockMetadataDatabase.getTablePrefix());
- ClpMetadataProvider metadataProvider = new ClpMySqlMetadataProvider(config);
+
+ URL resource = getClass().getClassLoader().getResource("test-mysql-split-metadata.json");
+ if (resource == null) {
+ throw new FileNotFoundException("test-mysql-split-metadata.json not found in resources");
+ }
+
+ config.setSplitMetadataConfigPath(Paths.get(resource.toURI()).toAbsolutePath().toString());
+ ClpMetadataProvider metadataProvider =
+ new ClpMySqlMetadataProvider(config, new ClpSplitMetadataConfig(config, createTestFunctionAndTypeManager()));
metadata = new ClpMetadata(config, metadataProvider);
}
@@ -115,6 +129,7 @@ public void testGetTableMetadata()
ClpTableHandle clpTableHandle = (ClpTableHandle) metadata.getTableHandle(SESSION, new SchemaTableName(DEFAULT_SCHEMA_NAME, TABLE_NAME));
ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(SESSION, clpTableHandle);
ImmutableSet columnMetadata = ImmutableSet.builder()
+ // data columns
.add(ColumnMetadata.builder()
.setName("a_bigint")
.setType(BIGINT)
@@ -150,6 +165,17 @@ public void testGetTableMetadata()
RowType.field("h", new ArrayType(VARCHAR))))))))
.setNullable(true)
.build())
+ // metadata columns
+ .add(ColumnMetadata.builder()
+ .setName("level")
+ .setType(BIGINT)
+ .setNullable(true)
+ .build())
+ .add(ColumnMetadata.builder()
+ .setName("author")
+ .setType(VARCHAR)
+ .setNullable(true)
+ .build())
.build();
assertEquals(columnMetadata, ImmutableSet.copyOf(tableMetadata.getColumns()));
}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMySqlSplitMetadataExpressionConverter.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMySqlSplitMetadataExpressionConverter.java
new file mode 100644
index 0000000000000..784d0938cdc38
--- /dev/null
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMySqlSplitMetadataExpressionConverter.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.clp;
+
+import com.facebook.presto.plugin.clp.split.ClpMySqlSplitMetadataExpressionConverter;
+import com.facebook.presto.plugin.clp.split.ClpSplitMetadataConfig;
+import com.facebook.presto.spi.PrestoException;
+import com.facebook.presto.spi.SchemaTableName;
+import com.facebook.presto.sql.planner.TypeProvider;
+import com.google.common.collect.ImmutableMap;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
+
+import static com.facebook.presto.common.type.BigintType.BIGINT;
+import static java.lang.String.format;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertThrows;
+
+@Test(singleThreaded = true)
+public class TestClpMySqlSplitMetadataExpressionConverter
+ extends TestClpQueryBase
+{
+ private TypeProvider typeProvider;
+ private ClpSplitMetadataConfig splitMetadataConfig;
+ private ClpMySqlSplitMetadataExpressionConverter converter;
+
+ @BeforeMethod
+ public void setUp() throws IOException, URISyntaxException
+ {
+ URL resource = getClass().getClassLoader().getResource("test-mysql-split-metadata.json");
+ if (resource == null) {
+ throw new FileNotFoundException("test-mysql-split-metadata.json not found in resources");
+ }
+
+ typeProvider = TypeProvider.viewOf(
+ ImmutableMap.of("msg.timestamp", BIGINT, "begin_timestamp", BIGINT, "end_timestamp", BIGINT));
+
+ ClpConfig config = new ClpConfig();
+ config.setSplitMetadataConfigPath(Paths.get(resource.toURI()).toAbsolutePath().toString());
+ splitMetadataConfig = new ClpSplitMetadataConfig(config, functionAndTypeManager);
+
+ SchemaTableName schemaTableName = new SchemaTableName("default", "table_1");
+ converter = new ClpMySqlSplitMetadataExpressionConverter(
+ functionAndTypeManager,
+ standardFunctionResolution,
+ splitMetadataConfig.getExposedToOriginalMapping(schemaTableName),
+ splitMetadataConfig.getDataColumnRangeMapping(schemaTableName),
+ splitMetadataConfig.getRequiredColumns(schemaTableName));
+ }
+
+ @Test
+ public void checkRequiredColumns()
+ {
+ SessionHolder sessionHolder = new SessionHolder();
+ SchemaTableName schemaTableName = new SchemaTableName("default", "table_1");
+
+ ClpMySqlSplitMetadataExpressionConverter converter = new ClpMySqlSplitMetadataExpressionConverter(
+ functionAndTypeManager,
+ standardFunctionResolution,
+ splitMetadataConfig.getExposedToOriginalMapping(schemaTableName),
+ splitMetadataConfig.getDataColumnRangeMapping(schemaTableName),
+ splitMetadataConfig.getRequiredColumns(schemaTableName));
+
+ TypeProvider typeProvider = TypeProvider.viewOf(ImmutableMap.of("level", BIGINT, "msg.timestamp", BIGINT));
+ assertThrows(PrestoException.class, ()
+ -> converter.transform(getRowExpression("(\"level\" >= 1 AND \"level\" <= 3)", typeProvider, sessionHolder)));
+
+ converter.transform(
+ getRowExpression("(\"msg.timestamp\" > 1234 AND \"msg.timestamp\" < 5678)", typeProvider, sessionHolder));
+ }
+
+ @Test
+ public void remapSplitFilterPushDownExpression()
+ {
+ // Integer
+ testRange(1234, 5678);
+ testRange(-5678, -1234);
+
+ // Decimal
+ testRange(1234.001, 5678.999);
+ testRange(-5678.999, -1234.001);
+
+ // Scientific
+ testRange("1.234E3", "5.678e3", 1234.0, 5678.0);
+ testRange("-1.234e-3", "-5.678E-3", -0.001234, -0.005678);
+ }
+
+ private void testRange(T lowerBound, T upperBound)
+ {
+ testRange(lowerBound, upperBound, lowerBound, upperBound);
+ }
+
+ private void testRange(T lowerBound, T upperBound, T expectedLowerBound, T expectedUpperBound)
+ {
+ SessionHolder sessionHolder = new SessionHolder();
+
+ String remappedSql1 = converter.transform(
+ getRowExpression(
+ format("(\"msg.timestamp\" > %s AND \"msg.timestamp\" < %s)", lowerBound, upperBound),
+ typeProvider,
+ sessionHolder));
+ assertEquals(
+ remappedSql1,
+ format("(end_timestamp > %s) AND (begin_timestamp < %s)", expectedLowerBound, expectedUpperBound));
+
+ String remappedSql2 = converter.transform(
+ getRowExpression(
+ format("(\"msg.timestamp\" >= %s AND \"msg.timestamp\" <= %s)", lowerBound, upperBound),
+ typeProvider,
+ sessionHolder));
+ assertEquals(
+ remappedSql2,
+ format("(end_timestamp >= %s) AND (begin_timestamp <= %s)", expectedLowerBound, expectedUpperBound));
+
+ String remappedSql3 = converter.transform(
+ getRowExpression(
+ format("(\"msg.timestamp\" > %s AND \"msg.timestamp\" <= %s)", lowerBound, upperBound),
+ typeProvider,
+ sessionHolder));
+ assertEquals(
+ remappedSql3,
+ format("(end_timestamp > %s) AND (begin_timestamp <= %s)", expectedLowerBound, expectedUpperBound));
+
+ String remappedSql4 = converter.transform(
+ getRowExpression(
+ format("(\"msg.timestamp\" >= %s AND \"msg.timestamp\" < %s)", lowerBound, upperBound),
+ typeProvider,
+ sessionHolder));
+ assertEquals(
+ remappedSql4,
+ format("(end_timestamp >= %s) AND (begin_timestamp < %s)", expectedLowerBound, expectedUpperBound));
+
+ String remappedSql5 = converter.transform(
+ getRowExpression(format("(\"msg.timestamp\" = %s)", lowerBound), typeProvider, sessionHolder));
+ assertEquals(
+ remappedSql5,
+ format("(begin_timestamp <= %s) AND (end_timestamp >= %s)", expectedLowerBound, expectedLowerBound));
+ }
+}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java
index ee259d67e17c9..adebc714b91e6 100644
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java
@@ -115,6 +115,11 @@ protected RowExpression toRowExpression(Expression expression, TypeProvider type
return SqlToRowExpressionTranslator.translate(expression, expressionTypes, ImmutableMap.of(), functionAndTypeManager, session);
}
+ protected RowExpression getRowExpression(String sqlExpression, TypeProvider typeProvider, SessionHolder sessionHolder)
+ {
+ return toRowExpression(expression(sqlExpression), typeProvider, sessionHolder.getSession());
+ }
+
protected RowExpression getRowExpression(String sqlExpression, SessionHolder sessionHolder)
{
return toRowExpression(expression(sqlExpression), typeProvider, sessionHolder.getSession());
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java
index 60ce10b871cb5..4ee73f9680de6 100644
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java
@@ -16,8 +16,10 @@
import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
import com.facebook.presto.plugin.clp.mockdb.table.ArchivesTableRows;
import com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider;
+import com.facebook.presto.plugin.clp.split.ClpSplitMetadataConfig;
import com.facebook.presto.plugin.clp.split.ClpSplitProvider;
import com.facebook.presto.spi.SchemaTableName;
+import com.facebook.presto.sql.planner.TypeProvider;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.AfterMethod;
@@ -28,17 +30,19 @@
import java.util.Map;
import java.util.Optional;
+import static com.facebook.presto.common.type.BigintType.BIGINT;
import static com.facebook.presto.plugin.clp.ClpMetadata.DEFAULT_SCHEMA_NAME;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.ARCHIVES_STORAGE_DIRECTORY_BASE;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.lang.String.format;
import static org.testng.Assert.assertEquals;
@Test(singleThreaded = true)
public class TestClpSplit
+ extends TestClpQueryBase
{
private ClpMockMetadataDatabase mockMetadataDatabase;
private ClpSplitProvider clpSplitProvider;
+ private TypeProvider typeProvider;
private Map tableSplits;
@BeforeMethod
@@ -80,7 +84,12 @@ public void setUp()
.setMetadataDbUser(mockMetadataDatabase.getUsername())
.setMetadataDbPassword(mockMetadataDatabase.getPassword())
.setMetadataTablePrefix(mockMetadataDatabase.getTablePrefix());
- clpSplitProvider = new ClpMySqlSplitProvider(config);
+ clpSplitProvider = new ClpMySqlSplitProvider(
+ config,
+ functionAndTypeManager,
+ standardFunctionResolution,
+ new ClpSplitMetadataConfig(config, functionAndTypeManager));
+ typeProvider = TypeProvider.viewOf(ImmutableMap.of("begin_timestamp", BIGINT, "end_timestamp", BIGINT));
}
@AfterMethod
@@ -135,12 +144,13 @@ private void compareListSplitsResult(
Optional metadataSql,
List expectedSplitIndexes)
{
+ final String storageBase = "/tmp/archives/";
String tableName = entry.getKey();
- String tablePath = ARCHIVES_STORAGE_DIRECTORY_BASE + tableName;
+ String tablePath = storageBase + tableName;
ClpTableLayoutHandle layoutHandle = new ClpTableLayoutHandle(
new ClpTableHandle(new SchemaTableName(DEFAULT_SCHEMA_NAME, tableName), tablePath),
Optional.empty(),
- metadataSql);
+ metadataSql.map(sql -> getRowExpression(sql, typeProvider, new SessionHolder())));
List expectedSplitPaths = expectedSplitIndexes.stream()
.map(expectedSplitIndex -> format("%s/%s", tablePath, entry.getValue().getIds().get(expectedSplitIndex)))
.collect(toImmutableList());
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplitMetadataConfig.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplitMetadataConfig.java
new file mode 100644
index 0000000000000..7f8d2dd483411
--- /dev/null
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplitMetadataConfig.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.clp;
+
+import com.facebook.presto.common.type.Type;
+import com.facebook.presto.plugin.clp.split.ClpMySqlSplitMetadataExpressionConverter;
+import com.facebook.presto.plugin.clp.split.ClpSplitMetadataConfig;
+import com.facebook.presto.spi.PrestoException;
+import com.facebook.presto.spi.SchemaTableName;
+import com.facebook.presto.sql.planner.TypeProvider;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Set;
+
+import static com.facebook.presto.common.type.BigintType.BIGINT;
+import static com.facebook.presto.common.type.VarcharType.VARCHAR;
+import static java.util.UUID.randomUUID;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+@Test(singleThreaded = true)
+public class TestClpSplitMetadataConfig
+ extends TestClpQueryBase
+{
+ private String splitMetadataConfigPath;
+
+ @BeforeMethod
+ public void setUp() throws IOException, URISyntaxException
+ {
+ URL resource = getClass().getClassLoader().getResource("test-mysql-split-metadata.json");
+ if (resource == null) {
+ throw new FileNotFoundException("test-mysql-split-metadata.json not found in resources");
+ }
+
+ splitMetadataConfigPath = Paths.get(resource.toURI()).toAbsolutePath().toString();
+ }
+
+ @Test
+ public void checkRequiredColumns()
+ {
+ SessionHolder sessionHolder = new SessionHolder();
+ ClpConfig config = new ClpConfig();
+ config.setSplitMetadataConfigPath(splitMetadataConfigPath);
+ ClpSplitMetadataConfig splitMetadataConfig = new ClpSplitMetadataConfig(config, functionAndTypeManager);
+ SchemaTableName schemaTableName = new SchemaTableName("default", "table_1");
+
+ ClpMySqlSplitMetadataExpressionConverter converter = new ClpMySqlSplitMetadataExpressionConverter(
+ functionAndTypeManager,
+ standardFunctionResolution,
+ splitMetadataConfig.getExposedToOriginalMapping(schemaTableName),
+ splitMetadataConfig.getDataColumnRangeMapping(schemaTableName),
+ splitMetadataConfig.getRequiredColumns(schemaTableName));
+
+ TypeProvider typeProvider = TypeProvider.viewOf(ImmutableMap.of("level", BIGINT, "msg.timestamp", BIGINT));
+ assertThrows(PrestoException.class, ()
+ -> converter.transform(getRowExpression("(\"level\" >= 1 AND \"level\" <= 3)", typeProvider, sessionHolder)));
+
+ converter.transform(
+ getRowExpression("(\"msg.timestamp\" > 1234 AND \"msg.timestamp\" < 5678)", typeProvider, sessionHolder));
+ }
+
+ @Test
+ public void getMetadataColumnNames()
+ {
+ ClpConfig config = new ClpConfig();
+ config.setSplitMetadataConfigPath(splitMetadataConfigPath);
+ ClpSplitMetadataConfig splitMetadataConfig = new ClpSplitMetadataConfig(config, functionAndTypeManager);
+
+ Map columns = splitMetadataConfig.getMetadataColumns(new SchemaTableName("other", "test"));
+ assertEquals(ImmutableMap.of("level", BIGINT), columns);
+ columns = splitMetadataConfig.getMetadataColumns(new SchemaTableName("default", "table_2"));
+ assertEquals(ImmutableMap.of("level", BIGINT, "author", VARCHAR), columns);
+ columns = splitMetadataConfig.getMetadataColumns(new SchemaTableName("default", "table_1"));
+ assertEquals(columns, ImmutableMap.of(
+ "level", BIGINT,
+ "author", VARCHAR,
+ "begin_timestamp", BIGINT,
+ "end_timestamp", BIGINT,
+ "file_name", VARCHAR));
+ Set dataColumnsWithRangeBounds =
+ splitMetadataConfig.getDataColumnsWithRangeBounds(new SchemaTableName("default", "table_1"));
+ assertEquals(dataColumnsWithRangeBounds, ImmutableSet.of("msg.timestamp"));
+ }
+
+ @Test
+ public void handleEmptyAndInvalidSplitMetadataConfig()
+ {
+ ClpConfig config = new ClpConfig();
+
+ // Empty config
+ ClpSplitMetadataConfig splitMetadataConfig = new ClpSplitMetadataConfig(config, functionAndTypeManager);
+ assertTrue(splitMetadataConfig.getMetadataColumns(new SchemaTableName("default", "clp")).isEmpty());
+ assertTrue(splitMetadataConfig.getMetadataColumns(new SchemaTableName("abc", "xyz")).isEmpty());
+ assertTrue(splitMetadataConfig.getMetadataColumns(new SchemaTableName("abc.opq", "xyz")).isEmpty());
+
+ // Invalid config
+ config.setSplitMetadataConfigPath(randomUUID().toString());
+ assertThrows(PrestoException.class, () -> new ClpSplitMetadataConfig(config, functionAndTypeManager));
+ }
+}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpUdfRewriter.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpUdfRewriter.java
index 3b83d1c95d10b..ddfacbc285f8e 100644
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpUdfRewriter.java
+++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpUdfRewriter.java
@@ -21,10 +21,11 @@
import com.facebook.presto.cost.StatsProvider;
import com.facebook.presto.metadata.FunctionAndTypeManager;
import com.facebook.presto.metadata.Metadata;
+import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
+import com.facebook.presto.plugin.clp.mockdb.table.ColumnMetadataTableRows;
import com.facebook.presto.plugin.clp.optimization.ClpComputePushDown;
import com.facebook.presto.plugin.clp.optimization.ClpUdfRewriter;
-import com.facebook.presto.plugin.clp.split.filter.ClpMySqlSplitFilterProvider;
-import com.facebook.presto.plugin.clp.split.filter.ClpSplitFilterProvider;
+import com.facebook.presto.plugin.clp.split.ClpSplitMetadataConfig;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.VariableAllocator;
@@ -45,7 +46,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import org.apache.commons.math3.util.Pair;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -60,13 +60,6 @@
import static com.facebook.presto.common.type.DoubleType.DOUBLE;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.ARCHIVES_STORAGE_DIRECTORY_BASE;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.METADATA_DB_PASSWORD;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.METADATA_DB_TABLE_PREFIX;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.METADATA_DB_URL_TEMPLATE;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.METADATA_DB_USER;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.getDbHandle;
-import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.setupMetadata;
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Boolean;
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.ClpString;
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Float;
@@ -80,7 +73,6 @@
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node;
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
-import static java.lang.String.format;
@Test(singleThreaded = true)
public class TestClpUdfRewriter
@@ -91,44 +83,49 @@ public class TestClpUdfRewriter
.setSchema(ClpMetadata.DEFAULT_SCHEMA_NAME)
.build();
- private ClpMetadataDbSetUp.DbHandle dbHandle;
+ private ClpMockMetadataDatabase mockMetadataDatabase;
ClpTableHandle table;
private LocalQueryRunner localQueryRunner;
private FunctionAndTypeManager functionAndTypeManager;
private FunctionResolution functionResolution;
- private ClpSplitFilterProvider splitFilterProvider;
+ private ClpSplitMetadataConfig splitMetadataConfig;
private PlanNodeIdAllocator planNodeIdAllocator;
private VariableAllocator variableAllocator;
@BeforeMethod
public void setUp()
{
- dbHandle = getDbHandle("metadata_query_testdb");
final String tableName = "test";
- final String tablePath = ARCHIVES_STORAGE_DIRECTORY_BASE + tableName;
- table = new ClpTableHandle(new SchemaTableName("default", tableName), tablePath);
-
- setupMetadata(dbHandle,
- ImmutableMap.of(
- tableName,
+ table = new ClpTableHandle(new SchemaTableName("default", tableName), "test");
+ mockMetadataDatabase = ClpMockMetadataDatabase.builder().build();
+ mockMetadataDatabase.addTableToDatasetsTableIfNotExist(ImmutableList.of(tableName));
+ mockMetadataDatabase.addColumnMetadata(ImmutableMap.of(
+ tableName,
+ new ColumnMetadataTableRows(
+ ImmutableList.of(
+ "city.Name",
+ "city.Region.Id",
+ "city.Region.Name",
+ "fare",
+ "isHoliday"),
ImmutableList.of(
- new Pair<>("city.Name", ClpString),
- new Pair<>("city.Region.Id", Integer),
- new Pair<>("city.Region.Name", VarString),
- new Pair<>("fare", Float),
- new Pair<>("isHoliday", Boolean))));
+ ClpString,
+ Integer,
+ VarString,
+ Float,
+ Boolean))));
localQueryRunner = new LocalQueryRunner(defaultSession);
localQueryRunner.createCatalog("clp", new ClpConnectorFactory(), ImmutableMap.of(
- "clp.metadata-db-url", format(METADATA_DB_URL_TEMPLATE, dbHandle.getDbPath()),
- "clp.metadata-db-user", METADATA_DB_USER,
- "clp.metadata-db-password", METADATA_DB_PASSWORD,
- "clp.metadata-table-prefix", METADATA_DB_TABLE_PREFIX));
+ "clp.metadata-db-url", mockMetadataDatabase.getUrl(),
+ "clp.metadata-db-user", mockMetadataDatabase.getUsername(),
+ "clp.metadata-db-password", mockMetadataDatabase.getPassword(),
+ "clp.metadata-table-prefix", mockMetadataDatabase.getTablePrefix()));
localQueryRunner.getMetadata().registerBuiltInFunctions(extractFunctions(new ClpPlugin().getFunctions()));
functionAndTypeManager = localQueryRunner.getMetadata().getFunctionAndTypeManager();
functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver());
- splitFilterProvider = new ClpMySqlSplitFilterProvider(new ClpConfig());
+ splitMetadataConfig = new ClpSplitMetadataConfig(new ClpConfig(), functionAndTypeManager);
planNodeIdAllocator = new PlanNodeIdAllocator();
variableAllocator = new VariableAllocator();
}
@@ -137,7 +134,9 @@ public void setUp()
public void tearDown()
{
localQueryRunner.close();
- ClpMetadataDbSetUp.tearDown(dbHandle);
+ if (null != mockMetadataDatabase) {
+ mockMetadataDatabase.teardown();
+ }
}
@Test
@@ -153,7 +152,7 @@ public void testClpGetScanFilter()
WarningCollector.NOOP);
ClpUdfRewriter udfRewriter = new ClpUdfRewriter(functionAndTypeManager);
PlanNode optimizedPlan = udfRewriter.optimize(plan.getRoot(), session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
- ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitFilterProvider);
+ ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitMetadataConfig);
optimizedPlan = optimizer.optimize(optimizedPlan, session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
PlanAssert.assertPlan(
@@ -192,7 +191,7 @@ public void testClpGetScanProject()
WarningCollector.NOOP);
ClpUdfRewriter udfRewriter = new ClpUdfRewriter(functionAndTypeManager);
PlanNode optimizedPlan = udfRewriter.optimize(plan.getRoot(), session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
- ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitFilterProvider);
+ ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitMetadataConfig);
optimizedPlan = optimizer.optimize(optimizedPlan, session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
PlanAssert.assertPlan(
@@ -241,7 +240,7 @@ public void testClpGetScanProjectFilter()
WarningCollector.NOOP);
ClpUdfRewriter udfRewriter = new ClpUdfRewriter(functionAndTypeManager);
PlanNode optimizedPlan = udfRewriter.optimize(plan.getRoot(), session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
- ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitFilterProvider);
+ ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitMetadataConfig);
optimizedPlan = optimizer.optimize(optimizedPlan, session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
PlanAssert.assertPlan(
@@ -278,7 +277,7 @@ public void testClpGetJsonString()
WarningCollector.NOOP);
ClpUdfRewriter udfRewriter = new ClpUdfRewriter(functionAndTypeManager);
PlanNode optimizedPlan = udfRewriter.optimize(plan.getRoot(), session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
- ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitFilterProvider);
+ ClpComputePushDown optimizer = new ClpComputePushDown(functionAndTypeManager, functionResolution, splitMetadataConfig);
optimizedPlan = optimizer.optimize(optimizedPlan, session.toConnectorSession(), variableAllocator, planNodeIdAllocator);
PlanAssert.assertPlan(
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpMySqlSplitFilterConfig.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpMySqlSplitFilterConfig.java
deleted file mode 100644
index 27d9c86ee0092..0000000000000
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpMySqlSplitFilterConfig.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.facebook.presto.plugin.clp.ClpConfig;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Paths;
-
-import static java.lang.String.format;
-import static org.testng.Assert.assertEquals;
-
-@Test(singleThreaded = true)
-public class TestClpMySqlSplitFilterConfig
-{
- private String filterConfigPath;
-
- @BeforeMethod
- public void setUp() throws IOException, URISyntaxException
- {
- URL resource = getClass().getClassLoader().getResource("test-mysql-split-filter.json");
- if (resource == null) {
- throw new FileNotFoundException("test-mysql-split-filter.json not found in resources");
- }
-
- filterConfigPath = Paths.get(resource.toURI()).toAbsolutePath().toString();
- }
-
- @Test
- public void remapSplitFilterPushDownExpression()
- {
- ClpConfig config = new ClpConfig();
- config.setSplitFilterConfig(filterConfigPath);
- ClpMySqlSplitFilterProvider filterProvider = new ClpMySqlSplitFilterProvider(config);
-
- // Integer
- testRange(1234, 5678, filterProvider);
- testRange(-5678, -1234, filterProvider);
-
- // Decimal
- testRange(1234.001, 5678.999, filterProvider);
- testRange(-5678.999, -1234.001, filterProvider);
-
- // Scientific
- testRange("1.234E3", "5.678e3", filterProvider);
- testRange("-1.234e-3", "-5.678E-3", filterProvider);
- }
-
- private void testRange(T lowerBound, T upperBound, ClpMySqlSplitFilterProvider filterProvider)
- {
- String splitFilterSql1 = format("(\"msg.timestamp\" > %s AND \"msg.timestamp\" < %s)", lowerBound, upperBound);
- String remappedSql1 = filterProvider.remapSplitFilterPushDownExpression("clp.default.table_1", splitFilterSql1);
- assertEquals(remappedSql1, format("(end_timestamp > %s AND begin_timestamp < %s)", lowerBound, upperBound));
-
- String splitFilterSql2 = format("(\"msg.timestamp\" >= %s AND \"msg.timestamp\" <= %s)", lowerBound, upperBound);
- String remappedSql2 = filterProvider.remapSplitFilterPushDownExpression("clp.default.table_1", splitFilterSql2);
- assertEquals(remappedSql2, format("(end_timestamp >= %s AND begin_timestamp <= %s)", lowerBound, upperBound));
-
- String splitFilterSql3 = format("(\"msg.timestamp\" > %s AND \"msg.timestamp\" <= %s)", lowerBound, upperBound);
- String remappedSql3 = filterProvider.remapSplitFilterPushDownExpression("clp.default.table_1", splitFilterSql3);
- assertEquals(remappedSql3, format("(end_timestamp > %s AND begin_timestamp <= %s)", lowerBound, upperBound));
-
- String splitFilterSql4 = format("(\"msg.timestamp\" >= %s AND \"msg.timestamp\" < %s)", lowerBound, upperBound);
- String remappedSql4 = filterProvider.remapSplitFilterPushDownExpression("clp.default.table_1", splitFilterSql4);
- assertEquals(remappedSql4, format("(end_timestamp >= %s AND begin_timestamp < %s)", lowerBound, upperBound));
-
- String splitFilterSql5 = format("(\"msg.timestamp\" = %s)", lowerBound);
- String remappedSql5 = filterProvider.remapSplitFilterPushDownExpression("clp.default.table_1", splitFilterSql5);
- assertEquals(remappedSql5, format("((begin_timestamp <= %s AND end_timestamp >= %s))", lowerBound, lowerBound));
- }
-}
diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpSplitFilterConfigCommon.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpSplitFilterConfigCommon.java
deleted file mode 100644
index 7a4058f617d0c..0000000000000
--- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/split/filter/TestClpSplitFilterConfigCommon.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.facebook.presto.plugin.clp.split.filter;
-
-import com.facebook.presto.plugin.clp.ClpConfig;
-import com.facebook.presto.spi.PrestoException;
-import com.facebook.presto.spi.SchemaTableName;
-import com.google.common.collect.ImmutableSet;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Paths;
-import java.util.Set;
-
-import static com.facebook.presto.plugin.clp.ClpConnectorFactory.CONNECTOR_NAME;
-import static java.lang.String.format;
-import static java.util.UUID.randomUUID;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertThrows;
-import static org.testng.Assert.assertTrue;
-
-@Test(singleThreaded = true)
-public class TestClpSplitFilterConfigCommon
-{
- private String filterConfigPath;
-
- @BeforeMethod
- public void setUp() throws IOException, URISyntaxException
- {
- URL resource = getClass().getClassLoader().getResource("test-mysql-split-filter.json");
- if (resource == null) {
- throw new FileNotFoundException("test-mysql-split-filter.json not found in resources");
- }
-
- filterConfigPath = Paths.get(resource.toURI()).toAbsolutePath().toString();
- }
-
- @Test
- public void checkRequiredFilters()
- {
- ClpConfig config = new ClpConfig();
- config.setSplitFilterConfig(filterConfigPath);
- ClpMySqlSplitFilterProvider filterProvider = new ClpMySqlSplitFilterProvider(config);
- Set testTableScopeSet = ImmutableSet.of(format("%s.%s", CONNECTOR_NAME, new SchemaTableName("default", "table_1")));
- assertThrows(PrestoException.class, () -> filterProvider.checkContainsRequiredFilters(
- testTableScopeSet,
- "(\"level\" >= 1 AND \"level\" <= 3)"));
- filterProvider.checkContainsRequiredFilters(
- testTableScopeSet,
- "(\"msg.timestamp\" > 1234 AND \"msg.timestamp\" < 5678)");
- }
-
- @Test
- public void getFilterNames()
- {
- ClpConfig config = new ClpConfig();
- config.setSplitFilterConfig(filterConfigPath);
- ClpMySqlSplitFilterProvider filterProvider = new ClpMySqlSplitFilterProvider(config);
- Set catalogFilterNames = filterProvider.getColumnNames("clp");
- assertEquals(ImmutableSet.of("level"), catalogFilterNames);
- Set schemaFilterNames = filterProvider.getColumnNames("clp.default");
- assertEquals(ImmutableSet.of("level", "author"), schemaFilterNames);
- Set tableFilterNames = filterProvider.getColumnNames("clp.default.table_1");
- assertEquals(ImmutableSet.of("level", "author", "msg.timestamp", "file_name"), tableFilterNames);
- }
-
- @Test
- public void handleEmptyAndInvalidSplitFilterConfig()
- {
- ClpConfig config = new ClpConfig();
-
- // Empty config
- ClpMySqlSplitFilterProvider filterProvider = new ClpMySqlSplitFilterProvider(config);
- assertTrue(filterProvider.getColumnNames("clp").isEmpty());
- assertTrue(filterProvider.getColumnNames("abc.xyz").isEmpty());
- assertTrue(filterProvider.getColumnNames("abc.opq.xyz").isEmpty());
-
- // Invalid config
- config.setSplitFilterConfig(randomUUID().toString());
- assertThrows(PrestoException.class, () -> new ClpMySqlSplitFilterProvider(config));
- }
-}
diff --git a/presto-clp/src/test/resources/test-mysql-split-filter.json b/presto-clp/src/test/resources/test-mysql-split-filter.json
deleted file mode 100644
index ea5be310aff63..0000000000000
--- a/presto-clp/src/test/resources/test-mysql-split-filter.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "clp": [
- {
- "columnName": "level"
- }
- ],
- "clp.default": [
- {
- "columnName": "author"
- }
- ],
- "clp.default.table_1": [
- {
- "columnName": "msg.timestamp",
- "customOptions": {
- "rangeMapping": {
- "lowerBound": "begin_timestamp",
- "upperBound": "end_timestamp"
- }
- },
- "required": true
- },
- {
- "columnName": "file_name"
- }
- ]
-}
diff --git a/presto-clp/src/test/resources/test-mysql-split-metadata.json b/presto-clp/src/test/resources/test-mysql-split-metadata.json
new file mode 100644
index 0000000000000..94c1e234ff717
--- /dev/null
+++ b/presto-clp/src/test/resources/test-mysql-split-metadata.json
@@ -0,0 +1,44 @@
+{
+ "": {
+ "metaColumns": {
+ "level": {
+ "type": "BIGINT"
+ }
+ }
+ },
+ "default": {
+ "metaColumns": {
+ "author": {
+ "type": "VARCHAR"
+ }
+ }
+ },
+ "default.table_1": {
+ "metaColumns": {
+ "begin_timestamp": {
+ "type": "BIGINT",
+ "filter": {
+ "asRangeBoundOf": "msg.timestamp",
+ "boundType": "lower"
+ }
+ },
+ "end_timestamp": {
+ "type": "BIGINT",
+ "filter": {
+ "asRangeBoundOf": "msg.timestamp",
+ "boundType": "upper"
+ }
+ },
+ "file_name": {
+ "type": "VARCHAR"
+ }
+ },
+ "filterRules": [
+ {
+ "column": "msg.timestamp",
+ "required": true,
+ "reason": "Full scan would be too expensive without timestamp filtering."
+ }
+ ]
+ }
+}
diff --git a/presto-docs/src/main/sphinx/connector/clp.rst b/presto-docs/src/main/sphinx/connector/clp.rst
index fc1631c3766f8..ab1e41930deb6 100644
--- a/presto-docs/src/main/sphinx/connector/clp.rst
+++ b/presto-docs/src/main/sphinx/connector/clp.rst
@@ -78,8 +78,8 @@ Property Name Description
``clp.metadata-refresh-interval`` Specifies how frequently metadata is refreshed from the source, in 60
seconds. Set this to a lower value for frequently changing datasets or
to a higher value to reduce load.
-``clp.split-filter-config`` The absolute path to an optional split filter config file. See the
- :ref:`Split Filter Config File` section for
+``clp.split-metadata-config-path`` The absolute path to an optional split metadata config file. See the
+ :ref:`Split Metadata Config File` section for
details.
``clp.split-filter-provider-type`` Specifies the split filter provider type. Currently, the only supported ``mysql``
type is a MySQL database, which is also used by the CLP package to
@@ -103,120 +103,176 @@ If you prefer to use a different source--or the same source with a custom implem
implementations of the ``ClpMetadataProvider`` and ``ClpSplitProvider`` interfaces, and configure the connector
accordingly.
-.. _split-filter-config-file:
+.. _split-metadata-config-file:
-***************************
-Split Filter Config File
-***************************
+**************************
+Split Metadata Config File
+**************************
-The split filter config file allows you to configure the set of columns that can be used to filter out irrelevant
-splits (CLP archives) when querying CLP's metadata database. This can significantly improve performance by reducing the
-amount of data that needs to be scanned. For a given query, the connector will translate any supported filter predicates
-that involve the configured columns into a query against CLP's metadata database.
+The split metadata config file allows you to configure the set of metadata columns that can be used in ``WHERE`` clause
+to filter out irrelevant splits (CLP archives) when querying CLP's metadata database. This can significantly improve
+performance by reducing the amount of data that needs to be scanned. For a given query, the connector will translate
+any supported filter predicates that involve the configured columns into a query against CLP's metadata database.
-The configuration is a JSON object where each key under the root represents a :ref:`scope` and each scope maps
-to an array of :ref:`filter configs`.
+Structure Overview
+==================
-.. _scopes:
+The configuration is a JSON object where each top-level key represents a namespace. Each namespace defines metadata and
+filtering behavior for one or more tables.
-Scopes
-======
+A *namespace* can be one of the following:
-A *scope* can be one of the following:
+- An empty string ``""`` (applies globally to all schemas and tables)
+- A schema name (applies to all tables in the schema)
+- A schema and table name delimited by ``"."`` (applies only to that table)
-- A catalog name
-- A fully-qualified schema name
-- A fully-qualified table name
+When resolving configuration for a specific table:
-Filter configs under a particular scope will apply to all child scopes. For example, filter configs at the schema level
-will apply to all tables within that schema.
+1. The connector merges definitions from matching namespaces in this order:
-.. _filter-configs:
+ - ``""`` (global)
+ - ``"schema"``
+ - ``"schema.table"``
-Filter Configs
-==============
+2. Later (more specific) namespaces **override** or **extend** parent configurations.
-Each filter config indicates how a *data column*---i.e., a column in the Presto table---should be mapped to one or
-more *metadata columns*---i.e., columns in CLP's metadata database.
+3. Duplicate filter rules are deduplicated by ``column`` name.
-For example, an integer data column (e.g., ``timestamp``), may be remapped to a pair of metadata columns that represent
-the range of possible values (e.g., ``begin_timestamp`` and ``end_timestamp``) of the data column within a split.
+Namespace Configuration Format
+==============================
-Each filter config has the following options:
+Each namespace maps to an object containing the following fields:
-- ``columnName``: The data column's name.
+- ``metaColumns``: an object defining metadata columns for this namespace.
+- ``filterRules``: an array specifying filters that must or can be used in queries.
-- ``customOptions`` *(optional)*: Custom options for a split filter provider. Options for the default split filter
- provider (``ClpMySqlSplitFilterProvider``) are :ref:`below`.
+Meta Columns
+------------
-- ``required`` *(optional, defaults to false)*: Whether the filter **must** be present in the generated metadata query.
- If a required filter is missing or cannot be added to the metadata query, the original query will be rejected.
+Each entry under ``metaColumns`` describes a single metadata column. Metadata columns represent attributes of data
+splits, such as file path, partition date, or timestamp range. The key of each entry is the column name used to filter
+splits (e.g. in a metadata filter SQL query). However, we can expose a different name to Presto.
-.. _clp-mysql-split-filter-provider-config:
+Supported fields:
-ClpMySqlSplitFilterProvider-Specific Filter Config
------------------------------------------------
+- ``type``: string, the Presto type of the metadata column (e.g., ``VARCHAR``, ``DATE``, ``BIGINT``)
+- ``exposedAs``: string, optional, the name exposed to queries (defaults to the original column name)
+- ``description``: string, optional, human-readable description
+- ``filter.asRangeBoundOf``: string, optional, the data column name this metadata column bounds
+- ``filter.boundType``: string, optional, must be ``"lower"`` or ``"upper"``, indicates bound type
-For ``ClpMySqlSplitFilterProvider``, the ``customOptions`` option of the filter config has the following sub-options:
+Example::
-- ``rangeMapping`` *(optional)*: an object with the following properties:
+ "metaColumns": {
+ "partition_date": {
+ "type": "DATE",
+ "exposedAs": "partition_date",
+ "description": "Logical partition of the data file"
+ },
+ "$ir_path": {
+ "type": "STRING",
+ "exposedAs": "tpath",
+ "description": "Internal file path"
+ },
+ "begin_timestamp": {
+ "type": "LONG",
+ "description": "Start of the timestamp range for the file",
+ "filter": {
+ "asRangeBoundOf": "msg.timestamp",
+ "boundType": "lower"
+ }
+ },
+ "end_timestamp": {
+ "type": "LONG",
+ "description": "End of the timestamp range for the file",
+ "filter": {
+ "asRangeBoundOf": "msg.timestamp",
+ "boundType": "upper"
+ }
+ }
+ }
+
+Filter Rules
+------------
- .. note:: This option is only valid if the column has a numeric type.
+Each entry in ``filterRules`` specifies either a data column that has a range mapping to a metadata column or a metadata
+column that must appear in query filters. These rules ensure that queries contain sufficient constraints for efficient
+split selection.
- - ``lowerBound``: The metadata column that represents the lower bound of values in a split for the data column.
- - ``upperBound``: The metadata column that represents the upper bound of values in a split for the data column.
+Supported fields:
-Filter Config Example
----------------------
+- ``column``: string, the name of the data column or metadata column (as seen by Presto)
+- ``required``: boolean, optional, whether this column **must** appear in query filters (default: ``false``)
+- ``reason``: string, optional, explanation of why the column is required
-The code block shows an example filter config file:
+Example::
+
+ "filterRules": [
+ {
+ "column": "msg.timestamp",
+ "required": true,
+ "reason": "Full scan would be too expensive without timestamp filtering."
+ }
+ ]
+
+Complete Example
+----------------
+
+The code block shows an example of split metadata config file:
.. code-block:: json
{
- "clp": [
- {
- "columnName": "level"
- }
- ],
- "clp.default": [
- {
- "columnName": "author"
- }
- ],
- "clp.default.table_1": [
- {
- "columnName": "msg.timestamp",
- "customOptions": {
- "rangeMapping": {
- "lowerBound": "begin_timestamp",
- "upperBound": "end_timestamp"
- }
+ "": {
+ "metaColumns": {
+ "global_meta": {
+ "type": "STRING",
+ "description": "Global metadata column"
+ }
+ },
+ "filterRules": []
+ },
+ "default": {
+ "metaColumns": {
+ "partition_date": {
+ "type": "DATE",
+ "exposedAs": "partition_date",
+ "description": "Logical partition of the data file"
+ }
+ },
+ },
+ "default.table_1": {
+ "metaColumns": {
+ "begin_timestamp": {
+ "type": "LONG",
+ "filter": { "asRangeBoundOf": "msg.timestamp", "boundType": "lower" }
},
- "required": true
+ "end_timestamp": {
+ "type": "LONG",
+ "filter": { "asRangeBoundOf": "msg.timestamp", "boundType": "upper" }
+ }
},
- {
- "columnName": "file_name"
- }
- ]
+ "filterRules": [
+ {
+ "column": "msg.timestamp",
+ "required": true,
+ "reason": "Full scan would be too expensive without timestamp filtering."
+ }
+ ]
+ }
}
-- The first key-value pair adds the following filter configs for all schemas and tables under the ``clp`` catalog:
-
- - The column ``level`` is used as-is without remapping.
-
-- The second key-value pair adds the following filter configs for all tables under the ``clp.default`` schema:
-
- - The column ``author`` is used as-is without remapping.
-
-- The third key-value pair adds two filter configs for the table ``clp.default.table_1``:
+- The empty string namespace ``""`` defines **global metadata** that applies to all schemas and tables. In this case,
+ ``global_meta`` column is available everywhere.
- - The column ``msg.timestamp`` is remapped via a ``rangeMapping`` to the metadata columns ``begin_timestamp`` and
- ``end_timestamp``, and is required to exist in every query.
- - The column ``file_name`` is used as-is without remapping.
+- The schema namespace ``"default"`` defines metadata columns and filters for **all tables in the `default` schema**.
+ - ``partition_date`` is exposed as ``partition_date`` for query usage.
-If you prefer to use a different format for ``customOptions``, you can provide your own implementation of the
-``ClpSplitFilterProvider`` interface, and configure the connector accordingly.
+- The table-specific namespace ``"default.table_1"`` defines metadata and filter rules for a **single table**:
+ - ``begin_timestamp`` and ``end_timestamp`` map to the data column ``msg.timestamp`` as lower and upper bounds.
+ This enables range-based filtering on timestamp values when generating split queries.
+ - The filter rule for ``msg.timestamp`` is marked as **required**, ensuring that queries without timestamp constraints
+ are rejected.
Supported SQL Expressions
=========================