From db2d39c4cd6fa70914058376eb525f5467cd78e7 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Wed, 11 Feb 2026 22:26:42 -0600 Subject: [PATCH 1/9] Add Docker integration tests. --- .bin/pre-release-docker | 5 + .github/workflows/pre-release.yaml | 25 ++- ice-rest-catalog/pom.xml | 1 - .../rest/catalog/DockerScenarioBasedIT.java | 183 ++++++++++++++++++ .../ice/rest/catalog/RESTCatalogTestBase.java | 97 ++++++++++ .../ice/rest/catalog/ScenarioBasedIT.java | 111 +---------- .../ice/rest/catalog/ScenarioConfig.java | 13 +- .../ice/rest/catalog/ScenarioTestRunner.java | 4 +- .../src/test/resources/scenarios/README.md | 20 -- .../scenarios/basic-operations/input.parquet | Bin 0 -> 2446 bytes .../scenarios/basic-operations/run.sh.tmpl | 8 - .../scenarios/basic-operations/scenario.yaml | 19 -- .../insert-partitioned/scenario.yaml | 19 -- .../scenarios/insert-scan/scenario.yaml | 17 -- 14 files changed, 315 insertions(+), 207 deletions(-) create mode 100644 ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java create mode 100644 ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet diff --git a/.bin/pre-release-docker b/.bin/pre-release-docker index 2671de32..990a5597 100755 --- a/.bin/pre-release-docker +++ b/.bin/pre-release-docker @@ -8,6 +8,11 @@ export SKIP_VERIFY=1 export PATH="$(pwd)/.bin:$PATH" +echo >&2 'Building ice Docker image' +docker-build-ice +echo >&2 'Building ice-rest-catalog Docker image' +docker-build-ice-rest-catalog + echo >&2 'Pushing ice Docker image' docker-build-ice --push echo >&2 'Pushing ice-rest-catalog Docker image' diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml index b1ec2086..fb893650 100644 --- a/.github/workflows/pre-release.yaml +++ b/.github/workflows/pre-release.yaml @@ -49,4 +49,27 @@ jobs: - uses: actions/checkout@v4 - - run: .bin/pre-release-docker + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'graalvm' + cache: maven + + - name: Build Docker images + run: | + export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD) + export IMAGE_TAG="latest-master" + export SKIP_VERIFY=1 + export PATH="$(pwd)/.bin:$PATH" + docker-build-ice + docker-build-ice-rest-catalog + + - name: Run Docker integration tests + run: > + ./mvnw -pl ice-rest-catalog install -Dmaven.test.skip=true -Pno-check && + ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify + -Dit.test=DockerScenarioBasedIT + -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64 + + - name: Push Docker images + run: .bin/pre-release-docker diff --git a/ice-rest-catalog/pom.xml b/ice-rest-catalog/pom.xml index 43a8121e..fda0550b 100644 --- a/ice-rest-catalog/pom.xml +++ b/ice-rest-catalog/pom.xml @@ -393,7 +393,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml ${jackson.version} - test io.etcd diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java new file mode 100644 index 00000000..2916589a --- /dev/null +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; + +/** + * Docker-based integration tests for ICE REST Catalog. + * + *

Runs the ice-rest-catalog Docker image (specified via system property {@code docker.image}) + * alongside a MinIO container, then executes scenario-based tests against it. + */ +public class DockerScenarioBasedIT extends RESTCatalogTestBase { + + private Network network; + + private GenericContainer minio; + + private GenericContainer catalog; + + @Override + @BeforeClass + public void setUp() throws Exception { + String dockerImage = + System.getProperty("docker.image", "altinity/ice-rest-catalog:debug-with-ice-0.12.0"); + logger.info("Using Docker image: {}", dockerImage); + + network = Network.newNetwork(); + + // Start MinIO + minio = + new GenericContainer<>("minio/minio:latest") + .withNetwork(network) + .withNetworkAliases("minio") + .withExposedPorts(9000) + .withEnv("MINIO_ACCESS_KEY", "minioadmin") + .withEnv("MINIO_SECRET_KEY", "minioadmin") + .withCommand("server", "/data") + .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000)); + minio.start(); + + // Create test bucket via MinIO's host-mapped port + String minioHostEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000); + try (var s3Client = + software.amazon.awssdk.services.s3.S3Client.builder() + .endpointOverride(java.net.URI.create(minioHostEndpoint)) + .region(software.amazon.awssdk.regions.Region.US_EAST_1) + .credentialsProvider( + software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( + software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create( + "minioadmin", "minioadmin"))) + .forcePathStyle(true) + .build()) { + s3Client.createBucket( + software.amazon.awssdk.services.s3.model.CreateBucketRequest.builder() + .bucket("test-bucket") + .build()); + logger.info("Created test-bucket in MinIO"); + } + + // Build YAML config for the catalog container (using the Docker network alias for MinIO) + String catalogConfig = + String.join( + "\n", + "uri: \"jdbc:sqlite::memory:\"", + "warehouse: \"s3://test-bucket/warehouse\"", + "s3:", + " endpoint: \"http://minio:9000\"", + " pathStyleAccess: true", + " accessKeyID: \"minioadmin\"", + " secretAccessKey: \"minioadmin\"", + " region: \"us-east-1\"", + "anonymousAccess:", + " enabled: true", + " accessConfig:", + " readOnly: false", + ""); + + Path scenariosDir = getScenariosDirectory().toAbsolutePath(); + if (!Files.exists(scenariosDir) || !Files.isDirectory(scenariosDir)) { + throw new IllegalStateException( + "Scenarios directory must exist at " + + scenariosDir + + ". Run 'mvn test-compile' or run the test from Maven (e.g. mvn failsafe:integration-test)."); + } + Path insertScanInput = scenariosDir.resolve("insert-scan").resolve("input.parquet"); + if (!Files.exists(insertScanInput)) { + throw new IllegalStateException( + "Scenario input not found at " + + insertScanInput + + ". Ensure test resources are on the classpath and scenarios/insert-scan/input.parquet exists."); + } + + // Start the ice-rest-catalog container (debug-with-ice has ice CLI at /usr/local/bin/ice) + GenericContainer catalogContainer = + new GenericContainer<>(dockerImage) + .withNetwork(network) + .withExposedPorts(5000) + .withEnv("ICE_REST_CATALOG_CONFIG", "") + .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig) + .withFileSystemBind(scenariosDir.toString(), "/scenarios") + .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200)); + + catalog = catalogContainer; + try { + catalog.start(); + } catch (Exception e) { + if (catalog != null) { + logger.error("Catalog container logs (stdout): {}", catalog.getLogs()); + } + throw e; + } + + // Copy CLI config into container so ice CLI can talk to co-located REST server + File cliConfigHost = File.createTempFile("ice-docker-cli-", ".yaml"); + try { + Files.write(cliConfigHost.toPath(), "uri: http://localhost:5000\n".getBytes()); + catalog.copyFileToContainer( + MountableFile.forHostPath(cliConfigHost.toPath()), "/tmp/ice-cli.yaml"); + } finally { + cliConfigHost.delete(); + } + + logger.info( + "Catalog container started at {}:{}", catalog.getHost(), catalog.getMappedPort(5000)); + } + + @Override + @AfterClass + public void tearDown() { + if (catalog != null && catalog.isRunning()) { + catalog.stop(); + } + if (minio != null && minio.isRunning()) { + minio.stop(); + } + if (network != null) { + network.close(); + } + } + + @Override + protected ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception { + Path scenariosDir = getScenariosDirectory(); + + String containerId = catalog.getContainerId(); + + // Wrapper script on host: docker exec ice "$@" (CLI runs inside container) + File wrapperScript = File.createTempFile("ice-docker-exec-", ".sh"); + wrapperScript.deleteOnExit(); + String wrapperContent = "#!/bin/sh\n" + "exec docker exec " + containerId + " ice \"$@\"\n"; + Files.write(wrapperScript.toPath(), wrapperContent.getBytes()); + if (!wrapperScript.setExecutable(true)) { + throw new IllegalStateException("Could not set wrapper script executable: " + wrapperScript); + } + + Map templateVars = new HashMap<>(); + templateVars.put("ICE_CLI", wrapperScript.getAbsolutePath()); + templateVars.put("CLI_CONFIG", "/tmp/ice-cli.yaml"); + templateVars.put("SCENARIO_DIR", "/scenarios/" + scenarioName); + templateVars.put("MINIO_ENDPOINT", ""); + templateVars.put("CATALOG_URI", "http://localhost:5000"); + + return new ScenarioTestRunner(scenariosDir, templateVars); + } +} diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java index d7ffcf85..54c94d40 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java @@ -14,7 +14,12 @@ import com.altinity.ice.rest.catalog.internal.config.Config; import java.io.File; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.Map; import org.apache.iceberg.catalog.Catalog; import org.eclipse.jetty.server.Server; @@ -23,6 +28,8 @@ import org.testcontainers.containers.GenericContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -152,4 +159,94 @@ protected String getMinioEndpoint() { protected String getCatalogUri() { return "http://localhost:8080"; } + + /** + * Get the path to the scenarios directory. + * + * @return Path to scenarios directory + * @throws URISyntaxException If the resource URL cannot be converted to a path + */ + protected Path getScenariosDirectory() throws URISyntaxException { + URL scenariosUrl = getClass().getClassLoader().getResource("scenarios"); + if (scenariosUrl == null) { + return Paths.get("src/test/resources/scenarios"); + } + return Paths.get(scenariosUrl.toURI()); + } + + /** + * Create a ScenarioTestRunner for the given scenario. Subclasses provide host or container-based + * CLI and config. + * + * @param scenarioName Name of the scenario (e.g. for container path resolution) + * @return Configured ScenarioTestRunner + * @throws Exception If there's an error creating the runner + */ + protected abstract ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception; + + /** Data provider that discovers all test scenarios. */ + @DataProvider(name = "scenarios") + public Object[][] scenarioProvider() throws Exception { + Path scenariosDir = getScenariosDirectory(); + ScenarioTestRunner runner = new ScenarioTestRunner(scenariosDir, Map.of()); + List scenarios = runner.discoverScenarios(); + + if (scenarios.isEmpty()) { + logger.warn("No test scenarios found in: {}", scenariosDir); + return new Object[0][0]; + } + + logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios); + + Object[][] data = new Object[scenarios.size()][1]; + for (int i = 0; i < scenarios.size(); i++) { + data[i][0] = scenarios.get(i); + } + return data; + } + + /** Parameterized test that executes a single scenario. */ + @Test(dataProvider = "scenarios") + public void testScenario(String scenarioName) throws Exception { + logger.info("====== Starting scenario test: {} ======", scenarioName); + + ScenarioTestRunner runner = createScenarioRunner(scenarioName); + ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName); + + if (result.runScriptResult() != null) { + logger.info("Run script exit code: {}", result.runScriptResult().exitCode()); + } + if (result.verifyScriptResult() != null) { + logger.info("Verify script exit code: {}", result.verifyScriptResult().exitCode()); + } + + assertScenarioSuccess(scenarioName, result); + logger.info("====== Scenario test passed: {} ======", scenarioName); + } + + /** Assert that the scenario result indicates success; otherwise throw AssertionError. */ + protected void assertScenarioSuccess( + String scenarioName, ScenarioTestRunner.ScenarioResult result) { + if (result.isSuccess()) { + return; + } + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n"); + + if (result.runScriptResult() != null && result.runScriptResult().exitCode() != 0) { + errorMessage.append("\nRun script failed with exit code: "); + errorMessage.append(result.runScriptResult().exitCode()); + errorMessage.append("\nStdout:\n").append(result.runScriptResult().stdout()); + errorMessage.append("\nStderr:\n").append(result.runScriptResult().stderr()); + } + + if (result.verifyScriptResult() != null && result.verifyScriptResult().exitCode() != 0) { + errorMessage.append("\nVerify script failed with exit code: "); + errorMessage.append(result.verifyScriptResult().exitCode()); + errorMessage.append("\nStdout:\n").append(result.verifyScriptResult().stdout()); + errorMessage.append("\nStderr:\n").append(result.verifyScriptResult().stderr()); + } + + throw new AssertionError(errorMessage.toString()); + } } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java index b4351344..24b098e7 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java @@ -10,15 +10,10 @@ package com.altinity.ice.rest.catalog; import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; /** * Scenario-based integration tests for ICE REST Catalog. @@ -28,92 +23,8 @@ */ public class ScenarioBasedIT extends RESTCatalogTestBase { - /** - * Data provider that discovers all test scenarios. - * - * @return Array of scenario names to be used as test parameters - * @throws Exception If there's an error discovering scenarios - */ - @DataProvider(name = "scenarios") - public Object[][] scenarioProvider() throws Exception { - Path scenariosDir = getScenariosDirectory(); - ScenarioTestRunner runner = createScenarioRunner(); - - List scenarios = runner.discoverScenarios(); - - if (scenarios.isEmpty()) { - logger.warn("No test scenarios found in: {}", scenariosDir); - return new Object[0][0]; - } - - logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios); - - // Convert to Object[][] for TestNG data provider - Object[][] data = new Object[scenarios.size()][1]; - for (int i = 0; i < scenarios.size(); i++) { - data[i][0] = scenarios.get(i); - } - return data; - } - - /** - * Parameterized test that executes a single scenario. - * - * @param scenarioName Name of the scenario to execute - * @throws Exception If the scenario execution fails - */ - @Test(dataProvider = "scenarios") - public void testScenario(String scenarioName) throws Exception { - logger.info("====== Starting scenario test: {} ======", scenarioName); - - ScenarioTestRunner runner = createScenarioRunner(); - ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName); - - // Log results - if (result.runScriptResult() != null) { - logger.info("Run script exit code: {}", result.runScriptResult().exitCode()); - } - - if (result.verifyScriptResult() != null) { - logger.info("Verify script exit code: {}", result.verifyScriptResult().exitCode()); - } - - // Assert success - if (!result.isSuccess()) { - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n"); - - if (result.runScriptResult() != null && result.runScriptResult().exitCode() != 0) { - errorMessage.append("\nRun script failed with exit code: "); - errorMessage.append(result.runScriptResult().exitCode()); - errorMessage.append("\nStdout:\n"); - errorMessage.append(result.runScriptResult().stdout()); - errorMessage.append("\nStderr:\n"); - errorMessage.append(result.runScriptResult().stderr()); - } - - if (result.verifyScriptResult() != null && result.verifyScriptResult().exitCode() != 0) { - errorMessage.append("\nVerify script failed with exit code: "); - errorMessage.append(result.verifyScriptResult().exitCode()); - errorMessage.append("\nStdout:\n"); - errorMessage.append(result.verifyScriptResult().stdout()); - errorMessage.append("\nStderr:\n"); - errorMessage.append(result.verifyScriptResult().stderr()); - } - - throw new AssertionError(errorMessage.toString()); - } - - logger.info("====== Scenario test passed: {} ======", scenarioName); - } - - /** - * Create a ScenarioTestRunner with the appropriate template variables. - * - * @return Configured ScenarioTestRunner - * @throws Exception If there's an error creating the runner - */ - private ScenarioTestRunner createScenarioRunner() throws Exception { + @Override + protected ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception { Path scenariosDir = getScenariosDirectory(); // Create CLI config file @@ -143,22 +54,4 @@ private ScenarioTestRunner createScenarioRunner() throws Exception { return new ScenarioTestRunner(scenariosDir, templateVars); } - - /** - * Get the path to the scenarios directory. - * - * @return Path to scenarios directory - * @throws URISyntaxException If the resource URL cannot be converted to a path - */ - private Path getScenariosDirectory() throws URISyntaxException { - // Get the scenarios directory from test resources - URL scenariosUrl = getClass().getClassLoader().getResource("scenarios"); - - if (scenariosUrl == null) { - // If not found in resources, try relative to project - return Paths.get("src/test/resources/scenarios"); - } - - return Paths.get(scenariosUrl.toURI()); - } } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java index 17d5b169..3d6ae098 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java @@ -9,7 +9,6 @@ */ package com.altinity.ice.rest.catalog; -import java.util.List; import java.util.Map; /** @@ -21,17 +20,7 @@ public record ScenarioConfig( String name, String description, CatalogConfig catalogConfig, - Map env, - CloudResources cloudResources, - List phases) { + Map env) { public record CatalogConfig(String warehouse, String name, String uri) {} - - public record CloudResources(S3Resources s3, SqsResources sqs) {} - - public record S3Resources(List buckets) {} - - public record SqsResources(List queues) {} - - public record Phase(String name, String description) {} } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java index 83a6b19d..c74d22b0 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java @@ -105,7 +105,9 @@ public ScenarioResult executeScenario(String scenarioName) throws Exception { // Build template variables map Map templateVars = new HashMap<>(globalTemplateVars); - templateVars.put("SCENARIO_DIR", scenarioDir.toAbsolutePath().toString()); + if (!templateVars.containsKey("SCENARIO_DIR")) { + templateVars.put("SCENARIO_DIR", scenarioDir.toAbsolutePath().toString()); + } // Add environment variables from scenario config if (config.env() != null) { diff --git a/ice-rest-catalog/src/test/resources/scenarios/README.md b/ice-rest-catalog/src/test/resources/scenarios/README.md index 0b0ba237..feb6de3e 100644 --- a/ice-rest-catalog/src/test/resources/scenarios/README.md +++ b/ice-rest-catalog/src/test/resources/scenarios/README.md @@ -32,26 +32,6 @@ env: NAMESPACE_NAME: "test_ns" TABLE_NAME: "test_ns.table1" INPUT_FILE: "input.parquet" - -# Optional: Cloud resources needed (for future provisioning) -cloudResources: - s3: - buckets: - - "test-bucket" - sqs: - queues: - - "test-queue" - -# Optional: Test execution phases -phases: - - name: "setup" - description: "Initialize resources" - - name: "run" - description: "Execute main test logic" - - name: "verify" - description: "Verify results" - - name: "cleanup" - description: "Clean up resources" ``` ## Script Templates diff --git a/ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet b/ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet new file mode 100644 index 0000000000000000000000000000000000000000..028c64cf9f7d9ea46bfeeaf940482aaee90a1b54 GIT binary patch literal 2446 zcmZ`*L1-Ii7JgDj^8efVlXj z+h~)tReRFWhaP+iY7a$+Qgkq^*-PEa;-d=<2zpU(g6+Yl#l^)qB!vBb$#tBx9gt_{ z{qO(gz4v|Z{Vu=u=TMXW->5#yzNoj-c+To9(buwYj)=C>skKZhHy(ZdSt^wqi|&7y zN?D^(=3;JbG@?w3J<{(uM64qj3mr{oQZ#l|z@TipsH&9LfvnNEcG%`J+fs0g##Ncd zbUX7exWy$nkLrLEd4Po?Zml3GL2Z)^amx`evS2yK8Lb3r>3ybuE+Fm-Q=hc3= zNd<)4#fLqv(M9da4o6rocpN%8X>h^1rgkZHk(+Rj{l-b2p`%XBV1fC_E;2_-tzKG(RB zxnWH|)LsdNuF!!t4fhz?K0wudcULwRJSzEwmaEg@e2aLSeYTC_Yuo}$M_1AGM>@{zU?#KLo&SQ`ZEz>>IwEsSoA>Df=};m6B7Ba zYPh`*Hwha|&mO5BWFjo18v1$OCRwHj$n`+`nLnXJ50r-ichGNo)XJZ@VbL)d_zA^& zgo(2W(^`t(p^AoQy!rplOZJpnJOxz=gF0Hp^ypT|-Ly+QQoK(kQ1y)Z&I9Jjbf985Lr!D=ae)}z)e`vso zO&o&sm({zs?ko@E>94D|@2p?Hb@SG3Y2Mav-&lWt{rWOyZ9#kegVP=yoag^|!e@Gg zR#Gczopx`&49To~aQ*Kq?_Ildy}J63Wj#!Kcp~lO?yT09Z@zSM_5B-ns{_->!1U4j z$}g5&ZT0T6P6N}4Qw~T&A1>csU%mVB1W|q~?H1g>P2%Vob~O&ur@o@oSkK=Z%YQL6 zDPDEN;MBMrIbynhag-}9|3F6rxPXRw2n*3_$6p?|5J?1)tc-D%QCx}lac?$W} z(B!|*pF&9H>%5SA6e=JPiD-;}gstG4MkWHTQ+21e&5i48iNkM-0b*SR?d74NQmGPh0{{e!n B Date: Wed, 11 Feb 2026 22:31:04 -0600 Subject: [PATCH 2/9] Fixed lint error --- .../java/com/altinity/ice/rest/catalog/ScenarioConfig.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java index 3d6ae098..e7163456 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java @@ -17,10 +17,7 @@ *

This class uses Jackson/SnakeYAML annotations for YAML deserialization. */ public record ScenarioConfig( - String name, - String description, - CatalogConfig catalogConfig, - Map env) { + String name, String description, CatalogConfig catalogConfig, Map env) { public record CatalogConfig(String warehouse, String name, String uri) {} } From 1949fbe3d2178ff59eccec088a676782f3899157 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 12:53:15 -0600 Subject: [PATCH 3/9] - fixed imports, moved config to a separate file, modified withFileSystemBind to copyToContainer. --- .bin/pre-release-docker | 5 -- .github/workflows/verify.yaml | 18 +++++- .../rest/catalog/DockerScenarioBasedIT.java | 59 +++++++++---------- 3 files changed, 45 insertions(+), 37 deletions(-) diff --git a/.bin/pre-release-docker b/.bin/pre-release-docker index 990a5597..2671de32 100755 --- a/.bin/pre-release-docker +++ b/.bin/pre-release-docker @@ -8,11 +8,6 @@ export SKIP_VERIFY=1 export PATH="$(pwd)/.bin:$PATH" -echo >&2 'Building ice Docker image' -docker-build-ice -echo >&2 'Building ice-rest-catalog Docker image' -docker-build-ice-rest-catalog - echo >&2 'Pushing ice Docker image' docker-build-ice --push echo >&2 'Pushing ice-rest-catalog Docker image' diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 6eff3114..67edde5f 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -15,9 +15,23 @@ jobs: distribution: 'graalvm' cache: maven - run: ./mvnw clean verify - - name: Install + - name: Install run: ./mvnw install # TODO: check native-image can build ice - name: Run Scenario-Based Integration Tests run: ../mvnw test -Dtest=ScenarioBasedIT - working-directory: ice-rest-catalog \ No newline at end of file + working-directory: ice-rest-catalog + - name: Build Docker images + run: | + export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD) + export IMAGE_TAG="latest-master" + export SKIP_VERIFY=1 + export PATH="$(pwd)/.bin:$PATH" + docker-build-ice + docker-build-ice-rest-catalog + - name: Run Docker integration tests + run: > + ./mvnw -pl ice-rest-catalog install -Dmaven.test.skip=true -Pno-check && + ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify + -Dit.test=DockerScenarioBasedIT + -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64 diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java index 2916589a..39d0b251 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java @@ -10,8 +10,11 @@ package com.altinity.ice.rest.catalog; import java.io.File; +import java.net.URI; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import org.testcontainers.containers.GenericContainer; @@ -20,6 +23,11 @@ import org.testcontainers.utility.MountableFile; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; /** * Docker-based integration tests for ICE REST Catalog. @@ -37,6 +45,7 @@ public class DockerScenarioBasedIT extends RESTCatalogTestBase { @Override @BeforeClass + @SuppressWarnings("resource") public void setUp() throws Exception { String dockerImage = System.getProperty("docker.image", "altinity/ice-rest-catalog:debug-with-ice-0.12.0"); @@ -59,39 +68,28 @@ public void setUp() throws Exception { // Create test bucket via MinIO's host-mapped port String minioHostEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000); try (var s3Client = - software.amazon.awssdk.services.s3.S3Client.builder() - .endpointOverride(java.net.URI.create(minioHostEndpoint)) - .region(software.amazon.awssdk.regions.Region.US_EAST_1) + S3Client.builder() + .endpointOverride(URI.create(minioHostEndpoint)) + .region(Region.US_EAST_1) .credentialsProvider( - software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( - software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( "minioadmin", "minioadmin"))) .forcePathStyle(true) .build()) { s3Client.createBucket( - software.amazon.awssdk.services.s3.model.CreateBucketRequest.builder() + CreateBucketRequest.builder() .bucket("test-bucket") .build()); logger.info("Created test-bucket in MinIO"); } - // Build YAML config for the catalog container (using the Docker network alias for MinIO) - String catalogConfig = - String.join( - "\n", - "uri: \"jdbc:sqlite::memory:\"", - "warehouse: \"s3://test-bucket/warehouse\"", - "s3:", - " endpoint: \"http://minio:9000\"", - " pathStyleAccess: true", - " accessKeyID: \"minioadmin\"", - " secretAccessKey: \"minioadmin\"", - " region: \"us-east-1\"", - "anonymousAccess:", - " enabled: true", - " accessConfig:", - " readOnly: false", - ""); + // Load YAML config for the catalog container (MinIO via Docker network alias "minio") + URL configResource = getClass().getClassLoader().getResource("docker-catalog-config.yaml"); + if (configResource == null) { + throw new IllegalStateException("docker-catalog-config.yaml not found on classpath"); + } + String catalogConfig = Files.readString(Paths.get(configResource.toURI())); Path scenariosDir = getScenariosDirectory().toAbsolutePath(); if (!Files.exists(scenariosDir) || !Files.isDirectory(scenariosDir)) { @@ -109,16 +107,17 @@ public void setUp() throws Exception { } // Start the ice-rest-catalog container (debug-with-ice has ice CLI at /usr/local/bin/ice) - GenericContainer catalogContainer = + catalog = new GenericContainer<>(dockerImage) .withNetwork(network) .withExposedPorts(5000) .withEnv("ICE_REST_CATALOG_CONFIG", "") .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig) - .withFileSystemBind(scenariosDir.toString(), "/scenarios") + .withCopyFileToContainer( + MountableFile.forHostPath(scenariosDir), + "/scenarios") .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200)); - catalog = catalogContainer; try { catalog.start(); } catch (Exception e) { @@ -145,11 +144,11 @@ public void setUp() throws Exception { @Override @AfterClass public void tearDown() { - if (catalog != null && catalog.isRunning()) { - catalog.stop(); + if (catalog != null) { + catalog.close(); } - if (minio != null && minio.isRunning()) { - minio.stop(); + if (minio != null) { + minio.close(); } if (network != null) { network.close(); From 63ff7cb6cddf94493506dcd489b2355864a7eec7 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 12:55:58 -0600 Subject: [PATCH 4/9] Fixed lint error --- .../ice/rest/catalog/DockerScenarioBasedIT.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java index 39d0b251..f3fc0889 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java @@ -73,14 +73,10 @@ public void setUp() throws Exception { .region(Region.US_EAST_1) .credentialsProvider( StaticCredentialsProvider.create( - AwsBasicCredentials.create( - "minioadmin", "minioadmin"))) + AwsBasicCredentials.create("minioadmin", "minioadmin"))) .forcePathStyle(true) .build()) { - s3Client.createBucket( - CreateBucketRequest.builder() - .bucket("test-bucket") - .build()); + s3Client.createBucket(CreateBucketRequest.builder().bucket("test-bucket").build()); logger.info("Created test-bucket in MinIO"); } @@ -113,9 +109,7 @@ public void setUp() throws Exception { .withExposedPorts(5000) .withEnv("ICE_REST_CATALOG_CONFIG", "") .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig) - .withCopyFileToContainer( - MountableFile.forHostPath(scenariosDir), - "/scenarios") + .withCopyFileToContainer(MountableFile.forHostPath(scenariosDir), "/scenarios") .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200)); try { From 958efdba95f9fee2208ad4b14af760fa462fd205 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 13:01:44 -0600 Subject: [PATCH 5/9] Add docker-catalog-config.yaml --- .../src/test/resources/docker-catalog-config.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ice-rest-catalog/src/test/resources/docker-catalog-config.yaml diff --git a/ice-rest-catalog/src/test/resources/docker-catalog-config.yaml b/ice-rest-catalog/src/test/resources/docker-catalog-config.yaml new file mode 100644 index 00000000..d7c07ecd --- /dev/null +++ b/ice-rest-catalog/src/test/resources/docker-catalog-config.yaml @@ -0,0 +1,12 @@ +uri: "jdbc:sqlite::memory:" +warehouse: "s3://test-bucket/warehouse" +s3: + endpoint: "http://minio:9000" + pathStyleAccess: true + accessKeyID: "minioadmin" + secretAccessKey: "minioadmin" + region: "us-east-1" +anonymousAccess: + enabled: true + accessConfig: + readOnly: false From 79356c2a2d5dd1a1315b237844d937e046c9cba9 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 14:04:00 -0600 Subject: [PATCH 6/9] Added docker login step --- .github/workflows/verify.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 67edde5f..f4161d2b 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -8,6 +8,19 @@ jobs: test: runs-on: ubuntu-24.04 steps: + - uses: docker/login-action@v3 + - with: + - username: ${{ secrets.DOCKERHUB_USERNAME }} + - password: ${{ secrets.DOCKERHUB_TOKEN }} + - # https://github.com/docker/setup-buildx-action + - name: Set up QEMU + - uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + - uses: docker/setup-buildx-action@v3 + - name: Install regctl + - uses: regclient/actions/regctl-installer@ce5fd131e371ffcdd7508b478cb223b3511a9183 + - name: regctl login + - uses: regclient/actions/regctl-login@ce5fd131e371ffcdd7508b478cb223b3511a9183 - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: From 57e4f0a66d20f96ec0088c2c9fe0231980c0590e Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 14:17:58 -0600 Subject: [PATCH 7/9] fixed syntax in verify.yaml --- .github/workflows/verify.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index f4161d2b..76769a3d 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -9,18 +9,18 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: docker/login-action@v3 - - with: - - username: ${{ secrets.DOCKERHUB_USERNAME }} - - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # https://github.com/docker/setup-buildx-action + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + # https://github.com/docker/setup-buildx-action - name: Set up QEMU - - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3 - name: Install regctl - - uses: regclient/actions/regctl-installer@ce5fd131e371ffcdd7508b478cb223b3511a9183 + uses: regclient/actions/regctl-installer@ce5fd131e371ffcdd7508b478cb223b3511a9183 - name: regctl login - - uses: regclient/actions/regctl-login@ce5fd131e371ffcdd7508b478cb223b3511a9183 + uses: regclient/actions/regctl-login@ce5fd131e371ffcdd7508b478cb223b3511a9183 - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: From cd4fc13f25a6980206026efa646600ed74e02668 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 15:21:24 -0600 Subject: [PATCH 8/9] move docker integration to a separate job in github action. --- .github/workflows/verify.yaml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 76769a3d..746c32ca 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -6,6 +6,22 @@ on: - master jobs: test: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'graalvm' + cache: maven + - run: ./mvnw clean verify + - name: Install + run: ./mvnw install + # TODO: check native-image can build ice + - name: Run Scenario-Based Integration Tests + run: ../mvnw test -Dtest=ScenarioBasedIT + working-directory: ice-rest-catalog + docker-integration: runs-on: ubuntu-24.04 steps: - uses: docker/login-action@v3 @@ -27,13 +43,6 @@ jobs: java-version: '21' distribution: 'graalvm' cache: maven - - run: ./mvnw clean verify - - name: Install - run: ./mvnw install - # TODO: check native-image can build ice - - name: Run Scenario-Based Integration Tests - run: ../mvnw test -Dtest=ScenarioBasedIT - working-directory: ice-rest-catalog - name: Build Docker images run: | export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD) From f7329eb5c8dcc2aefb4c1d4fe73de5b042a47f8e Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 12 Feb 2026 15:35:31 -0600 Subject: [PATCH 9/9] fix skipping test logic. --- .github/workflows/verify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 746c32ca..c75844db 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -53,7 +53,7 @@ jobs: docker-build-ice-rest-catalog - name: Run Docker integration tests run: > - ./mvnw -pl ice-rest-catalog install -Dmaven.test.skip=true -Pno-check && + ./mvnw -pl ice-rest-catalog install -DskipTests=true -Pno-check && ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify -Dit.test=DockerScenarioBasedIT -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64