diff --git a/.github/workflows/e2e-tests-flink-1.x.yml b/.github/workflows/e2e-tests-flink-1.x.yml index 63e6c368ccd8..91f15ab397c3 100644 --- a/.github/workflows/e2e-tests-flink-1.x.yml +++ b/.github/workflows/e2e-tests-flink-1.x.yml @@ -45,6 +45,7 @@ jobs: matrix: # Only Test Latest Version flink_version: [ '1.20' ] + hadoop_profile: ['', 'hadoop3'] steps: - name: Checkout code uses: actions/checkout@v6 @@ -56,7 +57,12 @@ jobs: distribution: 'temurin' - name: Build Flink - run: mvn -T 2C -B clean install -DskipTests -Pflink1,spark3 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }} + run: | + profile="flink1,spark3,flink-${{ matrix.flink_version }}" + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi + mvn -T 2C -B clean install -DskipTests -P$profile -pl paimon-e2e-tests -am - name: Test Flink run: | @@ -64,11 +70,14 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - profile="flink-${{ matrix.flink_version }}" - if [ "${{ matrix.flink_version }}" = "${{ matrix.flink_version[-1] }}" ]; then - mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone - else - mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile} + profile="flink1,spark3" + if ! [ "${{ matrix.flink_version }}" = "${{ matrix.flink_version[-1] }}" ]; then + profile="$profile,flink-${{ matrix.flink_version }}" fi + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi + profile="flink-${{ matrix.flink_version }}" + mvn -T 1C -B test -P$profile -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone env: MAVEN_OPTS: -Xmx4096m diff --git a/.github/workflows/utitcase-flink-1.x-common.yml b/.github/workflows/utitcase-flink-1.x-common.yml index c41cdbfd7099..cb88598d6d30 100644 --- a/.github/workflows/utitcase-flink-1.x-common.yml +++ b/.github/workflows/utitcase-flink-1.x-common.yml @@ -39,7 +39,9 @@ jobs: build_test: runs-on: ubuntu-latest timeout-minutes: 60 - + strategy: + matrix: + hadoop_profile: ['', 'hadoop3'] steps: - name: Checkout code uses: actions/checkout@v6 @@ -52,18 +54,26 @@ jobs: - name: Build Flink run: | + profile="flink1,spark3" + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi COMPILE_MODULE="org.apache.paimon:paimon-flink-common" echo "Start compiling modules: $COMPILE_MODULE" - mvn -T 2C -B clean install -DskipTests -Pflink1,spark3 -pl "${COMPILE_MODULE}" -am + mvn -T 2C -B clean install -DskipTests -P$profile -pl "${COMPILE_MODULE}" -am - name: Test Flink run: | . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" + profile="flink1,spark3" + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi TEST_MODULE="org.apache.paimon:paimon-flink-common" echo "Start testing module: $TEST_MODULE" - mvn -T 2C -B test verify -Pflink1,spark3 -pl "${TEST_MODULE}" -Duser.timezone=$jvm_timezone + mvn -T 2C -B test verify -P$profile -pl "${TEST_MODULE}" -Duser.timezone=$jvm_timezone echo "All modules tested" env: - MAVEN_OPTS: -Xmx4096m -XX:+UseG1GC -XX:CICompilerCount=2 \ No newline at end of file + MAVEN_OPTS: -Xmx4096m -XX:+UseG1GC -XX:CICompilerCount=2 diff --git a/.github/workflows/utitcase-flink-1.x-others.yml b/.github/workflows/utitcase-flink-1.x-others.yml index e5a4036a736b..d6852bce714a 100644 --- a/.github/workflows/utitcase-flink-1.x-others.yml +++ b/.github/workflows/utitcase-flink-1.x-others.yml @@ -39,6 +39,9 @@ jobs: build_test: runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + matrix: + hadoop_profile: ['', 'hadoop3'] steps: - name: Checkout code @@ -52,18 +55,26 @@ jobs: - name: Build Flink run: | - mvn -T 2C -B clean install -DskipTests -Pflink1,spark3 + profile="flink1,spark3" + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi + mvn -T 2C -B clean install -DskipTests -P$profile - name: Test Flink run: | . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" + profile="flink1,spark3" + if ! [ "${{ matrix.hadoop_profile }}" = "" ]; then + profile="$profile,${{ matrix.hadoop_profile }}" + fi test_modules="" for suffix in cdc 1.16 1.17 1.18 1.19 1.20; do test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" - mvn -T 2C -B test verify -Pflink1,spark3 -pl "${test_modules}" -Duser.timezone=$jvm_timezone + mvn -T 2C -B test verify -P$profile -pl "${test_modules}" -Duser.timezone=$jvm_timezone env: - MAVEN_OPTS: -Xmx4096m -XX:+UseG1GC -XX:CICompilerCount=2 \ No newline at end of file + MAVEN_OPTS: -Xmx4096m -XX:+UseG1GC -XX:CICompilerCount=2 diff --git a/docs/content/flink/quick-start.md b/docs/content/flink/quick-start.md index 0317ce1bad5c..4fd8563dd82c 100644 --- a/docs/content/flink/quick-start.md +++ b/docs/content/flink/quick-start.md @@ -70,7 +70,17 @@ You can also manually build bundled jar from the source code. To build from source code, [clone the git repository]({{< github_repo >}}). Build bundled jar with the following command. -- `mvn clean install -DskipTests` + +```bash +# build paimon flink 1.x +mvn clean install -DskipTests + +# build paimon flink 1.x with hadoop 3.x +mvn clean package -DskipTests -Pflink1,hadoop3 + +# build paimon flink 2.x (Java 11 required) +mvn clean package -DskipTests -Pflink2 +``` You can find the bundled jar in `./paimon-flink/paimon-flink-/target/paimon-flink--{{< version >}}.jar`, and the action jar in `./paimon-flink/paimon-flink-action/target/paimon-flink-action-{{< version >}}.jar`. diff --git a/paimon-common/pom.xml b/paimon-common/pom.xml index 198c2fe76969..26cf1b233fb0 100644 --- a/paimon-common/pom.xml +++ b/paimon-common/pom.xml @@ -258,6 +258,20 @@ under the License. + + + hadoop3 + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + provided + + + + + diff --git a/paimon-flink/paimon-flink-cdc/pom.xml b/paimon-flink/paimon-flink-cdc/pom.xml index 5bb119b4989f..a16b381730a9 100644 --- a/paimon-flink/paimon-flink-cdc/pom.xml +++ b/paimon-flink/paimon-flink-cdc/pom.xml @@ -339,15 +339,15 @@ under the License. - - - - - - - - - + + + + com.fasterxml.jackson.core + jackson-annotations + 2.15.2 + + + diff --git a/pom.xml b/pom.xml index d020c79f0bea..cab7a31fb256 100644 --- a/pom.xml +++ b/pom.xml @@ -532,6 +532,12 @@ under the License. true + + hadoop3 + + 3.4.2 + +