[WIP]feat: Support apache uniffle remote shuffle service

.github/workflows/pr_build_linux.yml

@@ -583,3 +583,131 @@ jobs:
           SPARK_TPCDS_JOIN_CONF: |
             spark.sql.autoBroadcastJoinThreshold=-1
             spark.sql.join.forceApplyShuffledHashJoin=true
+
+  # Uniffle integration test - verifies Comet shuffle against a local Uniffle cluster
+  uniffle-integration-test:
+    needs: build-native
+    name: Uniffle Integration Test
+    runs-on: ubuntu-24.04
+    container:
+      image: amd64/rust
+    env:
+      JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+      UNIFFLE_VERSION: 0.10.0
+      HADOOP_VERSION: 2.10.2
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.RUST_VERSION }}
+          jdk-version: 17
+
+      - name: Download native library
+        uses: actions/download-artifact@v8
+        with:
+          name: native-lib-linux
+          path: native/target/release/
+
+      - name: Cache Maven dependencies
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.m2/repository
+            /root/.m2/repository
+          key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-java-maven-
+
+      - name: Cache TPC-H data
+        id: cache-uniffle-tpch
+        uses: actions/cache@v6
+        with:
+          path: ./tpch
+          key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
+
+      - name: Cache TPC-DS data
+        id: cache-uniffle-tpcds
+        uses: actions/cache@v6
+        with:
+          path: ./tpcds-sf-1
+          key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
+
+      - name: Install Uniffle
+        run: |
+          wget "https://archive.apache.org/dist/uniffle/${UNIFFLE_VERSION}/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz" \
+            -O "/opt/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz"
+          wget "https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz" \
+            -O "/opt/hadoop-${HADOOP_VERSION}.tar.gz"
+          mkdir /opt/uniffle
+          tar xzf "/opt/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz" \
+            -C /opt/uniffle --strip-components=1
+          tar xzf "/opt/hadoop-${HADOOP_VERSION}.tar.gz" -C /opt/
+          mkdir /opt/uniffle/shuffle_data
+          printf 'XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-%s\n' "${HADOOP_VERSION}" \
+            > /opt/uniffle/conf/rss-env.sh
+          printf 'rss.coordinator.shuffle.nodes.max 1\nrss.rpc.server.port 19999\n' \
+            > /opt/uniffle/conf/coordinator.conf
+          printf '%s\n' \
+            'rss.server.app.expired.withoutHeartbeat 7200000' \
+            'rss.server.heartbeat.delay 3000' \
+            'rss.rpc.server.port 19997' \
+            'rss.rpc.server.type GRPC_NETTY' \
+            'rss.jetty.http.port 19996' \
+            'rss.server.netty.port 19995' \
+            'rss.storage.basePath /opt/uniffle/shuffle_data' \
+            'rss.storage.type MEMORY_LOCALFILE' \
+            'rss.coordinator.quorum localhost:19999' \
+            'rss.server.flush.thread.alive 10' \
+            'rss.server.single.buffer.flush.threshold 64m' \
+            > /opt/uniffle/conf/server.conf
+          cd /opt/uniffle
+          bash ./bin/start-coordinator.sh
+          bash ./bin/start-shuffle-server.sh
+
+      - name: Build project
+        run: |
+          ./mvnw -B -Prelease install -DskipTests -Pspark-3.5,uniffle
+
+      - name: Generate TPC-H data (SF=1)
+        if: steps.cache-uniffle-tpch.outputs.cache-hit != 'true'
+        run: |
+          cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease -Pspark-3.5 exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--location `pwd`/.. --scaleFactor 1 --numPartitions 1 --overwrite"
+
+      - name: Checkout tpcds-kit
+        if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+        uses: actions/checkout@v7
+        with:
+          repository: databricks/tpcds-kit
+          path: ./tpcds-kit
+
+      - name: Build tpcds-kit
+        if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+        run: |
+          apt-get update && apt-get install -y yacc bison flex gcc-12 g++-12
+          update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 120 --slave /usr/bin/g++ g++ /usr/bin/g++-12
+          cd tpcds-kit/tools && make OS=LINUX
+
+      - name: Generate TPC-DS data (SF=1)
+        if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+        run: |
+          cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease -Pspark-3.5 exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1"
+
+      - name: Run TPC-H queries with Uniffle
+        run: |
+          SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Pspark-3.5,uniffle,uniffle-comet-test -Dtest=none -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test
+        env:
+          COMET_SHUFFLE_MANAGER: uniffle
+          SPARK_TPCH_JOIN_CONF: |
+            spark.sql.autoBroadcastJoinThreshold=-1
+            spark.sql.join.preferSortMergeJoin=true
+
+      - name: Run TPC-DS queries with Uniffle
+        run: |
+          SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Pspark-3.5,uniffle,uniffle-comet-test -Dtest=none -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
+        env:
+          COMET_SHUFFLE_MANAGER: uniffle
+          SPARK_TPCDS_JOIN_CONF: |
+            spark.sql.autoBroadcastJoinThreshold=-1
+            spark.sql.join.preferSortMergeJoin=true