[CELEBORN-2379] Keep sbt and maven dependency lists in sync and cross-check both in CI

.github/workflows/deps.yml

@@ -34,40 +34,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  sbt:
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        module:
-          - 'service'
-          - 'spark-3.0'
-          - 'spark-3.1'
-          - 'spark-3.2'
-          - 'spark-3.3'
-          - 'spark-3.4'
-          - 'spark-3.5'
-          - 'flink-1.18'
-          - 'flink-1.19'
-          - 'flink-1.20'
-          - 'mr'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Setup JDK 8
-      uses: actions/setup-java@v4
-      with:
-        distribution: zulu
-        java-version: 8
-        check-latest: false
-    - name: SBT Check dependency list
-      run: |
-        ./dev/dependencies.sh --sbt --module ${{ matrix.module }} --check
-
-  maven-jdk8:
+  jdk8:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
+        tool: [maven, sbt]
         module:
           - 'service'
           - 'spark-3.0'
@@ -81,24 +53,28 @@ jobs:
           - 'flink-1.20'
           - 'mr'
           - 'tez'
+        exclude:
+          - tool: sbt
+            module: 'tez'
     steps:
-    - uses: actions/checkout@v4
-    - name: Setup JDK 8
-      uses: actions/setup-java@v4
-      with:
-        distribution: zulu
-        java-version: 8
-        cache: maven
-        check-latest: false
-    - name: Maven Check dependency list
-      run: |
-        ./dev/dependencies.sh --module ${{ matrix.module }} --check
+      - uses: actions/checkout@v4
+      - name: Setup JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+          cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
+          check-latest: false
+      - name: ${{ matrix.tool }} Check dependency list
+        run: |
+          ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check
 
-  maven-jdk11:
+  jdk11:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
+        tool: [maven, sbt]
         module:
           - 'flink-2.0'
           - 'flink-2.1'
@@ -111,17 +87,18 @@ jobs:
         with:
           distribution: zulu
           java-version: 11
-          cache: maven
+          cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
           check-latest: false
-      - name: Maven Check dependency list
+      - name: ${{ matrix.tool }} Check dependency list
         run: |
-          ./dev/dependencies.sh --module ${{ matrix.module }} --check
+          ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check
 
-  maven-jdk17:
+  jdk17:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
+        tool: [maven, sbt]
         module:
           - 'spark-4.0'
           - 'spark-4.1'
@@ -133,8 +110,8 @@ jobs:
         with:
           distribution: zulu
           java-version: 17
-          cache: maven
+          cache: ${{ matrix.tool == 'maven' && 'maven' || '' }}
           check-latest: false
-      - name: Maven Check dependency list
+      - name: ${{ matrix.tool }} Check dependency list
         run: |
-          ./dev/dependencies.sh --module ${{ matrix.module }} --check
+          ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check

dev/dependencies.sh

@@ -19,10 +19,6 @@
 
 set -ex
 
-# Explicitly set locale in order to make `sort` output consistent across machines.
-# See https://stackoverflow.com/questions/28881 for more details.
-export LC_ALL=C
-
 PWD=$(cd "$(dirname "$0")"/.. || exit; pwd)
 
 MVN="${PWD}/build/mvn"
@@ -52,7 +48,7 @@ function mvn_build_classpath() {
       classifier_end_index=index(jar_name, ".jar") - 1;
       classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1);
       print artifact_id"/"version"/"classifier"/"jar_name
-    }' | grep -v "celeborn" | sort -u >> "${DEP_PR}"
+    }' | grep -v "celeborn" | LC_ALL=C sort -u >> "${DEP_PR}"
 }
 
 function sbt_build_client_classpath() {
@@ -108,7 +104,7 @@ function sbt_process_classpath() {
 
   result=("${result1[@]}" "${result2[@]}")
 
-  echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}"
+  echo "${result[@]}" | tr ' ' '\n' | LC_ALL=C sort -u >> "${DEP_PR}"
 }
 
 function check_diff() {

dev/deps/dependencies-client-spark-4.1

@@ -73,7 +73,6 @@ netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu
 netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
 netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar
 paranamer/2.8.3//paranamer-2.8.3.jar
-paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.25.5//protobuf-java-3.25.5.jar
 scala-library/2.13.17//scala-library-2.13.17.jar
 scala-reflect/2.13.17//scala-reflect-2.13.17.jar

dev/deps/dependencies-client-spark-4.2

@@ -73,7 +73,6 @@ netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu
 netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
 netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar
 paranamer/2.8.3//paranamer-2.8.3.jar
-paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.25.5//protobuf-java-3.25.5.jar
 scala-library/2.13.18//scala-library-2.13.18.jar
 scala-reflect/2.13.18//scala-reflect-2.13.18.jar

pom.xml

@@ -1638,6 +1638,15 @@
         <spark.version>4.1.2</spark.version>
         <zstd-jni.version>1.5.7-6</zstd-jni.version>
       </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>com.thoughtworks.paranamer</groupId>
+            <artifactId>paranamer</artifactId>
+            <version>2.8.3</version>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
     </profile>
 
     <profile>
@@ -1658,6 +1667,15 @@
         <spark.version>4.2.0</spark.version>
         <zstd-jni.version>1.5.7-7</zstd-jni.version>
       </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>com.thoughtworks.paranamer</groupId>
+            <artifactId>paranamer</artifactId>
+            <version>2.8.3</version>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
     </profile>
 
     <profile>

project/CelebornBuild.scala

@@ -688,7 +688,6 @@ object CelebornCommon {
         Dependencies.commonsLang3,
         Dependencies.hadoopClientApi,
         Dependencies.hadoopClientRuntime,
-        Dependencies.jdkTools,
         Dependencies.leveldbJniAll,
         Dependencies.roaringBitmap,
         Dependencies.scalaReflect,
@@ -707,6 +706,14 @@ object CelebornCommon {
         Dependencies.bouncycastleBcprovJdk18on,
         Dependencies.bouncycastleBcpkixJdk18on
       ) ++ commonUnitTestDependencies,
+      // maven-jdk-tools-wrapper (jdkTools) only provides jdk.tools (tools.jar) on
+      // JDK 8; on JDK 9+ the tools API is built into the JDK, so the wrapper is a
+      // no-op there. Gate it to JDK 8 to match the `JDKTools` plugin
+      // (project/JDKTools.scala) and the Maven `jdk-8` profile, and to keep the
+      // sbt/maven classpaths in sync (see dev/dependencies.sh).
+      libraryDependencies ++=
+        (if (System.getProperty("java.specification.version").startsWith("1."))
+          Seq(Dependencies.jdkTools) else Nil),
 
       Compile / sourceGenerators += Def.task {
         val file = (Compile / sourceManaged).value / "org" / "apache" / "celeborn" / "package.scala"
@@ -1004,6 +1011,7 @@ object Spark41 extends SparkClientProjects {
   val scalaBinaryVersion = "2.13"
 
   override val sparkColumnarShuffleVersion: String = "4"
+  override val paranamerVersionOverride: Option[String] = Some("2.8.3")
 }
 
 object Spark42 extends SparkClientProjects {
@@ -1022,6 +1030,7 @@ object Spark42 extends SparkClientProjects {
 
   override val lz4JavaGroup = "at.yawk.lz4"
   override val sparkColumnarShuffleVersion: String = "4"
+  override val paranamerVersionOverride: Option[String] = Some("2.8.3")
 }
 
 trait SparkClientProjects {
@@ -1037,6 +1046,8 @@ trait SparkClientProjects {
   val sparkVersion: String
   val zstdJniVersion: String
 
+  val paranamerVersionOverride: Option[String] = None
+
   val includeColumnarShuffle: Boolean = true
 
   def modules: Seq[Project] = {
@@ -1082,7 +1093,10 @@ trait SparkClientProjects {
           "org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
           Dependencies.javaxServletApi % "test",
           Dependencies.jakartaServletApi % "test"
-        ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline % "test")
+        ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline % "test"),
+        dependencyOverrides ++= paranamerVersionOverride
+          .map(v => Seq("com.thoughtworks.paranamer" % "paranamer" % v))
+          .getOrElse(Seq.empty)
       )
   }