diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 9451fc7c5d7..df3d23dd254 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -34,40 +34,12 @@ concurrency: cancel-in-progress: true jobs: - sbt: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - module: - - 'service' - - 'spark-3.0' - - 'spark-3.1' - - 'spark-3.2' - - 'spark-3.3' - - 'spark-3.4' - - 'spark-3.5' - - 'flink-1.18' - - 'flink-1.19' - - 'flink-1.20' - - 'mr' - steps: - - uses: actions/checkout@v4 - - name: Setup JDK 8 - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 8 - check-latest: false - - name: SBT Check dependency list - run: | - ./dev/dependencies.sh --sbt --module ${{ matrix.module }} --check - - maven-jdk8: + jdk8: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: + tool: [maven, sbt] module: - 'service' - 'spark-3.0' @@ -81,24 +53,28 @@ jobs: - 'flink-1.20' - 'mr' - 'tez' + exclude: + - tool: sbt + module: 'tez' steps: - - uses: actions/checkout@v4 - - name: Setup JDK 8 - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 8 - cache: maven - check-latest: false - - name: Maven Check dependency list - run: | - ./dev/dependencies.sh --module ${{ matrix.module }} --check + - uses: actions/checkout@v4 + - name: Setup JDK 8 + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 8 + cache: ${{ matrix.tool == 'maven' && 'maven' || '' }} + check-latest: false + - name: ${{ matrix.tool }} Check dependency list + run: | + ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check - maven-jdk11: + jdk11: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: + tool: [maven, sbt] module: - 'flink-2.0' - 'flink-2.1' @@ -111,17 +87,18 @@ jobs: with: distribution: zulu java-version: 11 - cache: maven + cache: ${{ matrix.tool == 'maven' && 'maven' || '' }} check-latest: false - - name: Maven Check dependency list + - name: ${{ matrix.tool }} Check dependency list run: | - ./dev/dependencies.sh --module ${{ matrix.module }} --check + ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check - maven-jdk17: + jdk17: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: + tool: [maven, sbt] module: - 'spark-4.0' - 'spark-4.1' @@ -133,8 +110,8 @@ jobs: with: distribution: zulu java-version: 17 - cache: maven + cache: ${{ matrix.tool == 'maven' && 'maven' || '' }} check-latest: false - - name: Maven Check dependency list + - name: ${{ matrix.tool }} Check dependency list run: | - ./dev/dependencies.sh --module ${{ matrix.module }} --check + ./dev/dependencies.sh ${{ matrix.tool == 'sbt' && '--sbt' || '' }} --module ${{ matrix.module }} --check diff --git a/dev/dependencies.sh b/dev/dependencies.sh index 832be53f982..4cfbb7d1f42 100755 --- a/dev/dependencies.sh +++ b/dev/dependencies.sh @@ -19,10 +19,6 @@ set -ex -# Explicitly set locale in order to make `sort` output consistent across machines. -# See https://stackoverflow.com/questions/28881 for more details. -export LC_ALL=C - PWD=$(cd "$(dirname "$0")"/.. || exit; pwd) MVN="${PWD}/build/mvn" @@ -52,7 +48,7 @@ function mvn_build_classpath() { classifier_end_index=index(jar_name, ".jar") - 1; classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1); print artifact_id"/"version"/"classifier"/"jar_name - }' | grep -v "celeborn" | sort -u >> "${DEP_PR}" + }' | grep -v "celeborn" | LC_ALL=C sort -u >> "${DEP_PR}" } function sbt_build_client_classpath() { @@ -108,7 +104,7 @@ function sbt_process_classpath() { result=("${result1[@]}" "${result2[@]}") - echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}" + echo "${result[@]}" | tr ' ' '\n' | LC_ALL=C sort -u >> "${DEP_PR}" } function check_diff() { diff --git a/dev/deps/dependencies-client-spark-4.1 b/dev/deps/dependencies-client-spark-4.1 index 8ad85bec05f..aa0460df433 100644 --- a/dev/deps/dependencies-client-spark-4.1 +++ b/dev/deps/dependencies-client-spark-4.1 @@ -73,7 +73,6 @@ netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar paranamer/2.8.3//paranamer-2.8.3.jar -paranamer/2.8//paranamer-2.8.jar protobuf-java/3.25.5//protobuf-java-3.25.5.jar scala-library/2.13.17//scala-library-2.13.17.jar scala-reflect/2.13.17//scala-reflect-2.13.17.jar diff --git a/dev/deps/dependencies-client-spark-4.2 b/dev/deps/dependencies-client-spark-4.2 index 2b62a29fed2..ca992cea38d 100644 --- a/dev/deps/dependencies-client-spark-4.2 +++ b/dev/deps/dependencies-client-spark-4.2 @@ -73,7 +73,6 @@ netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqu netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar paranamer/2.8.3//paranamer-2.8.3.jar -paranamer/2.8//paranamer-2.8.jar protobuf-java/3.25.5//protobuf-java-3.25.5.jar scala-library/2.13.18//scala-library-2.13.18.jar scala-reflect/2.13.18//scala-reflect-2.13.18.jar diff --git a/pom.xml b/pom.xml index b7577e991d3..9632fd398ca 100644 --- a/pom.xml +++ b/pom.xml @@ -1638,6 +1638,15 @@ 4.1.2 1.5.7-6 + + + + com.thoughtworks.paranamer + paranamer + 2.8.3 + + + @@ -1658,6 +1667,15 @@ 4.2.0 1.5.7-7 + + + + com.thoughtworks.paranamer + paranamer + 2.8.3 + + + diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index 2d9bb62c676..4aa615d969a 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -688,7 +688,6 @@ object CelebornCommon { Dependencies.commonsLang3, Dependencies.hadoopClientApi, Dependencies.hadoopClientRuntime, - Dependencies.jdkTools, Dependencies.leveldbJniAll, Dependencies.roaringBitmap, Dependencies.scalaReflect, @@ -707,6 +706,14 @@ object CelebornCommon { Dependencies.bouncycastleBcprovJdk18on, Dependencies.bouncycastleBcpkixJdk18on ) ++ commonUnitTestDependencies, + // maven-jdk-tools-wrapper (jdkTools) only provides jdk.tools (tools.jar) on + // JDK 8; on JDK 9+ the tools API is built into the JDK, so the wrapper is a + // no-op there. Gate it to JDK 8 to match the `JDKTools` plugin + // (project/JDKTools.scala) and the Maven `jdk-8` profile, and to keep the + // sbt/maven classpaths in sync (see dev/dependencies.sh). + libraryDependencies ++= + (if (System.getProperty("java.specification.version").startsWith("1.")) + Seq(Dependencies.jdkTools) else Nil), Compile / sourceGenerators += Def.task { val file = (Compile / sourceManaged).value / "org" / "apache" / "celeborn" / "package.scala" @@ -1004,6 +1011,7 @@ object Spark41 extends SparkClientProjects { val scalaBinaryVersion = "2.13" override val sparkColumnarShuffleVersion: String = "4" + override val paranamerVersionOverride: Option[String] = Some("2.8.3") } object Spark42 extends SparkClientProjects { @@ -1022,6 +1030,7 @@ object Spark42 extends SparkClientProjects { override val lz4JavaGroup = "at.yawk.lz4" override val sparkColumnarShuffleVersion: String = "4" + override val paranamerVersionOverride: Option[String] = Some("2.8.3") } trait SparkClientProjects { @@ -1037,6 +1046,8 @@ trait SparkClientProjects { val sparkVersion: String val zstdJniVersion: String + val paranamerVersionOverride: Option[String] = None + val includeColumnarShuffle: Boolean = true def modules: Seq[Project] = { @@ -1082,7 +1093,10 @@ trait SparkClientProjects { "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", Dependencies.javaxServletApi % "test", Dependencies.jakartaServletApi % "test" - ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline % "test") + ) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline % "test"), + dependencyOverrides ++= paranamerVersionOverride + .map(v => Seq("com.thoughtworks.paranamer" % "paranamer" % v)) + .getOrElse(Seq.empty) ) }