From 1cc28d79631c47ee47b253ee2289f86e770a98a5 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 10 Sep 2024 11:16:20 +0200 Subject: [PATCH] spark: test on latest version 3.5.2 (#3056) Signed-off-by: Maciej Obuchowski --- .circleci/workflows/openlineage-spark.yml | 18 +++++++++--------- integration/README.md | 2 +- integration/spark-docker/manifest.json | 16 ++++++++-------- integration/spark/app/build.gradle | 8 ++++---- integration/spark/gradle.properties | 2 +- .../spark/scala-fixtures/build.gradle.kts | 6 +++--- .../spark/vendor/snowflake/build.gradle | 2 +- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.circleci/workflows/openlineage-spark.yml b/.circleci/workflows/openlineage-spark.yml index 4060cdf268..623773d4cf 100644 --- a/.circleci/workflows/openlineage-spark.yml +++ b/.circleci/workflows/openlineage-spark.yml @@ -31,10 +31,10 @@ workflows: 'java:17-spark:3.3.4-scala:2.13', 'java:8-spark:3.4.3-scala:2.12', 'java:8-spark:3.4.3-scala:2.13', - 'java:8-spark:3.5.1-scala:2.12', - 'java:8-spark:3.5.1-scala:2.13', - 'java:17-spark:3.5.1-scala:2.12', - 'java:17-spark:3.5.1-scala:2.13', + 'java:8-spark:3.5.2-scala:2.12', + 'java:8-spark:3.5.2-scala:2.13', + 'java:17-spark:3.5.2-scala:2.12', + 'java:17-spark:3.5.2-scala:2.13', 'java:17-spark:4.0.0-scala:2.13' ] requires: @@ -92,7 +92,7 @@ workflows: context: integration-tests matrix: parameters: - spark-version: [ '3.4.2', '3.5.0' ] + spark-version: [ '3.4.2', '3.5.2' ] requires: - approval-integration-spark - integration-test-integration-spark: @@ -112,10 +112,10 @@ workflows: 'java:17-spark:3.3.4-scala:2.13', 'java:8-spark:3.4.3-scala:2.12', 'java:8-spark:3.4.3-scala:2.13', - 'java:8-spark:3.5.1-scala:2.12', - 'java:8-spark:3.5.1-scala:2.13', - 'java:17-spark:3.5.1-scala:2.12', - 'java:17-spark:3.5.1-scala:2.13', + 'java:8-spark:3.5.2-scala:2.12', + 'java:8-spark:3.5.2-scala:2.13', + 'java:17-spark:3.5.2-scala:2.12', + 'java:17-spark:3.5.2-scala:2.13', 'java:17-spark:4.0.0-scala:2.13' ] requires: diff --git a/integration/README.md b/integration/README.md index 3a1fe322fe..fe2e37c51c 100644 --- a/integration/README.md +++ b/integration/README.md @@ -4,7 +4,7 @@ |Apache Airflow| 2.1.*, 2.2.*, 2.3.* |https://github.com/apache/airflow/releases|[README](./airflow/README.md)|Support for Airflow 1.x is deprecated| |Dagster| 0.13.8+ |https://github.com/dagster-io/dagster/releases|[README](./dagster/README.md)| | |dbt| 0.20+, 1.3 |https://github.com/dbt-labs/dbt-core/releases|[README](./dbt/README.md)| | -|Apache Spark| 2.4.6, 3.1.2, 3.2.1+, 3.3.1+, 3.4.3, 3.5.1 |https://github.com/apache/spark/tags|[README](./spark/README.md)| | +|Apache Spark| 2.4.6, 3.1.2, 3.2.1+, 3.3.1+, 3.4.3, 3.5.2 |https://github.com/apache/spark/tags|[README](./spark/README.md)| | |Apache Flink| 1.15.4, 1.16.2, 1.17.1, 1.18.1 |https://github.com/apache/flink/tags|[README](./flink/README.md)| | ---- diff --git a/integration/spark-docker/manifest.json b/integration/spark-docker/manifest.json index 56dbad7ef9..d9b578f59d 100644 --- a/integration/spark-docker/manifest.json +++ b/integration/spark-docker/manifest.json @@ -56,19 +56,19 @@ "sparkVersion": "3.4.3" }, { - "baseImageTag": "3.5.1", + "baseImageTag": "3.5.2", "scalaBinaryVersion": "2.12", "sparkPgpKeys": "https://www.apache.org/dist/spark/KEYS", - "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz", - "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz.asc", - "sparkVersion": "3.5.1" + "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz", + "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz.asc", + "sparkVersion": "3.5.2" }, { - "baseImageTag": "3.5.1", + "baseImageTag": "3.5.2", "scalaBinaryVersion": "2.13", "sparkPgpKeys": "https://www.apache.org/dist/spark/KEYS", - "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz", - "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz.asc", - "sparkVersion": "3.5.1" + "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3-scala2.13.tgz", + "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3-scala2.13.tgz.asc", + "sparkVersion": "3.5.2" } ] \ No newline at end of file diff --git a/integration/spark/app/build.gradle b/integration/spark/app/build.gradle index 097a82f3b2..913d139aa2 100644 --- a/integration/spark/app/build.gradle +++ b/integration/spark/app/build.gradle @@ -495,7 +495,7 @@ List icebergDependencies(String spark, String scala) { dependencies.create("org.scala-lang:scala-reflect:${scalaVersion}"), dependencies.create("org.scala-lang.modules:scala-collection-compat_${scala}:2.11.0"), ], - "3.5.1": [ + "3.5.2": [ dependencies.create("org.apache.iceberg:iceberg-spark-runtime-3.5_${scala}:1.6.0"), dependencies.create("org.scala-lang:scala-library:${scalaVersion}"), dependencies.create("org.scala-lang:scala-reflect:${scalaVersion}"), @@ -528,7 +528,7 @@ List gcsDependencies(String spark, String scala) { "3.2.4": "hadoop3-2.2.9", "3.3.4": "hadoop3-2.2.9", "3.4.3": "hadoop3-2.2.9", - "3.5.1": "hadoop3-2.2.9", + "3.5.2": "hadoop3-2.2.9", ] def gcs = registry.get(spark) @@ -548,7 +548,7 @@ List hadoopClientDependencies(String spark, String scala) { "3.2.4": "3.3.4", "3.3.4": "3.3.2", "3.4.3": "3.3.4", - "3.5.1": "3.3.4", + "3.5.2": "3.3.4", ] def hadoopClient = registry.get(spark) @@ -608,7 +608,7 @@ List additionalJars(String spark, String scala) { }), dependencies.create("org.apache.iceberg:iceberg-spark-runtime-3.4_${scala}:1.6.0") ], - "3.5.1": [ + "3.5.2": [ dependencies.create("org.slf4j:slf4j-api:2.0.10"), dependencies.create("org.slf4j:slf4j-reload4j:2.0.16"), dependencies.create("org.apache.spark:spark-mllib_${scala}:${spark}", { diff --git a/integration/spark/gradle.properties b/integration/spark/gradle.properties index f6bfdd8986..6a056b0a0a 100644 --- a/integration/spark/gradle.properties +++ b/integration/spark/gradle.properties @@ -11,5 +11,5 @@ spark31.spark.version=3.1.3 spark32.spark.version=3.2.4 spark33.spark.version=3.3.4 spark34.spark.version=3.4.3 -spark35.spark.version=3.5.1 +spark35.spark.version=3.5.2 spark40.spark.version=4.0.0 \ No newline at end of file diff --git a/integration/spark/scala-fixtures/build.gradle.kts b/integration/spark/scala-fixtures/build.gradle.kts index a4726c767e..824e51f2c2 100644 --- a/integration/spark/scala-fixtures/build.gradle.kts +++ b/integration/spark/scala-fixtures/build.gradle.kts @@ -8,11 +8,11 @@ repositories { } configureScalaVariant("2.11", "2.4.8") -configureScalaVariant("2.12", "3.5.1") -configureScalaVariant("2.13", "3.5.1") +configureScalaVariant("2.12", "3.5.2") +configureScalaVariant("2.13", "3.5.2") dependencies { - compileOnly("org.apache.spark:spark-sql_2.12:3.5.1") + compileOnly("org.apache.spark:spark-sql_2.12:3.5.2") } fun configureScalaVariant(scalaBinaryVersion: String, sparkVersion: String) { diff --git a/integration/spark/vendor/snowflake/build.gradle b/integration/spark/vendor/snowflake/build.gradle index 5ee7d1530d..64cfda089c 100644 --- a/integration/spark/vendor/snowflake/build.gradle +++ b/integration/spark/vendor/snowflake/build.gradle @@ -23,7 +23,7 @@ ext { sparkProp = project.findProperty('spark.version').toString() // because snowflake doesn't have a 3.5.0 connector yet - spark = (sparkProp == "3.5.1" || sparkProp.toString().startsWith("4")) ? "3.4.3" : sparkProp + spark = (sparkProp == "3.5.2" || sparkProp.toString().startsWith("4")) ? "3.4.3" : sparkProp series = spark.substring(0, 3) scala = project.findProperty('scala.binary.version').toString()