spark: test on latest version 3.5.2 (OpenLineage#3056)

Signed-off-by: Maciej Obuchowski <[email protected]>
nubank · Sep 10, 2024 · 1cc28d7 · 1cc28d7
1 parent 8bceec8
commit 1cc28d7
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 27 deletions.
diff --git a/.circleci/workflows/openlineage-spark.yml b/.circleci/workflows/openlineage-spark.yml
@@ -31,10 +31,10 @@ workflows:
                 'java:17-spark:3.3.4-scala:2.13',
                 'java:8-spark:3.4.3-scala:2.12',
                 'java:8-spark:3.4.3-scala:2.13',
-                'java:8-spark:3.5.1-scala:2.12',
-                'java:8-spark:3.5.1-scala:2.13',
-                'java:17-spark:3.5.1-scala:2.12',
-                'java:17-spark:3.5.1-scala:2.13',
+                'java:8-spark:3.5.2-scala:2.12',
+                'java:8-spark:3.5.2-scala:2.13',
+                'java:17-spark:3.5.2-scala:2.12',
+                'java:17-spark:3.5.2-scala:2.13',
                 'java:17-spark:4.0.0-scala:2.13'
               ]
           requires:
@@ -92,7 +92,7 @@ workflows:
           context: integration-tests
           matrix:
             parameters:
-              spark-version: [ '3.4.2', '3.5.0' ]
+              spark-version: [ '3.4.2', '3.5.2' ]
           requires:
             - approval-integration-spark
       - integration-test-integration-spark:
@@ -112,10 +112,10 @@ workflows:
                 'java:17-spark:3.3.4-scala:2.13',
                 'java:8-spark:3.4.3-scala:2.12',
                 'java:8-spark:3.4.3-scala:2.13',
-                'java:8-spark:3.5.1-scala:2.12',
-                'java:8-spark:3.5.1-scala:2.13',
-                'java:17-spark:3.5.1-scala:2.12',
-                'java:17-spark:3.5.1-scala:2.13',
+                'java:8-spark:3.5.2-scala:2.12',
+                'java:8-spark:3.5.2-scala:2.13',
+                'java:17-spark:3.5.2-scala:2.12',
+                'java:17-spark:3.5.2-scala:2.13',
                 'java:17-spark:4.0.0-scala:2.13'
               ]
           requires:

diff --git a/integration/README.md b/integration/README.md
@@ -4,7 +4,7 @@
 |Apache Airflow| 2.1.*, 2.2.*, 2.3.*                        |https://github.com/apache/airflow/releases|[README](./airflow/README.md)|Support for Airflow 1.x is deprecated|
 |Dagster| 0.13.8+                                    |https://github.com/dagster-io/dagster/releases|[README](./dagster/README.md)| |
 |dbt| 0.20+, 1.3                                 |https://github.com/dbt-labs/dbt-core/releases|[README](./dbt/README.md)| |
-|Apache Spark| 2.4.6, 3.1.2, 3.2.1+, 3.3.1+, 3.4.3, 3.5.1 |https://github.com/apache/spark/tags|[README](./spark/README.md)| |
+|Apache Spark| 2.4.6, 3.1.2, 3.2.1+, 3.3.1+, 3.4.3, 3.5.2 |https://github.com/apache/spark/tags|[README](./spark/README.md)| |
 |Apache Flink| 1.15.4, 1.16.2, 1.17.1, 1.18.1             |https://github.com/apache/flink/tags|[README](./flink/README.md)|  |
 
 ----

diff --git a/integration/spark-docker/manifest.json b/integration/spark-docker/manifest.json
@@ -56,19 +56,19 @@
     "sparkVersion": "3.4.3"
   },
   {
-    "baseImageTag": "3.5.1",
+    "baseImageTag": "3.5.2",
     "scalaBinaryVersion": "2.12",
     "sparkPgpKeys": "https://www.apache.org/dist/spark/KEYS",
-    "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz",
-    "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz.asc",
-    "sparkVersion": "3.5.1"
+    "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz",
+    "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz.asc",
+    "sparkVersion": "3.5.2"
   },
   {
-    "baseImageTag": "3.5.1",
+    "baseImageTag": "3.5.2",
     "scalaBinaryVersion": "2.13",
     "sparkPgpKeys": "https://www.apache.org/dist/spark/KEYS",
-    "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz",
-    "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz.asc",
-    "sparkVersion": "3.5.1"
+    "sparkSourceBinaries": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3-scala2.13.tgz",
+    "sparkSourceBinariesAsc": "https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3-scala2.13.tgz.asc",
+    "sparkVersion": "3.5.2"
   }
 ]
diff --git a/integration/spark/app/build.gradle b/integration/spark/app/build.gradle
@@ -495,7 +495,7 @@ List<Dependency> icebergDependencies(String spark, String scala) {
                     dependencies.create("org.scala-lang:scala-reflect:${scalaVersion}"),
                     dependencies.create("org.scala-lang.modules:scala-collection-compat_${scala}:2.11.0"),
             ],
-            "3.5.1": [
+            "3.5.2": [
                     dependencies.create("org.apache.iceberg:iceberg-spark-runtime-3.5_${scala}:1.6.0"),
                     dependencies.create("org.scala-lang:scala-library:${scalaVersion}"),
                     dependencies.create("org.scala-lang:scala-reflect:${scalaVersion}"),
@@ -528,7 +528,7 @@ List<Dependency> gcsDependencies(String spark, String scala) {
             "3.2.4": "hadoop3-2.2.9",
             "3.3.4": "hadoop3-2.2.9",
             "3.4.3": "hadoop3-2.2.9",
-            "3.5.1": "hadoop3-2.2.9",
+            "3.5.2": "hadoop3-2.2.9",
     ]
 
     def gcs = registry.get(spark)
@@ -548,7 +548,7 @@ List<Dependency> hadoopClientDependencies(String spark, String scala) {
             "3.2.4": "3.3.4",
             "3.3.4": "3.3.2",
             "3.4.3": "3.3.4",
-            "3.5.1": "3.3.4",
+            "3.5.2": "3.3.4",
     ]
 
     def hadoopClient = registry.get(spark)
@@ -608,7 +608,7 @@ List<Dependency> additionalJars(String spark, String scala) {
                     }),
                     dependencies.create("org.apache.iceberg:iceberg-spark-runtime-3.4_${scala}:1.6.0")
             ],
-            "3.5.1": [
+            "3.5.2": [
                     dependencies.create("org.slf4j:slf4j-api:2.0.10"),
                     dependencies.create("org.slf4j:slf4j-reload4j:2.0.16"),
                     dependencies.create("org.apache.spark:spark-mllib_${scala}:${spark}", {

diff --git a/integration/spark/gradle.properties b/integration/spark/gradle.properties
@@ -11,5 +11,5 @@ spark31.spark.version=3.1.3
 spark32.spark.version=3.2.4
 spark33.spark.version=3.3.4
 spark34.spark.version=3.4.3
-spark35.spark.version=3.5.1
+spark35.spark.version=3.5.2
 spark40.spark.version=4.0.0
diff --git a/integration/spark/scala-fixtures/build.gradle.kts b/integration/spark/scala-fixtures/build.gradle.kts
@@ -8,11 +8,11 @@ repositories {
 }
 
 configureScalaVariant("2.11", "2.4.8")
-configureScalaVariant("2.12", "3.5.1")
-configureScalaVariant("2.13", "3.5.1")
+configureScalaVariant("2.12", "3.5.2")
+configureScalaVariant("2.13", "3.5.2")
 
 dependencies {
-    compileOnly("org.apache.spark:spark-sql_2.12:3.5.1")
+    compileOnly("org.apache.spark:spark-sql_2.12:3.5.2")
 }
 
 fun configureScalaVariant(scalaBinaryVersion: String, sparkVersion: String) {

diff --git a/integration/spark/vendor/snowflake/build.gradle b/integration/spark/vendor/snowflake/build.gradle
@@ -23,7 +23,7 @@ ext {
 
     sparkProp = project.findProperty('spark.version').toString()
     // because snowflake doesn't have a 3.5.0 connector yet
-    spark = (sparkProp == "3.5.1" || sparkProp.toString().startsWith("4")) ? "3.4.3" : sparkProp
+    spark = (sparkProp == "3.5.2" || sparkProp.toString().startsWith("4")) ? "3.4.3" : sparkProp
 
     series = spark.substring(0, 3)
     scala = project.findProperty('scala.binary.version').toString()