From 8cf09cfb5582202c68d60e9217e773622d99d8c5 Mon Sep 17 00:00:00 2001 From: Luis Yamada Date: Mon, 10 Feb 2025 13:08:08 -0300 Subject: [PATCH] logging confs of spark context and session --- .../io/openlineage/spark/agent/NuEventEmitter.java | 3 +-- .../agent/lifecycle/SparkSQLExecutionContext.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java b/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java index d91aa1529b..3689d79cf1 100644 --- a/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java +++ b/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java @@ -28,8 +28,7 @@ public class NuEventEmitter { private static final Set WANTED_EVENT_NAME_SUBSTRINGS = new HashSet<>( Arrays.asList( ".execute_insert_into_hadoop_fs_relation_command.", - ".adaptive_spark_plan.", - "." + ".adaptive_spark_plan." ) ); diff --git a/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java b/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java index 95ff0c21fc..f5b457f3be 100644 --- a/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java +++ b/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java @@ -25,12 +25,16 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Stack; import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; +import scala.Tuple2; + import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.JobFailed; import org.apache.spark.scheduler.SparkListenerApplicationEnd; @@ -80,8 +84,12 @@ public SparkSQLExecutionContext( @Override public void start(SparkListenerSQLExecutionStart startEvent) { log.info("SparkListenerSQLExecutionStart - executionId: {}", startEvent.executionId()); + Tuple2[] allConfs = olContext.getSparkContext().get().getConf().getAll(); + for (Tuple2 conf : allConfs) { + log.info("SparkListenerSQLExecutionStart - conf: {}", conf.toString()); + } + log.info("SparkListenerSQLExecutionStart - mapOfConfsFromContext: {}", olContext.getSparkSession().get().conf().getAll().toString()); log.info("SparkListenerSQLExecutionStart - event: {}", startEvent.toString()); - log.info("SparkListenerSQLExecutionStart - event.sparkPlanInfo: {}", startEvent.sparkPlanInfo()); if (log.isDebugEnabled()) { log.debug("SparkListenerSQLExecutionStart - executionId: {}", startEvent.executionId()); } @@ -120,7 +128,6 @@ public void start(SparkListenerSQLExecutionStart startEvent) { @Override public void end(SparkListenerSQLExecutionEnd endEvent) { log.info("SparkListenerSQLExecutionEnd - executionId: {}", endEvent.executionId()); - log.info("SparkListenerSQLExecutionEnd - event: {}", endEvent.toString()); if (log.isDebugEnabled()) { log.debug("SparkListenerSQLExecutionEnd - executionId: {}", endEvent.executionId()); }