From 156ab0ed34141500f8389fa73d96d9d5732ae6aa Mon Sep 17 00:00:00 2001 From: Manpreet Singh Date: Thu, 3 Aug 2023 10:12:33 -0700 Subject: [PATCH] Resolve conflicts --- service_configuration_lib/spark_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 9aaccba..9d1389f 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -1109,6 +1109,10 @@ def get_spark_conf( else: raise UnsupportedClusterManagerException(cluster_manager) + # Enabling auto-decommission feature to move data around makes sense for only spot nodes + if 'batch' != paasta_pool and 'spark.decommission.enabled' in spark_conf: + spark_conf.update({'spark.decommission.enabled': 'false'}) + # configure dynamic resource allocation configs spark_conf = self.get_dra_configs(spark_conf)