Resolve conflicts

Yelp · Aug 3, 2023 · 156ab0e · 156ab0e
1 parent 87624ac
commit 156ab0e
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py
@@ -1109,6 +1109,10 @@ def get_spark_conf(
         else:
             raise UnsupportedClusterManagerException(cluster_manager)
 
+        # Enabling auto-decommission feature to move data around makes sense for only spot nodes
+        if 'batch' != paasta_pool and 'spark.decommission.enabled' in spark_conf:
+            spark_conf.update({'spark.decommission.enabled': 'false'})
+
         # configure dynamic resource allocation configs
         spark_conf = self.get_dra_configs(spark_conf)