Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "lakebench"
version = "0.13.2"
version = "0.13.3"
authors = [
{ name="Miles Cole" },
]
Expand Down
4 changes: 2 additions & 2 deletions src/lakebench/engines/spark.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,8 @@ def get_compute_size(self) -> str:
vm_host_count = len(set(executor.host() for executor in self.spark.sparkContext._jsc.sc().statusTracker().getExecutorInfos()))
worker_count = vm_host_count - 1
worker_cores = os.cpu_count()
as_min_workers = sc_conf_dict['spark.dynamicAllocation.initialExecutors'] if sc_conf_dict.get('spark.autoscale.executorResourceInfoTag.enabled', 'false') == 'true' else None
as_max_workers = sc_conf_dict['spark.dynamicAllocation.maxExecutors'] if sc_conf_dict.get('spark.autoscale.executorResourceInfoTag.enabled', 'false') == 'true' else None
as_min_workers = sc_conf_dict.get('spark.dynamicAllocation.initialExecutors') if sc_conf_dict.get('spark.autoscale.executorResourceInfoTag.enabled', 'false') == 'true' else None
as_max_workers = sc_conf_dict.get('spark.dynamicAllocation.maxExecutors') if sc_conf_dict.get('spark.autoscale.executorResourceInfoTag.enabled', 'false') == 'true' else None
as_enabled = True if as_min_workers != as_max_workers and sc_conf_dict.get('spark.dynamicAllocation.minExecutors', None) != sc_conf_dict.get('spark.dynamicAllocation.maxExecutors', None) else False
type = "SingleNode" if vm_host_count == 1 and not as_enabled else 'MultiNode'
workers_word = 'Workers' if worker_count > 1 or (as_max_workers is not None and int(as_max_workers) > 1) else 'Worker'
Expand Down
2 changes: 1 addition & 1 deletion src/lakebench/engines/synapse_spark.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def __init__(
compute_stats_all_cols=False
)

if not self.runtime != 'synapse':
if self.runtime != 'synapse':
raise RuntimeError("This engine is only supports Synapse Spark Pools.")
self.version: str = f"{self.spark.sparkContext.version} (vhd_name=={self.spark.conf.get('spark.synapse.vhd.name')})"
region = self.spark.conf.get('spark.cluster.region')
Expand Down