Skip to content

Commit f5933ad

Browse files
tazendtazend
committed
Backport: Rework Partition API (#296)
- Rework whole Partition API - avoid some duplicate logic in utils/uint.pyx - make the string "UNLIMITED" a constant - include new Partition API in the docs - add tests for new partition API
1 parent 3c46537 commit f5933ad

21 files changed

+1478
-83
lines changed

‎docs/reference/constants.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
title: constants
3+
---
4+
5+
::: pyslurm.constants
6+
handler: python
7+

‎docs/reference/old/partition.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
title: Partition
3+
---
4+
5+
!!! warning
6+
This class is superseded by [pyslurm.Partition](../partition.md) and will
7+
be removed in a future release.
8+
9+
::: pyslurm.partition
10+
handler: python

‎docs/reference/partition.md

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
title: Partition
33
---
44

5-
!!! warning
6-
This API is currently being completely reworked, and is subject to be
7-
removed in the future when a replacement is introduced
5+
!!! note
6+
This supersedes the [pyslurm.partition](old/partition.md) class, which
7+
will be removed in a future release
88

9-
::: pyslurm.partition
9+
::: pyslurm.Partition
10+
handler: python
11+
12+
::: pyslurm.Partitions
1013
handler: python

‎docs/reference/utilities.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
title: Utilities
2+
title: utils
33
---
44

55
::: pyslurm.utils

‎pyslurm/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
from pyslurm import utils
1616
from pyslurm import db
17+
from pyslurm import constants
1718

1819
from pyslurm.core.job import (
1920
Job,
@@ -23,6 +24,7 @@
2324
JobSubmitDescription,
2425
)
2526
from pyslurm.core.node import Node, Nodes
27+
from pyslurm.core.partition import Partition, Partitions
2628
from pyslurm.core import error
2729
from pyslurm.core.error import (
2830
PyslurmError,

‎pyslurm/constants.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#########################################################################
2+
# constants.py - pyslurm constants used throughout the project
3+
#########################################################################
4+
# Copyright (C) 2023 Toni Harzendorf <toni.harzendorf@gmail.com>
5+
# Copyright (C) 2023 PySlurm Developers
6+
#
7+
# This file is part of PySlurm
8+
#
9+
# PySlurm is free software; you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation; either version 2 of the License, or
12+
# (at your option) any later version.
13+
14+
# PySlurm is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License along
20+
# with PySlurm; if not, write to the Free Software Foundation, Inc.,
21+
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22+
#
23+
# cython: c_string_type=unicode, c_string_encoding=default
24+
# cython: language_level=3
25+
"""pyslurm common Constants"""
26+
27+
28+
UNLIMITED = "UNLIMITED"
29+
"""
30+
Represents an infinite/unlimited value. This is sometimes returned for
31+
specific attributes as a value to indicate that there is no restriction for it.
32+
"""

‎pyslurm/core/partition.pxd

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
#########################################################################
2+
# partition.pxd - interface to work with partitions in slurm
3+
#########################################################################
4+
# Copyright (C) 2023 Toni Harzendorf <toni.harzendorf@gmail.com>
5+
# Copyright (C) 2023 PySlurm Developers
6+
#
7+
# This file is part of PySlurm
8+
#
9+
# PySlurm is free software; you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation; either version 2 of the License, or
12+
# (at your option) any later version.
13+
14+
# PySlurm is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License along
20+
# with PySlurm; if not, write to the Free Software Foundation, Inc.,
21+
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22+
#
23+
# cython: c_string_type=unicode, c_string_encoding=default
24+
# cython: language_level=3
25+
26+
from libc.string cimport memcpy, memset
27+
from pyslurm cimport slurm
28+
from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t
29+
from pyslurm.slurm cimport (
30+
partition_info_msg_t,
31+
job_defaults_t,
32+
delete_part_msg_t,
33+
partition_info_t,
34+
update_part_msg_t,
35+
slurm_free_partition_info_members,
36+
slurm_free_partition_info_msg,
37+
slurm_free_update_part_msg,
38+
slurm_init_part_desc_msg,
39+
slurm_load_partitions,
40+
slurm_sprint_cpu_bind_type,
41+
cpu_bind_type_t,
42+
slurm_preempt_mode_string,
43+
slurm_preempt_mode_num,
44+
slurm_create_partition,
45+
slurm_update_partition,
46+
slurm_delete_partition,
47+
xfree,
48+
try_xmalloc,
49+
)
50+
from pyslurm.db.util cimport (
51+
SlurmList,
52+
SlurmListItem,
53+
)
54+
from pyslurm.utils cimport cstr
55+
from pyslurm.utils cimport ctime
56+
from pyslurm.utils.ctime cimport time_t
57+
from pyslurm.utils.uint cimport *
58+
from pyslurm.core cimport slurmctld
59+
60+
61+
cdef class Partitions(dict):
62+
"""A collection of [pyslurm.Partition][] objects.
63+
64+
Args:
65+
partitions (Union[list[str], dict[str, Partition], str], optional=None):
66+
Partitions to initialize this collection with.
67+
68+
Attributes:
69+
total_cpus (int):
70+
Total amount of CPUs the Partitions in a Collection have
71+
total_nodes (int):
72+
Total amount of Nodes the Partitions in a Collection have
73+
"""
74+
cdef:
75+
partition_info_msg_t *info
76+
partition_info_t tmp_info
77+
78+
79+
cdef class Partition:
80+
"""A Slurm partition.
81+
82+
??? info "Setting Memory related attributes"
83+
84+
Unless otherwise noted, all attributes in this class representing a
85+
memory value, like `default_memory_per_cpu`, may also be set with a
86+
string that contains suffixes like "K", "M", "G" or "T".
87+
88+
For example:
89+
90+
default_memory_per_cpu = "10G"
91+
92+
This will internally be converted to 10240 (how the Slurm API expects
93+
it)
94+
95+
Args:
96+
name (str, optional=None):
97+
Name of a Partition
98+
**kwargs (Any, optional=None):
99+
Every attribute of a Partition can be set, except for:
100+
101+
* total_cpus
102+
* total_nodes
103+
* select_type_parameters
104+
* consumable_resource
105+
106+
Attributes:
107+
name (str):
108+
Name of the Partition.
109+
allowed_submit_nodes (list[str]):
110+
List of Nodes from which Jobs can be submitted to the partition.
111+
allowed_accounts (list[str]):
112+
List of Accounts which are allowed to execute Jobs
113+
allowed_groups (list[str]):
114+
List of Groups which are allowed to execute Jobs
115+
allowed_qos (list[str]):
116+
List of QoS which are allowed to execute Jobs
117+
alternate (str):
118+
Name of the alternate Partition in case a Partition is down.
119+
consumable_resource (str):
120+
The type of consumable resource used in the Partition.
121+
select_type_parameters (list[str]):
122+
List of additional parameters passed to the select plugin used.
123+
cpu_binding (str):
124+
Default CPU-binding for Jobs that execute in a Partition.
125+
default_memory_per_cpu (int):
126+
Default Memory per CPU for Jobs in this Partition, in Mebibytes.
127+
Mutually exclusive with `default_memory_per_node`.
128+
129+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
130+
default_memory_per_node (int):
131+
Default Memory per Node for Jobs in this Partition, in Mebibytes.
132+
Mutually exclusive with `default_memory_per_cpu`.
133+
134+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
135+
max_memory_per_cpu (int):
136+
Max Memory per CPU allowed for Jobs in this Partition, in
137+
Mebibytes. Mutually exclusive with `max_memory_per_node`.
138+
139+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
140+
max_memory_per_node (int):
141+
Max Memory per Node allowed for Jobs in this Partition, in
142+
Mebibytes. Mutually exclusive with `max_memory_per_cpu`
143+
144+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
145+
default_time (int):
146+
Default run time-limit in minutes for Jobs that don't specify one.
147+
148+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
149+
denied_qos (list[str]):
150+
List of QoS that cannot be used in a Partition
151+
denied_accounts (list[str]):
152+
List of Accounts that cannot use a Partition
153+
preemption_grace_time (int):
154+
Grace Time in seconds when a Job is selected for Preemption.
155+
default_cpus_per_gpu (int):
156+
Default CPUs per GPU for Jobs in this Partition
157+
default_memory_per_gpu (int):
158+
Default Memory per GPU, in Mebibytes, for Jobs in this Partition
159+
max_cpus_per_node (int):
160+
Max CPUs per Node allowed for Jobs in this Partition
161+
162+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
163+
max_cpus_per_socket (int):
164+
Max CPUs per Socket allowed for Jobs in this Partition
165+
166+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
167+
max_nodes (int):
168+
Max number of Nodes allowed for Jobs
169+
170+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
171+
min_nodes (int):
172+
Minimum number of Nodes that must be requested by Jobs
173+
max_time_limit (int):
174+
Max Time-Limit in minutes that Jobs can request
175+
176+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
177+
oversubscribe (str):
178+
The oversubscribe mode for this Partition
179+
nodes (str):
180+
Nodes that are in a Partition
181+
nodesets (list[str]):
182+
List of Nodesets that a Partition has configured
183+
over_time_limit (int):
184+
Limit in minutes that Jobs can exceed their time-limit
185+
186+
This can also return [UNLIMITED][pyslurm.constants.UNLIMITED]
187+
preempt_mode (str):
188+
Preemption Mode in a Partition
189+
priority_job_factor (int):
190+
The Priority Job Factor for a partition
191+
priority_tier (int):
192+
The priority tier for a Partition
193+
qos (str):
194+
A QoS associated with a Partition, used to extend possible limits
195+
total_cpus (int):
196+
Total number of CPUs available in a Partition
197+
total_nodes (int):
198+
Total number of nodes available in a Partition
199+
state (str):
200+
State the Partition is in
201+
is_default (bool):
202+
Whether this Partition is the default partition or not
203+
allow_root_jobs (bool):
204+
Whether Jobs by the root user are allowed
205+
is_user_exclusive (bool):
206+
Whether nodes will be exclusively allocated to users
207+
is_hidden (bool):
208+
Whether the partition is hidden or not
209+
least_loaded_nodes_scheduling (bool):
210+
Whether Least-Loaded-Nodes scheduling algorithm is used on a
211+
Partition
212+
is_root_only (bool):
213+
Whether only root is able to use a Partition
214+
requires_reservation (bool):
215+
Whether a reservation is required to use a Partition
216+
"""
217+
cdef:
218+
partition_info_t *ptr
219+
int power_save_enabled
220+
slurmctld.Config slurm_conf
221+
222+
@staticmethod
223+
cdef Partition from_ptr(partition_info_t *in_ptr)

0 commit comments

Comments
 (0)