Skip to content

Commit 71cf611

Browse files
JoshKarpelJoshKarpel
authored
Merge pull request #6 from htcondor/new-user-tutorial
Add new user-focused job submission and management tutorial
2 parents c354139 + fef06fa commit 71cf611

File tree

3 files changed

+771
-171
lines changed

3 files changed

+771
-171
lines changed

‎tutorials/index.ipynb

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,15 @@
1818
"They utilize the same C++ libraries as HTCondor itself, meaning they have nearly the same behavior as the command line tools.\n",
1919
"\n",
2020
"Here, you will learn the basics of the Python bindings and how to use them. \n",
21-
"This tutorial is broken down into two major sections:\n",
21+
"The tutorials are broken down into a few major sections:\n",
2222
"\n",
23-
"- **Introduction**, a quick overview of the major components. Each learning module is meant to be done in sequence. Start here if you have never used the bindings before.\n",
24-
"- **Advanced**, an in-depth examination of the nooks and crannies of the system. Each module is standalone; read only those that look interesting to you.\n",
23+
"- **Users**, an guide to the features that users who want to submit and manage jobs will find most useful. Start here if you're a user who wants to submit jobs from Python.\n",
24+
"- **Introduction**, a quick overview of the major components for administrators and advanced users. Start here if you want to know where all the bits and pieces are. Each learning module is meant to be done in sequence.\n",
25+
"- **Advanced**, an in-depth examination of the nooks and crannies of the system. Each module is standalone; read only those that look interesting to you.\n",
26+
"\n",
27+
"## Users\n",
28+
"\n",
29+
"1. [Submitting and Managing Jobs](users/Submitting-and-Managing-Jobs.ipynb) - How to submit and manage HTCondor jobs from Python.\n",
2530
"\n",
2631
"## Introduction\n",
2732
"\n",
@@ -35,6 +40,13 @@
3540
"1. [Interacting with daemons](advanced/Interacting-With-Daemons.ipynb) - Generic commands that work with any HTCondor daemon.\n",
3641
"1. [Scalable job tracking](advanced/Scalable-Job-Tracking.ipynb) - Techniques for keeping close track of many jobs without overloading the schedd.\n"
3742
]
43+
},
44+
{
45+
"cell_type": "code",
46+
"execution_count": null,
47+
"metadata": {},
48+
"outputs": [],
49+
"source": []
3850
}
3951
],
4052
"metadata": {
@@ -54,8 +66,17 @@
5466
"nbconvert_exporter": "python",
5567
"pygments_lexer": "ipython3",
5668
"version": "3.7.3"
69+
},
70+
"pycharm": {
71+
"stem_cell": {
72+
"cell_type": "raw",
73+
"metadata": {
74+
"collapsed": false
75+
},
76+
"source": []
77+
}
5778
}
5879
},
5980
"nbformat": 4,
60-
"nbformat_minor": 2
81+
"nbformat_minor": 4
6182
}

‎tutorials/introductory/Submitting-and-Managing-Jobs.ipynb

Lines changed: 23 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
},
1616
{
1717
"cell_type": "code",
18-
"execution_count": 1,
18+
"execution_count": null,
1919
"metadata": {
2020
"pycharm": {}
2121
},
@@ -42,26 +42,11 @@
4242
},
4343
{
4444
"cell_type": "code",
45-
"execution_count": 2,
45+
"execution_count": null,
4646
"metadata": {
4747
"pycharm": {}
4848
},
49-
"outputs": [
50-
{
51-
"name": "stdout",
52-
"output_type": "stream",
53-
"text": [
54-
"=== START ===\n",
55-
"foo = 1\n",
56-
"bar = 2\n",
57-
"baz = $(foo)\n",
58-
"qux = 3\n",
59-
"\n",
60-
"=== END ===\n",
61-
"1\n"
62-
]
63-
}
64-
],
49+
"outputs": [],
6550
"source": [
6651
"sub = htcondor.Submit({\"foo\": \"1\", \"bar\": \"2\", \"baz\": \"$(foo)\"})\n",
6752
"sub.setdefault(\"qux\", \"3\")\n",
@@ -80,7 +65,7 @@
8065
},
8166
{
8267
"cell_type": "code",
83-
"execution_count": 3,
68+
"execution_count": null,
8469
"metadata": {
8570
"pycharm": {}
8671
},
@@ -106,19 +91,11 @@
10691
},
10792
{
10893
"cell_type": "code",
109-
"execution_count": 4,
94+
"execution_count": null,
11095
"metadata": {
11196
"pycharm": {}
11297
},
113-
"outputs": [
114-
{
115-
"name": "stdout",
116-
"output_type": "stream",
117-
"text": [
118-
"2\n"
119-
]
120-
}
121-
],
98+
"outputs": [],
12299
"source": [
123100
"schedd = htcondor.Schedd() # Create a schedd object using default settings.\n",
124101
"with schedd.transaction() as txn: # txn will now represent the transaction.\n",
@@ -138,17 +115,9 @@
138115
},
139116
{
140117
"cell_type": "code",
141-
"execution_count": 5,
118+
"execution_count": null,
142119
"metadata": {},
143-
"outputs": [
144-
{
145-
"name": "stdout",
146-
"output_type": "stream",
147-
"text": [
148-
"3\n"
149-
]
150-
}
151-
],
120+
"outputs": [],
152121
"source": [
153122
"schedd = htcondor.Schedd() # Create a fresh Schedd object, pointint at the current schedd.\n",
154123
"with schedd.transaction() as txn: # Start a new transaction\n",
@@ -165,29 +134,9 @@
165134
},
166135
{
167136
"cell_type": "code",
168-
"execution_count": 6,
137+
"execution_count": null,
169138
"metadata": {},
170-
"outputs": [
171-
{
172-
"data": {
173-
"text/plain": [
174-
"[[ ClusterId = 3; ProcId = 0; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
175-
" [ ClusterId = 3; ProcId = 1; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
176-
" [ ClusterId = 3; ProcId = 2; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
177-
" [ ClusterId = 3; ProcId = 3; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
178-
" [ ClusterId = 3; ProcId = 4; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
179-
" [ ClusterId = 3; ProcId = 5; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
180-
" [ ClusterId = 3; ProcId = 6; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
181-
" [ ClusterId = 3; ProcId = 7; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
182-
" [ ClusterId = 3; ProcId = 8; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ],\n",
183-
" [ ClusterId = 3; ProcId = 9; EnteredCurrentStatus = 1563826270; JobStatus = 1; ServerTime = 1563826270 ]]"
184-
]
185-
},
186-
"execution_count": 6,
187-
"metadata": {},
188-
"output_type": "execute_result"
189-
}
190-
],
139+
"outputs": [],
191140
"source": [
192141
"schedd.query(constraint='ClusterId=?={}'.format(cluster_id),\n",
193142
" attr_list=[\"ClusterId\", \"ProcId\", \"JobStatus\", \"EnteredCurrentStatus\"])"
@@ -204,7 +153,7 @@
204153
},
205154
{
206155
"cell_type": "code",
207-
"execution_count": 7,
156+
"execution_count": null,
208157
"metadata": {},
209158
"outputs": [],
210159
"source": [
@@ -220,36 +169,9 @@
220169
},
221170
{
222171
"cell_type": "code",
223-
"execution_count": 8,
172+
"execution_count": null,
224173
"metadata": {},
225-
"outputs": [
226-
{
227-
"name": "stdout",
228-
"output_type": "stream",
229-
"text": [
230-
"4\n"
231-
]
232-
},
233-
{
234-
"data": {
235-
"text/plain": [
236-
"[[ Args = \"0m\"; ClusterId = 4; ProcId = 0; JobStatus = 1; ServerTime = 1563826271 ],\n",
237-
" [ Args = \"1m\"; ClusterId = 4; ProcId = 1; JobStatus = 1; ServerTime = 1563826271 ],\n",
238-
" [ Args = \"2m\"; ClusterId = 4; ProcId = 2; JobStatus = 1; ServerTime = 1563826271 ],\n",
239-
" [ Args = \"3m\"; ClusterId = 4; ProcId = 3; JobStatus = 1; ServerTime = 1563826271 ],\n",
240-
" [ Args = \"4m\"; ClusterId = 4; ProcId = 4; JobStatus = 1; ServerTime = 1563826271 ],\n",
241-
" [ Args = \"5m\"; ClusterId = 4; ProcId = 5; JobStatus = 1; ServerTime = 1563826271 ],\n",
242-
" [ Args = \"6m\"; ClusterId = 4; ProcId = 6; JobStatus = 1; ServerTime = 1563826271 ],\n",
243-
" [ Args = \"7m\"; ClusterId = 4; ProcId = 7; JobStatus = 1; ServerTime = 1563826271 ],\n",
244-
" [ Args = \"8m\"; ClusterId = 4; ProcId = 8; JobStatus = 1; ServerTime = 1563826271 ],\n",
245-
" [ Args = \"9m\"; ClusterId = 4; ProcId = 9; JobStatus = 1; ServerTime = 1563826271 ]]"
246-
]
247-
},
248-
"execution_count": 8,
249-
"metadata": {},
250-
"output_type": "execute_result"
251-
}
252-
],
174+
"outputs": [],
253175
"source": [
254176
"with schedd.transaction() as txn: # Start a new transaction\n",
255177
" cluster_id = sub.queue(txn, count=10) # Submit 10 identical jobs\n",
@@ -275,7 +197,7 @@
275197
},
276198
{
277199
"cell_type": "code",
278-
"execution_count": 9,
200+
"execution_count": null,
279201
"metadata": {},
280202
"outputs": [],
281203
"source": [
@@ -298,26 +220,9 @@
298220
},
299221
{
300222
"cell_type": "code",
301-
"execution_count": 10,
223+
"execution_count": null,
302224
"metadata": {},
303-
"outputs": [
304-
{
305-
"name": "stdout",
306-
"output_type": "stream",
307-
"text": [
308-
"{'filename': 'job_0.txt'}\n",
309-
"{'filename': 'job_1.txt'}\n",
310-
"{'filename': 'job_2.txt'}\n",
311-
"{'filename': 'job_3.txt'}\n",
312-
"{'filename': 'job_4.txt'}\n",
313-
"{'filename': 'job_5.txt'}\n",
314-
"{'filename': 'job_6.txt'}\n",
315-
"{'filename': 'job_7.txt'}\n",
316-
"{'filename': 'job_8.txt'}\n",
317-
"{'filename': 'job_9.txt'}\n"
318-
]
319-
}
320-
],
225+
"outputs": [],
321226
"source": [
322227
"sub = htcondor.Submit({\"executable\": \"/bin/cat\"})\n",
323228
"sub[\"arguments\"] = \"$(filename)\"\n",
@@ -333,17 +238,9 @@
333238
},
334239
{
335240
"cell_type": "code",
336-
"execution_count": 11,
241+
"execution_count": null,
337242
"metadata": {},
338-
"outputs": [
339-
{
340-
"name": "stdout",
341-
"output_type": "stream",
342-
"text": [
343-
"5\n"
344-
]
345-
}
346-
],
243+
"outputs": [],
347244
"source": [
348245
"with schedd.transaction() as txn:\n",
349246
" # Submit one job per entry in the iterator.\n",
@@ -366,29 +263,9 @@
366263
},
367264
{
368265
"cell_type": "code",
369-
"execution_count": 12,
266+
"execution_count": null,
370267
"metadata": {},
371-
"outputs": [
372-
{
373-
"data": {
374-
"text/plain": [
375-
"[[ Out = \"results.0\"; JobStatus = 1; TransferInput = \"input_directory/job_0.txt\"; ServerTime = 1563826277; Args = \"job_0.txt\"; ClusterId = 5; ProcId = 0 ],\n",
376-
" [ Out = \"results.1\"; JobStatus = 1; TransferInput = \"input_directory/job_1.txt\"; ServerTime = 1563826277; Args = \"job_1.txt\"; ClusterId = 5; ProcId = 1 ],\n",
377-
" [ Out = \"results.2\"; JobStatus = 1; TransferInput = \"input_directory/job_2.txt\"; ServerTime = 1563826277; Args = \"job_2.txt\"; ClusterId = 5; ProcId = 2 ],\n",
378-
" [ Out = \"results.3\"; JobStatus = 1; TransferInput = \"input_directory/job_3.txt\"; ServerTime = 1563826277; Args = \"job_3.txt\"; ClusterId = 5; ProcId = 3 ],\n",
379-
" [ Out = \"results.4\"; JobStatus = 1; TransferInput = \"input_directory/job_4.txt\"; ServerTime = 1563826277; Args = \"job_4.txt\"; ClusterId = 5; ProcId = 4 ],\n",
380-
" [ Out = \"results.5\"; JobStatus = 1; TransferInput = \"input_directory/job_5.txt\"; ServerTime = 1563826277; Args = \"job_5.txt\"; ClusterId = 5; ProcId = 5 ],\n",
381-
" [ Out = \"results.6\"; JobStatus = 1; TransferInput = \"input_directory/job_6.txt\"; ServerTime = 1563826277; Args = \"job_6.txt\"; ClusterId = 5; ProcId = 6 ],\n",
382-
" [ Out = \"results.7\"; JobStatus = 1; TransferInput = \"input_directory/job_7.txt\"; ServerTime = 1563826277; Args = \"job_7.txt\"; ClusterId = 5; ProcId = 7 ],\n",
383-
" [ Out = \"results.8\"; JobStatus = 1; TransferInput = \"input_directory/job_8.txt\"; ServerTime = 1563826277; Args = \"job_8.txt\"; ClusterId = 5; ProcId = 8 ],\n",
384-
" [ Out = \"results.9\"; JobStatus = 1; TransferInput = \"input_directory/job_9.txt\"; ServerTime = 1563826277; Args = \"job_9.txt\"; ClusterId = 5; ProcId = 9 ]]"
385-
]
386-
},
387-
"execution_count": 12,
388-
"metadata": {},
389-
"output_type": "execute_result"
390-
}
391-
],
268+
"outputs": [],
392269
"source": [
393270
"schedd.query(constraint='ClusterId=?={}'.format(results.cluster()),\n",
394271
" attr_list=[\"ClusterId\", \"ProcId\", \"JobStatus\", \"TransferInput\", \"Out\", \"Args\"])"
@@ -428,32 +305,11 @@
428305
},
429306
{
430307
"cell_type": "code",
431-
"execution_count": 13,
308+
"execution_count": null,
432309
"metadata": {
433310
"pycharm": {}
434311
},
435-
"outputs": [
436-
{
437-
"name": "stdout",
438-
"output_type": "stream",
439-
"text": [
440-
"=== START JOB STATUS ===\n",
441-
"0: foo=bar, job_status = 1\n",
442-
"1: foo=bar, job_status = 1\n",
443-
"2: foo=default_string, job_status = 1\n",
444-
"3: foo=default_string, job_status = 1\n",
445-
"4: foo=default_string, job_status = 1\n",
446-
"=== END ===\n",
447-
"=== START JOB STATUS ===\n",
448-
"0: foo=bar, job_status = 1\n",
449-
"1: foo=bar, job_status = 1\n",
450-
"2: foo=default_string, job_status = 5\n",
451-
"3: foo=default_string, job_status = 5\n",
452-
"4: foo=default_string, job_status = 5\n",
453-
"=== END ===\n"
454-
]
455-
}
456-
],
312+
"outputs": [],
457313
"source": [
458314
"with schedd.transaction() as txn:\n",
459315
" clusterId = sub.queue(txn, 5) # Queues 5 copies of this job.\n",
@@ -515,5 +371,5 @@
515371
}
516372
},
517373
"nbformat": 4,
518-
"nbformat_minor": 2
374+
"nbformat_minor": 4
519375
}

0 commit comments

Comments
 (0)