first commit

Author: SuperFastPython

.gitignore

@@ -0,0 +1,3 @@
+# git ignore
+**/.DS_Store
+.DS_Store

README.md

@@ -0,0 +1,43 @@
+# Python Multiprocessing Pool Jump-Start
+
+![Python Multiprocessing Pool Jump-Start](cover.png)
+
+* <https://github.com/SuperFastPython/PythonMultiprocessingPoolJumpStart>
+
+This repository provides all source code for the book:
+
+* *Python Multiprocessing Pool Jump-Start*: _Run Your Python Functions In Parallel With Just A Few Lines Of Code_, Jason Brownlee, 2022.
+
+You can access all Python .py files directly here:
+
+* [src/])(src/)
+
+You can learn more about the book here:
+
+* [GumRoad](https://superfastpython.gumroad.com/l/pmpj)
+* [GoodReads](https://www.goodreads.com/book/show/61606103-python-multiprocessing-pool-jump-start)
+
+Blurb
+
+	How much faster can your python code run (if it used all CPU cores)?
+
+	The multiprocessing.Pool class provides easy-to-use process-based concurrency.
+
+	This is not some random third-party library, this is a class provided in the Python standard library (already installed on your system).
+
+	This is the class you need to use to make your code run faster.
+
+	There's just one problem. No one knows about it (or how to use it well).
+
+	Introducing: "Python Multiprocessing Pool Jump-Start". A new book designed to teach you multiprocessing pools in Python, super fast!
+
+	You will get a fast-paced, 7-part course to get you started and make you awesome at using the multiprocessing pool.
+
+	Each of the 7 lessons was carefully designed to teach one critical aspect of the multiprocessing pool, with explanations, code snippets and worked examples.
+
+	Each lesson ends with an exercise for you to complete to confirm you understood the topic, a summary of what was learned, and links for further reading if you want to go deeper.
+
+	Stop copy-pasting code from outdated blog posts.
+	Stop trying to parse messy StackOverflow answers.
+
+	Learn Python concurrency correctly, step-by-step.

cover.png

@@ -0,0 +1,18 @@
+# SuperFastPython.com
+# example of running a function in the multiprocessing pool
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task():
+    print('This is another process', flush=True)
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    pool = Pool()
+    # issue the task
+    async_result = pool.apply_async(task)
+    # wait for the task to finish
+    async_result.wait()
+    # close the multiprocessing pool
+    pool.close()

src/lesson01_pool.py

@@ -0,0 +1,16 @@
+# SuperFastPython.com
+# example of running a function in a new process
+from multiprocessing import Process
+
+# a task to execute in another process
+def task():
+    print('This is another process', flush=True)
+
+# entry point for the program
+if __name__ == '__main__':
+    # define a task to run in a new process
+    process = Process(target=task)
+    # start the task in a new process
+    process.start()
+    # wait for the task to complete
+    process.join()

src/lesson01_process.py

@@ -0,0 +1,12 @@
+# SuperFastPython.com
+# example of reporting the details of a default multiprocessing pool
+from multiprocessing import Pool
+
+# protect the entry point
+if __name__ == '__main__':
+    # create a multiprocessing pool with the default number of workers
+    pool = Pool()
+    # report the status of the multiprocessing pool
+    print(pool)
+    # close the multiprocessing pool
+    pool.close()

src/lesson02_default.py

@@ -0,0 +1,28 @@
+# SuperFastPython.com
+# example of initializing worker processes in the multiprocessing pool
+from time import sleep
+from multiprocessing import Pool
+
+# task executed in a worker process
+def task():
+    # report a message
+    print('Worker executing task...', flush=True)
+    # block for a moment
+    sleep(1)
+
+# initialize a worker in the multiprocessing pool
+def initialize_worker():
+    # report a message
+    print('Initializing worker...', flush=True)
+
+# protect the entry point
+if __name__ == '__main__':
+    # create and configure the multiprocessing pool
+    with Pool(2, initializer=initialize_worker) as pool:
+        # issue tasks to the multiprocessing pool
+        for _ in range(4):
+            pool.apply_async(task)
+        # close the multiprocessing pool
+        pool.close()
+        # wait for all tasks to complete
+        pool.join()

src/lesson02_initalizer.py

@@ -0,0 +1,14 @@
+# SuperFastPython.com
+# example of executing a task with Pool.apply()
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task():
+    print('This is another process', flush=True)
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # issue a task and wait for it to complete
+        pool.apply(task)

src/lesson03_apply.py

@@ -0,0 +1,18 @@
+# SuperFastPython.com
+# example of executing multiple task with Pool.map()
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg):
+    # report a message
+    print(f'This is another process with {arg}', flush=True)
+    # return a value
+    return arg * 2
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # issue multiple tasks to the pool and process return values
+        for result in pool.map(task, range(10)):
+            print(result)

src/lesson03_map.py

@@ -0,0 +1,20 @@
+# SuperFastPython.com
+# example of executing multiple tasks with multiple arguments
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg1, arg2, arg3):
+    # report a message
+    print(f'This is another process with {arg1}, {arg2}, {arg3}', flush=True)
+    # return a value
+    return arg1 + arg2 + arg3
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # prepare task arguments
+        args = [(i, i*2, i*3) for i in range(10)]
+        # issue multiple tasks to the pool and process return values
+        for result in pool.starmap(task, args):
+            print(result)

src/lesson03_starmap.py

@@ -0,0 +1,16 @@
+# SuperFastPython.com
+# example of executing a task with Pool.apply_async()
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task():
+    print('This is another process', flush=True)
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # issue a task asynchronously
+        async_result = pool.apply_async(task)
+        # wait for the task to complete
+        async_result.wait()

src/lesson04_apply_async.py

@@ -0,0 +1,20 @@
+# SuperFastPython.com
+# example of executing multiple task with Pool.map_async()
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg):
+    # report a message
+    print(f'This is another process with {arg}', flush=True)
+    # return a value
+    return arg * 2
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # issue multiple tasks to the pool
+        async_result = pool.map_async(task, range(10))
+        # process return values once all issued tasks have completed
+        for result in async_result.get():
+            print(result)

src/lesson04_map_async.py

@@ -0,0 +1,22 @@
+# SuperFastPython.com
+# example of executing multiple tasks with multiple arguments
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg1, arg2, arg3):
+    # report a message
+    print(f'This is another process with {arg1}, {arg2}, {arg3}', flush=True)
+    # return a value
+    return arg1 + arg2 + arg3
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # prepare task arguments
+        args = [(i, i*2, i*3) for i in range(10)]
+        # issue multiple tasks to the pool
+        async_result = pool.starmap_async(task, args)
+        # process return values
+        for result in async_result.get():
+            print(result)

src/lesson04_starmap_async.py

@@ -0,0 +1,22 @@
+# SuperFastPython.com
+# example of executing multiple task with Pool.imap()
+from time import sleep
+from random import random
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg):
+    # block for a random fraction of a second
+    sleep(random())
+    # report a message
+    print(f'This is another process with {arg}', flush=True)
+    # return a value
+    return arg * 2
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool(4) as pool:
+        # issue multiple tasks to the pool and process return values
+        for result in pool.imap(task, range(10)):
+            print(result)

src/lesson05_imap.py

@@ -0,0 +1,22 @@
+# SuperFastPython.com
+# example of executing multiple task with Pool.imap_unordered()
+from time import sleep
+from random import random
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task(arg):
+    # block for a random fraction of a second
+    sleep(random())
+    # report a message
+    print(f'This is another process with {arg}', flush=True)
+    # return a value
+    return arg * 2
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool(4) as pool:
+        # issue multiple tasks to the pool and process return values
+        for result in pool.imap_unordered(task, range(10)):
+            print(result)

src/lesson05_imap_unordered.py

@@ -0,0 +1,32 @@
+# SuperFastPython.com
+# example of checking the status and handling the result of an async task
+from time import sleep
+from random import random
+from multiprocessing import Pool
+
+# a task to execute in another process
+def task():
+    # loop a few times to simulate a slow task
+    for i in range(10):
+        # generate a random value between 0 and 1
+        value = random()
+        # block for a fraction of a second
+        sleep(value)
+        # report a message
+        print(f'>{i} got {value}', flush=True)
+
+# entry point for the program
+if __name__ == '__main__':
+    # create the multiprocessing pool
+    with Pool() as pool:
+        # issue a task asynchronously
+        async_result = pool.apply_async(task)
+        # wait until the task is complete
+        while not async_result.ready():
+            # report a message
+            print('Main process waiting...')
+            # block for a moment
+            async_result.wait(timeout=1)
+        # report if the task was successful
+        if async_result.successful():
+            print('Task was successful.')

src/lesson06_asyncresult.py

@@ -0,0 +1,31 @@
+# SuperFastPython.com
+# example of a callback function for Pool.apply_async()
+from random import random
+from time import sleep
+from multiprocessing.pool import Pool
+
+# result callback function
+def result_callback(return_value):
+    print(f'Callback got: {return_value}', flush=True)
+
+# task executed in a worker process
+def task(identifier):
+    # generate a value
+    value = random()
+    # report a message
+    print(f'Task {identifier} executing with {value}', flush=True)
+    # block for a moment
+    sleep(value)
+    # return the generated value
+    return value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create and configure the multiprocessing pool
+    with Pool() as pool:
+        # issue tasks to the multiprocessing pool
+        result = pool.apply_async(task, args=(0,), callback=result_callback)
+        # close the multiprocessing pool
+        pool.close()
+        # wait for all tasks to complete
+        pool.join()