[stable-2.9] Explicitly use multiprocessing fork start method (#63581) (#63591)

* Explicitly use multiprocessing fork start method

* Remove unused import

* Remove unused import

* Fallback to just multiprocessing on py2
(cherry picked from commit 82ee341)

Co-authored-by: Matt Martz <matt@sivel.net>

changelogs/fragments/python38-macos.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- TaskQueueManager - Explicitly set the mutliprocessing start method to ``fork`` to avoid issues with the default on macOS now being ``spawn``.

lib/ansible/executor/process/worker.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import multiprocessing
 import os
 import sys
 import traceback
@@ -41,13 +40,14 @@ from ansible.executor.task_executor import TaskExecutor
 from ansible.executor.task_result import TaskResult
 from ansible.module_utils._text import to_text
 from ansible.utils.display import Display
+from ansible.utils.multiprocessing import context as multiprocessing_context
 
 __all__ = ['WorkerProcess']
 
 display = Display()
 
 
-class WorkerProcess(multiprocessing.Process):
+class WorkerProcess(multiprocessing_context.Process):
     '''
     The worker thread class, which uses TaskExecutor to run tasks
     read from a job queue and pushes results into a results queue

lib/ansible/executor/task_queue_manager.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import multiprocessing
 import os
 import tempfile
 
@@ -41,6 +40,7 @@ from ansible.utils.helpers import pct_to_int
 from ansible.vars.hostvars import HostVars
 from ansible.vars.reserved import warn_if_reserved
 from ansible.utils.display import Display
+from ansible.utils.multiprocessing import context as multiprocessing_context
 
 
 __all__ = ['TaskQueueManager']
@@ -97,7 +97,7 @@ class TaskQueueManager:
         self._unreachable_hosts = dict()
 
         try:
-            self._final_q = multiprocessing.Queue()
+            self._final_q = multiprocessing_context.Queue()
         except OSError as e:
             raise AnsibleError("Unable to use multiprocessing, this is normally caused by lack of access to /dev/shm: %s" % to_native(e))
 

lib/ansible/utils/multiprocessing.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2019 Matt Martz <matt@sivel.net>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import multiprocessing
+
+# Explicit multiprocessing context using the fork start method
+# This exists as a compat layer now that Python3.8 has changed the default
+# start method for macOS to ``spawn`` which is incompatible with our
+# code base currently
+#
+# This exists in utils to allow it to be easily imported into various places
+# without causing circular import or dependency problems
+try:
+    context = multiprocessing.get_context('fork')
+except AttributeError:
+    # Py2 has no context functionality, and only supports fork
+    context = multiprocessing