Fix multiprocessing pool usage and remove stackoverflow hack

README.md

@@ -85,10 +85,14 @@ The API is simple and returns basic datastructures.
     )
     data = runner.run()
 
-    {
+    { 
-        'xyz.example.com' : [ 'any kind of datastructure is returnable' ],
+        'successful' : {
-        'foo.example.com' : None, # failed to connect,
+            'xyz.example.com' : [ 'any kind of datastructure is returnable' ],
-        ...
+            'foo.example.com' : [ '...' ]
+        },
+        'failed' : {
+            'bar.example.com' : [ 'failure message' ]
+        }
     }
 
 Additional options to Runner include the number of forks, hostname

bin/ansible-inventory

@@ -1,80 +0,0 @@
-#!/usr/bin/python
-
-# Copyright (c) 2012 Michael DeHaan <michael.dehaan@gmail.com>
-#
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation
-# files (the "Software"), to deal in the Software without restriction,
-# including without limitation the rights to use, copy, modify, merge,
-# publish, distribute, sublicense, and/or sell copies of the Software,
-# and to permit persons to whom the Software is furnished to do so,
-# subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
-# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
-# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-from optparse import OptionParser
-import json
-import os
-import ansible
-
-DEFAULT_HOST_LIST      = '/etc/ansible/hosts'
-DEFAULT_MODULE_PATH    = '/usr/share/ansible'
-DEFAULT_MODULE_NAME    = 'ping'
-DEFAULT_PATTERN        = '*'
-DEFAULT_FORKS          = 3
-DEFAULT_MODULE_ARGS    = ''
-DEFAULT_REMOTE_USER    = 'root'
-
-class Cli(object):
-
-    def __init__(self):
-        pass
-
-    def runner(self):
-        parser = OptionParser()
-        parser.add_option("-H", "--host-list", dest="host_list",
-            help="path to hosts list", default=DEFAULT_HOST_LIST)
-        parser.add_option("-L", "--library", dest="module_path",
-            help="path to module library", default=DEFAULT_MODULE_PATH)
-        parser.add_option("-f", "--forks", dest="forks",
-            help="level of parallelism", default=DEFAULT_FORKS)
-        parser.add_option("-n", "--name", dest="module_name",
-            help="module name to execute", default=DEFAULT_MODULE_NAME)
-        parser.add_option("-a", "--args", dest="module_args",
-            help="module arguments", default=DEFAULT_MODULE_ARGS)
-        parser.add_option("-p", "--pattern", dest="pattern",
-            help="hostname pattern", default=DEFAULT_PATTERN)
-        parser.add_option("-u", "--remote-user", dest="remote_user",
-            help="remote username", default=DEFAULT_REMOTE_USER)
-
-        options, args = parser.parse_args()
-
-        # TODO: more shell like splitting on module_args would
-        # be a good idea
-
-        return ansible.Runner(
-            module_name=options.module_name,
-            module_path=options.module_path,
-            module_args=options.module_args.split(' '),
-            remote_user=options.remote_user,
-            host_list=options.host_list,
-            forks=options.forks,
-            pattern=options.pattern,
-            verbose=False,
-        )
-
-if __name__ == '__main__':
-
-    result = Cli().runner().run()
-    print json.dumps(result, sort_keys=True, indent=4)
-
-

lib/ansible/__init__.py

@@ -20,7 +20,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import fnmatch
-from multiprocessing import Process, Pipe
+import multiprocessing
 from itertools import izip
 import os
 import json
@@ -39,24 +39,9 @@ DEFAULT_MODULE_ARGS    = ''
 DEFAULT_TIMEOUT        = 60
 DEFAULT_REMOTE_USER    = 'root'
 
-class Pooler(object):
+def _executor_hook(x):
-
+    (runner, host) = x
-    # credit: http://stackoverflow.com/questions/3288595/multiprocessing-using-pool-map-on-a-function-defined-in-a-class
+    return runner._executor(host)
-
-    @classmethod
-    def spawn(cls, f):
-        def fun(pipe,x):
-            pipe.send(f(x))
-            pipe.close()
-        return fun
-
-    @classmethod
-    def parmap(cls, f, X):
-        pipe=[Pipe() for x in X]
-        proc=[Process(target=cls.spawn(f),args=(c,x)) for x,(p,c) in izip(X,pipe)]
-        [p.start() for p in proc]
-        [p.join() for p in proc]
-        return [p.recv() for (p,c) in pipe]
 
 class Runner(object):
 
@@ -166,9 +151,9 @@ class Runner(object):
    def run(self):
        ''' xfer & run module on all matched hosts '''
        hosts = [ h for h in self.host_list if self._matches(h) ]
-       def executor(x):
+       pool = multiprocessing.Pool(self.forks)
-           return self._executor(x)
+       hosts = [ (self,x) for x in hosts ]
-       results = Pooler.parmap(executor, hosts)
+       results = pool.map(_executor_hook, hosts)
        results2 = {
           "successful" : {},
           "failed"     : {}