Merge pull request #9451 from bcoca/load_aliases

Load aliases

bin/ansible-doc

@@ -34,6 +34,7 @@ import traceback
 MODULEDIR = C.DEFAULT_MODULE_PATH
 
 BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm')
+IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README" ]
 
 _ITALIC = re.compile(r"I\(([^)]+)\)")
 _BOLD   = re.compile(r"B\(([^)]+)\)")
@@ -164,7 +165,11 @@ def get_snippet_text(doc):
     return "\n".join(text)
 
 def get_module_list_text(module_list):
+    columns = max(60, int(os.popen('stty size', 'r').read().split()[1]))
+    displace = max(len(x) for x in module_list)
+    linelimit = columns - displace - 5
     text = []
+    deprecated = []
     for module in sorted(set(module_list)):
 
         if module in module_docs.BLACKLIST_MODULES:
@@ -181,15 +186,45 @@ def get_module_list_text(module_list):
 
         try:
             doc, plainexamples = module_docs.get_docstring(filename)
-            desc = tty_ify(doc.get('short_description', '?'))
+            desc = tty_ify(doc.get('short_description', '?')).strip()
-            if len(desc) > 55:
+            if len(desc) > linelimit:
-                desc = desc + '...'
+                desc = desc[:linelimit] + '...'
-            text.append("%-20s %-60.60s" % (module, desc))
+
+            if module.startswith('_'): # Handle deprecated
+                deprecated.append("%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc))
+            else:
+                text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc))
         except:
             traceback.print_exc()
             sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module)
+
+    if len(deprecated) > 0:
+        text.append("\nDEPRECATED:")
+        text.extend(deprecated)
     return "\n".join(text)
 
+def find_modules(path, module_list):
+
+    if os.path.isdir(path):
+        for module in os.listdir(path):
+            if module.startswith('.'):
+                continue
+            elif os.path.isdir(module):
+                find_modules(module, module_list)
+            elif any(module.endswith(x) for x in BLACKLIST_EXTS):
+                continue
+            elif module.startswith('__'):
+                continue
+            elif module in IGNORE_FILES:
+                continue
+            elif module.startswith('_'):
+                fullpath = '/'.join([path,module])
+                if os.path.islink(fullpath): # avoids aliases
+                    continue
+
+            module = os.path.splitext(module)[0] # removes the extension
+            module_list.append(module)
+
 def main():
 
     p = optparse.OptionParser(
@@ -222,16 +257,11 @@ def main():
             utils.plugins.module_finder.add_directory(i)
 
     if options.list_dir:
-        # list all modules
+        # list modules
         paths = utils.plugins.module_finder._get_paths()
         module_list = []
         for path in paths:
-            # os.system("ls -C %s" % (path))
+            find_modules(path, module_list)
-            if os.path.isdir(path):
-                for module in os.listdir(path):
-                    if any(module.endswith(x) for x in BLACKLIST_EXTS):
-                        continue
-                    module_list.append(module)
 
         pager(get_module_list_text(module_list))
         sys.exit()
@@ -254,8 +284,7 @@ def main():
 
         filename = utils.plugins.module_finder.find_plugin(module)
         if filename is None:
-            sys.stderr.write("module %s not found in %s\n" % (module,
+            sys.stderr.write("module %s not found in %s\n" % (module, print_paths(utils.plugins.module_finder)))
-                    print_paths(utils.plugins.module_finder)))
             continue
 
         if any(filename.endswith(x) for x in BLACKLIST_EXTS):

docsite/rst/developing_modules.rst

@@ -465,6 +465,23 @@ a github pull request to the `extras <https://github.com/ansible/ansible-modules
 Included modules will ship with ansible, and also have a change to be promoted to 'core' status, which
 gives them slightly higher development priority (though they'll work in exactly the same way).
 
+
+Deprecating and making module aliases
+``````````````````````````````````````
+
+Starting in 1.8 you can deprecate modules by renaming them with a preceeding _, i.e. old_cloud.py to 
+_old_cloud.py, This will keep the module available but hide it from the primary docs and listing.
+
+You can also rename modules and keep an alias to the old name by using a symlink that starts with _.
+This example allows the stat module to be called with fileinfo, making the following examples equivalent
+
+    EXAMPLES = '''
+    ln -s stat.py _fileinfo.py
+    ansible -m stat -a "path=/tmp" localhost
+    ansible -m fileinfo -a "path=/tmp" localhost
+    '''
+
+
 .. seealso::
 
    :doc:`modules`

hacking/module_formatter.py

@@ -59,6 +59,8 @@ _MODULE = re.compile(r"M\(([^)]+)\)")
 _URL    = re.compile(r"U\(([^)]+)\)")
 _CONST  = re.compile(r"C\(([^)]+)\)")
 
+DEPRECATED = " (D)"
+NOTCORE    = " (E)"
 #####################################################################################
 
 def rst_ify(text):
@@ -118,28 +120,53 @@ def write_data(text, options, outputname, module):
 #####################################################################################
 
 
-def list_modules(module_dir):
+def list_modules(module_dir, depth=0):
     ''' returns a hash of categories, each category being a hash of module names to file paths '''
 
-    categories = dict(all=dict())
+    categories = dict(all=dict(),_aliases=dict())
-    files = glob.glob("%s/*/*" % module_dir)
+    if depth <= 3: # limit # of subdirs
-    for d in files:
+
-        if os.path.isdir(d):
+        files = glob.glob("%s/*" % module_dir)
-            files2 = glob.glob("%s/*" % d)
+        for d in files:
-            for f in files2:
+
-
+            category = os.path.splitext(os.path.basename(d))[0]
-                if not f.endswith(".py") or f.endswith('__init__.py'):
+            if os.path.isdir(d):
+
+                res = list_modules(d, depth + 1)
+                for key in res.keys():
+                    if key in categories:
+                        categories[key].update(res[key])
+                        res.pop(key, None)
+
+                if depth < 2:
+                    categories.update(res)
+                else:
+                    category = module_dir.split("/")[-1]
+                    if not category in categories:
+                        categories[category] = res
+                    else:
+                        categories[category].update(res)
+            else:
+                module = category
+                category = os.path.basename(module_dir)
+                if not d.endswith(".py") or d.endswith('__init__.py'):
                     # windows powershell modules have documentation stubs in python docstring
                     # format (they are not executed) so skip the ps1 format files
                     continue
+                elif module.startswith("_") and os.path.islink(d):
+                    source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0]
+                    module = module.replace("_","",1)
+                    if not d in categories['_aliases']:
+                        categories['_aliases'][source] = [module]
+                    else:
+                        categories['_aliases'][source].update(module)
+                    continue
 
-                tokens = f.split("/")
-                module = tokens[-1].replace(".py","")
-                category = tokens[-2]
                 if not category in categories:
                     categories[category] = {}
-                categories[category][module] = f
+                categories[category][module] = d
-                categories['all'][module] = f
+                categories['all'][module] = d
+
     return categories
 
 #####################################################################################
@@ -188,33 +215,48 @@ def jinja2_environment(template_dir, typ):
 
 #####################################################################################
 
-def process_module(module, options, env, template, outputname, module_map):
+def process_module(module, options, env, template, outputname, module_map, aliases):
-
-    print "rendering: %s" % module
-
 
     fname = module_map[module]
+    if isinstance(fname, dict):
+        return "SKIPPED"
+
+    basename = os.path.basename(fname)
+    deprecated = False
 
     # ignore files with extensions
-    if not os.path.basename(fname).endswith(".py"):
+    if not basename.endswith(".py"):
         return
+    elif module.startswith("_"):
+        if os.path.islink(fname):
+            return  # ignore, its an alias
+        deprecated = True
+        module = module.replace("_","",1)
+
+    print "rendering: %s" % module
 
     # use ansible core library to parse out doc metadata YAML and plaintext examples
     doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)
 
     # crash if module is missing documentation and not explicitly hidden from docs index
-    if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES:
-        sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
-        sys.exit(1)
-
     if doc is None:
-        return "SKIPPED"
+        if module in ansible.utils.module_docs.BLACKLIST_MODULES:
+            return "SKIPPED"
+        else:
+            sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
+            sys.exit(1)
+
+    if deprecated and 'deprecated' not in doc:
+        sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
+        sys.exit(1)
 
     if "/core/" in fname:
         doc['core'] = True
     else:
         doc['core'] = False
 
+    if module in aliases:
+        doc['aliases'] = aliases[module]
 
     all_keys = []
 
@@ -238,9 +280,10 @@ def process_module(module, options, env, template, outputname, module_map):
 
     for (k,v) in doc['options'].iteritems():
         all_keys.append(k)
+
     all_keys = sorted(all_keys)
-    doc['option_keys'] = all_keys
 
+    doc['option_keys']      = all_keys
     doc['filename']         = fname
     doc['docuri']           = doc['module'].replace('_', '-')
     doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
@@ -251,13 +294,32 @@ def process_module(module, options, env, template, outputname, module_map):
 
     text = template.render(doc)
     write_data(text, options, outputname, module)
+    return doc['short_description']
 
 #####################################################################################
 
+def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases):
+    modstring = module
+    modname = module
+    if module in deprecated:
+        modstring = modstring + DEPRECATED
+        modname = "_" + module
+    elif module not in core:
+        modstring = modstring + NOTCORE
+
+    result = process_module(modname, options, env, template, outputname, module_map, aliases)
+
+    if result != "SKIPPED":
+        category_file.write("  %s - %s <%s_module>\n" % (modstring, result, module))
+
 def process_category(category, categories, options, env, template, outputname):
 
     module_map = categories[category]
 
+    aliases = {}
+    if '_aliases' in categories:
+        aliases = categories['_aliases']
+
     category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
     category_file = open(category_file_path, "w")
     print "*** recording category %s in %s ***" % (category, category_file_path)
@@ -267,7 +329,27 @@ def process_category(category, categories, options, env, template, outputname):
     category = category.replace("_"," ")
     category = category.title()
 
-    modules = module_map.keys()
+    modules = []
+    deprecated = []
+    core = []
+    for module in module_map.keys():
+
+        if isinstance(module_map[module], dict):
+            for mod in module_map[module].keys():
+                if mod.startswith("_"):
+                    mod = mod.replace("_","",1)
+                    deprecated.append(mod)
+                elif '/core/' in module_map[module][mod]:
+                    core.append(mod)
+        else:
+            if module.startswith("_"):
+                module = module.replace("_","",1)
+                deprecated.append(module)
+            elif '/core/' in module_map[module]:
+                core.append(module)
+
+        modules.append(module)
+
     modules.sort()
 
     category_header = "%s Modules" % (category.title())
@@ -277,17 +359,33 @@ def process_category(category, categories, options, env, template, outputname):
 %s
 %s
 
-.. toctree::
+.. toctree:: :maxdepth: 1
-   :maxdepth: 1
 
 """ % (category_header, underscores))
-
+    sections = []
     for module in modules:
-        result = process_module(module, options, env, template, outputname, module_map)
+        if module in module_map and isinstance(module_map[module], dict):
-        if result != "SKIPPED":
+            sections.append(module)
-            category_file.write("   %s_module\n" % module)
+            continue
-
+        else:
-
+            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases)
+
+    sections.sort()
+    for section in sections:
+        category_file.write("%s\n%s\n\n" % (section.replace("_"," ").title(),'-' * len(section)))
+        category_file.write(".. toctree:: :maxdepth: 1\n\n")
+
+        section_modules = module_map[section].keys()
+        section_modules.sort()
+        #for module in module_map[section]:
+        for module in section_modules:
+            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases)
+
+    category_file.write("""\n\n
+.. note::
+    - %s: This marks a module as deprecated, kept for backwards compatibility but use is discouraged
+    - %s: Denotes that this module is not part of core, it can be found in the extras or some other external repo
+""" % (DEPRECATED, NOTCORE))
     category_file.close()
 
     # TODO: end a new category file
@@ -332,6 +430,8 @@ def main():
     category_list_file.write("   :maxdepth: 1\n\n")
 
     for category in category_names:
+        if category.startswith("_"):
+            continue
         category_list_file.write("   list_of_%s_modules\n" % category)
         process_category(category, categories, options, env, template, outputname)
 

hacking/templates/rst.j2

@@ -21,6 +21,17 @@
  #
  --------------------------------------------#}
 
+{% if aliases is defined -%}
+Aliases: @{ ','.join(aliases) }@
+{% endif %}
+
+{% if deprecated is defined -%}
+DEPRECATED
+----------
+
+@{ deprecated }@
+{% endif %}
+
 Synopsis
 --------
 
@@ -102,7 +113,8 @@ Examples
 {% endif %}
 
 
-{% if core %}
+{% if not deprecated %}
+    {% if core %}
 
 This is a Core Module
 ---------------------
@@ -117,7 +129,7 @@ Documentation updates for this module can also be edited directly by submitting
 
 This is a "core" ansible module, which means it will receive slightly higher priority for all requests than those in the "extras" repos.
 
-{% else %}
+    {% else %}
 
 This is an Extras Module
 ------------------------
@@ -133,6 +145,7 @@ Documentation updates for this module can also be edited directly by submitting
 Note that this module is designated a "extras" module.  Non-core modules are still fully usable, but may receive slightly lower response rates for issues and pull requests.
 Popular "extras" modules may be promoted to core modules over time.
 
+    {% endif %}
 {% endif %}
 
 For help in developing on modules, should you be so inclined, please read :doc:`community`, :doc:`developing_test_pr` and :doc:`developing_modules`.

lib/ansible/utils/plugins.py

@@ -178,6 +178,9 @@ class PluginLoader(object):
                     self._plugin_path_cache[full_name] = path
                     return path
 
+        if not name.startswith('_'):
+            return self.find_plugin('_' + name, suffixes, transport)
+
         return None
 
     def has_plugin(self, name):