diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py
index 9dba4cc2fab..7ec35d3062e 100644
--- a/lib/ansible/plugins/loader.py
+++ b/lib/ansible/plugins/loader.py
@@ -348,9 +348,6 @@ class PluginLoader:
 
     def _update_object(self, obj, name, path):
 
-        # load plugin config data
-        self._load_config_defs(name, path)
-
         # set extra info on the module, in case we want it later
         setattr(obj, '_original_path', path)
         setattr(obj, '_load_name', name)
@@ -394,6 +391,10 @@ class PluginLoader:
                     return None
                 raise
 
+        # load plugin config data
+        if not found_in_cache:
+            self._load_config_defs(name, path)
+
         self._update_object(obj, name, path)
         return obj
 
@@ -467,6 +468,10 @@ class PluginLoader:
                 except TypeError as e:
                     display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e)))
 
+            # load plugin config data
+            if not found_in_cache:
+                self._load_config_defs(name, path)
+
             self._update_object(obj, name, path)
             yield obj