Bug fixes for GCP modules (#60724)

lib/ansible/module_utils/gcp_utils.py

@@ -78,53 +78,118 @@ class GcpSession(object):
         self._validate()
 
     def get(self, url, body=None, **kwargs):
-        kwargs.update({'json': body, 'headers': self._headers()})
-        try:
-            return self.session().get(url, **kwargs)
-        except getattr(requests.exceptions, 'RequestException') as inst:
-            self.module.fail_json(msg=inst.message)
+        """
+        This method should be avoided in favor of full_get
+        """
+        kwargs.update({'json': body})
+        return self.full_get(url, **kwargs)
 
     def post(self, url, body=None, headers=None, **kwargs):
-        if headers:
-            headers = self._merge_dictionaries(headers, self._headers())
-        else:
-            headers = self._headers()
+        """
+        This method should be avoided in favor of full_post
+        """
+        kwargs.update({'json': body, 'headers': headers})
+        return self.full_post(url, **kwargs)
+
+    def post_contents(self, url, file_contents=None, headers=None, **kwargs):
+        """
+        This method should be avoided in favor of full_post
+        """
+        kwargs.update({'data': file_contents, 'headers': headers})
+        return self.full_post(url, **kwargs)
+
+    def delete(self, url, body=None):
+        """
+        This method should be avoided in favor of full_delete
+        """
+        kwargs = {'json': body}
+        return self.full_delete(url, **kwargs)
+
+    def put(self, url, body=None):
+        """
+        This method should be avoided in favor of full_put
+        """
+        kwargs = {'json': body}
+        return self.full_put(url, **kwargs)
+
+    def patch(self, url, body=None, **kwargs):
+        """
+        This method should be avoided in favor of full_patch
+        """
+        kwargs.update({'json': body})
+        return self.full_patch(url, **kwargs)
+
+    def list(self, url, callback, params=None, array_name='items',
+             pageToken='nextPageToken', **kwargs):
+        """
+        This should be used for calling the GCP list APIs. It will return
+        an array of items
+
+        This takes a callback to a `return_if_object(module, response)`
+        function that will decode the response + return a dictionary. Some
+        modules handle the decode + error processing differently, so we should
+        defer to the module to handle this.
+        """
+        resp = callback(self.module, self.full_get(url, params, **kwargs))
+        items = resp.get(array_name) if resp.get(array_name) else []
+        while resp.get(pageToken):
+            if params:
+                params['pageToken'] = resp.get(pageToken)
+            else:
+                params = {'pageToken': resp[pageToken]}
 
+            resp = callback(self.module, self.full_get(url, params, **kwargs))
+            if resp.get(array_name):
+                items = items + resp.get(array_name)
+        return items
+
+    # The following methods fully mimic the requests API and should be used.
+    def full_get(self, url, params=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
         try:
-            return self.session().post(url, json=body, headers=headers)
+            return self.session().get(url, params=params, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
+            # Only log the message to avoid logging any sensitive info.
             self.module.fail_json(msg=inst.message)
 
-    def post_contents(self, url, file_contents=None, headers=None, **kwargs):
-        if headers:
-            headers = self._merge_dictionaries(headers, self._headers())
-        else:
-            headers = self._headers()
+    def full_post(self, url, data=None, json=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
 
         try:
-            return self.session().post(url, data=file_contents, headers=headers)
+            return self.session().post(url, data=data, json=json, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def delete(self, url, body=None):
+    def full_put(self, url, data=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().delete(url, json=body, headers=self._headers())
+            return self.session().put(url, data=data, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def put(self, url, body=None):
+    def full_patch(self, url, data=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().put(url, json=body, headers=self._headers())
+            return self.session().patch(url, data=data, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def patch(self, url, body=None, **kwargs):
-        kwargs.update({'json': body, 'headers': self._headers()})
+    def full_delete(self, url, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().patch(url, **kwargs)
+            return self.session().delete(url, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
+    def _set_headers(self, headers):
+        if headers:
+            return self._merge_dictionaries(headers, self._headers())
+        else:
+            return self._headers()
+
     def session(self):
         return AuthorizedSession(
             self._credentials())

lib/ansible/modules/cloud/google/gcp_sql_instance_info.py

@@ -391,12 +391,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/sqlservice.admin']
 
-    items = fetch_list(module, collection(module))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module))}
     module.exit_json(**return_value)
 
 
@@ -406,8 +401,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'sql')
-    response = auth.get(link)
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items')
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_sql_user_info.py

@@ -114,12 +114,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/sqlservice.admin']
 
-    items = fetch_list(module, collection(module))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module))}
     module.exit_json(**return_value)
 
 
@@ -130,8 +125,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'sql')
-    response = auth.get(link)
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items')
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_tpu_node_info.py

@@ -167,12 +167,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
-    if items.get('nodes'):
-        items = items.get('nodes')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module))}
     module.exit_json(**return_value)
 
 
@@ -182,8 +177,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'tpu')
-    response = auth.get(link)
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='nodes')
 
 
 def return_if_object(module, response):