Bug fixes for GCP modules (#60719)

lib/ansible/modules/cloud/google/gcp_appengine_firewall_rule_info.py

@@ -101,12 +101,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('ingressRules'):
-        items = items.get('ingressRules')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -116,8 +111,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'appengine')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='ingressRules')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_bigquery_dataset_info.py

@@ -208,12 +208,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/bigquery']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('datasets'):
-        items = items.get('datasets')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -223,8 +218,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'bigquery')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='datasets')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_bigquery_table.py

@@ -69,6 +69,15 @@ options:
         - The ID of the the table.
         required: false
         type: str
+  clustering:
+    description:
+    - One or more fields on which data should be clustered. Only top-level, non-repeated,
+      simple-type fields are supported. When you cluster a table using multiple columns,
+      the order of columns you specify is important. The order of the specified columns
+      determines the sort order of the data.
+    required: false
+    type: list
+    version_added: 2.9
   description:
     description:
     - A user-friendly description of the dataset.
@@ -90,6 +99,13 @@ options:
     - Name of the table.
     required: false
     type: str
+  num_rows:
+    description:
+    - The number of rows of data in this table, excluding any data in the streaming
+      buffer.
+    required: false
+    type: int
+    version_added: 2.9
   view:
     description:
     - The view definition.
@@ -130,6 +146,16 @@ options:
         - Number of milliseconds for which to keep the storage for a partition.
         required: false
         type: int
+      field:
+        description:
+        - If not set, the table is partitioned by pseudo column, referenced via either
+          '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE type. If
+          field is specified, the table is instead partitioned by this field. The
+          field must be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE
+          or REQUIRED.
+        required: false
+        type: str
+        version_added: 2.9
       type:
         description:
         - The only type supported is DAY, which will generate one partition per day.
@@ -485,6 +511,14 @@ tableReference:
       - The ID of the the table.
       returned: success
       type: str
+clustering:
+  description:
+  - One or more fields on which data should be clustered. Only top-level, non-repeated,
+    simple-type fields are supported. When you cluster a table using multiple columns,
+    the order of columns you specify is important. The order of the specified columns
+    determines the sort order of the data.
+  returned: success
+  type: list
 creationTime:
   description:
   - The time when this dataset was created, in milliseconds since the epoch.
@@ -543,6 +577,12 @@ numRows:
     buffer.
   returned: success
   type: int
+requirePartitionFilter:
+  description:
+  - If set to true, queries over this table require a partition filter that can be
+    used for partition elimination to be specified.
+  returned: success
+  type: bool
 type:
   description:
   - Describes the table type.
@@ -588,6 +628,14 @@ timePartitioning:
       - Number of milliseconds for which to keep the storage for a partition.
       returned: success
       type: int
+    field:
+      description:
+      - If not set, the table is partitioned by pseudo column, referenced via either
+        '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE type. If field
+        is specified, the table is instead partitioned by this field. The field must
+        be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or REQUIRED.
+      returned: success
+      type: str
     type:
       description:
       - The only type supported is DAY, which will generate one partition per day.
@@ -916,10 +964,12 @@ def main():
         argument_spec=dict(
             state=dict(default='present', choices=['present', 'absent'], type='str'),
             table_reference=dict(type='dict', options=dict(dataset_id=dict(type='str'), project_id=dict(type='str'), table_id=dict(type='str'))),
+            clustering=dict(type='list', elements='str'),
             description=dict(type='str'),
             friendly_name=dict(type='str'),
             labels=dict(type='dict'),
             name=dict(type='str'),
+            num_rows=dict(type='int'),
             view=dict(
                 type='dict',
                 options=dict(
@@ -929,7 +979,7 @@ def main():
                     ),
                 ),
             ),
-            time_partitioning=dict(type='dict', options=dict(expiration_ms=dict(type='int'), type=dict(type='str'))),
+            time_partitioning=dict(type='dict', options=dict(expiration_ms=dict(type='int'), field=dict(type='str'), type=dict(type='str'))),
             schema=dict(
                 type='dict',
                 options=dict(
@@ -1069,10 +1119,12 @@ def resource_to_request(module):
     request = {
         u'kind': 'bigquery#table',
         u'tableReference': TableTablereference(module.params.get('table_reference', {}), module).to_request(),
+        u'clustering': module.params.get('clustering'),
         u'description': module.params.get('description'),
         u'friendlyName': module.params.get('friendly_name'),
         u'labels': module.params.get('labels'),
         u'name': module.params.get('name'),
+        u'numRows': module.params.get('num_rows'),
         u'view': TableView(module.params.get('view', {}), module).to_request(),
         u'timePartitioning': TableTimepartitioning(module.params.get('time_partitioning', {}), module).to_request(),
         u'schema': TableSchema(module.params.get('schema', {}), module).to_request(),
@@ -1145,6 +1197,7 @@ def is_different(module, response):
 def response_to_hash(module, response):
     return {
         u'tableReference': TableTablereference(response.get(u'tableReference', {}), module).from_response(),
+        u'clustering': response.get(u'clustering'),
         u'creationTime': response.get(u'creationTime'),
         u'description': response.get(u'description'),
         u'friendlyName': response.get(u'friendlyName'),
@@ -1156,6 +1209,7 @@ def response_to_hash(module, response):
         u'numBytes': response.get(u'numBytes'),
         u'numLongTermBytes': response.get(u'numLongTermBytes'),
         u'numRows': response.get(u'numRows'),
+        u'requirePartitionFilter': response.get(u'requirePartitionFilter'),
         u'type': response.get(u'type'),
         u'view': TableView(response.get(u'view', {}), module).from_response(),
         u'timePartitioning': TableTimepartitioning(response.get(u'timePartitioning', {}), module).from_response(),
@@ -1251,10 +1305,14 @@ class TableTimepartitioning(object):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict({u'expirationMs': self.request.get('expiration_ms'), u'type': self.request.get('type')})
+        return remove_nones_from_dict(
+            {u'expirationMs': self.request.get('expiration_ms'), u'field': self.request.get('field'), u'type': self.request.get('type')}
+        )
 
     def from_response(self):
-        return remove_nones_from_dict({u'expirationMs': self.request.get(u'expirationMs'), u'type': self.request.get(u'type')})
+        return remove_nones_from_dict(
+            {u'expirationMs': self.request.get(u'expirationMs'), u'field': self.request.get(u'field'), u'type': self.request.get(u'type')}
+        )
 
 
 class TableStreamingbuffer(object):

lib/ansible/modules/cloud/google/gcp_bigquery_table_info.py

@@ -86,6 +86,14 @@ resources:
           - The ID of the the table.
           returned: success
           type: str
+    clustering:
+      description:
+      - One or more fields on which data should be clustered. Only top-level, non-repeated,
+        simple-type fields are supported. When you cluster a table using multiple
+        columns, the order of columns you specify is important. The order of the specified
+        columns determines the sort order of the data.
+      returned: success
+      type: list
     creationTime:
       description:
       - The time when this dataset was created, in milliseconds since the epoch.
@@ -144,6 +152,12 @@ resources:
         buffer.
       returned: success
       type: int
+    requirePartitionFilter:
+      description:
+      - If set to true, queries over this table require a partition filter that can
+        be used for partition elimination to be specified.
+      returned: success
+      type: bool
     type:
       description:
       - Describes the table type.
@@ -189,6 +203,15 @@ resources:
           - Number of milliseconds for which to keep the storage for a partition.
           returned: success
           type: int
+        field:
+          description:
+          - If not set, the table is partitioned by pseudo column, referenced via
+            either '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE
+            type. If field is specified, the table is instead partitioned by this
+            field. The field must be a top-level TIMESTAMP or DATE field. Its mode
+            must be NULLABLE or REQUIRED.
+          returned: success
+          type: str
         type:
           description:
           - The only type supported is DAY, which will generate one partition per
@@ -523,12 +546,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/bigquery']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('tables'):
-        items = items.get('tables')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -538,8 +556,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'bigquery')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='tables')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_cloudbuild_trigger_info.py

@@ -324,12 +324,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('triggers'):
-        items = items.get('triggers')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -339,8 +334,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'cloudbuild')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='triggers')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_cloudfunctions_cloud_function_info.py

@@ -224,12 +224,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('functions'):
-        items = items.get('functions')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -239,8 +234,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'cloudfunctions')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='functions')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_cloudscheduler_job_info.py

@@ -311,12 +311,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('jobs'):
-        items = items.get('jobs')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -326,8 +321,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'cloudscheduler')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='jobs')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_cloudtasks_queue_info.py

@@ -216,12 +216,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
+    return_value = {'resources': fetch_list(module, collection(module))}
-    if items.get('queues'):
-        items = items.get('queues')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -231,8 +226,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'cloudtasks')
-    response = auth.get(link)
+    return auth.list(link, return_if_object, array_name='queues')
-    return return_if_object(module, response)
 
 
 def return_if_object(module, response):

lib/ansible/modules/cloud/google/gcp_compute_address_info.py

@@ -159,12 +159,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -174,8 +169,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
-    return return_if_object(module, response)
 
 
 def query_options(filters):

lib/ansible/modules/cloud/google/gcp_compute_autoscaler_info.py

@@ -224,12 +224,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -239,8 +234,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
-    return return_if_object(module, response)
 
 
 def query_options(filters):

lib/ansible/modules/cloud/google/gcp_compute_backend_bucket_info.py

@@ -142,12 +142,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -157,8 +152,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
-    return return_if_object(module, response)
 
 
 def query_options(filters):

lib/ansible/modules/cloud/google/gcp_compute_backend_service_info.py

@@ -379,12 +379,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
     module.exit_json(**return_value)
 
 
@@ -394,8 +389,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
-    return return_if_object(module, response)
 
 
 def query_options(filters):