postgresql_user_obj_stat_info: new module (#66892)

* postgresql_user_obj_stat_info: new module

* fix sanity

* add schema parameter

* add CI tests

* add RETURN section

* add example with schema

* fix CI

* fix example comments

* skip aix

* fixes part 1

* fixes part 2

* fix CI

* fix CI

* fix doc formatting
pull/67068/head
Andrew Klychkov 5 years ago committed by GitHub
parent 622f1c4c01
commit 21ae66db2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,334 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright: (c) 2020, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {
'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'
}
DOCUMENTATION = r'''
---
module: postgresql_user_obj_stat_info
short_description: Gather statistics about PostgreSQL user objects
description:
- Gathers statistics about PostgreSQL user objects.
version_added: '2.10'
options:
filter:
description:
- Limit the collected information by comma separated string or YAML list.
- Allowable values are C(functions), C(indexes), C(tables).
- By default, collects all subsets.
- Unsupported values are ignored.
type: list
elements: str
schema:
description:
- Restrict the output by certain schema.
type: str
db:
description:
- Name of database to connect.
type: str
aliases:
- login_db
session_role:
description:
- Switch to session_role after connecting. The specified session_role must
be a role that the current login_user is a member of.
- Permissions checking for SQL commands is carried out as though
the session_role were the one that had logged in originally.
type: str
notes:
- C(size) and C(total_size) returned values are presented in bytes.
- For tracking function statistics the PostgreSQL C(track_functions) parameter must be enabled.
See U(https://www.postgresql.org/docs/current/runtime-config-statistics.html) for more information.
seealso:
- module: postgresql_info
- module: postgresql_ping
- name: PostgreSQL statistics collector reference
description: Complete reference of the PostgreSQL statistics collector documentation.
link: https://www.postgresql.org/docs/current/monitoring-stats.html
author:
- Andrew Klychkov (@Andersson007)
extends_documentation_fragment: postgres
'''
EXAMPLES = r'''
- name: Collect information about all supported user objects of the acme database
postgresql_user_obj_stat_info:
db: acme
- name: Collect information about all supported user objects in the custom schema of the acme database
postgresql_user_obj_stat_info:
db: acme
schema: custom
- name: Collect information about user tables and indexes in the acme database
postgresql_user_obj_stat_info:
db: acme
filter: tables, indexes
'''
RETURN = r'''
indexes:
description: User index statistics
returned: always
type: dict
sample: {"public": {"test_id_idx": {"idx_scan": 0, "idx_tup_fetch": 0, "idx_tup_read": 0, "relname": "test", "size": 8192, ...}}}
tables:
description: User table statistics.
returned: always
type: dict
sample: {"public": {"test": {"analyze_count": 3, "n_dead_tup": 0, "n_live_tup": 0, "seq_scan": 2, "size": 0, "total_size": 8192, ...}}}
functions:
description: User function statistics.
returned: always
type: dict
sample: {"public": {"inc": {"calls": 1, "funcid": 26722, "self_time": 0.23, "total_time": 0.23}}}
'''
try:
from psycopg2.extras import DictCursor
except ImportError:
# psycopg2 is checked by connect_to_db()
# from ansible.module_utils.postgres
pass
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.postgres import (
connect_to_db,
exec_sql,
get_conn_params,
postgres_common_argument_spec,
)
from ansible.module_utils.six import iteritems
# ===========================================
# PostgreSQL module specific support methods.
#
class PgUserObjStatInfo():
"""Class to collect information about PostgreSQL user objects.
Args:
module (AnsibleModule): Object of AnsibleModule class.
cursor (cursor): Cursor object of psycopg2 library to work with PostgreSQL.
Attributes:
module (AnsibleModule): Object of AnsibleModule class.
cursor (cursor): Cursor object of psycopg2 library to work with PostgreSQL.
executed_queries (list): List of executed queries.
info (dict): Statistics dictionary.
obj_func_mapping (dict): Mapping of object types to corresponding functions.
schema (str): Name of a schema to restrict stat collecting.
"""
def __init__(self, module, cursor):
self.module = module
self.cursor = cursor
self.info = {
'functions': {},
'indexes': {},
'tables': {},
}
self.obj_func_mapping = {
'functions': self.get_func_stat,
'indexes': self.get_idx_stat,
'tables': self.get_tbl_stat,
}
self.schema = None
def collect(self, filter_=None, schema=None):
"""Collect statistics information of user objects.
Kwargs:
filter_ (list): List of subsets which need to be collected.
schema (str): Restrict stat collecting by certain schema.
Returns:
``self.info``.
"""
if schema:
self.set_schema(schema)
if filter_:
for obj_type in filter_:
obj_type = obj_type.strip()
obj_func = self.obj_func_mapping.get(obj_type)
if obj_func is not None:
obj_func()
else:
self.module.warn("Unknown filter option '%s'" % obj_type)
else:
for obj_func in self.obj_func_mapping.values():
obj_func()
return self.info
def get_func_stat(self):
"""Get function statistics and fill out self.info dictionary."""
if not self.schema:
query = "SELECT * FROM pg_stat_user_functions"
result = exec_sql(self, query, add_to_executed=False)
else:
query = "SELECT * FROM pg_stat_user_functions WHERE schemaname = %s"
result = exec_sql(self, query, query_params=(self.schema,),
add_to_executed=False)
if not result:
return
self.__fill_out_info(result,
info_key='functions',
schema_key='schemaname',
name_key='funcname')
def get_idx_stat(self):
"""Get index statistics and fill out self.info dictionary."""
if not self.schema:
query = "SELECT * FROM pg_stat_user_indexes"
result = exec_sql(self, query, add_to_executed=False)
else:
query = "SELECT * FROM pg_stat_user_indexes WHERE schemaname = %s"
result = exec_sql(self, query, query_params=(self.schema,),
add_to_executed=False)
if not result:
return
self.__fill_out_info(result,
info_key='indexes',
schema_key='schemaname',
name_key='indexrelname')
def get_tbl_stat(self):
"""Get table statistics and fill out self.info dictionary."""
if not self.schema:
query = "SELECT * FROM pg_stat_user_tables"
result = exec_sql(self, query, add_to_executed=False)
else:
query = "SELECT * FROM pg_stat_user_tables WHERE schemaname = %s"
result = exec_sql(self, query, query_params=(self.schema,),
add_to_executed=False)
if not result:
return
self.__fill_out_info(result,
info_key='tables',
schema_key='schemaname',
name_key='relname')
def __fill_out_info(self, result, info_key=None, schema_key=None, name_key=None):
# Convert result to list of dicts to handle it easier:
result = [dict(row) for row in result]
for elem in result:
# Add schema name as a key if not presented:
if not self.info[info_key].get(elem[schema_key]):
self.info[info_key][elem[schema_key]] = {}
# Add object name key as a subkey
# (they must be uniq over a schema, so no need additional checks):
self.info[info_key][elem[schema_key]][elem[name_key]] = {}
# Add other other attributes to a certain index:
for key, val in iteritems(elem):
if key not in (schema_key, name_key):
self.info[info_key][elem[schema_key]][elem[name_key]][key] = val
if info_key in ('tables', 'indexes'):
relname = elem[name_key]
schemaname = elem[schema_key]
if not self.schema:
result = exec_sql(self, "SELECT pg_relation_size ('%s.%s')" % (schemaname, relname),
add_to_executed=False)
else:
relname = '%s.%s' % (self.schema, relname)
result = exec_sql(self, "SELECT pg_relation_size (%s)",
query_params=(relname,),
add_to_executed=False)
self.info[info_key][elem[schema_key]][elem[name_key]]['size'] = result[0][0]
if info_key == 'tables':
relname = elem[name_key]
schemaname = elem[schema_key]
if not self.schema:
result = exec_sql(self, "SELECT pg_total_relation_size ('%s.%s')" % (schemaname, relname),
add_to_executed=False)
else:
relname = '%s.%s' % (self.schema, relname)
result = exec_sql(self, "SELECT pg_total_relation_size (%s)",
query_params=(relname,),
add_to_executed=False)
self.info[info_key][elem[schema_key]][elem[name_key]]['total_size'] = result[0][0]
def set_schema(self, schema):
"""If schema exists, sets self.schema, otherwise fails."""
query = ("SELECT 1 FROM information_schema.schemata "
"WHERE schema_name = %s")
result = exec_sql(self, query, query_params=(schema,),
add_to_executed=False)
if result and result[0][0]:
self.schema = schema
else:
self.module.fail_json(msg="Schema '%s' does not exist" % (schema))
# ===========================================
# Module execution.
#
def main():
argument_spec = postgres_common_argument_spec()
argument_spec.update(
db=dict(type='str', aliases=['login_db']),
filter=dict(type='list', elements='str'),
session_role=dict(type='str'),
schema=dict(type='str'),
)
module = AnsibleModule(
argument_spec=argument_spec,
supports_check_mode=True,
)
filter_ = module.params["filter"]
schema = module.params["schema"]
# Connect to DB and make cursor object:
pg_conn_params = get_conn_params(module, module.params)
# We don't need to commit anything, so, set it to False:
db_connection = connect_to_db(module, pg_conn_params, autocommit=False)
cursor = db_connection.cursor(cursor_factory=DictCursor)
############################
# Create object and do work:
pg_obj_info = PgUserObjStatInfo(module, cursor)
info_dict = pg_obj_info.collect(filter_, schema)
# Clean up:
cursor.close()
db_connection.close()
# Return information:
module.exit_json(**info_dict)
if __name__ == '__main__':
main()

@ -0,0 +1,8 @@
destructive
shippable/posix/group1
skip/osx
skip/centos
skip/freebsd
skip/rhel
skip/opensuse
skip/aix

@ -0,0 +1,12 @@
pg_user: postgres
db_default: postgres
test_table1: acme1
test_table2: acme2
test_table3: acme3
test_idx1: idx1
test_idx2: idx2
test_func1: func1
test_func2: func2
test_func3: func3
test_schema1: schema1

@ -0,0 +1,3 @@
# Initial tests of postgresql_user_obj_stat_info module:
- import_tasks: postgresql_user_obj_stat_info.yml
when: postgres_version_resp.stdout is version('9.4', '>=')

@ -0,0 +1,189 @@
# Copyright: (c) 2019, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
- vars:
task_parameters: &task_parameters
become_user: '{{ pg_user }}'
become: yes
register: result
pg_parameters: &pg_parameters
login_user: '{{ pg_user }}'
login_db: '{{ db_default }}'
block:
# Preparation:
# 0. create test schema
# 1. create test tables
# 2. create test indexes
# 3. create test functions
# 4. enable track_functions and restart
- name: Create schema
<<: *task_parameters
postgresql_schema:
<<: *pg_parameters
name: '{{ test_schema1 }}'
- name: Create test tables
<<: *task_parameters
postgresql_table:
<<: *pg_parameters
name: '{{ item }}'
columns:
- id int
loop:
- '{{ test_table1 }}'
- '{{ test_table2 }}'
- name: Create test table in another schema
<<: *task_parameters
postgresql_table:
<<: *pg_parameters
name: '{{ test_schema1 }}.{{ test_table3 }}'
- name: Create test indexes
<<: *task_parameters
postgresql_idx:
<<: *pg_parameters
name: '{{ item }}'
table: '{{ test_table1 }}'
columns:
- id
loop:
- '{{ test_idx1 }}'
- '{{ test_idx2 }}'
- name: Set track_function (restart is required)
<<: *task_parameters
postgresql_set:
<<: *pg_parameters
name: track_functions
value: all
- name: Restart PostgreSQL
become: yes
service:
name: "{{ postgresql_service }}"
state: restarted
- name: Create test functions
<<: *task_parameters
postgresql_query:
<<: *pg_parameters
query: 'CREATE FUNCTION {{ item }}() RETURNS boolean AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL'
loop:
- '{{ test_func1 }}'
- '{{ test_func2 }}'
- '{{ test_schema1 }}.{{ test_func3 }}'
- name: Touch test functions
<<: *task_parameters
postgresql_query:
<<: *pg_parameters
query: 'SELECT {{ item }}()'
loop:
- '{{ test_func1 }}'
- '{{ test_func2 }}'
- '{{ test_schema1 }}.{{ test_func3 }}'
#######
# Tests
#######
# 0. Without filter
- name: Collect all stats
<<: *task_parameters
postgresql_user_obj_stat_info:
<<: *pg_parameters
- assert:
that:
- result is not changed
- result.tables.public.{{ test_table1 }}.size == 0
- result.tables.public.{{ test_table1 }}.size == 0
- result.tables.{{ test_schema1 }}.{{ test_table3 }}.size == 0
- result.functions.public.{{ test_func1 }}.calls == 1
- result.functions.public.{{ test_func2 }}.calls == 1
- result.functions.{{ test_schema1 }}.{{ test_func3 }}.calls == 1
- result.indexes.public.{{ test_idx1 }}.idx_scan == 0
- result.indexes.public.{{ test_idx2 }}.idx_scan == 0
# 1. With filter
- name: Collect stats with filter
<<: *task_parameters
postgresql_user_obj_stat_info:
<<: *pg_parameters
filter: tables, indexes
- assert:
that:
- result is not changed
- result.tables.public.{{ test_table1 }}.size == 0
- result.tables.public.{{ test_table1 }}.size == 0
- result.tables.{{ test_schema1 }}.{{ test_table3 }}.size == 0
- result.functions == {}
- result.indexes.public.{{ test_idx1 }}.idx_scan == 0
- result.indexes.public.{{ test_idx2 }}.idx_scan == 0
# 2. With schema
- name: Collect stats for objects in certain schema
<<: *task_parameters
postgresql_user_obj_stat_info:
<<: *pg_parameters
schema: public
- assert:
that:
- result is not changed
- result.tables.public.{{ test_table1 }}.size == 0
- result.tables.public.{{ test_table1 }}.size == 0
- result.indexes.public.{{ test_idx1 }}.idx_scan == 0
- result.indexes.public.{{ test_idx2 }}.idx_scan == 0
- result.functions.public.{{ test_func1 }}.calls == 1
- result.functions.public.{{ test_func2 }}.calls == 1
- result.tables.{{ test_schema1 }} is not defined
# 3. With wrong schema
- name: Try to collect data in nonexistent schema
<<: *task_parameters
postgresql_user_obj_stat_info:
<<: *pg_parameters
schema: nonexistent
ignore_errors: yes
- assert:
that:
- result is failed
- result.msg == "Schema 'nonexistent' does not exist"
##########
# Clean up
##########
- name: Drop schema
<<: *task_parameters
postgresql_schema:
<<: *pg_parameters
name: '{{ test_schema1 }}'
state: absent
cascade_drop: yes
- name: Drop test tables
<<: *task_parameters
postgresql_table:
<<: *pg_parameters
name: '{{ item }}'
state: absent
loop:
- '{{ test_table1 }}'
- '{{ test_table2 }}'
- name: Drop test functions
<<: *task_parameters
postgresql_query:
<<: *pg_parameters
query: 'DROP FUNCTION {{ item }}()'
loop:
- '{{ test_func1 }}'
- '{{ test_func2 }}'
- '{{ test_schema1 }}.{{ test_func3 }}'
ignore_errors: yes
Loading…
Cancel
Save