[stable-2.6] Migrate from MySQLdb to PyMySQL (#40123)

* Migrate from MySQLdb to PyMySQL

* Deduplicate driver loading and failure message

* Explain requirements

* Apply requirements docs change to proxysql too

* Add changelog.
(cherry picked from commit d34cf93f1a)

Co-authored-by: Daniel Speichert <DSpeichert@users.noreply.github.com>
pull/46146/head
Daniel Speichert 6 years ago committed by Matt Clay
parent 85626291bf
commit 6e90277c69

@ -0,0 +1,2 @@
bugfixes:
- mysql_*, proxysql_* - PyMySQL (a pure-Python MySQL driver) is now a preferred dependency also supporting Python 3.X.

@ -30,10 +30,14 @@
import os
try:
import MySQLdb
mysqldb_found = True
import pymysql as mysql_driver
except ImportError:
mysqldb_found = False
try:
import MySQLdb as mysql_driver
except ImportError:
mysql_driver = None
mysql_driver_fail_msg = 'The PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) module is required.'
def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None,
@ -69,8 +73,8 @@ def mysql_connect(module, login_user=None, login_password=None, config_file='',
if connect_timeout is not None:
config['connect_timeout'] = connect_timeout
db_connection = MySQLdb.connect(**config)
db_connection = mysql_driver.connect(**config)
if cursor_class is not None:
return db_connection.cursor(cursorclass=MySQLdb.cursors.DictCursor)
return db_connection.cursor(cursorclass=mysql_driver.cursors.DictCursor)
else:
return db_connection.cursor()

@ -61,7 +61,9 @@ requirements:
- mysql (command line binary)
- mysqldump (command line binary)
notes:
- Requires the python-mysqldb package on the remote host, as well as mysql and mysqldump binaries.
- Requires the PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) package on the remote host,
as well as mysql and mysqldump binaries.
- This module is B(not idempotent) when I(state) is C(import), and will import the dump file each time if run more than once.
extends_documentation_fragment: mysql
'''
@ -100,16 +102,9 @@ import pipes
import subprocess
import traceback
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import mysql_quote_identifier
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -268,8 +263,8 @@ def main():
supports_check_mode=True
)
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
db = module.params["name"]
encoding = module.params["encoding"]

@ -118,15 +118,8 @@ EXAMPLES = '''
import os
import warnings
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -239,16 +232,16 @@ def main():
connect_timeout = module.params['connect_timeout']
config_file = module.params['config_file']
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
else:
warnings.filterwarnings('error', category=MySQLdb.Warning)
warnings.filterwarnings('error', category=mysql_driver.Warning)
login_password = module.params["login_password"]
login_user = module.params["login_user"]
try:
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'MySQLdb.cursors.DictCursor',
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'mysql_driver.cursors.DictCursor',
connect_timeout=connect_timeout)
except Exception as e:
if os.path.exists(config_file):
@ -325,7 +318,7 @@ def main():
chm.append("MASTER_AUTO_POSITION = 1")
try:
changemaster(cursor, chm, chm_params)
except MySQLdb.Warning as e:
except mysql_driver.Warning as e:
result['warning'] = to_native(e)
except Exception as e:
module.fail_json(msg='%s. Query == CHANGE MASTER TO %s' % (to_native(e), chm))

@ -196,16 +196,9 @@ import re
import string
import traceback
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import SQLParseError
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
@ -578,8 +571,8 @@ def main():
db = 'mysql'
sql_log_bin = module.params["sql_log_bin"]
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
cursor = None
try:
@ -618,14 +611,14 @@ def main():
else:
changed = user_mod(cursor, user, host, host_all, None, encrypted, priv, append_privs, module)
except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
module.fail_json(msg=to_native(e), exception=traceback.format_exc())
else:
if host_all:
module.fail_json(msg="host_all parameter cannot be used when adding a user")
try:
changed = user_add(cursor, user, host, host_all, password, encrypted, priv, module.check_mode)
except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
module.fail_json(msg=to_native(e), exception=traceback.format_exc())
elif state == "absent":
if user_exists(cursor, user, host, host_all):

@ -50,16 +50,9 @@ import os
import warnings
from re import match
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import SQLParseError, mysql_quote_identifier
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -148,10 +141,10 @@ def main():
module.fail_json(msg="Cannot run without variable to operate with")
if match('^[0-9a-z_]+$', mysqlvar) is None:
module.fail_json(msg="invalid variable name \"%s\"" % mysqlvar)
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
else:
warnings.filterwarnings('error', category=MySQLdb.Warning)
warnings.filterwarnings('error', category=mysql_driver.Warning)
try:
cursor = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,

@ -148,18 +148,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -192,10 +184,8 @@ def perform_checks(module):
msg="max_replication_lag must be set between 0 and 102400"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -457,8 +447,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -487,7 +477,7 @@ def main():
" and doesn't need to be updated.")
result['server'] = \
proxysql_server.get_server_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify server.. %s" % to_native(e)
)
@ -502,7 +492,7 @@ def main():
result['changed'] = False
result['msg'] = ("The server is already absent from the" +
" mysql_hosts memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove server.. %s" % to_native(e)
)

@ -73,17 +73,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -96,10 +88,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(variable, cursor):
@ -211,8 +201,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -231,7 +221,7 @@ def main():
msg="The variable \"%s\" was not found" % variable
)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to get config.. %s" % to_native(e)
)
@ -264,7 +254,7 @@ def main():
msg="The variable \"%s\" was not found" % variable
)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to set config.. %s" % to_native(e)
)

@ -96,16 +96,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -140,10 +133,8 @@ def perform_checks(module):
" with the CONFIG config_layer")
module.fail_json(msg=msg_string % module.params["direction"])
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def manage_config(manage_config_settings, cursor):
@ -201,7 +192,7 @@ def main():
login_user,
login_password,
config_file)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -214,7 +205,7 @@ def main():
try:
result['changed'] = manage_config(manage_config_settings,
cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to manage config.. %s" % to_native(e)
)

@ -137,18 +137,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -161,10 +153,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -424,8 +414,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -454,7 +444,7 @@ def main():
" and doesn't need to be updated.")
result['user'] = \
proxysql_user.get_user_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify user.. %s" % to_native(e)
)
@ -469,7 +459,7 @@ def main():
result['changed'] = False
result['msg'] = ("The user is already absent from the" +
" mysql_users memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove user.. %s" % to_native(e)
)

@ -220,18 +220,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -244,10 +236,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -551,8 +541,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -582,7 +572,7 @@ def main():
result['rules'] = \
proxysql_query_rule.get_rule_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify rule.. %s" % to_native(e)
)
@ -605,7 +595,7 @@ def main():
result['changed'] = False
result['msg'] = ("The rule is already absent from the" +
" mysql_query_rules memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove rule.. %s" % to_native(e)
)

@ -95,17 +95,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -135,10 +127,8 @@ def perform_checks(module):
msg="reader_hostgroup cannot equal writer_hostgroup"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -328,8 +318,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -360,7 +350,7 @@ def main():
result['repl_group'] = \
proxysql_repl_group.get_repl_group_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify replication hostgroup.. %s" % to_native(e)
)
@ -378,7 +368,7 @@ def main():
" mysql_replication_hostgroups memory" +
" configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to delete replication hostgroup.. %s" % to_native(e)
)

@ -125,18 +125,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -155,10 +147,8 @@ def perform_checks(module):
msg="interval_ms must between 100ms & 100000000ms"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -366,8 +356,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -390,7 +380,7 @@ def main():
" need to be updated.")
result['schedules'] = \
proxysql_schedule.get_schedule_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify schedule.. %s" % to_native(e)
)
@ -413,7 +403,7 @@ def main():
result['changed'] = False
result['msg'] = ("The schedule is already absent from the" +
" memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove schedule.. %s" % to_native(e)
)

@ -52,7 +52,8 @@ options:
ssl_ca:
version_added: "2.0"
description:
- The path to a Certificate Authority (CA) certificate. This option, if used, must specify the same certificate as used by the server.
- The path to a Certificate Authority (CA) certificate. This option, if used, must specify the same certificate
as used by the server.
ssl_cert:
version_added: "2.0"
description:
@ -62,11 +63,12 @@ options:
description:
- The path to the client private key.
requirements:
- MySQLdb
- PyMySQL (Python 2.7 and Python 3.X), or
- MySQLdb (Python 2.x)
notes:
- Requires the MySQLdb Python package on the remote host. For Ubuntu, this
is as easy as apt-get install python-mysqldb. (See M(apt).) For CentOS/Fedora, this
is as easy as yum install MySQL-python. (See M(yum).)
- Requires the PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) Python package on the remote host.
For Ubuntu, this is as easy as apt-get install python-pymysql. (See M(apt).) For CentOS/Fedora, this
is as easy as yum install python2-PyMySQL. (See M(yum).)
- Both C(login_password) and C(login_user) are required when you are
passing credentials. If none are present, the module will attempt to read
the credentials from C(~/.my.cnf), and finally fall back to using the MySQL

@ -27,7 +27,8 @@ options:
are to be read.
default: ''
requirements:
- MySQLdb
- PyMySQL (Python 2.7 and Python 3.X), or
- MySQLdb (Python 2.x)
'''
# Documentation fragment for managing ProxySQL configuration

@ -2,5 +2,5 @@ mysql_service: 'mariadb'
mysql_packages:
- mariadb-server
- python3-mysql
- python3-PyMySQL
- bzip2

@ -2,5 +2,5 @@ mysql_service: 'mariadb'
mysql_packages:
- mariadb-server
- MySQL-python
- python-PyMySQL
- bzip2

@ -2,4 +2,4 @@ mysql_service: 'mysql-server'
mysql_packages:
- mariadb101-server
- py27-mysql-connector-python2
- py-pymysql

@ -2,5 +2,5 @@ mysql_service: 'mysql'
mysql_packages:
- mariadb
- python-MySQL-python
- python-PyMySQL
- bzip2

@ -2,5 +2,5 @@ mysql_service: 'mysql'
mysql_packages:
- mysql-server
- python3-mysqldb
- python3-pymysql
- bzip2

Loading…
Cancel
Save