Fix MariaDB integration tests for CentOS 8 (#64859)

* Fix setup_mariadb for CentOS 8
Switch to using systemd for multiple instances on CentOS 7 and 8

test/integration/targets/mariadb_replication/tasks/main.yml

@@ -3,13 +3,19 @@
 
 # Initial CI tests of mysql_replication module
 - import_tasks: mariadb_replication_initial.yml
-  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version >= '7'
+  when:
+    - ansible_facts.distribution == 'CentOS'
+    - ansible_facts.distribution_major_version is version('7', '>=')
 
 # Tests of master_use_gtid parameter
 # https://github.com/ansible/ansible/pull/62648
 - import_tasks: mariadb_master_use_gtid.yml
-  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version >= '7'
+  when:
+    - ansible_facts.distribution == 'CentOS'
+    - ansible_facts.distribution_major_version is version('7', '>=')
 
 # Tests of connection_name parameter
 - import_tasks: mariadb_replication_connection_name.yml
-  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version >= '7'
+  when:
+    - ansible_facts.distribution == 'CentOS'
+    - ansible_facts.distribution_major_version is version('7', '>=')

test/integration/targets/mariadb_replication/tasks/mariadb_master_use_gtid.yml

@@ -12,22 +12,22 @@
 - name: Get master status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: getmaster
-  register: master_status
+  register: primary_status
 
 # Set master_use_gtid disabled:
 - name: Run replication
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: changemaster
     master_host: 127.0.0.1
-    master_port: "{{ master_port }}"
+    master_port: "{{ primary_db.port }}"
     master_user: "{{ replication_user }}"
     master_password: "{{ replication_pass }}"
     master_log_file: mysql-bin.000001
-    master_log_pos: '{{ master_status.Position }}'
+    master_log_pos: '{{ primary_status.Position }}'
     master_use_gtid: disabled
   register: result
 
@@ -39,13 +39,13 @@
 - name: Start standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: startslave
 
 - name: Get standby status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: getslave
   register: slave_status
 
@@ -57,7 +57,7 @@
 - name: Stop standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave
 
 ################################
@@ -68,22 +68,22 @@
 - name: Get master status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: getmaster
-  register: master_status
+  register: primary_status
 
 # Set master_use_gtid current_pos:
 - name: Run replication
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: changemaster
     master_host: 127.0.0.1
-    master_port: "{{ master_port }}"
+    master_port: "{{ primary_db.port }}"
     master_user: "{{ replication_user }}"
     master_password: "{{ replication_pass }}"
     master_log_file: mysql-bin.000001
-    master_log_pos: '{{ master_status.Position }}'
+    master_log_pos: '{{ primary_status.Position }}'
     master_use_gtid: current_pos
   register: result
 
@@ -95,13 +95,13 @@
 - name: Start standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: startslave
 
 - name: Get standby status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: getslave
   register: slave_status
 
@@ -113,7 +113,7 @@
 - name: Stop standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave
 
 ##############################
@@ -124,22 +124,22 @@
 - name: Get master status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: getmaster
-  register: master_status
+  register: primary_status
 
 # Set master_use_gtid slave_pos:
 - name: Run replication
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: changemaster
     master_host: 127.0.0.1
-    master_port: "{{ master_port }}"
+    master_port: "{{ primary_db.port }}"
     master_user: "{{ replication_user }}"
     master_password: "{{ replication_pass }}"
     master_log_file: mysql-bin.000001
-    master_log_pos: '{{ master_status.Position }}'
+    master_log_pos: '{{ primary_status.Position }}'
     master_use_gtid: slave_pos
   register: result
 
@@ -151,13 +151,13 @@
 - name: Start standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: startslave
 
 - name: Get standby status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: getslave
   register: slave_status
 
@@ -169,5 +169,5 @@
 - name: Stop standby
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave

test/integration/targets/mariadb_replication/tasks/mariadb_replication_connection_name.yml

@@ -5,35 +5,35 @@
 - name: Stop slave
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave
 
 - name: Reset slave all
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: resetslaveall
 
 # Get master log pos:
 - name: Get master status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: getmaster
-  register: master_status
+  register: primary_status
 
 # Test changemaster mode:
 - name: Run replication with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: changemaster
     master_host: 127.0.0.1
-    master_port: "{{ master_port }}"
+    master_port: "{{ primary_db.port }}"
     master_user: "{{ replication_user }}"
     master_password: "{{ replication_pass }}"
     master_log_file: mysql-bin.000001
-    master_log_pos: '{{ master_status.Position }}'
+    master_log_pos: '{{ primary_status.Position }}'
     connection_name: '{{ conn_name }}'
   register: result
 
@@ -46,7 +46,7 @@
 - name: Start slave with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: startslave
     connection_name: "{{ conn_name }}"
   register: result
@@ -60,7 +60,7 @@
 - name: Get standby statu with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: getslave
     connection_name: "{{ conn_name }}"
   register: slave_status
@@ -69,8 +69,8 @@
     that:
     - slave_status.Is_Slave == true
     - slave_status.Master_Host == '127.0.0.1'
-    - slave_status.Exec_Master_Log_Pos == master_status.Position
+    - slave_status.Exec_Master_Log_Pos == primary_status.Position
-    - slave_status.Master_Port == {{ master_port }}
+    - slave_status.Master_Port == {{ primary_db.port }}
     - slave_status.Last_IO_Errno == 0
     - slave_status.Last_IO_Error == ''
     - slave_status is not changed
@@ -79,7 +79,7 @@
 - name: Stop slave with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave
     connection_name: "{{ conn_name }}"
   register: result
@@ -93,7 +93,7 @@
 - name: Reset slave with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: resetslave
     connection_name: "{{ conn_name }}"
   register: result
@@ -107,7 +107,7 @@
 - name: Reset slave all with connection_name
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: resetslaveall
     connection_name: "{{ conn_name }}"
   register: result

test/integration/targets/mariadb_replication/tasks/mariadb_replication_initial.yml

@@ -3,26 +3,26 @@
 
 # Preparation:
 - name: Create user for replication
-  shell: "echo \"GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'localhost' IDENTIFIED BY '{{ replication_pass }}'; FLUSH PRIVILEGES;\" | mysql -P {{ master_port }} -h 127.0.0.1"
+  shell: "echo \"GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'localhost' IDENTIFIED BY '{{ replication_pass }}'; FLUSH PRIVILEGES;\" | mysql -P {{ primary_db.port }} -h 127.0.0.1"
 
 - name: Create test database
   mysql_db:
     login_host: 127.0.0.1
-    login_port: '{{ master_port }}'
+    login_port: '{{ primary_db.port }}'
     state: present
     name: '{{ test_db }}'
 
 - name: Dump all databases from the master
-  shell: 'mysqldump -P {{ master_port }} -h 127.0.01 --all-databases --master-data=2 > {{ dump_path }}'
+  shell: 'mysqldump -P {{ primary_db.port }} -h 127.0.01 --all-databases --master-data=2 > {{ dump_path }}'
 
-- name: Restore the dump to the standby
+- name: Restore the dump to the replica
-  shell: 'mysql -P {{ standby_port }} -h 127.0.0.1 < {{ dump_path }}'
+  shell: 'mysql -P {{ replica_db.port }} -h 127.0.0.1 < {{ dump_path }}'
 
 # Test getmaster mode:
 - name: Get master status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ master_port }}"
+    login_port: "{{ primary_db.port }}"
     mode: getmaster
   register: master_status
 
@@ -36,10 +36,10 @@
 - name: Run replication
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: changemaster
     master_host: 127.0.0.1
-    master_port: "{{ master_port }}"
+    master_port: "{{ primary_db.port }}"
     master_user: "{{ replication_user }}"
     master_password: "{{ replication_pass }}"
     master_log_file: mysql-bin.000001
@@ -55,7 +55,7 @@
 - name: Start slave
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: startslave
   register: result
 
@@ -65,10 +65,10 @@
     - result.queries == ["START SLAVE"]
 
 # Test getslave mode:
-- name: Get standby status
+- name: Get replica status
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: getslave
   register: slave_status
 
@@ -77,7 +77,7 @@
     - slave_status.Is_Slave == true
     - slave_status.Master_Host == '127.0.0.1'
     - slave_status.Exec_Master_Log_Pos == master_status.Position
-    - slave_status.Master_Port == {{ master_port }}
+    - slave_status.Master_Port == {{ primary_db.port }}
     - slave_status.Last_IO_Errno == 0
     - slave_status.Last_IO_Error == ''
     - slave_status is not changed
@@ -86,7 +86,7 @@
 - name: Stop slave
   mysql_replication:
     login_host: 127.0.0.1
-    login_port: "{{ standby_port }}"
+    login_port: "{{ replica_db.port }}"
     mode: stopslave
   register: result
 

test/integration/targets/setup_mariadb/defaults/main.yml

@@ -2,32 +2,36 @@ repo_link: http://yum.mariadb.org/10.1/centos7-amd64
 repo_gpgkey: https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
 
 mariadb_packages:
-- MariaDB-server
+  - MariaDB-server
-- MariaDB-client
+  - MariaDB-client
 
 packages_to_cleanup:
-- MariaDB-server
+  - MariaDB-common
-- MariaDB-client
+  - boost-program-options
-- MariaDB-common
+  - galera
-- boost-program-options
+  - jemalloc
-- galera
-- jemalloc
 
-master_port: 3306
+primary_db:
-master_datadir: /var/lib/mysql_master
+  name: primary
+  port: 3306
+  datadir: /var/lib/mysql_primary
+  logdir: /var/log/mysql_primary
 
-standby_port: 3307
+replica_db:
-standby_datadir: /var/lib/mysql_standby
+  name: replica01
-standby_logdir: /var/log/mysql_standby
+  port: 3307
+  datadir: /var/lib/mysql_replica01
+  logdir: /var/log/mysql_replica01
 
 default_datadir: /var/lib/mysql
 default_logdir: /var/log/mariadb
 mysql_safe_err_log: /var/log/mariadb/mysql_safe-err.log
 
 fs_objects_to_clean_up:
-- "{{ master_datadir }}"
+  - "{{ primary_db.datadir }}"
-- "{{ standby_datadir }}"
+  - "{{ primary_db.logdir }}"
-- "{{ standby_logdir }}"
+  - "{{ replica_db.datadir }}"
-- "{{ default_logdir }}"
+  - "{{ replica_db.logdir }}"
-- "{{ mysql_safe_err_log }}"
+  - "{{ default_datadir }}"
-- "{{ default_datadir }}"
+  - "{{ default_logdir }}"
+  - "{{ mysql_safe_err_log }}"

test/integration/targets/setup_mariadb/handlers/main.yml

@@ -1,3 +1,12 @@
+- name: Stop MariaDB services
+  service:
+    name: mariadb@{{ item }}
+    state: stopped
+  loop:
+    - "{{ primary_db.name }}"
+    - "{{ replica_db.name }}"
+  listen: cleanup mariadb
+
 - name: Remove MariaDB repo
   yum_repository:
     name: MariaDB
@@ -6,7 +15,7 @@
 
 - name: Remove MariaDB related packages
   yum:
-    name: "{{ packages_to_cleanup }}"
+    name: "{{ packages_to_cleanup | union(mariadb_packages) }}"
     state: absent
   listen: cleanup mariadb
 

test/integration/targets/setup_mariadb/tasks/main.yml

@@ -3,5 +3,5 @@
 
 - import_tasks: setup_mariadb.yml
   when:
-  - ansible_distribution == 'CentOS'
+    - ansible_distribution == 'CentOS'
-  - ansible_distribution_major_version >= '7'
+    - ansible_distribution_major_version is version('7', '>=')

test/integration/targets/setup_mariadb/tasks/setup_mariadb.yml

@@ -1,5 +1,18 @@
 # We run two servers listening different ports
-# to be able to check replication (one server for master, another for standby).
+# to be able to check replication (one server for primary, another for replica).
+
+- name: Include distribution specific variables
+  include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_facts.distribution }}-{{ ansible_facts.distribution_version }}.yml"
+        - "{{ ansible_facts.os_family }}-{{ ansible_facts.distribution_major_version }}.yml"
+        - "{{ ansible_facts.distribution }}.yml"
+        - "{{ ansible_facts.os_family }}.yml"
+        - default.yml
+      paths:
+        - vars
 
 - name: Install MariaDB repo
   yum_repository:
@@ -10,11 +23,10 @@
     gpgcheck: yes
   notify: cleanup mariadb
 
-- name: Install MariaDB packages on RedHat family OS
+- name: Install MariaDB packages
   yum:
     name: "{{ mariadb_packages }}"
-    enablerepo: epel
+    enablerepo: "{{ mariadb_enable_repo | default(omit) }}"
-  when: ansible_os_family == 'RedHat'
   notify: cleanup mariadb
 
 - name: Create directories for instances
@@ -24,31 +36,49 @@
     owner: mysql
     group: mysql
   loop:
-  - "{{ master_datadir }}"
+    - "{{ primary_db.datadir }}"
-  - "{{ standby_datadir }}"
+    - "{{ primary_db.logdir }}"
-  - "{{ standby_logdir }}"
+    - "{{ replica_db.datadir }}"
-  - "{{ default_logdir }}"
+    - "{{ replica_db.logdir }}"
   notify: cleanup mariadb
 
-- name: Copy cnf template
+- name: Copy configuration templates
+  template:
+    src: "{{ 'my' ~ item ~ '.j2' }}"
+    dest: /etc/my.cnf.d/my{{ item }}.cnf
+    owner: mysql
+    group: mysql
+    force: yes
+  when: ansible_facts.distribution_major_version is version('7', '==')
+  loop:
+    - '{{ primary_db.name }}'
+    - '{{ replica_db.name }}'
+
+- name: Copy configuration template
   template:
     src: my.cnf.j2
     dest: /etc/my.cnf
     owner: mysql
     group: mysql
     force: yes
+  when: ansible_facts.distribution_major_version is version('8', '==')
 
 - name: Initialize DBs
   shell: 'mysql_install_db --user=mysql --datadir={{ item }}'
   loop:
-  - '{{ master_datadir }}'
+  - '{{ primary_db.datadir }}'
-  - '{{ standby_datadir }}'
+  - '{{ replica_db.datadir }}'
 
 - name: Start services
-  shell: 'mysqld_multi --log=/var/log/mariadb/mariadb.log start 1,2'
+  service:
+    name: mariadb@{{ item }}
+    state: started
+  loop:
+    - "{{ primary_db.name }}"
+    - "{{ replica_db.name }}"
 
-- name: Pause
+- pause:
-  pause: seconds=3
+    seconds: 3
 
 ########### For painful debug uncomment the lines below ##
 #- name: DEBUG Check stratup log
@@ -70,8 +100,8 @@
 #  shell: cat /etc/my.cnf
 ##########################################################
 
-- name: Check connection to the master
+- name: Check connection to the primary
-  shell: 'echo "SHOW DATABASES;" | mysql -P {{ master_port }} -h 127.0.0.1'
+  shell: 'echo "SHOW DATABASES;" | mysql -P {{ primary_db.port }} -h 127.0.0.1'
 
-- name: Check connection to the standby
+- name: Check connection to the replica
-  shell: "echo \"SHOW VARIABLES LIKE 'datadir';\" | mysql -P {{ standby_port }} -h 127.0.0.1"
+  shell: "echo \"SHOW VARIABLES LIKE 'datadir';\" | mysql -P {{ replica_db.port }} -h 127.0.0.1"

test/integration/targets/setup_mariadb/templates/my.cnf.j2

@@ -1,25 +1,25 @@
-[mysqld1]
+[mysqld.{{ primary_db.name }}]
 server_id           = 1
-port                = {{ master_port }}
+port                = {{ primary_db.port }}
-datadir             = {{ master_datadir }}
+datadir             = {{ primary_db.datadir }}
-socket              = {{ master_datadir }}/mysql.sock
+socket              = {{ primary_db.datadir }}/mysql.sock
-pid-file            = {{ master_datadir }}/mysql.pid
+pid-file            = {{ primary_db.datadir }}/mysql.pid
-log_bin             = /var/log/mariadb/mysql-bin.log
+log_bin             = {{ primary_db.logdir }}/mysql-bin.log
 sync_binlog         = 1
 binlog-format       = ROW
 innodb_flush_log_at_trx_commit  = 1
 
-[mysqld2]
+[mysqld.{{ replica_db.name }}]
 server_id           = 2
-port                = {{ standby_port }}
+port                = {{ replica_db.port }}
-socket              = /var/run/mysqld/mysqld_slave.sock
+datadir             = {{ replica_db.datadir }}
-pid-file            = /var/run/mysqld/mysqld_slave.pid
+socket              = {{ replica_db.datadir }}/mysqld_replica.sock
-datadir             = {{ standby_datadir }}
+pid-file            = {{ replica_db.datadir }}/mysqld_replica.pid
-log_bin             = {{ standby_logdir }}/mysql-bin.log
+log_bin             = {{ replica_db.logdir }}/mysql-bin.log
-relay-log           = {{ standby_logdir }}/relay-bin
+relay-log           = {{ replica_db.logdir }}/relay-bin
-relay-log-index     = {{ standby_logdir }}/relay-bin.index
+relay-log-index     = {{ replica_db.logdir }}/relay-bin.index
-master-info-file    = {{ standby_logdir }}/master.info
+master-info-file    = {{ replica_db.logdir }}/master.info
-relay-log-info-file = {{ standby_logdir }}/relay-log.info
+relay-log-info-file = {{ replica_db.logdir }}/relay-log.info
 
 [mysqld_multi]
 mysqld     = /usr/bin/mysqld_safe

test/integration/targets/setup_mariadb/templates/myprimary.j2

@@ -0,0 +1,10 @@
+[mysqld]
+server_id           = 1
+port                = {{ primary_db.port }}
+datadir             = {{ primary_db.datadir }}
+socket              = {{ primary_db.datadir }}/mysql.sock
+pid-file            = {{ primary_db.datadir }}/mysql.pid
+log_bin             = {{ primary_db.logdir }}/mysql-bin.log
+sync_binlog         = 1
+binlog-format       = ROW
+innodb_flush_log_at_trx_commit  = 1

test/integration/targets/setup_mariadb/templates/myreplica01.j2

@@ -0,0 +1,11 @@
+[mysqld]
+server_id           = 2
+port                = {{ replica_db.port }}
+datadir             = {{ replica_db.datadir }}
+socket              = {{ replica_db.datadir }}/mysqld_replica.sock
+pid-file            = {{ replica_db.datadir }}/mysqld_replica.pid
+log_bin             = {{ replica_db.logdir }}/mysql-bin.log
+relay-log           = {{ replica_db.logdir }}/relay-bin
+relay-log-index     = {{ replica_db.logdir }}/relay-bin.index
+master-info-file    = {{ replica_db.logdir }}/master.info
+relay-log-info-file = {{ replica_db.logdir }}/relay-log.info

test/integration/targets/setup_mariadb/vars/RedHat-7.yml

@@ -0,0 +1,4 @@
+mariadb_enable_repo: epel
+mariadb_packages:
+  - MariaDB-server
+  - MySQL-python

test/integration/targets/setup_mariadb/vars/RedHat-8.yml

@@ -0,0 +1,3 @@
+mariadb_packages:
+  - mariadb-server
+  - python3-PyMySQL