Add support for MySQL 8.4

Change-Id: If462e377d182a9a56614b37ce4258cfd692ab21b
Signed-off-by: wu.chunyang <wchy1001@gmail.com>

backup/install.sh

@@ -69,7 +69,14 @@ if [ "${OPT_DATASTORE}" = "mysql" ]; then
 	dpkg -i percona-release.deb
 	percona-release enable-only tools release
 	apt-get update
-	apt-get install ${APTOPTS} percona-xtrabackup-80
+	if [ "${OPT_DATASTORE_VERSION}" = "8.0" ]; then
+		apt-get install ${APTOPTS} percona-xtrabackup-80
+	elif [ "${OPT_DATASTORE_VERSION}" = "8.4" ]; then
+		apt-get install ${APTOPTS} percona-xtrabackup-84
+	else
+		echo "Unsupported MySQL version: ${OPT_DATASTORE_VERSION}"
+		exit 1
+	fi
 	rm -f percona-release.deb
 elif [ "${OPT_DATASTORE}" = "mariadb" ]; then
 	# See the url below about the supported version.

devstack/plugin.sh

@@ -527,18 +527,18 @@ function create_registry_container {
     container=$(sudo docker ps -a --format "{{.Names}}" --filter name=registry)
     if [ -z $container ]; then
         sudo docker run -d --net=host -e REGISTRY_HTTP_ADDR=0.0.0.0:4000 --restart=always -v /opt/trove_registry/:/var/lib/registry --name registry quay.io/openstack.trove/registry:2
-        for img in {"mysql:8.0","mariadb:11.4","postgres:17"};
+        for img in {"mysql:8.4","mariadb:11.4","postgres:17"};
         do
         sudo docker pull quay.io/openstack.trove/${img} && sudo docker tag quay.io/openstack.trove/${img} 127.0.0.1:4000/trove-datastores/${img} && sudo docker push 127.0.0.1:4000/trove-datastores/${img}
         done
         pushd $DEST/trove/backup
         # build backup images
-        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 .
+        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.4 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.4 .
         sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:11.4 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=11.4 .
         sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:17 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=17 .
         popd
         # push backup images
-        for backupimg in {"db-backup-mysql:8.0","db-backup-mariadb:11.4","db-backup-postgresql:17"};
+        for backupimg in {"db-backup-mysql:8.4","db-backup-mariadb:11.4","db-backup-postgresql:17"};
         do
         sudo docker push 127.0.0.1:4000/trove-datastores/${backupimg}
         done

devstack/settings

@@ -29,7 +29,7 @@ TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/polic
 TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"}
 TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"noble"}
 TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"}
-TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.0"}
+TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.4"}
 
 # Configuration values listed here for reference
 TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE}

releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add support of MySQL 8.4, currently, The supported versions
+    are MySQL 8.0 & 8.4

trove/guestagent/common/sql_query.py

@@ -20,7 +20,6 @@ Do not hard-code strings into the guest agent; use this module to build
 them for you.
 
 """
-import semantic_version
 
 
 class Query(object):
@@ -363,20 +362,8 @@ class SetPassword(object):
         return str(self)
 
     def __str__(self):
-        if self.ds == 'mysql':
+        return (f"ALTER USER '{self.user}'@'{self.host}' "
-            cur_version = semantic_version.Version.coerce(self.ds_version)
+                f"IDENTIFIED BY '{self.new_password}';")
-            mysql_575 = semantic_version.Version('5.7.5')
-            if cur_version <= mysql_575:
-                return (f"SET PASSWORD FOR '{self.user}'@'{self.host}' = "
-                        f"PASSWORD('{self.new_password}');")
-
-            return (f"ALTER USER '{self.user}'@'{self.host}' "
-                    f"IDENTIFIED WITH mysql_native_password "
-                    f"BY '{self.new_password}';")
-        elif self.ds == 'mariadb':
-            return (f"ALTER USER '{self.user}'@'{self.host}' IDENTIFIED VIA "
-                    f"mysql_native_password USING "
-                    f"PASSWORD('{self.new_password}');")
 
 
 class DropUser(object):

trove/guestagent/datastore/mariadb/manager.py

@@ -19,6 +19,7 @@ from trove.common import cfg
 from trove.common import exception
 from trove.common import utils
 from trove.guestagent.common import operating_system
+from trove.guestagent.datastore import service as base_service
 from trove.guestagent.utils import docker as docker_utils
 
 
@@ -32,7 +33,7 @@ LOG = logging.getLogger(__name__)
 
 class Manager(manager.MySqlManager):
     def __init__(self):
-        status = service.MariadbAppStatus(self.docker_client)
+        status = base_service.BaseDbStatus(self.docker_client)
         app = service.MariaDBApp(status, self.docker_client)
         adm = service.MariaDBAdmin(app)
 

trove/guestagent/datastore/mariadb/service.py

@@ -24,65 +24,12 @@ from trove.common import utils
 from trove.guestagent.datastore.mysql_common import service as mysql_service
 from trove.guestagent.utils import docker as docker_util
 from trove.guestagent.utils import mysql as mysql_util
-from trove.instance import service_status
 
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
-class MariadbAppStatus(mysql_service.BaseMySqlAppStatus):
-
-    def _get_container_status(self):
-        status = docker_util.get_container_status(self.docker_client)
-        if status == "running":
-            root_pass = mysql_util.BaseDbApp.get_auth_password(file="root.cnf")
-            cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass
-            try:
-                docker_util.run_command(self.docker_client, cmd)
-                return service_status.ServiceStatuses.HEALTHY
-            except Exception as exc:
-                LOG.warning('Failed to run docker command, error: %s',
-                            str(exc))
-                container_log = docker_util.get_container_logs(
-                    self.docker_client, tail='all')
-                LOG.debug('container log: \n%s', '\n'.join(container_log))
-                return service_status.ServiceStatuses.RUNNING
-        elif status == "not running":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "restarting":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "paused":
-            return service_status.ServiceStatuses.PAUSED
-        elif status == "exited":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "dead":
-            return service_status.ServiceStatuses.CRASHED
-        else:
-            return service_status.ServiceStatuses.UNKNOWN
-
-    def get_actual_db_status(self):
-        """Check database service status."""
-        health = docker_util.get_container_health(self.docker_client)
-        LOG.debug('container health status: %s', health)
-        if health == "healthy":
-            return service_status.ServiceStatuses.HEALTHY
-        elif health == "starting":
-            return service_status.ServiceStatuses.RUNNING
-        elif health == "unhealthy":
-            # In case the container was stopped
-            status = docker_util.get_container_status(self.docker_client)
-            if status == "exited":
-                return service_status.ServiceStatuses.SHUTDOWN
-            else:
-                return service_status.ServiceStatuses.CRASHED
-
-        # if the health status is one of unkown or None, let's check
-        # container status .this is for the compatibility with the
-        # old datastores.
-        return self._get_container_status()
-
-
 class MariaDBApp(mysql_service.BaseMySqlApp):
 
     HEALTHCHECK = {

trove/guestagent/datastore/mysql/manager.py

@@ -11,17 +11,15 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-import semantic_version
-
 
 from oslo_log import log as logging
 
-
 from trove.common import cfg
 from trove.common import exception
 from trove.guestagent.common import operating_system
 from trove.guestagent.datastore.mysql import service
 from trove.guestagent.datastore.mysql_common import manager
+from trove.guestagent.datastore import service as base_service
 
 
 CONF = cfg.CONF
@@ -30,40 +28,22 @@ LOG = logging.getLogger(__name__)
 
 class Manager(manager.MySqlManager):
     def __init__(self):
-        status = service.MySqlAppStatus(self.docker_client)
+        status = base_service.BaseDbStatus(self.docker_client)
         app = service.MySqlApp(status, self.docker_client)
         adm = service.MySqlAdmin(app)
 
         super(Manager, self).__init__(app, status, adm)
 
-    def get_start_db_params(self, data_dir):
-        """Get parameters for starting database.
-
-        Cinder volume initialization(after formatted) may leave a lost+found
-        folder.
-
-        The --ignore-db-dir option is deprecated in MySQL 5.7. With the
-        introduction of the data dictionary in MySQL 8.0, it became
-        superfluous and was removed in that version.
-        """
-        params = f'--datadir={data_dir}'
-
-        mysql_8 = semantic_version.Version('8.0.0')
-        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
-        params = f'--datadir={data_dir}'
-        if cur_ver < mysql_8:
-            params = (f"{params} --ignore-db-dir=lost+found "
-                      f"--ignore-db-dir=conf.d")
-
-        return params
-
     def pre_create_backup(self, context, **kwargs):
         LOG.info("Running pre_create_backup")
         status = {}
         try:
             INFO_FILE = "%s/xtrabackup_binlog_info" % self.app.get_data_dir()
             self.app.execute_sql("FLUSH TABLES WITH READ LOCK;")
-            stt = self.app.execute_sql("SHOW MASTER STATUS;")
+            if self.app._is_mysql84():
+                stt = self.app.execute_sql("SHOW BINARY LOG STATUS;")
+            else:
+                stt = self.app.execute_sql("SHOW MASTER STATUS;")
             for row in stt:
                 status = {
                     'log_file': row._mapping['File'],

trove/guestagent/datastore/mysql/service.py

@@ -28,12 +28,26 @@ CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
-class MySqlAppStatus(service.BaseMySqlAppStatus):
-    def __init__(self, docker_client):
-        super(MySqlAppStatus, self).__init__(docker_client)
-
-
 class MySqlApp(service.BaseMySqlApp):
+
+    HEALTHCHECK = {
+        "test": ["CMD", "mysqladmin", "ping", "-h",
+                 "127.0.0.1", "-u", "root",
+                 "--password=$MYSQL_ROOT_PASSWORD"],
+        "start_period": 10 * 1000000000,  # 10 seconds in nanoseconds
+        "interval": 10 * 1000000000,
+        "timeout": 5 * 1000000000,
+        "retries": 3
+    }
+
+    def _is_mysql84(self):
+        mysql_84 = semantic_version.Version('8.4.0')
+        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
+        if cur_ver >= mysql_84:
+            return True
+        else:
+            return False
+
     def __init__(self, status, docker_client):
         super(MySqlApp, self).__init__(status, docker_client)
 
@@ -44,11 +58,19 @@ class MySqlApp(service.BaseMySqlApp):
 
     def _get_slave_status(self):
         with mysql_util.SqlClient(self.get_engine()) as client:
-            return client.execute(text('SHOW SLAVE STATUS')).first()
+            if self._is_mysql84():
+                return client.execute(text('SHOW REPLICA STATUS')).first()
+            else:
+                return client.execute(text('SHOW SLAVE STATUS')).first()
 
     def _get_master_UUID(self):
         slave_status = self._get_slave_status()
-        return slave_status and slave_status._mapping['Master_UUID'] or None
+        if self._is_mysql84():
+            return slave_status and slave_status._mapping['Source_UUID'] \
+                or None
+        else:
+            return slave_status and slave_status._mapping['Master_UUID'] \
+                or None
 
     def get_latest_txn_id(self):
         return self._get_gtid_executed()
@@ -65,9 +87,19 @@ class MySqlApp(service.BaseMySqlApp):
         return master_UUID, int(last_txn_id)
 
     def wait_for_txn(self, txn):
+        if self._is_mysql84():
+            _sql = "SELECT WAIT_FOR_EXECUTED_GTID_SET('%s')" % txn
+        else:
+            _sql = "SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn
         with mysql_util.SqlClient(self.get_engine()) as client:
-            client.execute(
+            client.execute(text(_sql))
-                text("SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn))
+
+    def stop_master(self):
+        LOG.info("Stopping replication master.")
+        if not self._is_mysql84():
+            return super().stop_master()
+        with mysql_util.SqlClient(self.get_engine()) as client:
+            client.execute(text("RESET BINARY LOGS AND GTIDS"))
 
     def get_backup_image(self):
         """Get the actual container image based on datastore version.
@@ -93,22 +125,16 @@ class MySqlApp(service.BaseMySqlApp):
         innobackupex was removed in Percona XtraBackup 8.0, use xtrabackup
         instead.
         """
-        strategy = cfg.get_configuration_property('backup_strategy')
+        return 'xtrabackup'
-
-        mysql_8 = semantic_version.Version('8.0.0')
-        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
-        if cur_ver >= mysql_8:
-            strategy = 'xtrabackup'
-
-        return strategy
 
     def reset_data_for_restore_snapshot(self, data_dir):
-        """This function try remove slave status in database"""
+        """This function try remove replica status in database"""
-        mysql_8 = semantic_version.Version('8.0.0')
+        # '--skip-replica-start' was introduced in mysql 8.0.26 and the
-        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
+        # '--skip-slave-start' not be removed yet for mysql 8.0.x
-        command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir
+        if self._is_mysql84:
-        if cur_ver >= mysql_8:
             command = "mysqld --skip-replica-start=ON --datadir=%s" % data_dir
+        else:
+            command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir
 
         extra_volumes = {
             "/etc/mysql": {"bind": "/etc/mysql", "mode": "rw"},
@@ -133,6 +159,34 @@ class MySqlApp(service.BaseMySqlApp):
             except Exception as err:
                 LOG.error('Failed to remove container. error: %s', str(err))
 
+    def start_slave(self):
+        LOG.info("Starting slave replication.")
+        if not self._is_mysql84():
+            return super().start_slave()
+        with mysql_util.SqlClient(self.get_engine()) as client:
+            client.execute(text('START REPLICA'))
+            self.wait_for_slave_status("ON", client, 180)
+
+    def stop_slave(self, for_failover):
+        LOG.info("Stopping slave replication.")
+        if not self._is_mysql84():
+            return super().stop_slave(for_failover)
+        replication_user = None
+        with mysql_util.SqlClient(self.get_engine()) as client:
+            result = client.execute(
+                text('SHOW REPLICA STATUS')).mappings().first()
+            if result:
+                replication_user = result['Source_User']
+                client.execute(text('STOP REPLICA'))
+                client.execute(text('RESET REPLICA ALL'))
+            self.wait_for_slave_status('OFF', client, 180)
+            if not for_failover and replication_user:
+                client.execute(
+                    text('DROP USER IF EXISTS ' + replication_user))
+        return {
+            'replication_user': replication_user
+        }
+
 
 class MySqlRootAccess(service.BaseMySqlRootAccess):
     def __init__(self, app):

trove/guestagent/datastore/mysql_common/manager.py

@@ -15,7 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-import tempfile
+import os.path
 
 from oslo_log import log as logging
 
@@ -208,54 +208,29 @@ class MySqlManager(manager.Manager):
             root_pass = utils.generate_random_password()
             self.app.save_password('root', root_pass)
 
-        init_file = tempfile.NamedTemporaryFile(mode='w')
+        init_file = os.path.join(data_dir, "init.sql")
         operating_system.write_file(
-            init_file.name,
+            init_file,
-            f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';"
+            f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';",
+            as_root=True
+        )
+        # Change ownership so the database service user
+        # can read it in the container
+        operating_system.chown(
+            init_file,
+            user=self.app.database_service_uid,
+            group=self.app.database_service_gid,
+            as_root=True
         )
-        err_file = tempfile.NamedTemporaryFile(suffix='.err')
 
-        # Get the original file owner and group before changing the owner.
-        from pathlib import Path
-        init_file_path = Path(init_file.name)
-        init_file_owner = init_file_path.owner()
-        init_file_group = init_file_path.group()
-
-        # Allow database service user to access the temporary files.
-        try:
-            for file in [init_file.name, err_file.name]:
-                operating_system.chown(
-                    file,
-                    self.app.database_service_uid,
-                    self.app.database_service_gid,
-                    force=True, as_root=True)
-        except Exception as err:
-            LOG.error('Failed to change file owner, error: %s', str(err))
-            for file in [init_file.name, err_file.name]:
-                LOG.debug('Reverting the %s owner to %s '
-                          'before close it.', file, init_file_owner)
-                operating_system.chown(file, init_file_owner,
-                                       init_file_group, force=True,
-                                       as_root=True)
-            init_file.close()
-            err_file.close()
-            raise err
-
-        # Allow database service user to access the temporary files.
         command = (
-            f'mysqld --init-file={init_file.name} '
+            f'mysqld --init-file={init_file} '
-            f'--log-error={err_file.name} '
             f'--datadir={data_dir} '
         )
-        extra_volumes = {
-            init_file.name: {"bind": init_file.name, "mode": "rw"},
-            err_file.name: {"bind": err_file.name, "mode": "rw"},
-        }
 
         # Start the database container process.
         try:
-            self.app.start_db(ds_version=ds_version, command=command,
+            self.app.start_db(ds_version=ds_version, command=command)
-                              extra_volumes=extra_volumes)
         except Exception as err:
             LOG.error('Failed to reset password for restore, error: %s',
                       str(err))
@@ -267,19 +242,12 @@ class MySqlManager(manager.Manager):
                     docker_util.get_container_logs(self.app.docker_client)
                 )
                 docker_util.remove_container(self.app.docker_client)
+                # Remove init.sql file after password reset
+                operating_system.remove(init_file, force=True, as_root=True)
             except Exception as err:
-                LOG.error('Failed to remove container. error: %s',
+                LOG.error('Failed to remove container or init file. error: %s',
                           str(err))
                 pass
-            for file in [init_file.name, err_file.name]:
-                LOG.debug('Reverting the %s owner to %s '
-                          'before close it.', file, init_file_owner)
-                operating_system.chown(file, init_file_owner,
-                                       init_file_group, force=True,
-                                       as_root=True)
-            init_file.close()
-            err_file.close()
-
         LOG.info('Finished to reset password for restore')
 
     def _validate_slave_for_replication(self, context, replica_info):

trove/guestagent/datastore/mysql_common/service.py

@@ -62,41 +62,6 @@ BACKUP_LOG = re.compile(r'.*Backup successfully, checksum: (?P<checksum>.*), '
                         r'location: (?P<location>.*)')
 
 
-class BaseMySqlAppStatus(service.BaseDbStatus):
-
-    def __init__(self, docker_client):
-        super(BaseMySqlAppStatus, self).__init__(docker_client)
-
-    def get_actual_db_status(self):
-        """Check database service status."""
-        status = docker_util.get_container_status(self.docker_client)
-        if status == "running":
-            root_pass = service.BaseDbApp.get_auth_password(file="root.cnf")
-            cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass
-            try:
-                docker_util.run_command(self.docker_client, cmd)
-                return service_status.ServiceStatuses.HEALTHY
-            except Exception as exc:
-                LOG.warning('Failed to run docker command, error: %s',
-                            str(exc))
-                container_log = docker_util.get_container_logs(
-                    self.docker_client, tail='all')
-                LOG.debug('container log: \n%s', '\n'.join(container_log))
-                return service_status.ServiceStatuses.RUNNING
-        elif status == "not running":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "restarting":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "paused":
-            return service_status.ServiceStatuses.PAUSED
-        elif status == "exited":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "dead":
-            return service_status.ServiceStatuses.CRASHED
-        else:
-            return service_status.ServiceStatuses.UNKNOWN
-
-
 class BaseMySqlAdmin(object, metaclass=abc.ABCMeta):
     """Handles administrative tasks on the MySQL database."""
 

trove/guestagent/datastore/postgres/manager.py

@@ -26,6 +26,7 @@ from trove.common import utils
 from trove.guestagent.common import operating_system
 from trove.guestagent.datastore import manager
 from trove.guestagent.datastore.postgres import service
+from trove.guestagent.datastore import service as base_service
 from trove.guestagent import guest_log
 
 LOG = logging.getLogger(__name__)
@@ -37,7 +38,7 @@ class PostgresManager(manager.Manager):
     def __init__(self):
         super(PostgresManager, self).__init__('postgres')
 
-        self.status = service.PgSqlAppStatus(self.docker_client)
+        self.status = base_service.BaseDbStatus(self.docker_client)
         self.app = service.PgSqlApp(self.status, self.docker_client)
         self.adm = service.PgSqlAdmin(service.SUPER_USER_NAME)
 

trove/guestagent/datastore/postgres/service.py

@@ -44,57 +44,6 @@ HBA_CONFIG_FILE = '/etc/postgresql/pg_hba.conf'
 WAL_ARCHIVE_DIR = '/var/lib/postgresql/data/wal_archive'
 
 
-class PgSqlAppStatus(service.BaseDbStatus):
-    def __init__(self, docker_client):
-        super(PgSqlAppStatus, self).__init__(docker_client)
-
-    def _get_container_status(self):
-        """Check database service status."""
-        status = docker_util.get_container_status(self.docker_client)
-        if status == "running":
-            cmd = "psql -U postgres -c 'select 1;'"
-            try:
-                docker_util.run_command(self.docker_client, cmd)
-                return service_status.ServiceStatuses.HEALTHY
-            except Exception as exc:
-                LOG.warning('Failed to run docker command, error: %s',
-                            str(exc))
-                container_log = docker_util.get_container_logs(
-                    self.docker_client, tail='all')
-                LOG.debug('container log: \n%s', '\n'.join(container_log))
-                return service_status.ServiceStatuses.RUNNING
-        elif status == "not running":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "paused":
-            return service_status.ServiceStatuses.PAUSED
-        elif status == "exited":
-            return service_status.ServiceStatuses.SHUTDOWN
-        elif status == "dead":
-            return service_status.ServiceStatuses.CRASHED
-        else:
-            return service_status.ServiceStatuses.UNKNOWN
-
-    def get_actual_db_status(self):
-        health = docker_util.get_container_health(self.docker_client)
-        LOG.debug('container health status: %s', health)
-        if health == "healthy":
-            return service_status.ServiceStatuses.HEALTHY
-        elif health == "starting":
-            return service_status.ServiceStatuses.RUNNING
-        elif health == "unhealthy":
-            # In case the container was stopped
-            status = docker_util.get_container_status(self.docker_client)
-            if status == "exited":
-                return service_status.ServiceStatuses.SHUTDOWN
-            else:
-                return service_status.ServiceStatuses.CRASHED
-
-        # if the health status is one of unkown or None, let's check
-        # container status. this is for the compatibility with the
-        # old datastores.
-        return self._get_container_status()
-
-
 class PgSqlApp(service.BaseDbApp):
     _configuration_manager = None
 

trove/guestagent/datastore/service.py

@@ -119,7 +119,25 @@ class BaseDbStatus(object):
         self.set_status(real_status, force=force)
 
     def get_actual_db_status(self):
-        raise NotImplementedError()
+        """Check database service status."""
+        health = docker_util.get_container_health(self.docker_client)
+        LOG.debug('container health status: %s', health)
+        if health == "healthy":
+            return service_status.ServiceStatuses.HEALTHY
+        elif health == "starting":
+            return service_status.ServiceStatuses.RUNNING
+        # when the container is not found, it returns not running
+        elif health == "not running":
+            return service_status.ServiceStatuses.SHUTDOWN
+        elif health == "unhealthy":
+            # In case the container was stopped
+            status = docker_util.get_container_status(self.docker_client)
+            if status == "exited":
+                return service_status.ServiceStatuses.SHUTDOWN
+            else:
+                return service_status.ServiceStatuses.CRASHED
+
+        return service_status.ServiceStatuses.UNKNOWN
 
     @property
     def is_installed(self):

trove/guestagent/strategies/replication/mysql_gtid.py

@@ -13,17 +13,23 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-from oslo_log import log as logging
 
+from oslo_log import log as logging
+import semantic_version
+
+from trove.common import cfg
 from trove.guestagent.strategies.replication import mysql_base
 
 LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
 
 
 class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
     """MySql Replication coordinated by GTIDs."""
 
     def connect_to_master(self, service, master_info):
+        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
+        mysql_84 = semantic_version.Version('8.4.0')
         if 'dataset' in master_info:
             # pull the last executed GTID from the master via
             # the xtrabackup metadata file. If that value is
@@ -35,10 +41,12 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
                 last_gtid = self.read_last_master_gtid(service)
             LOG.info("last_gtid value is %s", last_gtid)
             if '-' in last_gtid:
-                # See
-                # https://avdeo.com/tag/error-1840-hy000-global-gtid_purged-can-only-be-set-when/
                 # Also, FLUSH PRIVILEGES will restore gtid_executed.
-                service.execute_sql('RESET MASTER')
+                if cur_ver >= mysql_84:
+                    service.execute_sql('RESET BINARY LOGS AND GTIDS')
+                else:
+                    # for mysql 8.0
+                    service.execute_sql('RESET MASTER')
                 set_gtid_cmd = "SET GLOBAL gtid_purged='%s'" % last_gtid
                 service.execute_sql(set_gtid_cmd)
 
@@ -49,21 +57,37 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
             master_info['master']['port'],
             replica_conf['replication_user']['name']
         )
+        if cur_ver >= mysql_84:
+            change_master_cmd = (
+                "CHANGE REPLICATION SOURCE TO "
+                "SOURCE_HOST='%(host)s', "
+                "SOURCE_PORT=%(port)s, "
+                "SOURCE_USER='%(user)s', "
+                "SOURCE_PASSWORD='%(password)s', "
+                "SOURCE_AUTO_POSITION=1, "
+                "GET_SOURCE_PUBLIC_KEY=1, "
+                "SOURCE_CONNECT_RETRY=15" %
+                {
+                    'host': master_info['master']['host'],
+                    'port': master_info['master']['port'],
+                    'user': replica_conf['replication_user']['name'],
+                    'password': replica_conf['replication_user']['password']
+                })
+        else:
+            change_master_cmd = (
+                "CHANGE MASTER TO "
+                "MASTER_HOST='%(host)s', "
+                "MASTER_PORT=%(port)s, "
+                "MASTER_USER='%(user)s', "
+                "MASTER_PASSWORD='%(password)s', "
+                "MASTER_AUTO_POSITION=1, "
+                "MASTER_CONNECT_RETRY=15" %
+                {
+                    'host': master_info['master']['host'],
+                    'port': master_info['master']['port'],
+                    'user': replica_conf['replication_user']['name'],
+                    'password': replica_conf['replication_user']['password']
+                })
 
-        change_master_cmd = (
-            "CHANGE MASTER TO "
-            "MASTER_HOST='%(host)s', "
-            "MASTER_PORT=%(port)s, "
-            "MASTER_USER='%(user)s', "
-            "MASTER_PASSWORD='%(password)s', "
-            "MASTER_AUTO_POSITION=1, "
-            "MASTER_CONNECT_RETRY=15" %
-            {
-                'host': master_info['master']['host'],
-                'port': master_info['master']['port'],
-                'user': replica_conf['replication_user']['name'],
-                'password': replica_conf['replication_user']['password']
-            })
         service.execute_sql(change_master_cmd)
-
         service.start_slave()

trove/templates/mysql/config.template

@@ -15,7 +15,12 @@ secure-file-priv = NULL
 tmpdir = /var/tmp
 pid-file = /var/run/mysqld/mysqld.pid
 socket = /var/run/mysqld/mysqld.sock
+{% if datastore.semantic_version.major < 8 %}
 default_authentication_plugin = mysql_native_password
+{% endif %}
+{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %}
+default_authentication_plugin = mysql_native_password
+{% endif %}
 skip-external-locking = 1
 key_buffer_size = {{ (50 * flavor['ram']/512)|int }}M
 max_allowed_packet = {{ (1024 * flavor['ram']/512)|int }}K

trove/templates/mysql/replica.config.template

@@ -2,7 +2,12 @@
 log_bin = /var/lib/mysql/data/mysql-bin.log
 binlog_format = MIXED
 relay_log = /var/lib/mysql/data/mysql-relay-bin.log
+{% if datastore.semantic_version.major < 8 %}
 relay_log_info_repository = TABLE
+{% endif %}
+{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %}
+relay_log_info_repository = TABLE
+{% endif %}
 relay_log_recovery = 1
 relay_log_purge = 1
 log_slave_updates = ON

trove/tests/unittests/common/test_template.py

@@ -64,8 +64,12 @@ class TemplateTest(trove_testtools.TestCase):
         self.assertGreater(len(server_id), 1)
 
     def test_rendering(self):
-        rendered = self.template.render(flavor=self.flavor_dict,
+        datastore = mock_datastore_version("mysql", "mysql", "mysql", "8.4")
-                                        server_id=self.server_id)
+        datastore.semantic_version = Version("8.4.0")
+        rendered = self.template.render(
+            flavor=self.flavor_dict,
+            server_id=self.server_id,
+            datastore=datastore)
         self.validate_template(rendered,
                                "innodb_buffer_pool_size",
                                self.flavor_dict,

zuul.d/jobs.yaml

@@ -217,119 +217,6 @@
         TROVE_ENABLE_LOCAL_REGISTRY: True
       tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_replication
 
-- job:
-    name: trove-tempest-ubuntu-base-mysql8.0
-    parent: trove-tempest-ubuntu-base
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^test-requirements.txt$
-      - ^tox.ini$
-      - ^LICENSE$
-      - ^contrib/
-      - ^zuul\.d/
-      - ^backup/
-      - ^\..+
-      - ^trove/guestagent/strategies/replication/
-      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
-    vars:
-        devstack_localrc:
-          TROVE_DATASTORE_VERSION: 8.0
-          TROVE_STATE_CHANGE_WAIT_TIME: 900
-        devstack_local_conf:
-          test-config:
-            $TEMPEST_CONFIG:
-              database:
-                default_datastore_versions: mysql:8.0
-
-- job:
-    name: trove-tempest-ubuntu-backup-mysql8.0
-    parent: trove-tempest-ubuntu-backup
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^test-requirements.txt$
-      - ^tox.ini$
-      - ^LICENSE$
-      - ^contrib/
-      - ^zuul\.d/
-      - ^\..+
-      - ^trove/guestagent/strategies/replication/
-      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
-    vars:
-        devstack_localrc:
-          TROVE_DATASTORE_VERSION: 8.0
-          TROVE_STATE_CHANGE_WAIT_TIME: 900
-        devstack_local_conf:
-          test-config:
-            $TEMPEST_CONFIG:
-              database:
-                backup_wait_timeout: 1200
-                default_datastore_versions: mysql:8.0
-
-- job:
-    name: trove-tempest-ubuntu-replication-mysql8.0
-    parent: trove-tempest-ubuntu-replication
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^test-requirements.txt$
-      - ^tox.ini$
-      - ^LICENSE$
-      - ^contrib/
-      - ^zuul\.d/
-      - ^backup/
-      - ^\..+
-      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
-      - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
-    vars:
-        devstack_localrc:
-          TROVE_DATASTORE_VERSION: 8.0
-          TROVE_STATE_CHANGE_WAIT_TIME: 900
-        devstack_local_conf:
-          test-config:
-            $TEMPEST_CONFIG:
-              database:
-                backup_wait_timeout: 1200
-                default_datastore_versions: mysql:8.0
-
-- job:
-    name: trove-tempest-cinder-storage-driver-mysql8.0
-    parent:  trove-tempest-snapshot
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^test-requirements.txt$
-      - ^tox.ini$
-      - ^LICENSE$
-      - ^contrib/
-      - ^zuul\.d/
-      - ^backup/
-      - ^\..+
-      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
-      - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
-    vars:
-        devstack_localrc:
-          TROVE_DATASTORE_VERSION: 8.0
-        devstack_local_conf:
-          test-config:
-            $TEMPEST_CONFIG:
-              database:
-                backup_wait_timeout: 1200
-                default_datastore_versions: mysql:8.0
-
 - job:
     name: publish-trove-guest-image
     parent: publish-openstack-artifacts

zuul.d/mysql_job.yaml

@@ -0,0 +1,113 @@
+#MySQL jobs
+- job:
+    name: trove-tempest-ubuntu-base-mysql8.4
+    parent: trove-tempest-ubuntu-base
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^backup/
+      - ^\..+
+      - ^trove/guestagent/strategies/replication/
+      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 8.4
+          TROVE_STATE_CHANGE_WAIT_TIME: 900
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                default_datastore_versions: mysql:8.4
+
+- job:
+    name: trove-tempest-ubuntu-backup-mysql8.4
+    parent: trove-tempest-ubuntu-backup
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^\..+
+      - ^trove/guestagent/strategies/replication/
+      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 8.4
+          TROVE_STATE_CHANGE_WAIT_TIME: 900
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                backup_wait_timeout: 1200
+                default_datastore_versions: mysql:8.4
+
+- job:
+    name: trove-tempest-ubuntu-replication-mysql8.4
+    parent: trove-tempest-ubuntu-replication
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^backup/
+      - ^\..+
+      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
+      - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 8.4
+          TROVE_STATE_CHANGE_WAIT_TIME: 900
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                backup_wait_timeout: 1200
+                default_datastore_versions: mysql:8.4
+
+- job:
+    name: trove-tempest-cinder-storage-driver-mysql8.4
+    parent:  trove-tempest-snapshot
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^backup/
+      - ^\..+
+      - ^trove/guestagent/datastore/(postgres|mariadb)/.*$
+      - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 8.4
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                backup_wait_timeout: 1200
+                default_datastore_versions: mysql:8.4

zuul.d/projects.yaml

@@ -11,12 +11,12 @@
       - release-notes-jobs-python3
     check:
       jobs:
-        - trove-tempest-ubuntu-base-mysql8.0
+        - trove-tempest-ubuntu-base-mysql8.4
-        - trove-tempest-ubuntu-backup-mysql8.0:
+        - trove-tempest-ubuntu-backup-mysql8.4:
             voting: false
-        - trove-tempest-ubuntu-replication-mysql8.0:
+        - trove-tempest-ubuntu-replication-mysql8.4:
             voting: false
-        - trove-tempest-cinder-storage-driver-mysql8.0:
+        - trove-tempest-cinder-storage-driver-mysql8.4:
             voting: false
         - trove-tempest-ubuntu-base-mariadb11.4:
             voting: false
@@ -42,7 +42,7 @@
             voting: true
     gate:
       jobs:
-        - trove-tempest-ubuntu-base-mysql8.0
+        - trove-tempest-ubuntu-base-mysql8.4
         - trove-tempest-ubuntu-base-mariadb11.4
         - trove-tempest-ubuntu-base-postgresql17
     experimental: