From 4f1f1bbc1d9840bb2d84b63228e0248883dcafd1 Mon Sep 17 00:00:00 2001 From: "wu.chunyang" Date: Wed, 25 Jun 2025 16:32:59 +0800 Subject: [PATCH] Add support for MySQL 8.4 Change-Id: If462e377d182a9a56614b37ce4258cfd692ab21b Signed-off-by: wu.chunyang --- backup/install.sh | 9 +- devstack/plugin.sh | 6 +- devstack/settings | 2 +- ...add-support-mysql8.4-2490b5c87f885b01.yaml | 5 + trove/guestagent/common/sql_query.py | 17 +-- trove/guestagent/datastore/mariadb/manager.py | 3 +- trove/guestagent/datastore/mariadb/service.py | 53 -------- trove/guestagent/datastore/mysql/manager.py | 32 +---- trove/guestagent/datastore/mysql/service.py | 98 +++++++++++---- .../datastore/mysql_common/manager.py | 68 +++-------- .../datastore/mysql_common/service.py | 35 ------ .../guestagent/datastore/postgres/manager.py | 3 +- .../guestagent/datastore/postgres/service.py | 51 -------- trove/guestagent/datastore/service.py | 20 +++- .../strategies/replication/mysql_gtid.py | 62 +++++++--- trove/templates/mysql/config.template | 5 + trove/templates/mysql/replica.config.template | 5 + trove/tests/unittests/common/test_template.py | 8 +- zuul.d/jobs.yaml | 113 ------------------ zuul.d/mysql_job.yaml | 113 ++++++++++++++++++ zuul.d/projects.yaml | 10 +- 21 files changed, 319 insertions(+), 399 deletions(-) create mode 100644 releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml create mode 100644 zuul.d/mysql_job.yaml diff --git a/backup/install.sh b/backup/install.sh index 429743e67e..7e5a189d0b 100755 --- a/backup/install.sh +++ b/backup/install.sh @@ -69,7 +69,14 @@ if [ "${OPT_DATASTORE}" = "mysql" ]; then dpkg -i percona-release.deb percona-release enable-only tools release apt-get update - apt-get install ${APTOPTS} percona-xtrabackup-80 + if [ "${OPT_DATASTORE_VERSION}" = "8.0" ]; then + apt-get install ${APTOPTS} percona-xtrabackup-80 + elif [ "${OPT_DATASTORE_VERSION}" = "8.4" ]; then + apt-get install ${APTOPTS} percona-xtrabackup-84 + else + echo "Unsupported MySQL version: ${OPT_DATASTORE_VERSION}" + exit 1 + fi rm -f percona-release.deb elif [ "${OPT_DATASTORE}" = "mariadb" ]; then # See the url below about the supported version. diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 4f09b99404..91ff2efdda 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -527,18 +527,18 @@ function create_registry_container { container=$(sudo docker ps -a --format "{{.Names}}" --filter name=registry) if [ -z $container ]; then sudo docker run -d --net=host -e REGISTRY_HTTP_ADDR=0.0.0.0:4000 --restart=always -v /opt/trove_registry/:/var/lib/registry --name registry quay.io/openstack.trove/registry:2 - for img in {"mysql:8.0","mariadb:11.4","postgres:17"}; + for img in {"mysql:8.4","mariadb:11.4","postgres:17"}; do sudo docker pull quay.io/openstack.trove/${img} && sudo docker tag quay.io/openstack.trove/${img} 127.0.0.1:4000/trove-datastores/${img} && sudo docker push 127.0.0.1:4000/trove-datastores/${img} done pushd $DEST/trove/backup # build backup images - sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 . + sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.4 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.4 . sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:11.4 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=11.4 . sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:17 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=17 . popd # push backup images - for backupimg in {"db-backup-mysql:8.0","db-backup-mariadb:11.4","db-backup-postgresql:17"}; + for backupimg in {"db-backup-mysql:8.4","db-backup-mariadb:11.4","db-backup-postgresql:17"}; do sudo docker push 127.0.0.1:4000/trove-datastores/${backupimg} done diff --git a/devstack/settings b/devstack/settings index 12685bc52e..b8daba1d42 100644 --- a/devstack/settings +++ b/devstack/settings @@ -29,7 +29,7 @@ TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/polic TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"} TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"noble"} TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"} -TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.0"} +TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.4"} # Configuration values listed here for reference TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE} diff --git a/releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml b/releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml new file mode 100644 index 0000000000..175a669daa --- /dev/null +++ b/releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support of MySQL 8.4, currently, The supported versions + are MySQL 8.0 & 8.4 \ No newline at end of file diff --git a/trove/guestagent/common/sql_query.py b/trove/guestagent/common/sql_query.py index 7a188471cb..06ec4cee3b 100644 --- a/trove/guestagent/common/sql_query.py +++ b/trove/guestagent/common/sql_query.py @@ -20,7 +20,6 @@ Do not hard-code strings into the guest agent; use this module to build them for you. """ -import semantic_version class Query(object): @@ -363,20 +362,8 @@ class SetPassword(object): return str(self) def __str__(self): - if self.ds == 'mysql': - cur_version = semantic_version.Version.coerce(self.ds_version) - mysql_575 = semantic_version.Version('5.7.5') - if cur_version <= mysql_575: - return (f"SET PASSWORD FOR '{self.user}'@'{self.host}' = " - f"PASSWORD('{self.new_password}');") - - return (f"ALTER USER '{self.user}'@'{self.host}' " - f"IDENTIFIED WITH mysql_native_password " - f"BY '{self.new_password}';") - elif self.ds == 'mariadb': - return (f"ALTER USER '{self.user}'@'{self.host}' IDENTIFIED VIA " - f"mysql_native_password USING " - f"PASSWORD('{self.new_password}');") + return (f"ALTER USER '{self.user}'@'{self.host}' " + f"IDENTIFIED BY '{self.new_password}';") class DropUser(object): diff --git a/trove/guestagent/datastore/mariadb/manager.py b/trove/guestagent/datastore/mariadb/manager.py index 691f146106..e2b749febd 100644 --- a/trove/guestagent/datastore/mariadb/manager.py +++ b/trove/guestagent/datastore/mariadb/manager.py @@ -19,6 +19,7 @@ from trove.common import cfg from trove.common import exception from trove.common import utils from trove.guestagent.common import operating_system +from trove.guestagent.datastore import service as base_service from trove.guestagent.utils import docker as docker_utils @@ -32,7 +33,7 @@ LOG = logging.getLogger(__name__) class Manager(manager.MySqlManager): def __init__(self): - status = service.MariadbAppStatus(self.docker_client) + status = base_service.BaseDbStatus(self.docker_client) app = service.MariaDBApp(status, self.docker_client) adm = service.MariaDBAdmin(app) diff --git a/trove/guestagent/datastore/mariadb/service.py b/trove/guestagent/datastore/mariadb/service.py index 71059a8977..2896a824af 100644 --- a/trove/guestagent/datastore/mariadb/service.py +++ b/trove/guestagent/datastore/mariadb/service.py @@ -24,65 +24,12 @@ from trove.common import utils from trove.guestagent.datastore.mysql_common import service as mysql_service from trove.guestagent.utils import docker as docker_util from trove.guestagent.utils import mysql as mysql_util -from trove.instance import service_status CONF = cfg.CONF LOG = logging.getLogger(__name__) -class MariadbAppStatus(mysql_service.BaseMySqlAppStatus): - - def _get_container_status(self): - status = docker_util.get_container_status(self.docker_client) - if status == "running": - root_pass = mysql_util.BaseDbApp.get_auth_password(file="root.cnf") - cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass - try: - docker_util.run_command(self.docker_client, cmd) - return service_status.ServiceStatuses.HEALTHY - except Exception as exc: - LOG.warning('Failed to run docker command, error: %s', - str(exc)) - container_log = docker_util.get_container_logs( - self.docker_client, tail='all') - LOG.debug('container log: \n%s', '\n'.join(container_log)) - return service_status.ServiceStatuses.RUNNING - elif status == "not running": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "restarting": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "paused": - return service_status.ServiceStatuses.PAUSED - elif status == "exited": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "dead": - return service_status.ServiceStatuses.CRASHED - else: - return service_status.ServiceStatuses.UNKNOWN - - def get_actual_db_status(self): - """Check database service status.""" - health = docker_util.get_container_health(self.docker_client) - LOG.debug('container health status: %s', health) - if health == "healthy": - return service_status.ServiceStatuses.HEALTHY - elif health == "starting": - return service_status.ServiceStatuses.RUNNING - elif health == "unhealthy": - # In case the container was stopped - status = docker_util.get_container_status(self.docker_client) - if status == "exited": - return service_status.ServiceStatuses.SHUTDOWN - else: - return service_status.ServiceStatuses.CRASHED - - # if the health status is one of unkown or None, let's check - # container status .this is for the compatibility with the - # old datastores. - return self._get_container_status() - - class MariaDBApp(mysql_service.BaseMySqlApp): HEALTHCHECK = { diff --git a/trove/guestagent/datastore/mysql/manager.py b/trove/guestagent/datastore/mysql/manager.py index 0ae5bae4b8..388295da4d 100644 --- a/trove/guestagent/datastore/mysql/manager.py +++ b/trove/guestagent/datastore/mysql/manager.py @@ -11,17 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import semantic_version - from oslo_log import log as logging - from trove.common import cfg from trove.common import exception from trove.guestagent.common import operating_system from trove.guestagent.datastore.mysql import service from trove.guestagent.datastore.mysql_common import manager +from trove.guestagent.datastore import service as base_service CONF = cfg.CONF @@ -30,40 +28,22 @@ LOG = logging.getLogger(__name__) class Manager(manager.MySqlManager): def __init__(self): - status = service.MySqlAppStatus(self.docker_client) + status = base_service.BaseDbStatus(self.docker_client) app = service.MySqlApp(status, self.docker_client) adm = service.MySqlAdmin(app) super(Manager, self).__init__(app, status, adm) - def get_start_db_params(self, data_dir): - """Get parameters for starting database. - - Cinder volume initialization(after formatted) may leave a lost+found - folder. - - The --ignore-db-dir option is deprecated in MySQL 5.7. With the - introduction of the data dictionary in MySQL 8.0, it became - superfluous and was removed in that version. - """ - params = f'--datadir={data_dir}' - - mysql_8 = semantic_version.Version('8.0.0') - cur_ver = semantic_version.Version.coerce(CONF.datastore_version) - params = f'--datadir={data_dir}' - if cur_ver < mysql_8: - params = (f"{params} --ignore-db-dir=lost+found " - f"--ignore-db-dir=conf.d") - - return params - def pre_create_backup(self, context, **kwargs): LOG.info("Running pre_create_backup") status = {} try: INFO_FILE = "%s/xtrabackup_binlog_info" % self.app.get_data_dir() self.app.execute_sql("FLUSH TABLES WITH READ LOCK;") - stt = self.app.execute_sql("SHOW MASTER STATUS;") + if self.app._is_mysql84(): + stt = self.app.execute_sql("SHOW BINARY LOG STATUS;") + else: + stt = self.app.execute_sql("SHOW MASTER STATUS;") for row in stt: status = { 'log_file': row._mapping['File'], diff --git a/trove/guestagent/datastore/mysql/service.py b/trove/guestagent/datastore/mysql/service.py index 392f5a0288..b3615594f8 100644 --- a/trove/guestagent/datastore/mysql/service.py +++ b/trove/guestagent/datastore/mysql/service.py @@ -28,12 +28,26 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) -class MySqlAppStatus(service.BaseMySqlAppStatus): - def __init__(self, docker_client): - super(MySqlAppStatus, self).__init__(docker_client) - - class MySqlApp(service.BaseMySqlApp): + + HEALTHCHECK = { + "test": ["CMD", "mysqladmin", "ping", "-h", + "127.0.0.1", "-u", "root", + "--password=$MYSQL_ROOT_PASSWORD"], + "start_period": 10 * 1000000000, # 10 seconds in nanoseconds + "interval": 10 * 1000000000, + "timeout": 5 * 1000000000, + "retries": 3 + } + + def _is_mysql84(self): + mysql_84 = semantic_version.Version('8.4.0') + cur_ver = semantic_version.Version.coerce(CONF.datastore_version) + if cur_ver >= mysql_84: + return True + else: + return False + def __init__(self, status, docker_client): super(MySqlApp, self).__init__(status, docker_client) @@ -44,11 +58,19 @@ class MySqlApp(service.BaseMySqlApp): def _get_slave_status(self): with mysql_util.SqlClient(self.get_engine()) as client: - return client.execute(text('SHOW SLAVE STATUS')).first() + if self._is_mysql84(): + return client.execute(text('SHOW REPLICA STATUS')).first() + else: + return client.execute(text('SHOW SLAVE STATUS')).first() def _get_master_UUID(self): slave_status = self._get_slave_status() - return slave_status and slave_status._mapping['Master_UUID'] or None + if self._is_mysql84(): + return slave_status and slave_status._mapping['Source_UUID'] \ + or None + else: + return slave_status and slave_status._mapping['Master_UUID'] \ + or None def get_latest_txn_id(self): return self._get_gtid_executed() @@ -65,9 +87,19 @@ class MySqlApp(service.BaseMySqlApp): return master_UUID, int(last_txn_id) def wait_for_txn(self, txn): + if self._is_mysql84(): + _sql = "SELECT WAIT_FOR_EXECUTED_GTID_SET('%s')" % txn + else: + _sql = "SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn with mysql_util.SqlClient(self.get_engine()) as client: - client.execute( - text("SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn)) + client.execute(text(_sql)) + + def stop_master(self): + LOG.info("Stopping replication master.") + if not self._is_mysql84(): + return super().stop_master() + with mysql_util.SqlClient(self.get_engine()) as client: + client.execute(text("RESET BINARY LOGS AND GTIDS")) def get_backup_image(self): """Get the actual container image based on datastore version. @@ -93,22 +125,16 @@ class MySqlApp(service.BaseMySqlApp): innobackupex was removed in Percona XtraBackup 8.0, use xtrabackup instead. """ - strategy = cfg.get_configuration_property('backup_strategy') - - mysql_8 = semantic_version.Version('8.0.0') - cur_ver = semantic_version.Version.coerce(CONF.datastore_version) - if cur_ver >= mysql_8: - strategy = 'xtrabackup' - - return strategy + return 'xtrabackup' def reset_data_for_restore_snapshot(self, data_dir): - """This function try remove slave status in database""" - mysql_8 = semantic_version.Version('8.0.0') - cur_ver = semantic_version.Version.coerce(CONF.datastore_version) - command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir - if cur_ver >= mysql_8: + """This function try remove replica status in database""" + # '--skip-replica-start' was introduced in mysql 8.0.26 and the + # '--skip-slave-start' not be removed yet for mysql 8.0.x + if self._is_mysql84: command = "mysqld --skip-replica-start=ON --datadir=%s" % data_dir + else: + command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir extra_volumes = { "/etc/mysql": {"bind": "/etc/mysql", "mode": "rw"}, @@ -133,6 +159,34 @@ class MySqlApp(service.BaseMySqlApp): except Exception as err: LOG.error('Failed to remove container. error: %s', str(err)) + def start_slave(self): + LOG.info("Starting slave replication.") + if not self._is_mysql84(): + return super().start_slave() + with mysql_util.SqlClient(self.get_engine()) as client: + client.execute(text('START REPLICA')) + self.wait_for_slave_status("ON", client, 180) + + def stop_slave(self, for_failover): + LOG.info("Stopping slave replication.") + if not self._is_mysql84(): + return super().stop_slave(for_failover) + replication_user = None + with mysql_util.SqlClient(self.get_engine()) as client: + result = client.execute( + text('SHOW REPLICA STATUS')).mappings().first() + if result: + replication_user = result['Source_User'] + client.execute(text('STOP REPLICA')) + client.execute(text('RESET REPLICA ALL')) + self.wait_for_slave_status('OFF', client, 180) + if not for_failover and replication_user: + client.execute( + text('DROP USER IF EXISTS ' + replication_user)) + return { + 'replication_user': replication_user + } + class MySqlRootAccess(service.BaseMySqlRootAccess): def __init__(self, app): diff --git a/trove/guestagent/datastore/mysql_common/manager.py b/trove/guestagent/datastore/mysql_common/manager.py index 1fa2ba49d6..235bcaa34b 100644 --- a/trove/guestagent/datastore/mysql_common/manager.py +++ b/trove/guestagent/datastore/mysql_common/manager.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. # -import tempfile +import os.path from oslo_log import log as logging @@ -208,54 +208,29 @@ class MySqlManager(manager.Manager): root_pass = utils.generate_random_password() self.app.save_password('root', root_pass) - init_file = tempfile.NamedTemporaryFile(mode='w') + init_file = os.path.join(data_dir, "init.sql") operating_system.write_file( - init_file.name, - f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';" + init_file, + f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';", + as_root=True + ) + # Change ownership so the database service user + # can read it in the container + operating_system.chown( + init_file, + user=self.app.database_service_uid, + group=self.app.database_service_gid, + as_root=True ) - err_file = tempfile.NamedTemporaryFile(suffix='.err') - # Get the original file owner and group before changing the owner. - from pathlib import Path - init_file_path = Path(init_file.name) - init_file_owner = init_file_path.owner() - init_file_group = init_file_path.group() - - # Allow database service user to access the temporary files. - try: - for file in [init_file.name, err_file.name]: - operating_system.chown( - file, - self.app.database_service_uid, - self.app.database_service_gid, - force=True, as_root=True) - except Exception as err: - LOG.error('Failed to change file owner, error: %s', str(err)) - for file in [init_file.name, err_file.name]: - LOG.debug('Reverting the %s owner to %s ' - 'before close it.', file, init_file_owner) - operating_system.chown(file, init_file_owner, - init_file_group, force=True, - as_root=True) - init_file.close() - err_file.close() - raise err - - # Allow database service user to access the temporary files. command = ( - f'mysqld --init-file={init_file.name} ' - f'--log-error={err_file.name} ' + f'mysqld --init-file={init_file} ' f'--datadir={data_dir} ' ) - extra_volumes = { - init_file.name: {"bind": init_file.name, "mode": "rw"}, - err_file.name: {"bind": err_file.name, "mode": "rw"}, - } # Start the database container process. try: - self.app.start_db(ds_version=ds_version, command=command, - extra_volumes=extra_volumes) + self.app.start_db(ds_version=ds_version, command=command) except Exception as err: LOG.error('Failed to reset password for restore, error: %s', str(err)) @@ -267,19 +242,12 @@ class MySqlManager(manager.Manager): docker_util.get_container_logs(self.app.docker_client) ) docker_util.remove_container(self.app.docker_client) + # Remove init.sql file after password reset + operating_system.remove(init_file, force=True, as_root=True) except Exception as err: - LOG.error('Failed to remove container. error: %s', + LOG.error('Failed to remove container or init file. error: %s', str(err)) pass - for file in [init_file.name, err_file.name]: - LOG.debug('Reverting the %s owner to %s ' - 'before close it.', file, init_file_owner) - operating_system.chown(file, init_file_owner, - init_file_group, force=True, - as_root=True) - init_file.close() - err_file.close() - LOG.info('Finished to reset password for restore') def _validate_slave_for_replication(self, context, replica_info): diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py index 4ff90000e2..1f80ea83eb 100644 --- a/trove/guestagent/datastore/mysql_common/service.py +++ b/trove/guestagent/datastore/mysql_common/service.py @@ -62,41 +62,6 @@ BACKUP_LOG = re.compile(r'.*Backup successfully, checksum: (?P.*), ' r'location: (?P.*)') -class BaseMySqlAppStatus(service.BaseDbStatus): - - def __init__(self, docker_client): - super(BaseMySqlAppStatus, self).__init__(docker_client) - - def get_actual_db_status(self): - """Check database service status.""" - status = docker_util.get_container_status(self.docker_client) - if status == "running": - root_pass = service.BaseDbApp.get_auth_password(file="root.cnf") - cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass - try: - docker_util.run_command(self.docker_client, cmd) - return service_status.ServiceStatuses.HEALTHY - except Exception as exc: - LOG.warning('Failed to run docker command, error: %s', - str(exc)) - container_log = docker_util.get_container_logs( - self.docker_client, tail='all') - LOG.debug('container log: \n%s', '\n'.join(container_log)) - return service_status.ServiceStatuses.RUNNING - elif status == "not running": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "restarting": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "paused": - return service_status.ServiceStatuses.PAUSED - elif status == "exited": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "dead": - return service_status.ServiceStatuses.CRASHED - else: - return service_status.ServiceStatuses.UNKNOWN - - class BaseMySqlAdmin(object, metaclass=abc.ABCMeta): """Handles administrative tasks on the MySQL database.""" diff --git a/trove/guestagent/datastore/postgres/manager.py b/trove/guestagent/datastore/postgres/manager.py index 45eccdf954..2b5aa04209 100644 --- a/trove/guestagent/datastore/postgres/manager.py +++ b/trove/guestagent/datastore/postgres/manager.py @@ -26,6 +26,7 @@ from trove.common import utils from trove.guestagent.common import operating_system from trove.guestagent.datastore import manager from trove.guestagent.datastore.postgres import service +from trove.guestagent.datastore import service as base_service from trove.guestagent import guest_log LOG = logging.getLogger(__name__) @@ -37,7 +38,7 @@ class PostgresManager(manager.Manager): def __init__(self): super(PostgresManager, self).__init__('postgres') - self.status = service.PgSqlAppStatus(self.docker_client) + self.status = base_service.BaseDbStatus(self.docker_client) self.app = service.PgSqlApp(self.status, self.docker_client) self.adm = service.PgSqlAdmin(service.SUPER_USER_NAME) diff --git a/trove/guestagent/datastore/postgres/service.py b/trove/guestagent/datastore/postgres/service.py index fe7b243a2a..305819b6c7 100644 --- a/trove/guestagent/datastore/postgres/service.py +++ b/trove/guestagent/datastore/postgres/service.py @@ -44,57 +44,6 @@ HBA_CONFIG_FILE = '/etc/postgresql/pg_hba.conf' WAL_ARCHIVE_DIR = '/var/lib/postgresql/data/wal_archive' -class PgSqlAppStatus(service.BaseDbStatus): - def __init__(self, docker_client): - super(PgSqlAppStatus, self).__init__(docker_client) - - def _get_container_status(self): - """Check database service status.""" - status = docker_util.get_container_status(self.docker_client) - if status == "running": - cmd = "psql -U postgres -c 'select 1;'" - try: - docker_util.run_command(self.docker_client, cmd) - return service_status.ServiceStatuses.HEALTHY - except Exception as exc: - LOG.warning('Failed to run docker command, error: %s', - str(exc)) - container_log = docker_util.get_container_logs( - self.docker_client, tail='all') - LOG.debug('container log: \n%s', '\n'.join(container_log)) - return service_status.ServiceStatuses.RUNNING - elif status == "not running": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "paused": - return service_status.ServiceStatuses.PAUSED - elif status == "exited": - return service_status.ServiceStatuses.SHUTDOWN - elif status == "dead": - return service_status.ServiceStatuses.CRASHED - else: - return service_status.ServiceStatuses.UNKNOWN - - def get_actual_db_status(self): - health = docker_util.get_container_health(self.docker_client) - LOG.debug('container health status: %s', health) - if health == "healthy": - return service_status.ServiceStatuses.HEALTHY - elif health == "starting": - return service_status.ServiceStatuses.RUNNING - elif health == "unhealthy": - # In case the container was stopped - status = docker_util.get_container_status(self.docker_client) - if status == "exited": - return service_status.ServiceStatuses.SHUTDOWN - else: - return service_status.ServiceStatuses.CRASHED - - # if the health status is one of unkown or None, let's check - # container status. this is for the compatibility with the - # old datastores. - return self._get_container_status() - - class PgSqlApp(service.BaseDbApp): _configuration_manager = None diff --git a/trove/guestagent/datastore/service.py b/trove/guestagent/datastore/service.py index 040cf1dbac..88ead56f7b 100644 --- a/trove/guestagent/datastore/service.py +++ b/trove/guestagent/datastore/service.py @@ -119,7 +119,25 @@ class BaseDbStatus(object): self.set_status(real_status, force=force) def get_actual_db_status(self): - raise NotImplementedError() + """Check database service status.""" + health = docker_util.get_container_health(self.docker_client) + LOG.debug('container health status: %s', health) + if health == "healthy": + return service_status.ServiceStatuses.HEALTHY + elif health == "starting": + return service_status.ServiceStatuses.RUNNING + # when the container is not found, it returns not running + elif health == "not running": + return service_status.ServiceStatuses.SHUTDOWN + elif health == "unhealthy": + # In case the container was stopped + status = docker_util.get_container_status(self.docker_client) + if status == "exited": + return service_status.ServiceStatuses.SHUTDOWN + else: + return service_status.ServiceStatuses.CRASHED + + return service_status.ServiceStatuses.UNKNOWN @property def is_installed(self): diff --git a/trove/guestagent/strategies/replication/mysql_gtid.py b/trove/guestagent/strategies/replication/mysql_gtid.py index 9076641b4a..13ce703e97 100644 --- a/trove/guestagent/strategies/replication/mysql_gtid.py +++ b/trove/guestagent/strategies/replication/mysql_gtid.py @@ -13,17 +13,23 @@ # License for the specific language governing permissions and limitations # under the License. # -from oslo_log import log as logging +from oslo_log import log as logging +import semantic_version + +from trove.common import cfg from trove.guestagent.strategies.replication import mysql_base LOG = logging.getLogger(__name__) +CONF = cfg.CONF class MysqlGTIDReplication(mysql_base.MysqlReplicationBase): """MySql Replication coordinated by GTIDs.""" def connect_to_master(self, service, master_info): + cur_ver = semantic_version.Version.coerce(CONF.datastore_version) + mysql_84 = semantic_version.Version('8.4.0') if 'dataset' in master_info: # pull the last executed GTID from the master via # the xtrabackup metadata file. If that value is @@ -35,10 +41,12 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase): last_gtid = self.read_last_master_gtid(service) LOG.info("last_gtid value is %s", last_gtid) if '-' in last_gtid: - # See - # https://avdeo.com/tag/error-1840-hy000-global-gtid_purged-can-only-be-set-when/ # Also, FLUSH PRIVILEGES will restore gtid_executed. - service.execute_sql('RESET MASTER') + if cur_ver >= mysql_84: + service.execute_sql('RESET BINARY LOGS AND GTIDS') + else: + # for mysql 8.0 + service.execute_sql('RESET MASTER') set_gtid_cmd = "SET GLOBAL gtid_purged='%s'" % last_gtid service.execute_sql(set_gtid_cmd) @@ -49,21 +57,37 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase): master_info['master']['port'], replica_conf['replication_user']['name'] ) + if cur_ver >= mysql_84: + change_master_cmd = ( + "CHANGE REPLICATION SOURCE TO " + "SOURCE_HOST='%(host)s', " + "SOURCE_PORT=%(port)s, " + "SOURCE_USER='%(user)s', " + "SOURCE_PASSWORD='%(password)s', " + "SOURCE_AUTO_POSITION=1, " + "GET_SOURCE_PUBLIC_KEY=1, " + "SOURCE_CONNECT_RETRY=15" % + { + 'host': master_info['master']['host'], + 'port': master_info['master']['port'], + 'user': replica_conf['replication_user']['name'], + 'password': replica_conf['replication_user']['password'] + }) + else: + change_master_cmd = ( + "CHANGE MASTER TO " + "MASTER_HOST='%(host)s', " + "MASTER_PORT=%(port)s, " + "MASTER_USER='%(user)s', " + "MASTER_PASSWORD='%(password)s', " + "MASTER_AUTO_POSITION=1, " + "MASTER_CONNECT_RETRY=15" % + { + 'host': master_info['master']['host'], + 'port': master_info['master']['port'], + 'user': replica_conf['replication_user']['name'], + 'password': replica_conf['replication_user']['password'] + }) - change_master_cmd = ( - "CHANGE MASTER TO " - "MASTER_HOST='%(host)s', " - "MASTER_PORT=%(port)s, " - "MASTER_USER='%(user)s', " - "MASTER_PASSWORD='%(password)s', " - "MASTER_AUTO_POSITION=1, " - "MASTER_CONNECT_RETRY=15" % - { - 'host': master_info['master']['host'], - 'port': master_info['master']['port'], - 'user': replica_conf['replication_user']['name'], - 'password': replica_conf['replication_user']['password'] - }) service.execute_sql(change_master_cmd) - service.start_slave() diff --git a/trove/templates/mysql/config.template b/trove/templates/mysql/config.template index acfa5e04fc..2a841f984b 100644 --- a/trove/templates/mysql/config.template +++ b/trove/templates/mysql/config.template @@ -15,7 +15,12 @@ secure-file-priv = NULL tmpdir = /var/tmp pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock +{% if datastore.semantic_version.major < 8 %} default_authentication_plugin = mysql_native_password +{% endif %} +{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %} +default_authentication_plugin = mysql_native_password +{% endif %} skip-external-locking = 1 key_buffer_size = {{ (50 * flavor['ram']/512)|int }}M max_allowed_packet = {{ (1024 * flavor['ram']/512)|int }}K diff --git a/trove/templates/mysql/replica.config.template b/trove/templates/mysql/replica.config.template index 548fc4916e..cf3ff78514 100644 --- a/trove/templates/mysql/replica.config.template +++ b/trove/templates/mysql/replica.config.template @@ -2,7 +2,12 @@ log_bin = /var/lib/mysql/data/mysql-bin.log binlog_format = MIXED relay_log = /var/lib/mysql/data/mysql-relay-bin.log +{% if datastore.semantic_version.major < 8 %} relay_log_info_repository = TABLE +{% endif %} +{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %} +relay_log_info_repository = TABLE +{% endif %} relay_log_recovery = 1 relay_log_purge = 1 log_slave_updates = ON diff --git a/trove/tests/unittests/common/test_template.py b/trove/tests/unittests/common/test_template.py index 1e24bf13c9..1188e0598e 100644 --- a/trove/tests/unittests/common/test_template.py +++ b/trove/tests/unittests/common/test_template.py @@ -64,8 +64,12 @@ class TemplateTest(trove_testtools.TestCase): self.assertGreater(len(server_id), 1) def test_rendering(self): - rendered = self.template.render(flavor=self.flavor_dict, - server_id=self.server_id) + datastore = mock_datastore_version("mysql", "mysql", "mysql", "8.4") + datastore.semantic_version = Version("8.4.0") + rendered = self.template.render( + flavor=self.flavor_dict, + server_id=self.server_id, + datastore=datastore) self.validate_template(rendered, "innodb_buffer_pool_size", self.flavor_dict, diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 7d104501c3..810bf75004 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -217,119 +217,6 @@ TROVE_ENABLE_LOCAL_REGISTRY: True tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_replication -- job: - name: trove-tempest-ubuntu-base-mysql8.0 - parent: trove-tempest-ubuntu-base - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^test-requirements.txt$ - - ^tox.ini$ - - ^LICENSE$ - - ^contrib/ - - ^zuul\.d/ - - ^backup/ - - ^\..+ - - ^trove/guestagent/strategies/replication/ - - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ - vars: - devstack_localrc: - TROVE_DATASTORE_VERSION: 8.0 - TROVE_STATE_CHANGE_WAIT_TIME: 900 - devstack_local_conf: - test-config: - $TEMPEST_CONFIG: - database: - default_datastore_versions: mysql:8.0 - -- job: - name: trove-tempest-ubuntu-backup-mysql8.0 - parent: trove-tempest-ubuntu-backup - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^test-requirements.txt$ - - ^tox.ini$ - - ^LICENSE$ - - ^contrib/ - - ^zuul\.d/ - - ^\..+ - - ^trove/guestagent/strategies/replication/ - - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ - vars: - devstack_localrc: - TROVE_DATASTORE_VERSION: 8.0 - TROVE_STATE_CHANGE_WAIT_TIME: 900 - devstack_local_conf: - test-config: - $TEMPEST_CONFIG: - database: - backup_wait_timeout: 1200 - default_datastore_versions: mysql:8.0 - -- job: - name: trove-tempest-ubuntu-replication-mysql8.0 - parent: trove-tempest-ubuntu-replication - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^test-requirements.txt$ - - ^tox.ini$ - - ^LICENSE$ - - ^contrib/ - - ^zuul\.d/ - - ^backup/ - - ^\..+ - - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ - - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$ - vars: - devstack_localrc: - TROVE_DATASTORE_VERSION: 8.0 - TROVE_STATE_CHANGE_WAIT_TIME: 900 - devstack_local_conf: - test-config: - $TEMPEST_CONFIG: - database: - backup_wait_timeout: 1200 - default_datastore_versions: mysql:8.0 - -- job: - name: trove-tempest-cinder-storage-driver-mysql8.0 - parent: trove-tempest-snapshot - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^test-requirements.txt$ - - ^tox.ini$ - - ^LICENSE$ - - ^contrib/ - - ^zuul\.d/ - - ^backup/ - - ^\..+ - - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ - - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$ - vars: - devstack_localrc: - TROVE_DATASTORE_VERSION: 8.0 - devstack_local_conf: - test-config: - $TEMPEST_CONFIG: - database: - backup_wait_timeout: 1200 - default_datastore_versions: mysql:8.0 - - job: name: publish-trove-guest-image parent: publish-openstack-artifacts diff --git a/zuul.d/mysql_job.yaml b/zuul.d/mysql_job.yaml new file mode 100644 index 0000000000..4b1774937a --- /dev/null +++ b/zuul.d/mysql_job.yaml @@ -0,0 +1,113 @@ +#MySQL jobs +- job: + name: trove-tempest-ubuntu-base-mysql8.4 + parent: trove-tempest-ubuntu-base + irrelevant-files: + - ^.*\.rst$ + - ^api-ref/.*$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^test-requirements.txt$ + - ^tox.ini$ + - ^LICENSE$ + - ^contrib/ + - ^zuul\.d/ + - ^backup/ + - ^\..+ + - ^trove/guestagent/strategies/replication/ + - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ + vars: + devstack_localrc: + TROVE_DATASTORE_VERSION: 8.4 + TROVE_STATE_CHANGE_WAIT_TIME: 900 + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + database: + default_datastore_versions: mysql:8.4 + +- job: + name: trove-tempest-ubuntu-backup-mysql8.4 + parent: trove-tempest-ubuntu-backup + irrelevant-files: + - ^.*\.rst$ + - ^api-ref/.*$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^test-requirements.txt$ + - ^tox.ini$ + - ^LICENSE$ + - ^contrib/ + - ^zuul\.d/ + - ^\..+ + - ^trove/guestagent/strategies/replication/ + - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ + vars: + devstack_localrc: + TROVE_DATASTORE_VERSION: 8.4 + TROVE_STATE_CHANGE_WAIT_TIME: 900 + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + database: + backup_wait_timeout: 1200 + default_datastore_versions: mysql:8.4 + +- job: + name: trove-tempest-ubuntu-replication-mysql8.4 + parent: trove-tempest-ubuntu-replication + irrelevant-files: + - ^.*\.rst$ + - ^api-ref/.*$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^test-requirements.txt$ + - ^tox.ini$ + - ^LICENSE$ + - ^contrib/ + - ^zuul\.d/ + - ^backup/ + - ^\..+ + - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ + - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$ + vars: + devstack_localrc: + TROVE_DATASTORE_VERSION: 8.4 + TROVE_STATE_CHANGE_WAIT_TIME: 900 + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + database: + backup_wait_timeout: 1200 + default_datastore_versions: mysql:8.4 + +- job: + name: trove-tempest-cinder-storage-driver-mysql8.4 + parent: trove-tempest-snapshot + irrelevant-files: + - ^.*\.rst$ + - ^api-ref/.*$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^test-requirements.txt$ + - ^tox.ini$ + - ^LICENSE$ + - ^contrib/ + - ^zuul\.d/ + - ^backup/ + - ^\..+ + - ^trove/guestagent/datastore/(postgres|mariadb)/.*$ + - ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$ + vars: + devstack_localrc: + TROVE_DATASTORE_VERSION: 8.4 + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + database: + backup_wait_timeout: 1200 + default_datastore_versions: mysql:8.4 diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 0936b187a3..9668411cf8 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -11,12 +11,12 @@ - release-notes-jobs-python3 check: jobs: - - trove-tempest-ubuntu-base-mysql8.0 - - trove-tempest-ubuntu-backup-mysql8.0: + - trove-tempest-ubuntu-base-mysql8.4 + - trove-tempest-ubuntu-backup-mysql8.4: voting: false - - trove-tempest-ubuntu-replication-mysql8.0: + - trove-tempest-ubuntu-replication-mysql8.4: voting: false - - trove-tempest-cinder-storage-driver-mysql8.0: + - trove-tempest-cinder-storage-driver-mysql8.4: voting: false - trove-tempest-ubuntu-base-mariadb11.4: voting: false @@ -42,7 +42,7 @@ voting: true gate: jobs: - - trove-tempest-ubuntu-base-mysql8.0 + - trove-tempest-ubuntu-base-mysql8.4 - trove-tempest-ubuntu-base-mariadb11.4 - trove-tempest-ubuntu-base-postgresql17 experimental: