openstack/trove
Code Issues Proposed changes

Add support for MySQL 8.4

Browse Source 
Change-Id: If462e377d182a9a56614b37ce4258cfd692ab21b
Signed-off-by: wu.chunyang <wchy1001@gmail.com>
This commit is contained in:
wu.chunyang
2025-06-25 16:32:59 +08:00
parent 57cbd95621
commit 4f1f1bbc1d
21 changed files with 319 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backup/install.sh
View File

@@ -69,7 +69,14 @@ if [ "${OPT_DATASTORE}" = "mysql" ]; then
dpkg -i percona-release.deb
percona-release enable-only tools release
apt-get update
apt-get install ${APTOPTS} percona-xtrabackup-80
if [ "${OPT_DATASTORE_VERSION}" = "8.0" ]; then
apt-get install ${APTOPTS} percona-xtrabackup-80
elif [ "${OPT_DATASTORE_VERSION}" = "8.4" ]; then
apt-get install ${APTOPTS} percona-xtrabackup-84
else
echo "Unsupported MySQL version: ${OPT_DATASTORE_VERSION}"
exit 1
fi
rm -f percona-release.deb
elif [ "${OPT_DATASTORE}" = "mariadb" ]; then
# See the url below about the supported version.

6
devstack/plugin.sh
View File

@@ -527,18 +527,18 @@ function create_registry_container {
container=$(sudo docker ps -a --format "{{.Names}}" --filter name=registry)
if [ -z $container ]; then
sudo docker run -d --net=host -e REGISTRY_HTTP_ADDR=0.0.0.0:4000 --restart=always -v /opt/trove_registry/:/var/lib/registry --name registry quay.io/openstack.trove/registry:2
for img in {"mysql:8.0","mariadb:11.4","postgres:17"};
for img in {"mysql:8.4","mariadb:11.4","postgres:17"};
do
sudo docker pull quay.io/openstack.trove/${img} && sudo docker tag quay.io/openstack.trove/${img} 127.0.0.1:4000/trove-datastores/${img} && sudo docker push 127.0.0.1:4000/trove-datastores/${img}
done
pushd $DEST/trove/backup
# build backup images
sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 .
sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.4 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.4 .
sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:11.4 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=11.4 .
sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:17 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=17 .
popd
# push backup images
for backupimg in {"db-backup-mysql:8.0","db-backup-mariadb:11.4","db-backup-postgresql:17"};
for backupimg in {"db-backup-mysql:8.4","db-backup-mariadb:11.4","db-backup-postgresql:17"};
do
sudo docker push 127.0.0.1:4000/trove-datastores/${backupimg}
done

2
devstack/settings
View File

@@ -29,7 +29,7 @@ TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/polic
TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"}
TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"noble"}
TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"}
TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.0"}
TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"8.4"}
# Configuration values listed here for reference
TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE}

5
releasenotes/notes/add-support-mysql8.4-2490b5c87f885b01.yaml Normal file
View File

@@ -0,0 +1,5 @@
---
features:
- |
Add support of MySQL 8.4, currently, The supported versions
are MySQL 8.0 & 8.4

17
trove/guestagent/common/sql_query.py
View File

@@ -20,7 +20,6 @@ Do not hard-code strings into the guest agent; use this module to build
them for you.
"""
import semantic_version
class Query(object):
@@ -363,20 +362,8 @@ class SetPassword(object):
return str(self)
def __str__(self):
if self.ds == 'mysql':
cur_version = semantic_version.Version.coerce(self.ds_version)
mysql_575 = semantic_version.Version('5.7.5')
if cur_version <= mysql_575:
return (f"SET PASSWORD FOR '{self.user}'@'{self.host}' = "
f"PASSWORD('{self.new_password}');")
return (f"ALTER USER '{self.user}'@'{self.host}' "
f"IDENTIFIED WITH mysql_native_password "
f"BY '{self.new_password}';")
elif self.ds == 'mariadb':
return (f"ALTER USER '{self.user}'@'{self.host}' IDENTIFIED VIA "
f"mysql_native_password USING "
f"PASSWORD('{self.new_password}');")
return (f"ALTER USER '{self.user}'@'{self.host}' "
f"IDENTIFIED BY '{self.new_password}';")
class DropUser(object):

3
trove/guestagent/datastore/mariadb/manager.py
View File

@@ -19,6 +19,7 @@ from trove.common import cfg
from trove.common import exception
from trove.common import utils
from trove.guestagent.common import operating_system
from trove.guestagent.datastore import service as base_service
from trove.guestagent.utils import docker as docker_utils
@@ -32,7 +33,7 @@ LOG = logging.getLogger(__name__)
class Manager(manager.MySqlManager):
def __init__(self):
status = service.MariadbAppStatus(self.docker_client)
status = base_service.BaseDbStatus(self.docker_client)
app = service.MariaDBApp(status, self.docker_client)
adm = service.MariaDBAdmin(app)

53
trove/guestagent/datastore/mariadb/service.py
View File

@@ -24,65 +24,12 @@ from trove.common import utils
from trove.guestagent.datastore.mysql_common import service as mysql_service
from trove.guestagent.utils import docker as docker_util
from trove.guestagent.utils import mysql as mysql_util
from trove.instance import service_status
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class MariadbAppStatus(mysql_service.BaseMySqlAppStatus):
def _get_container_status(self):
status = docker_util.get_container_status(self.docker_client)
if status == "running":
root_pass = mysql_util.BaseDbApp.get_auth_password(file="root.cnf")
cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass
try:
docker_util.run_command(self.docker_client, cmd)
return service_status.ServiceStatuses.HEALTHY
except Exception as exc:
LOG.warning('Failed to run docker command, error: %s',
str(exc))
container_log = docker_util.get_container_logs(
self.docker_client, tail='all')
LOG.debug('container log: \n%s', '\n'.join(container_log))
return service_status.ServiceStatuses.RUNNING
elif status == "not running":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "restarting":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "paused":
return service_status.ServiceStatuses.PAUSED
elif status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "dead":
return service_status.ServiceStatuses.CRASHED
else:
return service_status.ServiceStatuses.UNKNOWN
def get_actual_db_status(self):
"""Check database service status."""
health = docker_util.get_container_health(self.docker_client)
LOG.debug('container health status: %s', health)
if health == "healthy":
return service_status.ServiceStatuses.HEALTHY
elif health == "starting":
return service_status.ServiceStatuses.RUNNING
elif health == "unhealthy":
# In case the container was stopped
status = docker_util.get_container_status(self.docker_client)
if status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
else:
return service_status.ServiceStatuses.CRASHED
# if the health status is one of unkown or None, let's check
# container status .this is for the compatibility with the
# old datastores.
return self._get_container_status()
class MariaDBApp(mysql_service.BaseMySqlApp):
HEALTHCHECK = {

32
trove/guestagent/datastore/mysql/manager.py
View File

@@ -11,17 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import semantic_version
from oslo_log import log as logging
from trove.common import cfg
from trove.common import exception
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.mysql import service
from trove.guestagent.datastore.mysql_common import manager
from trove.guestagent.datastore import service as base_service
CONF = cfg.CONF
@@ -30,40 +28,22 @@ LOG = logging.getLogger(__name__)
class Manager(manager.MySqlManager):
def __init__(self):
status = service.MySqlAppStatus(self.docker_client)
status = base_service.BaseDbStatus(self.docker_client)
app = service.MySqlApp(status, self.docker_client)
adm = service.MySqlAdmin(app)
super(Manager, self).__init__(app, status, adm)
def get_start_db_params(self, data_dir):
"""Get parameters for starting database.
Cinder volume initialization(after formatted) may leave a lost+found
folder.
The --ignore-db-dir option is deprecated in MySQL 5.7. With the
introduction of the data dictionary in MySQL 8.0, it became
superfluous and was removed in that version.
"""
params = f'--datadir={data_dir}'
mysql_8 = semantic_version.Version('8.0.0')
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
params = f'--datadir={data_dir}'
if cur_ver < mysql_8:
params = (f"{params} --ignore-db-dir=lost+found "
f"--ignore-db-dir=conf.d")
return params
def pre_create_backup(self, context, **kwargs):
LOG.info("Running pre_create_backup")
status = {}
try:
INFO_FILE = "%s/xtrabackup_binlog_info" % self.app.get_data_dir()
self.app.execute_sql("FLUSH TABLES WITH READ LOCK;")
stt = self.app.execute_sql("SHOW MASTER STATUS;")
if self.app._is_mysql84():
stt = self.app.execute_sql("SHOW BINARY LOG STATUS;")
else:
stt = self.app.execute_sql("SHOW MASTER STATUS;")
for row in stt:
status = {
'log_file': row._mapping['File'],

98
trove/guestagent/datastore/mysql/service.py
View File

@@ -28,12 +28,26 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class MySqlAppStatus(service.BaseMySqlAppStatus):
def __init__(self, docker_client):
super(MySqlAppStatus, self).__init__(docker_client)
class MySqlApp(service.BaseMySqlApp):
HEALTHCHECK = {
"test": ["CMD", "mysqladmin", "ping", "-h",
"127.0.0.1", "-u", "root",
"--password=$MYSQL_ROOT_PASSWORD"],
"start_period": 10 * 1000000000, # 10 seconds in nanoseconds
"interval": 10 * 1000000000,
"timeout": 5 * 1000000000,
"retries": 3
}
def _is_mysql84(self):
mysql_84 = semantic_version.Version('8.4.0')
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
if cur_ver >= mysql_84:
return True
else:
return False
def __init__(self, status, docker_client):
super(MySqlApp, self).__init__(status, docker_client)
@@ -44,11 +58,19 @@ class MySqlApp(service.BaseMySqlApp):
def _get_slave_status(self):
with mysql_util.SqlClient(self.get_engine()) as client:
return client.execute(text('SHOW SLAVE STATUS')).first()
if self._is_mysql84():
return client.execute(text('SHOW REPLICA STATUS')).first()
else:
return client.execute(text('SHOW SLAVE STATUS')).first()
def _get_master_UUID(self):
slave_status = self._get_slave_status()
return slave_status and slave_status._mapping['Master_UUID'] or None
if self._is_mysql84():
return slave_status and slave_status._mapping['Source_UUID'] \
or None
else:
return slave_status and slave_status._mapping['Master_UUID'] \
or None
def get_latest_txn_id(self):
return self._get_gtid_executed()
@@ -65,9 +87,19 @@ class MySqlApp(service.BaseMySqlApp):
return master_UUID, int(last_txn_id)
def wait_for_txn(self, txn):
if self._is_mysql84():
_sql = "SELECT WAIT_FOR_EXECUTED_GTID_SET('%s')" % txn
else:
_sql = "SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn
with mysql_util.SqlClient(self.get_engine()) as client:
client.execute(
text("SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS('%s')" % txn))
client.execute(text(_sql))
def stop_master(self):
LOG.info("Stopping replication master.")
if not self._is_mysql84():
return super().stop_master()
with mysql_util.SqlClient(self.get_engine()) as client:
client.execute(text("RESET BINARY LOGS AND GTIDS"))
def get_backup_image(self):
"""Get the actual container image based on datastore version.
@@ -93,22 +125,16 @@ class MySqlApp(service.BaseMySqlApp):
innobackupex was removed in Percona XtraBackup 8.0, use xtrabackup
instead.
"""
strategy = cfg.get_configuration_property('backup_strategy')
mysql_8 = semantic_version.Version('8.0.0')
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
if cur_ver >= mysql_8:
strategy = 'xtrabackup'
return strategy
return 'xtrabackup'
def reset_data_for_restore_snapshot(self, data_dir):
"""This function try remove slave status in database"""
mysql_8 = semantic_version.Version('8.0.0')
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir
if cur_ver >= mysql_8:
"""This function try remove replica status in database"""
# '--skip-replica-start' was introduced in mysql 8.0.26 and the
# '--skip-slave-start' not be removed yet for mysql 8.0.x
if self._is_mysql84:
command = "mysqld --skip-replica-start=ON --datadir=%s" % data_dir
else:
command = "mysqld --skip-slave-start=ON --datadir=%s" % data_dir
extra_volumes = {
"/etc/mysql": {"bind": "/etc/mysql", "mode": "rw"},
@@ -133,6 +159,34 @@ class MySqlApp(service.BaseMySqlApp):
except Exception as err:
LOG.error('Failed to remove container. error: %s', str(err))
def start_slave(self):
LOG.info("Starting slave replication.")
if not self._is_mysql84():
return super().start_slave()
with mysql_util.SqlClient(self.get_engine()) as client:
client.execute(text('START REPLICA'))
self.wait_for_slave_status("ON", client, 180)
def stop_slave(self, for_failover):
LOG.info("Stopping slave replication.")
if not self._is_mysql84():
return super().stop_slave(for_failover)
replication_user = None
with mysql_util.SqlClient(self.get_engine()) as client:
result = client.execute(
text('SHOW REPLICA STATUS')).mappings().first()
if result:
replication_user = result['Source_User']
client.execute(text('STOP REPLICA'))
client.execute(text('RESET REPLICA ALL'))
self.wait_for_slave_status('OFF', client, 180)
if not for_failover and replication_user:
client.execute(
text('DROP USER IF EXISTS ' + replication_user))
return {
'replication_user': replication_user
}
class MySqlRootAccess(service.BaseMySqlRootAccess):
def __init__(self, app):

68
trove/guestagent/datastore/mysql_common/manager.py
View File

@@ -15,7 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import tempfile
import os.path
from oslo_log import log as logging
@@ -208,54 +208,29 @@ class MySqlManager(manager.Manager):
root_pass = utils.generate_random_password()
self.app.save_password('root', root_pass)
init_file = tempfile.NamedTemporaryFile(mode='w')
init_file = os.path.join(data_dir, "init.sql")
operating_system.write_file(
init_file.name,
f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';"
init_file,
f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';",
as_root=True
)
# Change ownership so the database service user
# can read it in the container
operating_system.chown(
init_file,
user=self.app.database_service_uid,
group=self.app.database_service_gid,
as_root=True
)
err_file = tempfile.NamedTemporaryFile(suffix='.err')
# Get the original file owner and group before changing the owner.
from pathlib import Path
init_file_path = Path(init_file.name)
init_file_owner = init_file_path.owner()
init_file_group = init_file_path.group()
# Allow database service user to access the temporary files.
try:
for file in [init_file.name, err_file.name]:
operating_system.chown(
file,
self.app.database_service_uid,
self.app.database_service_gid,
force=True, as_root=True)
except Exception as err:
LOG.error('Failed to change file owner, error: %s', str(err))
for file in [init_file.name, err_file.name]:
LOG.debug('Reverting the %s owner to %s '
'before close it.', file, init_file_owner)
operating_system.chown(file, init_file_owner,
init_file_group, force=True,
as_root=True)
init_file.close()
err_file.close()
raise err
# Allow database service user to access the temporary files.
command = (
f'mysqld --init-file={init_file.name} '
f'--log-error={err_file.name} '
f'mysqld --init-file={init_file} '
f'--datadir={data_dir} '
)
extra_volumes = {
init_file.name: {"bind": init_file.name, "mode": "rw"},
err_file.name: {"bind": err_file.name, "mode": "rw"},
}
# Start the database container process.
try:
self.app.start_db(ds_version=ds_version, command=command,
extra_volumes=extra_volumes)
self.app.start_db(ds_version=ds_version, command=command)
except Exception as err:
LOG.error('Failed to reset password for restore, error: %s',
str(err))
@@ -267,19 +242,12 @@ class MySqlManager(manager.Manager):
docker_util.get_container_logs(self.app.docker_client)
)
docker_util.remove_container(self.app.docker_client)
# Remove init.sql file after password reset
operating_system.remove(init_file, force=True, as_root=True)
except Exception as err:
LOG.error('Failed to remove container. error: %s',
LOG.error('Failed to remove container or init file. error: %s',
str(err))
pass
for file in [init_file.name, err_file.name]:
LOG.debug('Reverting the %s owner to %s '
'before close it.', file, init_file_owner)
operating_system.chown(file, init_file_owner,
init_file_group, force=True,
as_root=True)
init_file.close()
err_file.close()
LOG.info('Finished to reset password for restore')
def _validate_slave_for_replication(self, context, replica_info):

35
trove/guestagent/datastore/mysql_common/service.py
View File

@@ -62,41 +62,6 @@ BACKUP_LOG = re.compile(r'.*Backup successfully, checksum: (?P<checksum>.*), '
r'location: (?P<location>.*)')
class BaseMySqlAppStatus(service.BaseDbStatus):
def __init__(self, docker_client):
super(BaseMySqlAppStatus, self).__init__(docker_client)
def get_actual_db_status(self):
"""Check database service status."""
status = docker_util.get_container_status(self.docker_client)
if status == "running":
root_pass = service.BaseDbApp.get_auth_password(file="root.cnf")
cmd = 'mysql -uroot -p%s -e "select 1;"' % root_pass
try:
docker_util.run_command(self.docker_client, cmd)
return service_status.ServiceStatuses.HEALTHY
except Exception as exc:
LOG.warning('Failed to run docker command, error: %s',
str(exc))
container_log = docker_util.get_container_logs(
self.docker_client, tail='all')
LOG.debug('container log: \n%s', '\n'.join(container_log))
return service_status.ServiceStatuses.RUNNING
elif status == "not running":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "restarting":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "paused":
return service_status.ServiceStatuses.PAUSED
elif status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "dead":
return service_status.ServiceStatuses.CRASHED
else:
return service_status.ServiceStatuses.UNKNOWN
class BaseMySqlAdmin(object, metaclass=abc.ABCMeta):
"""Handles administrative tasks on the MySQL database."""

3
trove/guestagent/datastore/postgres/manager.py
View File

@@ -26,6 +26,7 @@ from trove.common import utils
from trove.guestagent.common import operating_system
from trove.guestagent.datastore import manager
from trove.guestagent.datastore.postgres import service
from trove.guestagent.datastore import service as base_service
from trove.guestagent import guest_log
LOG = logging.getLogger(__name__)
@@ -37,7 +38,7 @@ class PostgresManager(manager.Manager):
def __init__(self):
super(PostgresManager, self).__init__('postgres')
self.status = service.PgSqlAppStatus(self.docker_client)
self.status = base_service.BaseDbStatus(self.docker_client)
self.app = service.PgSqlApp(self.status, self.docker_client)
self.adm = service.PgSqlAdmin(service.SUPER_USER_NAME)

51
trove/guestagent/datastore/postgres/service.py
View File

@@ -44,57 +44,6 @@ HBA_CONFIG_FILE = '/etc/postgresql/pg_hba.conf'
WAL_ARCHIVE_DIR = '/var/lib/postgresql/data/wal_archive'
class PgSqlAppStatus(service.BaseDbStatus):
def __init__(self, docker_client):
super(PgSqlAppStatus, self).__init__(docker_client)
def _get_container_status(self):
"""Check database service status."""
status = docker_util.get_container_status(self.docker_client)
if status == "running":
cmd = "psql -U postgres -c 'select 1;'"
try:
docker_util.run_command(self.docker_client, cmd)
return service_status.ServiceStatuses.HEALTHY
except Exception as exc:
LOG.warning('Failed to run docker command, error: %s',
str(exc))
container_log = docker_util.get_container_logs(
self.docker_client, tail='all')
LOG.debug('container log: \n%s', '\n'.join(container_log))
return service_status.ServiceStatuses.RUNNING
elif status == "not running":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "paused":
return service_status.ServiceStatuses.PAUSED
elif status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
elif status == "dead":
return service_status.ServiceStatuses.CRASHED
else:
return service_status.ServiceStatuses.UNKNOWN
def get_actual_db_status(self):
health = docker_util.get_container_health(self.docker_client)
LOG.debug('container health status: %s', health)
if health == "healthy":
return service_status.ServiceStatuses.HEALTHY
elif health == "starting":
return service_status.ServiceStatuses.RUNNING
elif health == "unhealthy":
# In case the container was stopped
status = docker_util.get_container_status(self.docker_client)
if status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
else:
return service_status.ServiceStatuses.CRASHED
# if the health status is one of unkown or None, let's check
# container status. this is for the compatibility with the
# old datastores.
return self._get_container_status()
class PgSqlApp(service.BaseDbApp):
_configuration_manager = None

20
trove/guestagent/datastore/service.py
View File

@@ -119,7 +119,25 @@ class BaseDbStatus(object):
self.set_status(real_status, force=force)
def get_actual_db_status(self):
raise NotImplementedError()
"""Check database service status."""
health = docker_util.get_container_health(self.docker_client)
LOG.debug('container health status: %s', health)
if health == "healthy":
return service_status.ServiceStatuses.HEALTHY
elif health == "starting":
return service_status.ServiceStatuses.RUNNING
# when the container is not found, it returns not running
elif health == "not running":
return service_status.ServiceStatuses.SHUTDOWN
elif health == "unhealthy":
# In case the container was stopped
status = docker_util.get_container_status(self.docker_client)
if status == "exited":
return service_status.ServiceStatuses.SHUTDOWN
else:
return service_status.ServiceStatuses.CRASHED
return service_status.ServiceStatuses.UNKNOWN
@property
def is_installed(self):

62
trove/guestagent/strategies/replication/mysql_gtid.py
View File

@@ -13,17 +13,23 @@
# License for the specific language governing permissions and limitations
# under the License.
#
from oslo_log import log as logging
from oslo_log import log as logging
import semantic_version
from trove.common import cfg
from trove.guestagent.strategies.replication import mysql_base
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
"""MySql Replication coordinated by GTIDs."""
def connect_to_master(self, service, master_info):
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
mysql_84 = semantic_version.Version('8.4.0')
if 'dataset' in master_info:
# pull the last executed GTID from the master via
# the xtrabackup metadata file. If that value is
@@ -35,10 +41,12 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
last_gtid = self.read_last_master_gtid(service)
LOG.info("last_gtid value is %s", last_gtid)
if '-' in last_gtid:
# See
# https://avdeo.com/tag/error-1840-hy000-global-gtid_purged-can-only-be-set-when/
# Also, FLUSH PRIVILEGES will restore gtid_executed.
service.execute_sql('RESET MASTER')
if cur_ver >= mysql_84:
service.execute_sql('RESET BINARY LOGS AND GTIDS')
else:
# for mysql 8.0
service.execute_sql('RESET MASTER')
set_gtid_cmd = "SET GLOBAL gtid_purged='%s'" % last_gtid
service.execute_sql(set_gtid_cmd)
@@ -49,21 +57,37 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
master_info['master']['port'],
replica_conf['replication_user']['name']
)
if cur_ver >= mysql_84:
change_master_cmd = (
"CHANGE REPLICATION SOURCE TO "
"SOURCE_HOST='%(host)s', "
"SOURCE_PORT=%(port)s, "
"SOURCE_USER='%(user)s', "
"SOURCE_PASSWORD='%(password)s', "
"SOURCE_AUTO_POSITION=1, "
"GET_SOURCE_PUBLIC_KEY=1, "
"SOURCE_CONNECT_RETRY=15" %
{
'host': master_info['master']['host'],
'port': master_info['master']['port'],
'user': replica_conf['replication_user']['name'],
'password': replica_conf['replication_user']['password']
})
else:
change_master_cmd = (
"CHANGE MASTER TO "
"MASTER_HOST='%(host)s', "
"MASTER_PORT=%(port)s, "
"MASTER_USER='%(user)s', "
"MASTER_PASSWORD='%(password)s', "
"MASTER_AUTO_POSITION=1, "
"MASTER_CONNECT_RETRY=15" %
{
'host': master_info['master']['host'],
'port': master_info['master']['port'],
'user': replica_conf['replication_user']['name'],
'password': replica_conf['replication_user']['password']
})
change_master_cmd = (
"CHANGE MASTER TO "
"MASTER_HOST='%(host)s', "
"MASTER_PORT=%(port)s, "
"MASTER_USER='%(user)s', "
"MASTER_PASSWORD='%(password)s', "
"MASTER_AUTO_POSITION=1, "
"MASTER_CONNECT_RETRY=15" %
{
'host': master_info['master']['host'],
'port': master_info['master']['port'],
'user': replica_conf['replication_user']['name'],
'password': replica_conf['replication_user']['password']
})
service.execute_sql(change_master_cmd)
service.start_slave()

5
trove/templates/mysql/config.template
View File

@@ -15,7 +15,12 @@ secure-file-priv = NULL
tmpdir = /var/tmp
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
{% if datastore.semantic_version.major < 8 %}
default_authentication_plugin = mysql_native_password
{% endif %}
{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %}
default_authentication_plugin = mysql_native_password
{% endif %}
skip-external-locking = 1
key_buffer_size = {{ (50 * flavor['ram']/512)|int }}M
max_allowed_packet = {{ (1024 * flavor['ram']/512)|int }}K

5
trove/templates/mysql/replica.config.template
View File

@@ -2,7 +2,12 @@
log_bin = /var/lib/mysql/data/mysql-bin.log
binlog_format = MIXED
relay_log = /var/lib/mysql/data/mysql-relay-bin.log
{% if datastore.semantic_version.major < 8 %}
relay_log_info_repository = TABLE
{% endif %}
{% if datastore.semantic_version.major == 8 and datastore.semantic_version.minor < 4 %}
relay_log_info_repository = TABLE
{% endif %}
relay_log_recovery = 1
relay_log_purge = 1
log_slave_updates = ON

8
trove/tests/unittests/common/test_template.py
View File

@@ -64,8 +64,12 @@ class TemplateTest(trove_testtools.TestCase):
self.assertGreater(len(server_id), 1)
def test_rendering(self):
rendered = self.template.render(flavor=self.flavor_dict,
server_id=self.server_id)
datastore = mock_datastore_version("mysql", "mysql", "mysql", "8.4")
datastore.semantic_version = Version("8.4.0")
rendered = self.template.render(
flavor=self.flavor_dict,
server_id=self.server_id,
datastore=datastore)
self.validate_template(rendered,
"innodb_buffer_pool_size",
self.flavor_dict,

113
zuul.d/jobs.yaml
View File

@@ -217,119 +217,6 @@
TROVE_ENABLE_LOCAL_REGISTRY: True
tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_replication
- job:
name: trove-tempest-ubuntu-base-mysql8.0
parent: trove-tempest-ubuntu-base
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/strategies/replication/
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.0
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
default_datastore_versions: mysql:8.0
- job:
name: trove-tempest-ubuntu-backup-mysql8.0
parent: trove-tempest-ubuntu-backup
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^\..+
- ^trove/guestagent/strategies/replication/
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.0
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.0
- job:
name: trove-tempest-ubuntu-replication-mysql8.0
parent: trove-tempest-ubuntu-replication
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
- ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.0
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.0
- job:
name: trove-tempest-cinder-storage-driver-mysql8.0
parent: trove-tempest-snapshot
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
- ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.0
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.0
- job:
name: publish-trove-guest-image
parent: publish-openstack-artifacts

113
zuul.d/mysql_job.yaml Normal file
View File

@@ -0,0 +1,113 @@
#MySQL jobs
- job:
name: trove-tempest-ubuntu-base-mysql8.4
parent: trove-tempest-ubuntu-base
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/strategies/replication/
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.4
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
default_datastore_versions: mysql:8.4
- job:
name: trove-tempest-ubuntu-backup-mysql8.4
parent: trove-tempest-ubuntu-backup
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^\..+
- ^trove/guestagent/strategies/replication/
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.4
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.4
- job:
name: trove-tempest-ubuntu-replication-mysql8.4
parent: trove-tempest-ubuntu-replication
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
- ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.4
TROVE_STATE_CHANGE_WAIT_TIME: 900
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.4
- job:
name: trove-tempest-cinder-storage-driver-mysql8.4
parent: trove-tempest-snapshot
irrelevant-files:
- ^.*\.rst$
- ^api-ref/.*$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^test-requirements.txt$
- ^tox.ini$
- ^LICENSE$
- ^contrib/
- ^zuul\.d/
- ^backup/
- ^\..+
- ^trove/guestagent/datastore/(postgres|mariadb)/.*$
- ^trove/guestagent/strategies/replication/(postgresql.*|mariadb.*)\.py$
vars:
devstack_localrc:
TROVE_DATASTORE_VERSION: 8.4
devstack_local_conf:
test-config:
$TEMPEST_CONFIG:
database:
backup_wait_timeout: 1200
default_datastore_versions: mysql:8.4

10
zuul.d/projects.yaml
View File

@@ -11,12 +11,12 @@
- release-notes-jobs-python3
check:
jobs:
- trove-tempest-ubuntu-base-mysql8.0
- trove-tempest-ubuntu-backup-mysql8.0:
- trove-tempest-ubuntu-base-mysql8.4
- trove-tempest-ubuntu-backup-mysql8.4:
voting: false
- trove-tempest-ubuntu-replication-mysql8.0:
- trove-tempest-ubuntu-replication-mysql8.4:
voting: false
- trove-tempest-cinder-storage-driver-mysql8.0:
- trove-tempest-cinder-storage-driver-mysql8.4:
voting: false
- trove-tempest-ubuntu-base-mariadb11.4:
voting: false
@@ -42,7 +42,7 @@
voting: true
gate:
jobs:
- trove-tempest-ubuntu-base-mysql8.0
- trove-tempest-ubuntu-base-mysql8.4
- trove-tempest-ubuntu-base-mariadb11.4
- trove-tempest-ubuntu-base-postgresql17
experimental: