# Copyright 2015 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from distutils import version import logging from fuel_health.common import ssh from fuel_health import test LOG = logging.getLogger(__name__) class BaseMysqlTest(test.BaseTestCase): """Base methods for MySQL DB tests """ @classmethod def setUpClass(cls): super(BaseMysqlTest, cls).setUpClass() cls.nodes = cls.config.compute.nodes cls.controllers = cls.config.compute.online_controllers if cls.controllers: cls.controller_ip = cls.controllers[0] cls.node_key = cls.config.compute.path_to_private_key cls.node_user = cls.config.compute.ssh_user cls.mysql_user = 'root' cls.master_ip = [] cls.release_version = \ cls.config.compute.release_version.split('-')[1] cls.one_db_msg = "There is only one database online. Nothing to check" cls.no_db_msg = ("Can not find any online database. " "Check that at least one database is operable") def setUp(self): super(BaseMysqlTest, self).setUp() if 'ha' not in self.config.compute.deployment_mode: self.skipTest('Cluster is not HA mode, skipping tests') if not self.controllers: self.skipTest('All cluster controllers are offline') @classmethod def get_database_nodes(cls, controller_ip, username, key): if version.StrictVersion(cls.release_version)\ < version.StrictVersion('7.0'): return cls.config.compute.online_controllers # retrieve data from controller ssh_client = ssh.Client(controller_ip, username, key_filename=key, timeout=100) hiera_cmd = ('ruby -e \'require "hiera"; ' 'db_h = Hiera.new().lookup("database_nodes", {}, {}); ' 'db = db_h.keys.map{|k| db_h[k]["name"]}; ' 'if db != [] then puts db else puts "None" end\'') database_nodes = ssh_client.exec_command(hiera_cmd) # get online nodes database_nodes = database_nodes.splitlines() databases = [] for node in cls.config.compute.nodes: hostname = node['hostname'] if hostname in database_nodes and node['online']: databases.append(hostname) return databases class TestMysqlStatus(BaseMysqlTest): @classmethod def setUpClass(cls): super(TestMysqlStatus, cls).setUpClass() def setUp(self): super(TestMysqlStatus, self).setUp() if 'ha' not in self.config.compute.deployment_mode: self.skipTest('Cluster is not HA mode, skipping tests') def test_os_databases(self): """Check if amount of tables in databases is the same on each node Target Service: HA mysql Scenario: 1. Detect there are online database nodes. 2. Request list of tables for os databases on each node. 3. Check if amount of tables in databases is the same on each node Duration: 10 s. """ LOG.info("'Test OS Databases' started") dbs = ['nova', 'glance', 'keystone'] cmd = "mysql -h localhost -e 'SHOW TABLES FROM %(database)s'" databases = self.verify(20, self.get_database_nodes, 1, "Can not get database hostnames. Check that" " at least one controller is operable", "get database nodes", self.controller_ip, self.node_user, key=self.node_key) self.verify_response_body_not_equal(0, len(databases), self.no_db_msg, 1) if len(databases) == 1: self.skipTest(self.one_db_msg) for database in dbs: LOG.info('Current database name is %s' % database) temp_set = set() for node in databases: LOG.info('Current database node is %s' % node) cmd1 = cmd % {'database': database} LOG.info('Try to execute command %s' % cmd1) tables = ssh.Client( node, self.node_user, key_filename=self.node_key, timeout=self.config.compute.ssh_timeout) output = self.verify(40, tables.exec_command, 2, 'Can list tables', 'get amount of tables for each database', cmd1) tables = set(output.splitlines()) if len(temp_set) == 0: temp_set = tables self.verify_response_true( len(tables.symmetric_difference(temp_set)) == 0, "Step 3 failed: Tables in %s database are " "different" % database) del temp_set @staticmethod def get_variables_from_output(output, variables): """Return dict with variables and their values extracted from mysql Assume that output is "| Var_name | Value |" """ result = {} LOG.debug('Expected variables: "{0}"'.format(str(variables))) for line in output: try: var, value = line.strip("| ").split("|")[:2] except ValueError: continue var = var.strip() if var in variables: result[var] = value.strip() LOG.debug('Extracted values: "{0}"'.format(str(result))) return result def test_state_of_galera_cluster(self): """Check galera environment state Target Service: HA mysql Scenario: 1. Detect there are online database nodes. 2. Ssh on each node containing database and request state of galera node 3. For each node check cluster size 4. For each node check status is ready 5. For each node check that node is connected to cluster Duration: 10 s. """ databases = self.verify(20, self.get_database_nodes, 1, "Can not get database hostnames. Check that" " at least one controller is operable", "get database nodes", self.controller_ip, self.node_user, key=self.node_key) self.verify_response_body_not_equal(0, len(databases), self.no_db_msg, 1) if len(databases) == 1: self.skipTest(self.one_db_msg) for db_node in databases: command = "mysql -h localhost -e \"SHOW STATUS LIKE 'wsrep_%'\"" ssh_client = ssh.Client(db_node, self.node_user, key_filename=self.node_key, timeout=100) output = self.verify( 20, ssh_client.exec_command, 2, "Verification of galera cluster node status failed", 'get status from galera node', command).splitlines() LOG.debug('mysql output from node "{0}" is \n"{1}"'.format( db_node, output) ) mysql_vars = [ 'wsrep_cluster_size', 'wsrep_ready', 'wsrep_connected' ] result = self.get_variables_from_output(output, mysql_vars) self.verify_response_body_content( result.get('wsrep_cluster_size', 0), str(len(databases)), msg='Cluster size on %s less ' 'than databases count' % db_node, failed_step='3') self.verify_response_body_content( result.get('wsrep_ready', 'OFF'), 'ON', msg='wsrep_ready on %s is not ON' % db_node, failed_step='4') self.verify_response_body_content( result.get('wsrep_connected', 'OFF'), 'ON', msg='wsrep_connected on %s is not ON' % db_node, failed_step='5')