openstack/ceilometer
Code Issues Proposed changes
Files
fbb4b6d264c9d24e3f85f891ef765507be1f899a
ceilometer/ceilometer/tests/unit/polling/test_dynamic_pollster.py
Rafael Weingärtner fbb4b6d264 OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data 
Sometimes we want/need to add/gather extra metadata for the samples
being handled by Ceilometer Dynamic pollsters, such as the project
name, domain id, domain name, and other metadata that are not always
accessible via the OpenStack component where the sample is gathered.

For instance, when gathering the status of virtual machines (VMs) from
Nova, we only have the tenant_id, which must be used as the project_id.
However, for billing and later invoicing one might need/want the
project name, domain id, and other metadata that are available in
Keystone (and maybe some others that are scattered over other
components). To achieve that, one can use the OpenStack metadata
enrichment option. This feature is only available to OpenStack
pollsters, and can only gather extra metadata from OpenStack APIs.

This patch introduces a new option in the OpenStack Dynamic pollsters,
that enable operators to enrich sample's metadata with information that
can be captured in other OpenStack APIs such as Keystone, Nova, Neutron,
and many others.

Change-Id: I079bf26cf26fcbcf678dba10469516a1dcb52c0d
2022-01-20 11:55:41 -03:00

1008 lines
40 KiB
Python
Raw Blame History

#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for OpenStack dynamic pollster
"""
import copy
import logging
from unittest import mock
import requests
from urllib import parse as urlparse
from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.polling import dynamic_pollster
from ceilometer import sample
from oslotest import base
LOG = logging.getLogger(__name__)
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
'value_attribute', 'endpoint_type',
'url_path']
class SampleGenerator(object):
def __init__(self, samples_dict, turn_to_list=False):
self.turn_to_list = turn_to_list
self.samples_dict = {}
for k, v in samples_dict.items():
if isinstance(v, list):
self.samples_dict[k] = [0, v]
else:
self.samples_dict[k] = [0, [v]]
def get_next_sample_dict(self):
_dict = {}
for key in self.samples_dict.keys():
_dict[key] = self.get_next_sample(key)
if self.turn_to_list:
_dict = [_dict]
return _dict
def get_next_sample(self, key):
samples = self.samples_dict[key][1]
samples_next_iteration = self.samples_dict[key][0] % len(samples)
self.samples_dict[key][0] += 1
_sample = samples[samples_next_iteration]
if isinstance(_sample, SampleGenerator):
return _sample.get_next_sample_dict()
return _sample
class PagedSamplesGenerator(SampleGenerator):
def __init__(self, samples_dict, dict_name, page_link_name):
super(PagedSamplesGenerator, self).__init__(samples_dict)
self.dict_name = dict_name
self.page_link_name = page_link_name
self.response = {}
def generate_samples(self, page_base_link, page_links, last_page_size):
self.response.clear()
current_page_link = page_base_link
for page_link, page_size in page_links.items():
page_link = page_base_link + "/" + page_link
self.response[current_page_link] = {
self.page_link_name: [{'href': page_link, 'rel': 'next'}],
self.dict_name: self.populate_page(page_size)
}
current_page_link = page_link
self.response[current_page_link] = {
self.dict_name: self.populate_page(last_page_size)
}
def populate_page(self, page_size):
page = []
for item_number in range(0, page_size):
page.append(self.get_next_sample_dict())
return page
class PagedSamplesGeneratorHttpRequestMock(PagedSamplesGenerator):
def mock_request(self, url, **kwargs):
return_value = TestDynamicPollster.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = self.response[url]
return return_value
class TestDynamicPollster(base.BaseTestCase):
class FakeResponse(object):
status_code = None
json_object = None
def json(self):
return self.json_object
def raise_for_status(self):
raise requests.HTTPError("Mock HTTP error.", response=self)
class FakeManager(object):
def __init__(self, keystone=None):
self._keystone = keystone
def setUp(self):
super(TestDynamicPollster, self).setUp()
self.pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume", 'endpoint_type': "test",
'url_path': "v1/test/endpoint/fake"}
self.pollster_definition_all_fields = {
'metadata_fields': "metadata-field-name",
'skip_sample_values': ["I-do-not-want-entries-with-this-value"],
'value_mapping': {
'value-to-map': 'new-value', 'value-to-map-to-numeric': 12
},
'default_value_mapping': 0,
'metadata_mapping': {
'old-metadata-name': "new-metadata-name"
},
'preserve_mapped_metadata': False}
self.pollster_definition_all_fields.update(
self.pollster_definition_only_required_fields)
self.multi_metric_pollster_definition = {
'name': "test-pollster.{category}", 'sample_type': "gauge",
'unit': "test", 'value_attribute': "[categories].ops",
'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"}
def execute_basic_asserts(self, pollster, pollster_definition):
self.assertEqual(pollster, pollster.obj)
self.assertEqual(pollster_definition['name'], pollster.name)
for key in REQUIRED_POLLSTER_FIELDS:
self.assertEqual(pollster_definition[key],
pollster.pollster_definitions[key])
self.assertEqual(pollster_definition, pollster.pollster_definitions)
@mock.patch('keystoneclient.v2_0.client.Client')
def test_skip_samples_with_linked_samples(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
'volume': SampleGenerator(samples_dict={
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
'tmp': ['ra', 'rb', 'rc', 'rd', 're', 'rf', 'rg', 'rh']},
turn_to_list=True),
'id': [1, 2, 3, 4, 5, 6, 7, 8],
'name': ['a1', 'b2', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8']
}, dict_name='servers', page_link_name='server_link')
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
keystone_mock.session.get.side_effect = generator.mock_request
fake_manager = self.FakeManager(keystone=keystone_mock)
pollster_definition = dict(self.multi_metric_pollster_definition)
pollster_definition['name'] = 'test-pollster.{name}'
pollster_definition['value_attribute'] = '[volume].tmp'
pollster_definition['skip_sample_values'] = ['rb']
pollster_definition['url_path'] = 'v1/test-volumes'
pollster_definition['response_entries_key'] = 'servers'
pollster_definition['next_sample_url_attribute'] = \
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
'list(value) | value[0] | value.get("href")'
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
list(map(lambda s: s.volume, samples)))
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'name'
pollster_definition['skip_sample_values'] = ['b2']
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
list(map(lambda s: s.volume, samples)))
def test_all_required_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.execute_basic_asserts(
pollster, self.pollster_definition_only_required_fields)
self.assertEqual(
0, len(pollster.pollster_definitions['skip_sample_values']))
self.assertEqual(
0, len(pollster.pollster_definitions['value_mapping']))
self.assertEqual(
-1, pollster.pollster_definitions['default_value'])
self.assertEqual(
0, len(pollster.pollster_definitions['metadata_mapping']))
self.assertEqual(
True, pollster.pollster_definitions['preserve_mapped_metadata'])
def test_all_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_all_fields)
self.execute_basic_asserts(pollster,
self.pollster_definition_all_fields)
self.assertEqual(
1, len(pollster.pollster_definitions['skip_sample_values']))
self.assertEqual(
2, len(pollster.pollster_definitions['value_mapping']))
self.assertEqual(
0, pollster.pollster_definitions['default_value_mapping'])
self.assertEqual(
1, len(pollster.pollster_definitions['metadata_mapping']))
self.assertEqual(
False, pollster.pollster_definitions['preserve_mapped_metadata'])
def test_all_required_fields_exceptions(self):
for key in REQUIRED_POLLSTER_FIELDS:
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition.pop(key)
exception = self.assertRaises(DynamicPollsterDefinitionException,
dynamic_pollster.DynamicPollster,
pollster_definition)
self.assertEqual("Required fields ['%s'] not specified."
% key, exception.brief_message)
def test_invalid_sample_type(self):
self.pollster_definition_only_required_fields[
'sample_type'] = "invalid_sample_type"
exception = self.assertRaises(
DynamicPollsterDefinitionException,
dynamic_pollster.DynamicPollster,
self.pollster_definition_only_required_fields)
self.assertEqual("Invalid sample type [invalid_sample_type]. "
"Valid ones are [('gauge', 'delta', 'cumulative')].",
exception.brief_message)
def test_all_valid_sample_type(self):
for sample_type in sample.TYPES:
self.pollster_definition_only_required_fields[
'sample_type'] = sample_type
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.execute_basic_asserts(
pollster, self.pollster_definition_only_required_fields)
def test_default_discovery_method(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.assertEqual("endpoint:test", pollster.definitions.sample_gatherer
.default_discovery)
@mock.patch('keystoneclient.v2_0.client.Client')
def test_execute_request_get_samples_empty_response(self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {}
client_mock.session.get.return_value = return_value
samples = pollster.definitions.sample_gatherer. \
execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual(0, len(samples))
@mock.patch('keystoneclient.v2_0.client.Client')
def test_execute_request_get_samples_response_non_empty(
self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {"firstElement": [{}, {}, {}]}
client_mock.session.get.return_value = return_value
samples = pollster.definitions.sample_gatherer. \
execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual(3, len(samples))
@mock.patch('keystoneclient.v2_0.client.Client')
def test_execute_request_get_samples_exception_on_request(
self, client_mock):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
return_value.status_code = requests.codes.bad
client_mock.session.get.return_value = return_value
exception = self.assertRaises(requests.HTTPError,
pollster.definitions.sample_gatherer.
execute_request_get_samples,
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual("Mock HTTP error.", str(exception))
def test_generate_new_metadata_fields_no_metadata_mapping(self):
metadata = {'name': 'someName',
'value': 1}
metadata_before_call = copy.deepcopy(metadata)
self.pollster_definition_only_required_fields['metadata_mapping'] = {}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(metadata_before_call, metadata)
def test_generate_new_metadata_fields_preserve_old_key(self):
metadata = {'name': 'someName', 'value': 2}
expected_metadata = copy.deepcopy(metadata)
expected_metadata['balance'] = metadata['value']
self.pollster_definition_only_required_fields[
'metadata_mapping'] = {'value': 'balance'}
self.pollster_definition_only_required_fields[
'preserve_mapped_metadata'] = True
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(expected_metadata, metadata)
def test_generate_new_metadata_fields_preserve_old_key_equals_false(self):
metadata = {'name': 'someName', 'value': 1}
expected_clean_metadata = copy.deepcopy(metadata)
expected_clean_metadata['balance'] = metadata['value']
expected_clean_metadata.pop('value')
self.pollster_definition_only_required_fields[
'metadata_mapping'] = {'value': 'balance'}
self.pollster_definition_only_required_fields[
'preserve_mapped_metadata'] = False
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(expected_clean_metadata, metadata)
def test_execute_value_mapping_no_value_mapping(self):
self.pollster_definition_only_required_fields['value_mapping'] = {}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = value_to_be_mapped
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_execute_value_mapping_no_value_mapping_found_with_default(self):
self.pollster_definition_only_required_fields[
'value_mapping'] = {'some-possible-value': 15}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = -1
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_execute_value_mapping_no_value_mapping_found_with_custom_default(
self):
self.pollster_definition_only_required_fields[
'value_mapping'] = {'some-possible-value': 5}
self.pollster_definition_only_required_fields[
'default_value'] = 0
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = 0
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_execute_value_mapping(self):
self.pollster_definition_only_required_fields[
'value_mapping'] = {'test': 'new-value'}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
value_to_be_mapped = "test"
expected_value = 'new-value'
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
def test_get_samples_no_resources(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
samples = pollster.get_samples(None, None, None)
self.assertEqual(None, next(samples))
@mock.patch('ceilometer.polling.dynamic_pollster.'
'PollsterSampleGatherer.execute_request_get_samples')
def test_get_samples_empty_samples(self, execute_request_get_samples_mock):
execute_request_get_samples_mock.side_effect = []
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
fake_manager = self.FakeManager()
samples = pollster.get_samples(
fake_manager, None, ["https://endpoint.server.name.com/"])
samples_list = list()
try:
for s in samples:
samples_list.append(s)
except RuntimeError as e:
LOG.debug("Generator threw a StopIteration "
"and we need to catch it [%s]." % e)
self.assertEqual(0, len(samples_list))
def fake_sample_list(self, **kwargs):
samples_list = list()
samples_list.append(
{'name': "sample5", 'volume': 5, 'description': "desc-sample-5",
'user_id': "924d1f77-5d75-4b96-a755-1774d6be17af",
'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
'id': "e335c317-dfdd-4f22-809a-625bd9a5992d"
}
)
samples_list.append(
{'name': "sample1", 'volume': 2, 'description': "desc-sample-2",
'user_id': "20b5a704-b481-4603-a99e-2636c144b876",
'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
'id': "2e350554-6c05-4fda-8109-e47b595a714c"
}
)
return samples_list
@mock.patch.object(
dynamic_pollster.PollsterSampleGatherer,
'execute_request_get_samples',
fake_sample_list)
def test_get_samples(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
fake_manager = self.FakeManager()
samples = pollster.get_samples(
fake_manager, None, ["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(2, len(samples_list))
first_element = [
s for s in samples_list
if s.resource_id == "e335c317-dfdd-4f22-809a-625bd9a5992d"][0]
self.assertEqual(5, first_element.volume)
self.assertEqual(
"6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", first_element.project_id)
self.assertEqual(
"924d1f77-5d75-4b96-a755-1774d6be17af", first_element.user_id)
second_element = [
s for s in samples_list
if s.resource_id == "2e350554-6c05-4fda-8109-e47b595a714c"][0]
self.assertEqual(2, second_element.volume)
self.assertEqual(
"6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", second_element.project_id)
self.assertEqual(
"20b5a704-b481-4603-a99e-2636c144b876", second_element.user_id)
def test_retrieve_entries_from_response_response_is_a_list(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
response = [{"object1-attr1": 1}, {"object1-attr2": 2}]
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response, pollster.definitions)
self.assertEqual(response, entries)
def test_retrieve_entries_using_first_entry_from_response(self):
self.pollster_definition_only_required_fields[
'response_entries_key'] = "first"
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
first_entries_from_response = [{"object1-attr1": 1},
{"object1-attr2": 2}]
second_entries_from_response = [{"object1-attr3": 3},
{"object1-attr4": 33}]
response = {"first": first_entries_from_response,
"second": second_entries_from_response}
entries = pollster.definitions.sample_gatherer.\
retrieve_entries_from_response(
response, pollster.definitions.configurations)
self.assertEqual(first_entries_from_response, entries)
def test_retrieve_entries_using_second_entry_from_response(self):
self.pollster_definition_only_required_fields[
'response_entries_key'] = "second"
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
first_entries_from_response = [{"object1-attr1": 1},
{"object1-attr2": 2}]
second_entries_from_response = [{"object1-attr3": 3},
{"object1-attr4": 33}]
response = {"first": first_entries_from_response,
"second": second_entries_from_response}
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response,
pollster.definitions.configurations)
self.assertEqual(second_entries_from_response, entries)
def test_retrieve_attribute_nested_value_non_nested_key(self):
key = "key"
value = [{"d": 2}, {"g": {"h": "val"}}]
json_object = {"key": value}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(value, returned_value)
def test_retrieve_attribute_nested_value_nested_key(self):
key = "key.subKey"
value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value = [{"r": 245}, {"h": {"yu": "yu"}}]
json_object = {"key": {"subKey": sub_value, "subkey2": value1}}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor. \
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(sub_value, returned_value)
def test_retrieve_attribute_nested_value_with_operation_on_attribute(self):
# spaces here are added on purpose at the end to make sure we
# execute the strip in the code before the eval
key = "key.subKey | value + 1|value / 2 | value * 3"
value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value = 1
expected_value_after_operations = 3
json_object = {"key": {"subKey": sub_value, "subkey2": value1}}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
def test_retrieve_attribute_nested_value_simulate_radosgw_processing(self):
key = "user | value.split('$') | value[0] | value.strip()"
json_object = {"categories": [
{
"bytes_received": 0,
"bytes_sent": 357088,
"category": "complete_multipart",
"ops": 472,
"successful_ops": 472
}],
"total": {
"bytes_received": 206739531986,
"bytes_sent": 273793180,
"ops": 119690,
"successful_ops": 119682
},
"user":
" 00ab8d7e76fc4$00ab8d7e76fc45a37776732"
}
expected_value_after_operations = "00ab8d7e76fc4"
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
def fake_sample_multi_metric(self, **kwargs):
multi_metric_sample_list = [
{"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "create_bucket",
"ops": 2,
"successful_ops": 2
},
{
"bytes_received": 0,
"bytes_sent": 2120428,
"category": "get_obj",
"ops": 46,
"successful_ops": 46
},
{
"bytes_received": 0,
"bytes_sent": 21484,
"category": "list_bucket",
"ops": 8,
"successful_ops": 8
},
{
"bytes_received": 6889056,
"bytes_sent": 0,
"category": "put_obj",
"ops": 46,
"successful_ops": 6
}],
"total": {
"bytes_received": 6889056,
"bytes_sent": 2141912,
"ops": 102,
"successful_ops": 106
},
"user": "test-user"}]
return multi_metric_sample_list
@mock.patch.object(
dynamic_pollster.PollsterSampleGatherer,
'execute_request_get_samples',
fake_sample_multi_metric)
def test_get_samples_multi_metric_pollster(self):
pollster = dynamic_pollster.DynamicPollster(
self.multi_metric_pollster_definition)
fake_manager = self.FakeManager()
samples = pollster.get_samples(
fake_manager, None, ["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(4, len(samples_list))
create_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.create_bucket"][0]
get_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.get_obj"][0]
list_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.list_bucket"][0]
put_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.put_obj"][0]
self.assertEqual(2, create_bucket_sample.volume)
self.assertEqual(46, get_obj_sample.volume)
self.assertEqual(8, list_bucket_sample.volume)
self.assertEqual(46, put_obj_sample.volume)
def test_execute_request_get_samples_custom_ids(self):
sample = {'user_id_attribute': "1",
'project_id_attribute': "2",
'resource_id_attribute': "3",
'user_id': "234",
'project_id': "2334",
'id': "35"}
def internal_execute_request_get_samples_mock(self, **kwargs):
class Response:
def json(self):
return [sample]
return Response(), "url"
original_method = dynamic_pollster.PollsterSampleGatherer.\
_internal_execute_request_get_samples
try:
dynamic_pollster.PollsterSampleGatherer. \
_internal_execute_request_get_samples = \
internal_execute_request_get_samples_mock
self.pollster_definition_all_fields[
'user_id_attribute'] = 'user_id_attribute'
self.pollster_definition_all_fields[
'project_id_attribute'] = 'project_id_attribute'
self.pollster_definition_all_fields[
'resource_id_attribute'] = 'resource_id_attribute'
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_all_fields)
params = {"d": "d"}
response = pollster.definitions.sample_gatherer. \
execute_request_get_samples(**params)
self.assertEqual(sample['user_id_attribute'],
response[0]['user_id'])
self.assertEqual(sample['project_id_attribute'],
response[0]['project_id'])
self.assertEqual(sample['resource_id_attribute'],
response[0]['id'])
finally:
dynamic_pollster.PollsterSampleGatherer. \
_internal_execute_request_get_samples = original_method
def test_retrieve_attribute_self_reference_sample(self):
key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0"
sub_value1 = [{"d": 2}, {"g": {"h": "val"}}]
sub_value2 = [{"r": 245}, {"h": {"yu": "yu"}}]
json_object = {"key1": {"subKey1": sub_value1},
"key2": {"subkey2": sub_value2}}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(2, returned_value)
del json_object['key1']
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(0, returned_value)
def test_create_request_arguments_NonOpenStackApisSamplesGatherer(self):
pollster_definition = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
'url_path': "https://test.com/v1/test/endpoint/fake",
"module": "someModule",
"authentication_object": "objectAuthentication",
"authentication_parameters": "authParam",
"headers": [{"header1": "val1"}, {"header2": "val2"}]}
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
request_args = pollster.definitions.sample_gatherer\
.create_request_arguments(pollster.definitions.configurations)
self.assertTrue("headers" in request_args)
self.assertEqual(2, len(request_args["headers"]))
self.assertEqual(['header1', 'header2'],
list(map(lambda h: list(h.keys())[0],
request_args["headers"])))
self.assertEqual(['val1', 'val2'],
list(map(lambda h: list(h.values())[0],
request_args["headers"])))
self.assertTrue("authenticated" not in request_args)
def test_create_request_arguments_PollsterSampleGatherer(self):
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition["headers"] = [
{"x-openstack-nova-api-version": "2.46"},
{"custom_header": "custom"},
{"some_other_header": "something"}]
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
request_args = pollster.definitions.sample_gatherer\
.create_request_arguments(pollster.definitions.configurations)
self.assertTrue("headers" in request_args)
self.assertTrue("authenticated" in request_args)
self.assertTrue(request_args["authenticated"])
self.assertEqual(3, len(request_args["headers"]))
self.assertEqual(['x-openstack-nova-api-version', 'custom_header',
"some_other_header"],
list(map(lambda h: list(h.keys())[0],
request_args["headers"])))
self.assertEqual(['2.46', 'custom', 'something'],
list(map(lambda h: list(h.values())[0],
request_args["headers"])))
def test_create_request_arguments_PollsterSampleGatherer_no_headers(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
request_args =\
pollster.definitions.sample_gatherer.create_request_arguments(
pollster.definitions.configurations)
self.assertTrue("headers" not in request_args)
self.assertTrue("authenticated" in request_args)
self.assertTrue(request_args["authenticated"])
@mock.patch('keystoneclient.v2_0.client.Client')
def test_metadata_nested_objects(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
'flavor': [{"name": "a", "ram": 1}, {"name": "b", "ram": 2},
{"name": "c", "ram": 3}, {"name": "d", "ram": 4},
{"name": "e", "ram": 5}, {"name": "f", "ram": 6},
{"name": "g", "ram": 7}, {"name": "h", "ram": 8}],
'name': ['s1', 's2', 's3', 's4', 's5', 's6', 's7', 's8'],
'state': ['Active', 'Error', 'Down', 'Active', 'Active',
'Migrating', 'Active', 'Error']
}, dict_name='servers', page_link_name='server_link')
generator.generate_samples('http://test.com/v1/test-servers', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
keystone_mock.session.get.side_effect = generator.mock_request
fake_manager = self.FakeManager(keystone=keystone_mock)
pollster_definition = dict(self.multi_metric_pollster_definition)
pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'state'
pollster_definition['url_path'] = 'v1/test-servers'
pollster_definition['response_entries_key'] = 'servers'
pollster_definition['metadata_fields'] = ['flavor.name', 'flavor.ram']
pollster_definition['next_sample_url_attribute'] = \
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
'list(value)| value [0] | value.get("href")'
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
samples = list(samples)
self.assertEqual(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
list(map(lambda s: s.resource_metadata["flavor.name"],
samples)))
self.assertEqual(list(range(1, 9)),
list(map(lambda s: s.resource_metadata["flavor.ram"],
samples)))
def test_get_request_linked_samples_url_no_next_sample(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = urlparse.urljoin(
base_url, self.pollster_definition_only_required_fields[
'url_path'])
kwargs = {'resource': base_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(
kwargs, pollster.definitions.configurations)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_url(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = "http://test.com/next_page"
kwargs = {'resource': base_url,
'next_sample_url': expected_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs, pollster.definitions)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_only_url_path(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = "http://test.com/next_page"
kwargs = {'resource': base_url,
'next_sample_url': "/next_page"}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(
kwargs, pollster.definitions.configurations)
self.assertEqual(expected_url, url)
def test_generate_sample_and_extract_metadata(self):
definition = self.pollster_definition_only_required_fields.copy()
definition['metadata_fields'] = ["metadata1", 'metadata2']
pollster = dynamic_pollster.DynamicPollster(definition)
pollster_sample = {'metadata1': 'metadata1',
'metadata2': 'metadata2',
'value': 1}
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
manager=mock.Mock())
self.assertEqual(1, sample.volume)
self.assertEqual(2, len(sample.resource_metadata))
self.assertEqual('metadata1', sample.resource_metadata['metadata1'])
self.assertEqual('metadata2', sample.resource_metadata['metadata2'])
def test_generate_sample_and_extract_metadata_false_value(self):
definition = self.pollster_definition_only_required_fields.copy()
definition['metadata_fields'] = ["metadata1",
'metadata2',
'metadata3_false']
pollster = dynamic_pollster.DynamicPollster(definition)
pollster_sample = {'metadata1': 'metadata1',
'metadata2': 'metadata2',
'metadata3_false': False,
'value': 1}
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
manager=mock.Mock())
self.assertEqual(1, sample.volume)
self.assertEqual(3, len(sample.resource_metadata))
self.assertEqual('metadata1', sample.resource_metadata['metadata1'])
self.assertEqual('metadata2', sample.resource_metadata['metadata2'])
self.assertIs(False, sample.resource_metadata['metadata3_false'])
def test_generate_sample_and_extract_metadata_none_value(self):
definition = self.pollster_definition_only_required_fields.copy()
definition['metadata_fields'] = ["metadata1",
'metadata2',
'metadata3']
pollster = dynamic_pollster.DynamicPollster(definition)
pollster_sample = {'metadata1': 'metadata1',
'metadata2': 'metadata2',
'metadata3': None,
'value': 1}
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
manager=mock.Mock())
self.assertEqual(1, sample.volume)
self.assertEqual(3, len(sample.resource_metadata))
self.assertEqual('metadata1', sample.resource_metadata['metadata1'])
self.assertEqual('metadata2', sample.resource_metadata['metadata2'])
self.assertIsNone(sample.resource_metadata['metadata3'])
View Git Blame Copy Permalink