Add retry and debug logs to NoAuthClient of placement client

Based on logstash (see [0])
test_configurations_are_synced_towards_placement sporadically fails.
Based on the logs it seems that when the dummy placement wsgi server
restarts the client can't connect to it, to make the situation cleaner
I added some logs for the different possible requests exceptions and a
simple retry mechanism.

[0]: http://logstash.openstack.org/#dashboard/file/logstash.json?query=message%3A%5C%22test_agent_bandwidth_report.py*%20line%20249%2C%20in%20test_configurations_are_synced_towards_placement%5C%22

Change-Id: I283c4b159ad0b9ffef8a6f5a3bfe1fe2feac8210
This commit is contained in:
elajkat
2019-09-30 18:23:40 +02:00
parent 53f1847390
commit d07cb3675f

View File

@@ -15,6 +15,7 @@
import functools
import re
import time
import uuid
import requests
@@ -86,6 +87,7 @@ class NoAuthClient(object):
# TODO(lajoskatona): use perhaps http_connect_timeout from
# keystone_authtoken group
self.timeout = 5
self.retries = 2
def request(self, url, method, body=None, headers=None, **kwargs):
headers = headers or {}
@@ -98,20 +100,28 @@ class NoAuthClient(object):
# jsonification again, so better to create the json here and give it
# to requests with the data parameter.
body = jsonutils.dumps(body, cls=UUIDEncoder)
try:
resp = requests.request(
method,
url,
data=body,
headers=headers,
verify=False,
timeout=self.timeout,
**kwargs)
return resp
for i in range(self.retries):
try:
resp = requests.request(
method,
url,
data=body,
headers=headers,
verify=False,
timeout=self.timeout,
**kwargs)
return resp
except requests.Timeout:
LOG.exception('requests Timeout, let\'s retry it...')
except requests.ConnectionError:
LOG.exception('Connection Error appeared')
except requests.RequestException as e:
LOG.exception('Some really weird thing happened, let\'s '
'retry it')
time.sleep(self.timeout)
# Note(lajoskatona): requests raise ConnectionError, but
# PlacementReportPlugin expects keystonauth1 HttpError.
except requests.ConnectionError:
raise ks_exc.HttpError
raise ks_exc.HttpError
def get(self, url, endpoint_filter, **kwargs):
return self.request('%s%s' % (self.url, url), 'GET', **kwargs)