Add retry mechanisms to oslo.cache

This patch specifies a set of options required to setup a retry
context.  The context built from those options can later on be
passed to any of the oslo.cache backends that supports pymemcache's
retry mechanisms.

This patch also sets up the retry mechanisms context based on
the configuration option passed via oslo.config and adds it
as an argument to be passed to the selected oslo.cache backend.

Change-Id: I6c1a4872d7cf19e3a55c676ef4b4200f18e08f2c

oslo_cache/_opts.py

@@ -172,6 +172,22 @@ FILE_OPTIONS = {
             help='The maximum number of keepalive probes TCP should '
             'send before dropping the connection. Should be a '
             'positive integer greater than zero.'),
+        cfg.BoolOpt(
+            'enable_retry_client',
+            default=False,
+            help='Enable retry client mechanisms to handle failure. '
+            'Those mechanisms can be used to wrap all kind of pymemcache '
+            'clients. The wrapper allows you to define how many attempts '
+            'to make and how long to wait between attemots.'),
+        cfg.IntOpt(
+            'retry_attempts',
+            min=1,
+            default=2,
+            help='Number of times to attempt an action before failing.'),
+        cfg.FloatOpt(
+            'retry_delay',
+            default=0,
+            help='Number of seconds to sleep between each attempt.'),
     ],
 }
 

oslo_cache/core.py

@@ -211,6 +211,25 @@ def _build_cache_config(conf):
         # oslo.cache and pymemcache.
         conf_dict['%s.arguments.socket_keepalive' % prefix] = socket_keepalive
 
+    # NOTE(hberaud): The pymemcache library comes with retry mechanisms that
+    # can be used to wrap all kind of pymemcache clients. The retry wrapper
+    # allow you to define how many attempts to make and how long to wait
+    # between attempts. The section below will pass our config
+    # to dogpile.cache to setup the pymemcache retry client wrapper.
+    if conf.cache.enable_retry_client:
+        if conf.cache.backend != 'dogpile.cache.pymemcache':
+            msg = _(
+                "Retry client is only supported by the "
+                "'dogpile.cache.pymemcache' backend."
+            )
+            raise exception.ConfigurationError(msg)
+        import pymemcache
+        conf_dict['%s.arguments.enable_retry_client' % prefix] = True
+        conf_dict['%s.arguments.retry_attempts' % prefix] = \
+            conf.cache.retry_attempts
+        conf_dict['%s.arguments.retry_delay' % prefix] = \
+            conf.cache.retry_delay
+
     return conf_dict
 
 

oslo_cache/tests/unit/test_cache_basics.py

@@ -437,6 +437,83 @@ class CacheRegionTest(test_cache.BaseTestCase):
             config_dict['test_prefix.arguments.socket_keepalive'].cnt
         )
 
+    def test_cache_pymemcache_retry_enabled_with_wrong_backend(self):
+        """Validate we build a config without the retry option when retry
+        is disabled.
+        """
+        self.config_fixture.config(group='cache',
+                                   enabled=True,
+                                   config_prefix='test_prefix',
+                                   backend='oslo_cache.dict',
+                                   enable_retry_client=True,
+                                   retry_attempts=2,
+                                   retry_delay=2)
+
+        self.assertRaises(
+            exception.ConfigurationError,
+            cache._build_cache_config,
+            self.config_fixture.conf
+        )
+
+    def test_cache_pymemcache_retry_disabled(self):
+        """Validate we build a config without the retry option when retry
+        is disabled.
+        """
+        self.config_fixture.config(group='cache',
+                                   enabled=True,
+                                   config_prefix='test_prefix',
+                                   backend='dogpile.cache.pymemcache',
+                                   retry_attempts=2,
+                                   retry_delay=2)
+
+        config_dict = cache._build_cache_config(self.config_fixture.conf)
+
+        opts = ['enable_retry_client', 'retry_attempts', 'retry_delay']
+
+        for el in opts:
+            self.assertNotIn('test_prefix.arguments.{}'.format(el),
+                             config_dict)
+
+    def test_cache_pymemcache_retry_enabled(self):
+        """Validate we build a dogpile.cache dict config with retry."""
+        self.config_fixture.config(group='cache',
+                                   enabled=True,
+                                   config_prefix='test_prefix',
+                                   backend='dogpile.cache.pymemcache',
+                                   enable_retry_client=True)
+
+        config_dict = cache._build_cache_config(self.config_fixture.conf)
+
+        opts = ['enable_retry_client', 'retry_attempts', 'retry_delay']
+
+        for el in opts:
+            self.assertIn('test_prefix.arguments.{}'.format(el), config_dict)
+
+    def test_cache_pymemcache_retry_with_opts(self):
+        """Validate we build a valid config for the retry client."""
+        self.config_fixture.config(group='cache',
+                                   enabled=True,
+                                   config_prefix='test_prefix',
+                                   backend='dogpile.cache.pymemcache',
+                                   enable_retry_client=True,
+                                   retry_attempts=42,
+                                   retry_delay=42)
+
+        config_dict = cache._build_cache_config(self.config_fixture.conf)
+
+        self.assertTrue(
+            self.config_fixture.conf.cache.enable_retry_client)
+
+        self.assertEqual(
+            config_dict['test_prefix.arguments.retry_attempts'],
+            42
+        )
+
+        self.assertEqual(
+            config_dict['test_prefix.arguments.retry_delay'],
+            42
+        )
+
     def test_cache_dictionary_config_builder_flush_on_reconnect_enabled(self):
         """Validate we build a sane dogpile.cache dictionary config."""
         self.config_fixture.config(group='cache',

releasenotes/notes/pymemcache_retry_mecchanisms-fa969d1ac6f64096.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add new options (``enable_retry_client``, ``retry_attempts``,
+    ``retry_delay``) to add retry mechanisms to the pymemcache backend.