From 641b5016997d610493f6c4fd08bf416b676bede5 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 6 Nov 2018 15:33:40 +0100 Subject: [PATCH] Don't heartbeat manually in the cloudkitty orchestrator This fixes cloudkitty-processor randomly crashing when releasing locks. Until now, heartbeating was done manually after each collect period, once all tenants have been processed. However, the first collect period is generally much longer than the following ones, given that the tenant workers need to catch up with the current date. Because of that, too much times passes between the moment where the coordinator is created and the first heartbeat. This causes locks to be automatically released, which leads to an error when we try to release them manually. This issue is fixed by letting the tooz coordinator taking care of heartbeating. Change-Id: If82f5981645032dcc649ea6e29cfdf8b1054b01e Story: 2004270 Task: 27817 --- cloudkitty/orchestrator.py | 3 +-- releasenotes/notes/fix-lock-release-74d112c8599c9a59.yaml | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/fix-lock-release-74d112c8599c9a59.yaml diff --git a/cloudkitty/orchestrator.py b/cloudkitty/orchestrator.py index f5b22fc2..0cc7759b 100644 --- a/cloudkitty/orchestrator.py +++ b/cloudkitty/orchestrator.py @@ -253,7 +253,7 @@ class Orchestrator(object): self.coord = coordination.get_coordinator( CONF.orchestrator.coordination_url, uuidutils.generate_uuid().encode('ascii')) - self.coord.start() + self.coord.start(start_heart=True) def _lock(self, tenant_id): lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii') @@ -303,7 +303,6 @@ class Orchestrator(object): lock.release() - self.coord.heartbeat() # NOTE(sheeprine): Slow down looping if all tenants are # being processed eventlet.sleep(1) diff --git a/releasenotes/notes/fix-lock-release-74d112c8599c9a59.yaml b/releasenotes/notes/fix-lock-release-74d112c8599c9a59.yaml new file mode 100644 index 00000000..90c4cb58 --- /dev/null +++ b/releasenotes/notes/fix-lock-release-74d112c8599c9a59.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + cloudkitty-processor crashes which happened when using distributed + tooz locks have been fixed.