openstack/freezer
Code Issues Proposed changes

Merge "Add ability to restrict backup modes, actions, storages, engines"

Browse Source
This commit is contained in:
Zuul
2025-03-03 17:58:50 +00:00
committed by Gerrit Code Review
parent 8287dadb68 3a70336064
commit 27a01a7ca7
8 changed files with 364 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
doc/source/cli/freezer-scheduler.rst
View File

@@ -305,6 +305,37 @@ OPTIONS
Number of jobs that can be executed at the same time
.. oslo.config:group:: capabilities
.. oslo.config:option:: supported_actions
:Type: list
:Default: ``backup,restore,info,admin,exec``
List of supported actions separated by comma. Other actions will be ignored.
.. oslo.config:option:: supported_modes
:Type: list
:Default: ``fs,mongo,mysql,sqlserver,cinder,glance,cindernative,nova``
List of supported modes separated by comma. Other modes will be ignored.
.. oslo.config:option:: supported_storages
:Type: list
:Default: ``local,swift,ssh,s3,ftp,ftps``
List of supported storages separated by comma. Other storages will be ignored.
.. oslo.config:option:: supported_engines
:Type: list
:Default: ``tar,rsync,rsyncv2,nova,osbrick,glance``
List of supported engines separated by comma. Other engines will be ignored.
SEE ALSO
========

14
etc/scheduler.conf.sample
View File

@@ -146,3 +146,17 @@
# Enables or disables fatal status of deprecations. (boolean value)
#fatal_deprecations = false
[capabilities]
# List of supported actions separated by comma. Other actions will be ignored.
#supported_actions = backup,restore,info,admin,exec
# List of supported modes separated by comma. Other modes will be ignored.
#supported_modes = fs,mongo,mysql,sqlserver,cinder,glance,cindernative,nova
# List of supported storages separated by comma. Other storages will be ignored.
#supported_storages = local,swift,ssh,s3,ftp,ftps
# List of supported engines separated by comma. Other engines will be ignored.
#supported_engines = tar,rsync,rsyncv2,nova,osbrick,glance

50
freezer/scheduler/arguments.py
View File

@@ -28,6 +28,12 @@ if winutils.is_windows():
else:
DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d'
DEFAULT_SUPPORTED_ACTIONS = 'backup,restore,info,admin,exec'
DEFAULT_SUPPORTED_MODES = ('fs,mongo,mysql,sqlserver,cinder,glance,'
'cindernative,nova')
DEFAULT_SUPPORTED_STORAGES = 'local,swift,ssh,s3,ftp,ftps'
DEFAULT_SUPPORTED_ENGINES = 'tar,rsync,rsyncv2,nova,osbrick,glance'
def add_filter():
@@ -112,6 +118,41 @@ def get_common_opts():
return _COMMON
def get_capabilities_opts():
capabilities = [
cfg.ListOpt('supported-actions',
item_type=cfg.types.String(),
default=DEFAULT_SUPPORTED_ACTIONS,
dest='supported_actions',
help='List of supported actions separated by comma.'
'Other actions will be ignored. Default value is'
f' "{DEFAULT_SUPPORTED_ACTIONS}"'),
cfg.ListOpt('supported-modes',
item_type=cfg.types.String(),
default=DEFAULT_SUPPORTED_MODES,
dest='supported_modes',
help='List of supported modes separated by comma.'
'Other modes will be ignored. Default value is'
f' "{DEFAULT_SUPPORTED_MODES}"'),
cfg.ListOpt('supported-storages',
item_type=cfg.types.String(),
default=DEFAULT_SUPPORTED_STORAGES,
dest='supported_storages',
help='List of supported storages separated by comma.'
'Other storages will be ignored. Default value is'
f' "{DEFAULT_SUPPORTED_STORAGES}"'),
cfg.ListOpt('supported-engines',
item_type=cfg.types.String(),
default=DEFAULT_SUPPORTED_ENGINES,
dest='supported_engines',
help='List of supported engines separated by comma.'
'Other engines will be ignored. Default value is'
f' "{DEFAULT_SUPPORTED_ENGINES}"'),
]
return capabilities
def build_os_options():
osclient_opts = [
cfg.StrOpt('os-username',
@@ -200,8 +241,17 @@ def build_os_options():
return osclient_opts
def configure_capabilities_options():
capabilities_group = cfg.OptGroup(
name='capabilities', title='Capabilities of the freezer-scheduler')
CONF.register_group(capabilities_group)
CONF.register_cli_opts(get_capabilities_opts(), group=capabilities_group)
def parse_args(choices):
default_conf = cfg.find_config_files('freezer', 'scheduler', '.conf')
configure_capabilities_options()
CONF.register_cli_opts(get_common_opts())
CONF.register_cli_opts(build_os_options())
log.register_options(CONF)

19
freezer/scheduler/freezer_scheduler.py
View File

@@ -76,9 +76,26 @@ class FreezerScheduler(object):
self.remove_job = self.scheduler.remove_job
self.jobs = {}
def filter_jobs(self, job_doc_list):
"""Filter jobs by supported capabilities.
:param list[dict] job_doc_list: list of jobs
:return: list of jobs
:rtype: list[dict]
"""
jobs = []
for job_doc in job_doc_list:
job = scheduler_job.Job(self, self.freezerc_executable, job_doc)
if job.check_capabilities():
jobs.append(job_doc)
else:
LOG.debug(f'Job {job_doc["job_id"]} ignored: not supported')
return jobs
def get_jobs(self):
if self.client:
job_doc_list = utils.get_active_jobs_from_api(self.client)
job_doc_list = self.filter_jobs(
utils.get_active_jobs_from_api(self.client))
try:
utils.save_jobs_to_disk(job_doc_list, self.job_path)
except Exception as e:

21
freezer/scheduler/scheduler_job.py
View File

@@ -558,3 +558,24 @@ class Job(object):
def kill(self):
if self.process:
self.process.kill()
def check_capabilities(self):
"""Check if the requested capabilities are available.
:return: True if the job can be executed on this node
:rtype: bool
"""
capabilities = [
(CONF.capabilities.supported_actions, 'action'),
(CONF.capabilities.supported_modes, 'mode'),
(CONF.capabilities.supported_storages, 'storage'),
(CONF.capabilities.supported_engines, 'engine_name'),
]
actions = self.job_doc.get('job_actions')
for action in actions:
for supported_values, key in capabilities:
freezer_action = action.get('freezer_action')
requested_value = freezer_action.get(key, None)
if requested_value and requested_value not in supported_values:
return False
return True

31
freezer/tests/unit/scheduler/commons.py Normal file
View File

@@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from freezer.scheduler import arguments
from oslo_config import cfg
CONF = cfg.CONF
def set_test_capabilities():
arguments.configure_capabilities_options()
CONF.capabilities.supported_actions = ['backup']
CONF.capabilities.supported_modes = ['cindernative']
CONF.capabilities.supported_storages = ['swift']
CONF.capabilities.supported_engines = []
def set_default_capabilities():
CONF.capabilities.supported_actions = arguments.DEFAULT_SUPPORTED_ACTIONS
CONF.capabilities.supported_modes = arguments.DEFAULT_SUPPORTED_MODES
CONF.capabilities.supported_storages = arguments.DEFAULT_SUPPORTED_STORAGES
CONF.capabilities.supported_engines = arguments.DEFAULT_SUPPORTED_ENGINES

55
freezer/tests/unit/scheduler/test_freezer_scheduler.py Normal file
View File

@@ -0,0 +1,55 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from unittest import mock
from freezer.scheduler.freezer_scheduler import FreezerScheduler
from freezer.tests.unit.scheduler.commons import set_default_capabilities
from freezer.tests.unit.scheduler.commons import set_test_capabilities
SUPPORTED_JOB = {
'job_id': 'test2',
'job_schedule': {},
'job_actions': [
{'freezer_action': {'action': 'backup'}},
],
}
UNSUPPORTED_JOB = {
'job_id': 'test1',
'job_schedule': {},
'job_actions': [
{'freezer_action': {'action': 'exec'}},
],
}
class TestFreezerScheduler(unittest.TestCase):
def setUp(self):
self.scheduler = FreezerScheduler(
apiclient=mock.MagicMock(),
interval=1,
job_path='/tmp/test',
)
set_test_capabilities()
def tearDown(self):
set_default_capabilities()
def test_filter_jobs(self):
job_doc_list = [
SUPPORTED_JOB,
UNSUPPORTED_JOB,
]
expected_jobs = [SUPPORTED_JOB]
filtered_jobs = self.scheduler.filter_jobs(job_doc_list)
self.assertListEqual(filtered_jobs, expected_jobs)

144
freezer/tests/unit/scheduler/test_scheduler_job.py
View File

@@ -18,6 +18,8 @@ import tempfile
import unittest
from freezer.scheduler import scheduler_job
from freezer.tests.unit.scheduler.commons import set_default_capabilities
from freezer.tests.unit.scheduler.commons import set_test_capabilities
from oslo_config import cfg
from unittest import mock
@@ -366,3 +368,145 @@ class TestSchedulerJob1(unittest.TestCase):
self.job.process = process
self.assertIsNone(self.job.terminate())
self.assertIsNone(self.job.kill())
class TestSchedulerJobCapabilities(unittest.TestCase):
# Tests for SchedulerJob.check_capabilities()
def setUp(self):
self.scheduler = mock.MagicMock()
set_test_capabilities()
def tearDown(self):
set_default_capabilities()
def test_job_unsupported_action(self):
"""check_capabilities of a job that contains allowed and disallowed
actions. The job should be rejected.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {'action': 'backup'}},
{'freezer_action': {'action': 'exec'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertFalse(result)
def test_job_supported_action(self):
"""check_capabilities of a job that contains only allowed actions.
The job should be accepted.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {'action': 'backup'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertTrue(result)
def test_job_unsupported_mode(self):
"""check_capabilities of a job that contains allowed and disallowed
modes. The job should be rejected.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{
'freezer_action': {
'action': 'backup', 'mode': 'cindernative'}},
{'freezer_action': {'action': 'backup', 'mode': 'fs'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertFalse(result)
def test_job_supported_mode(self):
"""check_capabilities of a job that contains only allowed modes.
The job should be accepted.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {
'action': 'backup', 'mode': 'cindernative'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertTrue(result)
def test_job_unsupported_storage(self):
"""check_capabilities of a job that contains allowed and disallowed
storages. The job should be rejected.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {
'action': 'backup', 'storage': 'swift'}},
{'freezer_action': {'action': 'backup', 'storage': 'local'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertFalse(result)
def test_job_supported_storage(self):
"""check_capabilities of a job that contains only allowed storages.
The job should be accepted.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {'action': 'backup', 'storage': 'swift'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertTrue(result)
def test_job_unsupported_engine(self):
"""check_capabilities of a job that contains disallowed engines.
The job should be rejected.
"""
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {
'action': 'backup', 'engine_name': 'tar'}},
{'freezer_action': {
'action': 'backup', 'engine_name': 'rsync'}},
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertFalse(result)
def test_job_supported_engine(self):
"""check_capabilities of a job that contains only allowed engines.
The job should be accepted.
"""
CONF.capabilities.supported_engines = ['glance']
jobdoc = {
'job_id': 'test',
'job_schedule': {},
'job_actions': [
{'freezer_action': {
'action': 'backup', 'engine_name': 'glance'}}
],
}
job = scheduler_job.Job(self.scheduler, None, jobdoc)
result = job.check_capabilities()
self.assertTrue(result)