openstack/ironic
Code Issues Proposed changes

Add manual clean and automated verify steps to set BMC clock

Browse Source 
via Redfish Manager

This patch adds two new capabilities to the Redfish management
interface in Ironic for setting the BMC clock:

1. A manual cleaning step (`set_bmc_clock`) that allows operators
   to set the BMC clock explicitly by providing datetime and
   timezone offset.

2. An automated verify step (`verify_bmc_clock`) that, if enabled
   via configuration, sets the BMC clock during node verification
   using the current UTC time.

These steps aim to prevent certificate validation failures caused by
incorrect BMC time, particularly when dealing with TLS certificates.

A new configuration option `redfish.enable_verify_bmc_clock` has been
added to control the automated verify behavior.

The minimum version of `sushy` has also been updated to is 5.7.0
to support these features.

Related patches:
- https://review.opendev.org/c/openstack/sushy/+/950539
  (Add support for Manager DateTime fields in sushy)
- https://review.opendev.org/c/openstack/sushy-tools/+/950925
  (Fix Manager DateTime field handling in sushy-tools)

Partial-Bug: #2041904
Change-Id: I75cbd39a60f8470224dc5a2fe0a4f17c22acd1cd
Signed-off-by: Queensly Kyerewaa Acheampongmaa <qacheampong@gmail.com>
This commit is contained in:
Queensly Acheampongmaa
2025-06-26 16:34:28 +00:00
parent 069566ff6a
commit 94948bb194
5 changed files with 260 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ironic/conf/redfish.py
View File

@@ -129,6 +129,17 @@ opts = [
cfg.StrOpt('verify_ca',
help=_('The default verify_ca path when redfish_verify_ca '
'in driver_info is missing or set to True.')),
cfg.BoolOpt('enable_verify_bmc_clock',
default=False,
help=_('Whether to enable the automated verify step '
'that checks and sets the BMC clock. '
'When enabled, Ironic will automatically attempt '
'to set the BMC\'s clock during node registration '
'(in the verify phase) using Redfish\'s '
'DateTime fields. '
'This helps avoid TLS certificate issues '
'caused by incorrect BMC time.')),
]

97
ironic/drivers/modules/redfish/management.py
View File

@@ -14,9 +14,11 @@
# under the License.
import collections
from datetime import timezone
import time
from urllib.parse import urlparse
from dateutil import parser
from oslo_log import log
from oslo_utils import timeutils
import sushy
@@ -448,6 +450,101 @@ class RedfishManagement(base.ManagementInterface):
for attr in [getattr(resource, field)]
}
@base.clean_step(priority=0, abortable=False, argsinfo={
'target_datetime': {
'description': 'The datetime to set in ISO8601 format',
'required': True
},
'datetime_local_offset': {
'description': 'The local time offset from UTC',
'required': False
}
})
@task_manager.require_exclusive_lock
def set_bmc_clock(self, task, target_datetime, datetime_local_offset=None):
"""Set the BMC clock using Redfish Manager resource.
:param task: a TaskManager instance containing the node to act on.
:param target_datetime: The datetime to set in ISO8601 format
:param datetime_local_offset: The local time offset from UTC (optional)
:raises: RedfishError if the operation fails
"""
try:
system = redfish_utils.get_system(task.node)
manager = redfish_utils.get_manager(task.node, system)
LOG.debug("Setting BMC clock to %s (offset: %s)",
target_datetime, datetime_local_offset)
manager._conn.timeout = 30
manager.set_datetime(
target_datetime,
datetime_local_offset
)
manager.refresh()
if manager.datetime != target_datetime:
raise exception.RedfishError(
"BMC clock update failed: mismatch after setting datetime")
LOG.info(
"Successfully updated BMC clock for node %s",
task.node.uuid
)
except Exception as e:
LOG.exception("BMC clock update failed: %s", e)
raise exception.RedfishError(error=str(e))
@base.verify_step(priority=1)
@task_manager.require_exclusive_lock
def verify_bmc_clock(self, task):
"""Verify and auto-set the BMC clock to the current UTC time.
This step compares the system UTC time to the BMC's Redfish datetime.
If the difference exceeds 1 second, it attempts to sync the time.
Verification fails only if the BMC time remains incorrect
after the update.
"""
if not CONF.redfish.enable_verify_bmc_clock:
LOG.info("Skipping BMC clock verify step: disabled via config")
return
try:
system_time = timeutils.utcnow().replace(
tzinfo=timezone.utc).isoformat()
system = redfish_utils.get_system(task.node)
manager = redfish_utils.get_manager(task.node, system)
manager.refresh()
manager_time = parser.isoparse(manager.datetime)
local_time = parser.isoparse(system_time)
LOG.debug("BMC time: %s, Local time: %s",
manager_time, local_time)
LOG.debug("manager.datetime_local_offset: %s",
manager.datetimelocaloffset)
# Fail if the BMC clock differs from system time
# by more than 1 second
if abs((manager_time - local_time).total_seconds()) > 1:
LOG.info("BMC clock is out of sync. Updating...")
manager.set_datetime(system_time,
datetime_local_offset="+00:00")
manager.refresh()
updated_time = parser.isoparse(manager.datetime)
if abs((updated_time - local_time).total_seconds()) > 1:
raise exception.RedfishError(
"BMC clock still incorrect after update")
LOG.info("BMC clock update successful for node %s",
task.node.uuid)
except Exception as e:
LOG.exception("BMC clock auto-update failed during verify: %s", e)
raise exception.NodeVerifyFailure(
node=getattr(task.node, 'uuid', 'unknown'),
reason="BMC clock verify step failed: %s" % str(e)
)
@classmethod
def _get_sensors_fan(cls, chassis):
"""Get fan sensors reading.

134
ironic/tests/unit/drivers/modules/redfish/test_management.py
View File

@@ -555,6 +555,140 @@ class RedfishManagementTestCase(db_base.DbTestCase):
expected = boot_modes.LEGACY_BIOS
self.assertEqual(expected, response)
@mock.patch.object(redfish_utils, 'get_manager', autospec=True)
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
def test_set_bmc_clock_success(self, mock_get_system, mock_get_manager):
with task_manager.acquire(self.context, self.node.uuid,
shared=False) as task:
mock_system = mock.Mock()
mock_manager = mock.Mock()
mock_manager.datetime = '2025-06-27T12:00:00'
mock_get_system.return_value = mock_system
mock_get_manager.return_value = mock_manager
target_datetime = '2025-06-27T12:00:00'
task.driver.management.set_bmc_clock(
task, target_datetime=target_datetime)
mock_manager.set_datetime.assert_called_once_with(
target_datetime, None)
mock_manager.refresh.assert_called_once()
@mock.patch.object(redfish_utils, 'get_manager', autospec=True)
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
def test_set_bmc_clock_datetime_mismatch_raises(self, mock_get_system,
mock_get_manager):
with task_manager.acquire(self.context, self.node.uuid,
shared=False) as task:
mock_system = mock.Mock()
mock_manager = mock.Mock()
mock_manager.datetime = 'wrong-time'
mock_get_system.return_value = mock_system
mock_get_manager.return_value = mock_manager
with self.assertRaisesRegex(
exception.RedfishError,
'BMC clock update failed: mismatch after setting '
'datetime'):
task.driver.management.set_bmc_clock(
task, target_datetime='2025-06-27T12:00:00')
mock_manager.set_datetime.assert_called_once()
mock_manager.refresh.assert_called_once()
@mock.patch.object(redfish_utils, 'get_manager', autospec=True)
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
@mock.patch.object(CONF.redfish, 'enable_verify_bmc_clock', new=False)
def test_verify_bmc_clock_disabled_in_config(self, mock_get_system,
mock_get_manager):
with task_manager.acquire(self.context, self.node.uuid,
shared=False) as task:
task.driver.management.verify_bmc_clock(task)
# Ensure Redfish was never called
mock_get_system.assert_not_called()
mock_get_manager.assert_not_called()
@mock.patch.object(redfish_utils, 'get_manager', autospec=True)
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
@mock.patch.object(timeutils, 'utcnow', autospec=True)
def test_verify_bmc_clock_success_when_time_matches(self, mock_utcnow,
mock_get_system,
mock_get_manager):
self.config(enable_verify_bmc_clock=True, group='redfish')
current_datetime = datetime.datetime(2025, 7, 13, 3, 0, 0)
mock_utcnow.return_value = current_datetime
self.node.updated_at = datetime.datetime.utcnow()
self.node.save()
with task_manager.acquire(self.context, self.node.uuid,
shared=False) as task:
mock_system = mock.Mock()
mock_manager = mock.Mock()
mock_manager.datetime = current_datetime.replace(
tzinfo=datetime.timezone.utc).isoformat()
mock_manager.datetimelocaloffset = '+00:00'
mock_get_system.return_value = mock_system
mock_get_manager.return_value = mock_manager
task.driver.management.verify_bmc_clock(task)
mock_manager.set_datetime.assert_not_called()
mock_manager.refresh.assert_called_once()
@mock.patch.object(redfish_utils, 'get_manager', autospec=True)
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
@mock.patch.object(timeutils, 'utcnow', autospec=True)
def test_verify_bmc_clock_fails_on_mismatch(self, mock_utcnow,
mock_get_system,
mock_get_manager):
self.config(enable_verify_bmc_clock=True, group='redfish')
current_datetime = datetime.datetime(2025, 7, 13, 3, 0, 0)
mock_utcnow.return_value = current_datetime
self.node.updated_at = datetime.datetime.utcnow()
self.node.save()
with task_manager.acquire(self.context, self.node.uuid,
shared=False) as task:
if task.node is None:
task.node = self.node
if not getattr(task.node, 'uuid', None):
task.node.uuid = self.node.uuid
mock_system = mock.Mock()
mock_manager = mock.Mock()
# Mismatched time (e.g., 5 seconds later)
mock_manager.datetime = (
current_datetime + datetime.timedelta(seconds=5)).replace(
tzinfo=datetime.timezone.utc).isoformat()
mock_manager.datetimelocaloffset = "+00:00"
mock_get_system.return_value = mock_system
mock_get_manager.return_value = mock_manager
management = redfish_mgmt.RedfishManagement()
with self.assertRaisesRegex(exception.NodeVerifyFailure,
"BMC clock verify step failed"):
management.verify_bmc_clock(task)
self.assertEqual(mock_manager.refresh.call_count, 2)
mock_manager.set_datetime.assert_called_once_with(
current_datetime.replace(
tzinfo=datetime.timezone.utc).isoformat(),
datetime_local_offset="+00:00")
@mock.patch.object(redfish_utils, 'get_system', autospec=True)
def test_inject_nmi(self, mock_get_system):
fake_system = mock.Mock()

17
releasenotes/notes/add-bmc-clock-clean-verify-step-6b70b04a618bf6e1.yaml Normal file
View File

@@ -0,0 +1,17 @@
---
features:
- |
Adds two new capabilities to the Redfish managemnet interface
for managing the BMC clock.
1. A manual cleaning step ``set_bmc_clock`` that allows operators
to set the BMC's hardware clock to a specific datetime (in ISO8601 format),
optionally including a datetimelocaloffset.
2. An automated verify step ``verify_bmc_clock`` that compares the
BMC's Redfish datetime to the system UTC time, and automatically
updates the BMC clock if needed. Verification fails if the difference
exceeds 1 second after the update.
These steps helps ensure BMC clock synchronization in baremetal environments
where incorrect or drifting BMC clocks may lead to TLS certificate validation failures.

2
requirements.txt
View File

@@ -38,7 +38,7 @@ psutil>=3.2.2 # BSD
futurist>=1.2.0 # Apache-2.0
tooz>=2.7.0 # Apache-2.0
openstacksdk>=0.99.0 # Apache-2.0
sushy>=4.8.0
sushy>=5.7.0
construct>=2.9.39 # MIT
netaddr>=0.9.0 # BSD
microversion-parse>=1.0.1 # Apache-2.0