From 54d11e5f3966edfb80a3c419d1f8348688ad374a Mon Sep 17 00:00:00 2001 From: Andrew Vaillancourt Date: Thu, 7 Aug 2025 17:31:37 -0400 Subject: [PATCH] Add path_prefix support for registry separation Enables organizational separation within registry hosts by adding optional path_prefix configuration. This supports common use cases like Harbor projects, private registry namespaces, and organizational hierarchies. Key changes: - Add path_prefix field to Registry class with automatic slash normalization - Update DockerSyncImagesKeywords to construct URLs with path prefixes - Enhance configuration documentation with path_prefix examples - Add unit tests for path_prefix handling and URL construction Examples: - Harbor projects: "project-x/test" -> harbor.com/project-x/test/busybox - Team namespaces: "team-a" -> registry.com/team-a/my-image The path_prefix field is optional and trailing slashes are normalized automatically, making configuration flexible while ensuring correct URLs. Change-Id: I4784bc10e61840901baeaef2b69f323956bc23c3 Signed-off-by: Andrew Vaillancourt --- config/docker/files/default.json5 | 44 ++++++++++++--- config/docker/objects/docker_config.py | 1 + config/docker/objects/registry.py | 29 +++++++--- .../images/docker_sync_images_keywords.py | 19 +++++-- .../image_manifests/harbor-test-images.yaml | 13 +++-- .../image_manifests/stx-test-images.yaml | 13 +++-- .../stx-third-party-test-images.yaml | 11 ++-- .../docker/registry_path_prefix_test.py | 54 +++++++++++++++++++ 8 files changed, 154 insertions(+), 30 deletions(-) create mode 100644 unit_tests/config/docker/registry_path_prefix_test.py diff --git a/config/docker/files/default.json5 b/config/docker/files/default.json5 index 92752e86..b84dd1fd 100644 --- a/config/docker/files/default.json5 +++ b/config/docker/files/default.json5 @@ -35,7 +35,14 @@ // "manifest_registry" is set to null—for clarity and visibility. // // - "registries": -// A dictionary of registry definitions including URLs and credentials. +// A dictionary of registry definitions including URLs, credentials, and optional path prefixes. +// Each registry can define: +// - "path_prefix": Optional path prefix prepended to image names during sync. +// Enables organizational separation within the same registry host. +// Trailing slash is optional and will be normalized automatically. +// Examples: +// - Harbor projects: "project-x/test" -> harbor.com/project-x/test/busybox +// - Private registry namespaces: "team-a" -> registry.com/team-a/my-image // // ---------------------------------------------------------------------------- // Registry Resolution Behavior: @@ -76,6 +83,9 @@ // Use empty strings for "user_name" and "password" in these cases. // - Private registries or internal mirrors (including "local_registry") must be configured // with valid credentials if authentication is required. +// - "path_prefix" is optional and only used when the registry organizes images using +// path-based hierarchies (Harbor projects, private registry namespaces, etc.). +// Public registries like DockerHub, k8s.io typically don't need path prefixes. // ============================================================================ { @@ -100,8 +110,7 @@ "override": false, }, "resources/image_manifests/stx-third-party-images.yaml": { - "manifest_registry": "null", // No manifest fallback; each image uses its "source_registry" or "default_source_registry" - "override": false, + "manifest_registry": null, // No manifest fallback; each image uses its "source_registry" or "default_source_registry" }, // // Use Harbor as the default for images in this manifest that do not specify "source_registry" // "resources/image_manifests/stx-sanity-images.yaml": { @@ -111,7 +120,7 @@ // // No manifest fallback is defined; each image will use its "source_registry" if set, or "default_source_registry". // "resources/stx-networking-images.yaml": { // "manifest_registry": null, - // "override": false, + // // "override": false, // Not needed when manifest_registry is null (nothing to override with) // }, }, @@ -137,14 +146,35 @@ "password": "", }, - // Example entry for a private registry such as Harbor: - // "harbor": { - // "registry_name": "harbor", + // Examples of registries with path_prefix for organizational separation: + + // Harbor with project-based organization + // "harbor_project_x": { + // "registry_name": "harbor_project_x", // "registry_url": "harbor.example.org:5000", + // "path_prefix": "project-x/test", // "user_name": "robot_user", // "password": "robot_token", // }, + // Same Harbor host, different project + // "harbor_user": { + // "registry_name": "harbor_user", + // "registry_url": "harbor.example.org:5000", + // "path_prefix": "user-project", # Trailing slash is optional and will be normalized automatically + // "user_name": "harbor_username", + // "password": "harbor_password", + // }, + + // Private registry with namespace organization + // "private_team_a": { + // "registry_name": "private_team_a", + // "registry_url": "registry.company.com:5000", + // "path_prefix": "team-a/projects/", # Trailing slash is optional and will be normalized automatically + // "user_name": "team_user", + // "password": "team_token", + // }, + "local_registry": { "registry_name": "local_registry", "registry_url": "registry.local:9001", diff --git a/config/docker/objects/docker_config.py b/config/docker/objects/docker_config.py index 907cd438..e7cdfd47 100644 --- a/config/docker/objects/docker_config.py +++ b/config/docker/objects/docker_config.py @@ -49,6 +49,7 @@ class DockerConfig: registry_url=registry_dict["registry_url"], user_name=registry_dict["user_name"], password=registry_dict["password"], + path_prefix=registry_dict.get("path_prefix"), ) self.registry_list.append(reg) diff --git a/config/docker/objects/registry.py b/config/docker/objects/registry.py index a09f79a0..5b474c00 100644 --- a/config/docker/objects/registry.py +++ b/config/docker/objects/registry.py @@ -1,20 +1,22 @@ class Registry: """Represents a Docker registry configuration.""" - def __init__(self, registry_name: str, registry_url: str, user_name: str, password: str): + def __init__(self, registry_name: str, registry_url: str, user_name: str, password: str, path_prefix: str = None): """ Initializes a Registry object. Args: registry_name (str): Logical name of the registry (e.g., "source_registry"). - registry_url (str): Registry endpoint URL (e.g., "docker.io/starlingx"). + registry_url (str): Registry endpoint URL (e.g., "docker.io"). user_name (str): Username for authenticating with the registry. password (str): Password for authenticating with the registry. + path_prefix (str): Optional path prefix for registry projects (e.g., "project/namespace/"). """ self.registry_name = registry_name self.registry_url = registry_url self.user_name = user_name self.password = password + self.path_prefix = path_prefix or "" def get_registry_name(self) -> str: """ @@ -52,20 +54,35 @@ class Registry: """ return self.password + def get_path_prefix(self) -> str: + """ + Returns the path prefix prepended to image names during sync operations. + + The path prefix enables organizational separation within the same registry host, + such as Harbor projects or private registry namespaces. + + Returns: + str: Path prefix (e.g., "project/namespace") or empty string for registries + like DockerHub that don't require path-based organization. + """ + return self.path_prefix + def __str__(self) -> str: """ Returns a human-readable string representation of the registry. Returns: - str: Formatted string showing registry name and URL. + str: Formatted string showing registry name, URL, and path prefix if present. """ + if self.path_prefix: + return f"{self.registry_name} ({self.registry_url}/{self.path_prefix})" return f"{self.registry_name} ({self.registry_url})" def __repr__(self) -> str: """ - Returns the representation string of the registry. + Returns the representation string of the registry for debugging. Returns: - str: Registry representation. + str: Registry representation showing constructor parameters (excluding credentials). """ - return self.__str__() + return f"Registry(registry_name='{self.registry_name}', registry_url='{self.registry_url}', path_prefix='{self.path_prefix}')" diff --git a/keywords/docker/images/docker_sync_images_keywords.py b/keywords/docker/images/docker_sync_images_keywords.py index ec62141e..4cb203d1 100644 --- a/keywords/docker/images/docker_sync_images_keywords.py +++ b/keywords/docker/images/docker_sync_images_keywords.py @@ -108,7 +108,14 @@ class DockerSyncImagesKeywords(BaseKeyword): source_registry = docker_config.get_registry(source_registry_name) - source_image = f"{source_registry.get_registry_url()}/{name}:{tag}" + registry_url = source_registry.get_registry_url() + path_prefix = source_registry.get_path_prefix() + if path_prefix: + # Normalize path_prefix to ensure proper slash formatting + normalized_prefix = path_prefix.strip("/") + "/" + source_image = f"{registry_url}/{normalized_prefix}{name}:{tag}" + else: + source_image = f"{registry_url}/{name}:{tag}" target_image = f"{local_registry.get_registry_url()}/{name}:{tag}" get_logger().log_info(f"Pulling {source_image}") @@ -150,6 +157,7 @@ class DockerSyncImagesKeywords(BaseKeyword): source_registry = docker_config.get_registry(source_registry_name) source_url = source_registry.get_registry_url() + path_prefix = source_registry.get_path_prefix() # Always try to remove these two references refs = [ @@ -159,9 +167,14 @@ class DockerSyncImagesKeywords(BaseKeyword): # Optionally add full source registry tag if not DockerHub if "docker.io" not in source_url: - refs.insert(0, f"{source_url}/{image_name}:{image_tag}") + if path_prefix: + # Normalize path_prefix to ensure proper slash formatting + normalized_prefix = path_prefix.strip("/") + "/" + refs.insert(0, f"{source_url}/{normalized_prefix}{image_name}:{image_tag}") + else: + refs.insert(0, f"{source_url}/{image_name}:{image_tag}") else: - get_logger().log_debug(f"Skipping full docker.io-prefixed tag for {source_url}/{image_name}:{image_tag}") + get_logger().log_debug(f"Skipping full docker.io-prefixed tag for {source_url}/{path_prefix}{image_name}:{image_tag}") return refs diff --git a/resources/image_manifests/harbor-test-images.yaml b/resources/image_manifests/harbor-test-images.yaml index 49c63abd..8b29a939 100644 --- a/resources/image_manifests/harbor-test-images.yaml +++ b/resources/image_manifests/harbor-test-images.yaml @@ -18,11 +18,14 @@ # - Authentication details (username, password, etc.) are configured in # `config/docker/files/default.json5` under the corresponding `registries` entry. # - This file defaults to `config/docker/files/default.json5`, but can be overridden using `--docker_config_file`. -# - Registry resolution is handled dynamically via `ConfigurationManager`. -# - Resolution priority (from most to least specific): -# 1. `source_registry` field on the image entry (optional) -# 2. `manifest_registry_map` in `config/docker/files/default.json5` -# 3. `default_source_registry` in `config/docker/files/default.json5` +# - `source_registry` values must match a registry name defined in the Docker config file +# (e.g., "dockerhub", "k8s", "gcr" - not URLs like "docker.io") +# +# Registry resolution priority: +# 1. `manifest_registry_map` with `override: true` (uses registry specified in manifest mapping in config, ignores source_registry in this file) +# 2. `source_registry` field on individual image entry (if override=false or no manifest mapping exists, must match registry name in config) +# 3. `manifest_registry_map` with `override: false` (fallback for images without source_registry) +# 4. `default_source_registry` (final fallback) # ------------------------------------------------------------------------------ images: diff --git a/resources/image_manifests/stx-test-images.yaml b/resources/image_manifests/stx-test-images.yaml index 1f59d484..a2fb6c15 100644 --- a/resources/image_manifests/stx-test-images.yaml +++ b/resources/image_manifests/stx-test-images.yaml @@ -9,11 +9,14 @@ # - Image names must include their full namespace (e.g., `starlingx/stx-platformclients`). # - Registry URLs and credentials are not listed here. They are defined in: # `config/docker/files/default.json5` -# - Registry resolution is handled dynamically via `ConfigurationManager`. -# - Resolution priority (from most to least specific): -# 1. `source_registry` field on the individual image entry (optional) -# 2. `manifest_registry_map` entry in `config/docker/files/default.json5` -# 3. `default_source_registry` in `config/docker/files/default.json5` +# - `source_registry` values must match a registry name defined in the Docker config file +# (e.g., "dockerhub", "k8s", "gcr" - not URLs like "docker.io") +# +# Registry resolution priority: +# 1. `manifest_registry_map` with `override: true` (uses registry specified in manifest mapping in config, ignores source_registry in this file) +# 2. `source_registry` field on individual image entry (if override=false or no manifest mapping exists, must match registry name in config) +# 3. `manifest_registry_map` with `override: false` (fallback for images without source_registry) +# 4. `default_source_registry` (final fallback) # ------------------------------------------------------------------------------ images: - name: "starlingx/stx-platformclients" diff --git a/resources/image_manifests/stx-third-party-test-images.yaml b/resources/image_manifests/stx-third-party-test-images.yaml index 41cac9dc..9887b13f 100644 --- a/resources/image_manifests/stx-third-party-test-images.yaml +++ b/resources/image_manifests/stx-third-party-test-images.yaml @@ -13,11 +13,14 @@ # - Image names must include their full namespace (e.g., `google-samples/node-hello`). # - Registry URLs and credentials are defined in: # `config/docker/files/default.json5` +# - `source_registry` values must match a registry name defined in the Docker config file +# (e.g., "dockerhub", "k8s", "gcr" - not URLs like "docker.io") # -# Registry resolution priority (from most to least specific): -# 1. `source_registry` field on the individual image entry (recommended) -# 2. `manifest_registry_map` in the Docker config -# 3. `default_source_registry` fallback +# Registry resolution priority: +# 1. `manifest_registry_map` with `override: true` (uses registry specified in manifest mapping in config, ignores source_registry in this file) +# 2. `source_registry` field on individual image entry (if override=false or no manifest mapping exists, must match registry name in config) +# 3. `manifest_registry_map` with `override: false` (fallback for images without source_registry) +# 4. `default_source_registry` (final fallback) # ------------------------------------------------------------------------------ images: diff --git a/unit_tests/config/docker/registry_path_prefix_test.py b/unit_tests/config/docker/registry_path_prefix_test.py new file mode 100644 index 00000000..0348150a --- /dev/null +++ b/unit_tests/config/docker/registry_path_prefix_test.py @@ -0,0 +1,54 @@ +from config.docker.objects.registry import Registry + + +class TestRegistryPathPrefix: + """Tests for Registry path_prefix handling and URL construction.""" + + def test_path_prefix_with_trailing_slash(self): + """Test that path_prefix with trailing slash is returned as-is.""" + registry = Registry(registry_name="test_registry", registry_url="harbor.example.com", user_name="user", password="pass", path_prefix="project-x/test/") + assert registry.get_path_prefix() == "project-x/test/" + + def test_path_prefix_without_trailing_slash(self): + """Test that path_prefix without trailing slash is returned as-is.""" + registry = Registry(registry_name="test_registry", registry_url="harbor.example.com", user_name="user", password="pass", path_prefix="project-x/test") + assert registry.get_path_prefix() == "project-x/test" + + def test_path_prefix_with_leading_slash(self): + """Test that path_prefix with leading slash is returned as-is.""" + registry = Registry(registry_name="test_registry", registry_url="harbor.example.com", user_name="user", password="pass", path_prefix="/project-x/test/") + assert registry.get_path_prefix() == "/project-x/test/" + + def test_empty_path_prefix(self): + """Test that empty path_prefix returns empty string.""" + registry = Registry(registry_name="test_registry", registry_url="docker.io", user_name="user", password="pass", path_prefix="") + assert registry.get_path_prefix() == "" + + def test_none_path_prefix(self): + """Test that None path_prefix defaults to empty string.""" + registry = Registry(registry_name="test_registry", registry_url="docker.io", user_name="user", password="pass") + assert registry.get_path_prefix() == "" + + def test_url_construction_normalization(self): + """Test URL construction with various path_prefix formats.""" + test_cases = [ + ("project-x/test/", "harbor.com/project-x/test/busybox:1.0"), + ("project-x/test", "harbor.com/project-x/test/busybox:1.0"), + ("/project-x/test/", "harbor.com/project-x/test/busybox:1.0"), + ("/project-x/test", "harbor.com/project-x/test/busybox:1.0"), + ("", "harbor.com/busybox:1.0"), + ] + + for path_prefix, expected_url in test_cases: + registry = Registry(registry_name="test_registry", registry_url="harbor.com", user_name="user", password="pass", path_prefix=path_prefix) + + # Simulate the URL construction logic from the sync method + registry_url = registry.get_registry_url() + prefix = registry.get_path_prefix() + if prefix: + normalized_prefix = prefix.strip("/") + "/" + actual_url = f"{registry_url}/{normalized_prefix}busybox:1.0" + else: + actual_url = f"{registry_url}/busybox:1.0" + + assert actual_url == expected_url, f"Failed for path_prefix='{path_prefix}'"