Support subcloud deploy upload the common files

Add new CLI commands to upload and show the subcloud
deploy common files:
dcmanager subcloud-deploy upload \
--deploy-playbook <play book> \
--deploy-chart <helm chart> \
--deploy-overrides <overrides>

dcmanager subcloud-deploy show

Changes to the subcloud add commands
dcmanager subcloud add \
--bootstrap-address oam_ip_address_of_subclouds_controller-0 \
--bootstrap-values <file> \
--deploy-config <file> \
--sysadmin-password sysadmin_password \
--install-values <file> \
--bmc-password bmc_password

The password is base64 encoded in the REST API request.
The files are sent using multipart/form-data in the REST request.
The file contents are processed by the API server.

Depends-On: https://review.opendev.org/#/c/720589/
Closes-Bug: 1864508

Change-Id: Id92ee8b631789b4949b9682586060ce424983e88
Signed-off-by: Tao Liu <tao.liu@windriver.com>

distributedcloud-client/centos/distributedcloud-client.spec

@@ -45,6 +45,7 @@ BuildRequires: python-sphinxcontrib-httpdomain
 BuildRequires: pyOpenSSL
 BuildRequires: systemd
 BuildRequires: git
+BuildRequires: requests-toolbelt
 # Required to compile translation files
 BuildRequires: python-babel
 

distributedcloud-client/dcmanagerclient/api/v1/client.py

@@ -26,6 +26,7 @@ from keystoneauth1 import session as ks_session
 
 from dcmanagerclient.api import httpclient
 from dcmanagerclient.api.v1 import alarm_manager as am
+from dcmanagerclient.api.v1 import subcloud_deploy_manager as sdm
 from dcmanagerclient.api.v1 import subcloud_group_manager as gm
 from dcmanagerclient.api.v1 import subcloud_manager as sm
 from dcmanagerclient.api.v1 import sw_update_manager as sum
@@ -98,6 +99,8 @@ class Client(object):
         self.subcloud_manager = sm.subcloud_manager(self.http_client)
         self.subcloud_group_manager = \
             gm.subcloud_group_manager(self.http_client, self.subcloud_manager)
+        self.subcloud_deploy_manager = sdm.subcloud_deploy_manager(
+            self.http_client)
         self.alarm_manager = am.alarm_manager(self.http_client)
         self.sw_update_manager = sum.sw_update_manager(self.http_client)
         self.sw_update_options_manager = \

distributedcloud-client/dcmanagerclient/api/v1/subcloud_deploy_manager.py

@@ -0,0 +1,79 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+from requests_toolbelt import MultipartEncoder
+
+from dcmanagerclient.api import base
+from dcmanagerclient.api.base import get_json
+
+
+class SubcloudDeploy(base.Resource):
+    resource_name = 'subcloud_deploy'
+
+    def __init__(self, deploy_playbook, deploy_overrides, deploy_chart):
+        self.deploy_playbook = deploy_playbook
+        self.deploy_overrides = deploy_overrides
+        self.deploy_chart = deploy_chart
+
+
+class subcloud_deploy_manager(base.ResourceManager):
+    resource_class = SubcloudDeploy
+
+    def _subcloud_deploy_detail(self, url):
+        resp = self.http_client.get(url)
+        if resp.status_code != 200:
+            self._raise_api_exception(resp)
+        json_response_key = get_json(resp)
+        json_object = json_response_key['subcloud_deploy']
+        resource = list()
+        resource.append(
+            self.resource_class(
+                deploy_playbook=json_object['deploy_playbook'],
+                deploy_overrides=json_object['deploy_overrides'],
+                deploy_chart=json_object['deploy_chart']))
+        return resource
+
+    def _deploy_upload(self, url, data):
+        fields = dict()
+        for k, v in data.items():
+            fields.update({k: (v, open(v, 'rb'),)})
+        enc = MultipartEncoder(fields=fields)
+        headers = {'Content-Type': enc.content_type}
+        resp = self.http_client.post(url, enc, headers=headers)
+        if resp.status_code != 200:
+            self._raise_api_exception(resp)
+        json_object = get_json(resp)
+        resource = list()
+        resource.append(
+            self.resource_class(
+                deploy_playbook=json_object['deploy_playbook'],
+                deploy_overrides=json_object['deploy_overrides'],
+                deploy_chart=json_object['deploy_chart']))
+        return resource
+
+    def subcloud_deploy_show(self):
+        url = '/subcloud-deploy/'
+        return self._subcloud_deploy_detail(url)
+
+    def subcloud_deploy_upload(self, **kwargs):
+        data = kwargs
+        url = '/subcloud-deploy/'
+        return self._deploy_upload(url, data)

distributedcloud-client/dcmanagerclient/api/v1/subcloud_manager.py

@@ -22,6 +22,8 @@
 
 import json
 
+from requests_toolbelt import MultipartEncoder
+
 from dcmanagerclient.api import base
 from dcmanagerclient.api.base import get_json
 
@@ -82,9 +84,14 @@ class subcloud_manager(base.ResourceManager):
             updated_at=json_object['updated-at'],
             group_id=json_object['group_id'])
 
-    def subcloud_create(self, url, data):
+    def subcloud_create(self, url, body, data):
-        data = json.dumps(data)
+        fields = dict()
-        resp = self.http_client.post(url, data)
+        for k, v in body.items():
+            fields.update({k: (v, open(v, 'rb'),)})
+        fields.update(data)
+        enc = MultipartEncoder(fields=fields)
+        headers = {'Content-Type': enc.content_type}
+        resp = self.http_client.post(url, enc, headers=headers)
         if resp.status_code != 200:
             self._raise_api_exception(resp)
         json_object = get_json(resp)
@@ -185,9 +192,10 @@ class subcloud_manager(base.ResourceManager):
         return resource
 
     def add_subcloud(self, **kwargs):
-        data = kwargs
+        data = kwargs.get('data')
+        files = kwargs.get('files')
         url = '/subclouds/'
-        return self.subcloud_create(url, data)
+        return self.subcloud_create(url, files, data)
 
     def list_subclouds(self):
         url = '/subclouds/'

distributedcloud-client/dcmanagerclient/commands/v1/subcloud_deploy_manager.py

@@ -0,0 +1,130 @@
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+
+import os
+
+from dcmanagerclient.commands.v1 import base
+from dcmanagerclient import exceptions
+
+
+def _format(subcloud_deploy=None):
+    columns = (
+        'deploy_playbook',
+        'deploy_overrides',
+        'deploy_chart'
+    )
+
+    if subcloud_deploy:
+        data = (
+            subcloud_deploy.deploy_playbook,
+            subcloud_deploy.deploy_overrides,
+            subcloud_deploy.deploy_chart
+        )
+
+    else:
+        data = (tuple('<none>' for _ in range(len(columns))),)
+
+    return columns, data
+
+
+class SubcloudDeployUpload(base.DCManagerShowOne):
+    """Upload the subcloud deployment files"""
+
+    def _get_format_function(self):
+        return _format
+
+    def get_parser(self, prog_name):
+        parser = super(SubcloudDeployUpload, self).get_parser(prog_name)
+
+        parser.add_argument(
+            '--deploy-playbook',
+            required=True,
+            help='An ansible playbook to be run after the subcloud '
+                 'has been successfully bootstrapped. It will be run with the '
+                 'subcloud as the target and authentication is '
+                 'handled automatically. '
+                 'Must be a local file path'
+        )
+
+        parser.add_argument(
+            '--deploy-overrides',
+            required=True,
+            help='YAML file containing subcloud variables to be passed to the '
+                 'deploy playbook.'
+                 'Must be a local file path'
+        )
+
+        parser.add_argument(
+            '--deploy-chart',
+            required=True,
+            help='Deployment Manager helm chart to be passed to the '
+                 'deploy playbook.'
+                 'Must be a local file path'
+        )
+
+        return parser
+
+    def _get_resources(self, parsed_args):
+        dcmanager_client = self.app.client_manager.subcloud_deploy_manager
+        kwargs = dict()
+        if not os.path.isfile(parsed_args.deploy_playbook):
+            error_msg = "deploy-playbook does not exist: %s" % \
+                        parsed_args.deploy_playbook
+            raise exceptions.DCManagerClientException(error_msg)
+
+        kwargs['deploy_playbook'] = parsed_args.deploy_playbook
+
+        if not os.path.isfile(parsed_args.deploy_overrides):
+            error_msg = "deploy-overrides does not exist: %s" % \
+                        parsed_args.deploy_overrides
+            raise exceptions.DCManagerClientException(error_msg)
+
+        kwargs['deploy_overrides'] = parsed_args.deploy_overrides
+
+        if not os.path.isfile(parsed_args.deploy_chart):
+            error_msg = "deploy-chart does not exist: %s" % \
+                        parsed_args.deploy_chart
+            raise exceptions.DCManagerClientException(error_msg)
+
+        kwargs['deploy_chart'] = parsed_args.deploy_chart
+
+        try:
+            return dcmanager_client.subcloud_deploy_manager.\
+                subcloud_deploy_upload(**kwargs)
+        except Exception as e:
+            print(e)
+            error_msg = "Unable to upload subcloud deploy files"
+            raise exceptions.DCManagerClientException(error_msg)
+
+
+class SubcloudDeployShow(base.DCManagerShowOne):
+    """Show the uploaded deployment files."""
+
+    def _get_format_function(self):
+        return _format
+
+    def get_parser(self, prog_name):
+        parser = super(SubcloudDeployShow, self).get_parser(prog_name)
+
+        return parser
+
+    def _get_resources(self, parsed_args):
+        dcmanager_client = self.app.client_manager.subcloud_deploy_manager
+        return dcmanager_client.subcloud_deploy_manager.subcloud_deploy_show()

distributedcloud-client/dcmanagerclient/commands/v1/subcloud_manager.py

@@ -19,15 +19,14 @@
 # of an applicable Wind River license agreement.
 #
 
+import base64
 import getpass
 import os
-import yaml
 
 from osc_lib.command import command
 
 from dcmanagerclient.commands.v1 import base
 from dcmanagerclient import exceptions
-from dcmanagerclient import utils
 
 
 def format(subcloud=None):
@@ -136,17 +135,7 @@ class AddSubcloud(base.DCManagerShowOne):
         )
 
         parser.add_argument(
-            '--deploy-playbook',
+            '--deploy-config',
-            required=False,
-            help='An optional ansible playbook to be run after the subcloud '
-                 'has been successfully bootstrapped. It will be run with the '
-                 'subcloud as the target and authentication is '
-                 'handled automatically. '
-                 'Can be either a local file path or a URL.'
-        )
-
-        parser.add_argument(
-            '--deploy-values',
             required=False,
             help='YAML file containing subcloud variables to be passed to the '
                  'deploy playbook.'
@@ -182,52 +171,37 @@ class AddSubcloud(base.DCManagerShowOne):
 
     def _get_resources(self, parsed_args):
         dcmanager_client = self.app.client_manager.subcloud_manager
-        kwargs = dict()
+        files = dict()
-        kwargs['bootstrap-address'] = parsed_args.bootstrap_address
+        data = dict()
+        data['bootstrap-address'] = parsed_args.bootstrap_address
 
-        # Load the configuration from the install values yaml file
+        # Get the install values yaml file
         if parsed_args.install_values is not None:
-            filename = parsed_args.install_values
+            if not os.path.isfile(parsed_args.install_values):
-            stream = utils.get_contents_if_file(filename)
+                error_msg = "install-values does not exist: %s" % \
-            kwargs['install_values'] = yaml.safe_load(stream)
+                            parsed_args.install_values
-
-        # Load the configuration from the bootstrap yaml file
-        filename = parsed_args.bootstrap_values
-        stream = utils.get_contents_if_file(filename)
-        kwargs.update(yaml.safe_load(stream))
-
-        # Load the the deploy playbook yaml file
-        if parsed_args.deploy_playbook is not None:
-            if parsed_args.deploy_values is None:
-                error_msg = "Error: Deploy playbook cannot be specified " \
-                            "when the deploy values file has not been " \
-                            "specified."
                 raise exceptions.DCManagerClientException(error_msg)
-            filename = parsed_args.deploy_playbook
+            files['install_values'] = parsed_args.install_values
-            stream = utils.get_contents_if_file(filename)
-            kwargs['deploy_playbook'] = yaml.safe_load(stream)
 
-        # Load the configuration from the deploy values yaml file
+        # Get the bootstrap values yaml file
-        if parsed_args.deploy_values is not None:
+        if not os.path.isfile(parsed_args.bootstrap_values):
-            if parsed_args.deploy_playbook is None:
+            error_msg = "bootstrap-values does not exist: %s" % \
-                error_msg = "Error: Deploy values cannot be specified " \
+                        parsed_args.bootstrap_values
-                            "when a deploy playbook has not been specified."
+            raise exceptions.DCManagerClientException(error_msg)
-                raise exceptions.DCManagerClientException(error_msg)
+        files['bootstrap_values'] = parsed_args.bootstrap_values
 
-            filename = parsed_args.deploy_values
+        # Get the deploy config yaml file
-            if os.path.isdir(filename):
+        if parsed_args.deploy_config is not None:
-                error_msg = "Error: %s is a directory." % filename
+            if not os.path.isfile(parsed_args.deploy_config):
-                raise exceptions.DCManagerClientException(error_msg)
+                error_msg = "deploy-config does not exist: %s" % \
-            try:
+                            parsed_args.deploy_config
-                with open(filename, 'rb') as stream:
-                    kwargs['deploy_values'] = yaml.safe_load(stream)
-            except Exception:
-                error_msg = "Error: Could not open file %s." % filename
                 raise exceptions.DCManagerClientException(error_msg)
+            files['deploy_config'] = parsed_args.deploy_config
 
         # Prompt the user for the subcloud's password if it isn't provided
         if parsed_args.sysadmin_password is not None:
-            kwargs['sysadmin_password'] = parsed_args.sysadmin_password
+            data['sysadmin_password'] = base64.b64encode(
+                parsed_args.sysadmin_password.encode("utf-8"))
         else:
             while True:
                 password = getpass.getpass(
@@ -241,12 +215,14 @@ class AddSubcloud(base.DCManagerShowOne):
                 if password != confirm:
                     print("Passwords did not match")
                     continue
-                kwargs["sysadmin_password"] = password
+                data["sysadmin_password"] = base64.b64encode(
+                    password.encode("utf-8"))
                 break
 
         if parsed_args.install_values is not None:
             if parsed_args.bmc_password is not None:
-                kwargs['bmc_password'] = parsed_args.bmc_password
+                data['bmc_password'] = base64.b64encode(
+                    parsed_args.bmc_password.encode("utf-8"))
             else:
                 while True:
                     password = getpass.getpass(
@@ -260,13 +236,15 @@ class AddSubcloud(base.DCManagerShowOne):
                     if password != confirm:
                         print("Passwords did not match")
                         continue
-                    kwargs["bmc_password"] = password
+                    data["bmc_password"] = base64.b64encode(
+                        password.encode("utf-8"))
                     break
 
         if parsed_args.group is not None:
-            kwargs['group_id'] = parsed_args.group
+            data['group_id'] = parsed_args.group
 
-        return dcmanager_client.subcloud_manager.add_subcloud(**kwargs)
+        return dcmanager_client.subcloud_manager.add_subcloud(files=files,
+                                                              data=data)
 
 
 class ListSubcloud(base.DCManagerLister):

distributedcloud-client/dcmanagerclient/shell.py

@@ -37,6 +37,7 @@ from osc_lib.command import command
 
 import argparse
 from dcmanagerclient.commands.v1 import alarm_manager as am
+from dcmanagerclient.commands.v1 import subcloud_deploy_manager as sdm
 from dcmanagerclient.commands.v1 import subcloud_group_manager as gm
 from dcmanagerclient.commands.v1 import subcloud_manager as sm
 from dcmanagerclient.commands.v1 import sw_update_manager as sum
@@ -446,6 +447,7 @@ class DCManagerShell(app.App):
             (object,),
             dict(subcloud_manager=self.client,
                  subcloud_group_manager=self.client,
+                 subcloud_deploy_manager=self.client,
                  alarm_manager=self.client,
                  sw_update_manager=self.client,
                  strategy_step_manager=self.client,
@@ -488,6 +490,8 @@ class DCManagerShell(app.App):
             'subcloud-group list-subclouds': gm.ListSubcloudGroupSubclouds,
             'subcloud-group show': gm.ShowSubcloudGroup,
             'subcloud-group update': gm.UpdateSubcloudGroup,
+            'subcloud-deploy upload': sdm.SubcloudDeployUpload,
+            'subcloud-deploy show': sdm.SubcloudDeployShow,
             'alarm summary': am.ListAlarmSummary,
             'patch-strategy create': sum.CreatePatchStrategy,
             'patch-strategy delete': sum.DeletePatchStrategy,

distributedcloud-client/dcmanagerclient/tests/v1/test_subcloud_deploy_manager.py

@@ -0,0 +1,83 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+import os
+import tempfile
+
+from dcmanagerclient.api.v1 import subcloud_deploy_manager as sdm
+from dcmanagerclient.commands.v1 \
+    import subcloud_deploy_manager as subcloud_deploy_cmd
+from dcmanagerclient.tests import base
+
+
+DEPLOY_PLAYBOOK = 'deployment-manager-playbook.yaml'
+DEPLOY_OVERRIDES = 'deployment-manager-overrides-subcloud.yaml'
+DEPLOY_CHART = 'deployment-manager.tgz'
+
+SUBCLOUD_DEPLOY_DICT = {
+    'DEPLOY_PLAYBOOK': DEPLOY_PLAYBOOK,
+    'DEPLOY_OVERRIDES': DEPLOY_OVERRIDES,
+    'DEPLOY_CHART': DEPLOY_CHART
+}
+
+SUBCLOUD_DEPLOY = sdm.SubcloudDeploy(
+    deploy_playbook=SUBCLOUD_DEPLOY_DICT['DEPLOY_PLAYBOOK'],
+    deploy_overrides=SUBCLOUD_DEPLOY_DICT['DEPLOY_OVERRIDES'],
+    deploy_chart=SUBCLOUD_DEPLOY_DICT['DEPLOY_CHART']
+)
+
+
+class TestCLISubcloudDeployManagerV1(base.BaseCommandTest):
+
+    def setUp(self):
+        super(TestCLISubcloudDeployManagerV1, self).setUp()
+        # The client is the subcloud_deploy_manager
+        self.client = self.app.client_manager.subcloud_deploy_manager
+
+    def test_subcloud_deploy_show(self):
+        self.client.subcloud_deploy_manager.subcloud_deploy_show.\
+            return_value = [SUBCLOUD_DEPLOY]
+        actual_call = self.call(subcloud_deploy_cmd.SubcloudDeployShow)
+
+        self.assertEqual((DEPLOY_PLAYBOOK,
+                          DEPLOY_OVERRIDES,
+                          DEPLOY_CHART),
+                         actual_call[1])
+
+    def test_subcloud_deploy_upload(self):
+        self.client.subcloud_deploy_manager.subcloud_deploy_upload.\
+            return_value = [SUBCLOUD_DEPLOY]
+
+        with tempfile.NamedTemporaryFile() as f1,\
+                tempfile.NamedTemporaryFile() as f2,\
+                tempfile.NamedTemporaryFile() as f3:
+            file_path_1 = os.path.abspath(f1.name)
+            file_path_2 = os.path.abspath(f2.name)
+            file_path_3 = os.path.abspath(f3.name)
+            actual_call = self.call(
+                subcloud_deploy_cmd.SubcloudDeployUpload,
+                app_args=[
+                    '--deploy-playbook', file_path_1,
+                    '--deploy-overrides', file_path_2,
+                    '--deploy-chart', file_path_3])
+
+        self.assertEqual((DEPLOY_PLAYBOOK,
+                          DEPLOY_OVERRIDES,
+                          DEPLOY_CHART),
+                         actual_call[1])

distributedcloud-client/requirements.txt

@@ -10,3 +10,4 @@ PyYAML>=3.10.0 # MIT
 requests!=2.12.2,!=2.13.0,>=2.10.0 # Apache-2.0
 six>=1.9.0 # MIT
 beautifulsoup4
+requests-toolbelt