starlingx/distcloud-client
Code Issues Proposed changes

Support subcloud deploy upload the common files

Browse Source 
Add new CLI commands to upload and show the subcloud
deploy common files:
dcmanager subcloud-deploy upload \
--deploy-playbook <play book> \
--deploy-chart <helm chart> \
--deploy-overrides <overrides>

dcmanager subcloud-deploy show

Changes to the subcloud add commands
dcmanager subcloud add \
--bootstrap-address oam_ip_address_of_subclouds_controller-0 \
--bootstrap-values <file> \
--deploy-config <file> \
--sysadmin-password sysadmin_password \
--install-values <file> \
--bmc-password bmc_password

The password is base64 encoded in the REST API request.
The files are sent using multipart/form-data in the REST request.
The file contents are processed by the API server.

Depends-On: https://review.opendev.org/#/c/720589/
Closes-Bug: 1864508

Change-Id: Id92ee8b631789b4949b9682586060ce424983e88
Signed-off-by: Tao Liu <tao.liu@windriver.com>
This commit is contained in:
Tao Liu
2020-04-16 10:40:30 -04:00
parent 190c0c4558
commit 19f027179c
9 changed files with 347 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
distributedcloud-client/centos/distributedcloud-client.spec
View File

@@ -45,6 +45,7 @@ BuildRequires: python-sphinxcontrib-httpdomain
BuildRequires: pyOpenSSL
BuildRequires: systemd
BuildRequires: git
BuildRequires: requests-toolbelt
# Required to compile translation files
BuildRequires: python-babel

3
distributedcloud-client/dcmanagerclient/api/v1/client.py
View File

@@ -26,6 +26,7 @@ from keystoneauth1 import session as ks_session
from dcmanagerclient.api import httpclient
from dcmanagerclient.api.v1 import alarm_manager as am
from dcmanagerclient.api.v1 import subcloud_deploy_manager as sdm
from dcmanagerclient.api.v1 import subcloud_group_manager as gm
from dcmanagerclient.api.v1 import subcloud_manager as sm
from dcmanagerclient.api.v1 import sw_update_manager as sum
@@ -98,6 +99,8 @@ class Client(object):
self.subcloud_manager = sm.subcloud_manager(self.http_client)
self.subcloud_group_manager = \
gm.subcloud_group_manager(self.http_client, self.subcloud_manager)
self.subcloud_deploy_manager = sdm.subcloud_deploy_manager(
self.http_client)
self.alarm_manager = am.alarm_manager(self.http_client)
self.sw_update_manager = sum.sw_update_manager(self.http_client)
self.sw_update_options_manager = \

79
distributedcloud-client/dcmanagerclient/api/v1/subcloud_deploy_manager.py Normal file
View File

@@ -0,0 +1,79 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
from requests_toolbelt import MultipartEncoder
from dcmanagerclient.api import base
from dcmanagerclient.api.base import get_json
class SubcloudDeploy(base.Resource):
resource_name = 'subcloud_deploy'
def __init__(self, deploy_playbook, deploy_overrides, deploy_chart):
self.deploy_playbook = deploy_playbook
self.deploy_overrides = deploy_overrides
self.deploy_chart = deploy_chart
class subcloud_deploy_manager(base.ResourceManager):
resource_class = SubcloudDeploy
def _subcloud_deploy_detail(self, url):
resp = self.http_client.get(url)
if resp.status_code != 200:
self._raise_api_exception(resp)
json_response_key = get_json(resp)
json_object = json_response_key['subcloud_deploy']
resource = list()
resource.append(
self.resource_class(
deploy_playbook=json_object['deploy_playbook'],
deploy_overrides=json_object['deploy_overrides'],
deploy_chart=json_object['deploy_chart']))
return resource
def _deploy_upload(self, url, data):
fields = dict()
for k, v in data.items():
fields.update({k: (v, open(v, 'rb'),)})
enc = MultipartEncoder(fields=fields)
headers = {'Content-Type': enc.content_type}
resp = self.http_client.post(url, enc, headers=headers)
if resp.status_code != 200:
self._raise_api_exception(resp)
json_object = get_json(resp)
resource = list()
resource.append(
self.resource_class(
deploy_playbook=json_object['deploy_playbook'],
deploy_overrides=json_object['deploy_overrides'],
deploy_chart=json_object['deploy_chart']))
return resource
def subcloud_deploy_show(self):
url = '/subcloud-deploy/'
return self._subcloud_deploy_detail(url)
def subcloud_deploy_upload(self, **kwargs):
data = kwargs
url = '/subcloud-deploy/'
return self._deploy_upload(url, data)

18
distributedcloud-client/dcmanagerclient/api/v1/subcloud_manager.py
View File

@@ -22,6 +22,8 @@
import json
from requests_toolbelt import MultipartEncoder
from dcmanagerclient.api import base
from dcmanagerclient.api.base import get_json
@@ -82,9 +84,14 @@ class subcloud_manager(base.ResourceManager):
updated_at=json_object['updated-at'],
group_id=json_object['group_id'])
def subcloud_create(self, url, data):
data = json.dumps(data)
resp = self.http_client.post(url, data)
def subcloud_create(self, url, body, data):
fields = dict()
for k, v in body.items():
fields.update({k: (v, open(v, 'rb'),)})
fields.update(data)
enc = MultipartEncoder(fields=fields)
headers = {'Content-Type': enc.content_type}
resp = self.http_client.post(url, enc, headers=headers)
if resp.status_code != 200:
self._raise_api_exception(resp)
json_object = get_json(resp)
@@ -185,9 +192,10 @@ class subcloud_manager(base.ResourceManager):
return resource
def add_subcloud(self, **kwargs):
data = kwargs
data = kwargs.get('data')
files = kwargs.get('files')
url = '/subclouds/'
return self.subcloud_create(url, data)
return self.subcloud_create(url, files, data)
def list_subclouds(self):
url = '/subclouds/'

130
distributedcloud-client/dcmanagerclient/commands/v1/subcloud_deploy_manager.py Normal file
View File

@@ -0,0 +1,130 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
import os
from dcmanagerclient.commands.v1 import base
from dcmanagerclient import exceptions
def _format(subcloud_deploy=None):
columns = (
'deploy_playbook',
'deploy_overrides',
'deploy_chart'
)
if subcloud_deploy:
data = (
subcloud_deploy.deploy_playbook,
subcloud_deploy.deploy_overrides,
subcloud_deploy.deploy_chart
)
else:
data = (tuple('<none>' for _ in range(len(columns))),)
return columns, data
class SubcloudDeployUpload(base.DCManagerShowOne):
"""Upload the subcloud deployment files"""
def _get_format_function(self):
return _format
def get_parser(self, prog_name):
parser = super(SubcloudDeployUpload, self).get_parser(prog_name)
parser.add_argument(
'--deploy-playbook',
required=True,
help='An ansible playbook to be run after the subcloud '
'has been successfully bootstrapped. It will be run with the '
'subcloud as the target and authentication is '
'handled automatically. '
'Must be a local file path'
)
parser.add_argument(
'--deploy-overrides',
required=True,
help='YAML file containing subcloud variables to be passed to the '
'deploy playbook.'
'Must be a local file path'
)
parser.add_argument(
'--deploy-chart',
required=True,
help='Deployment Manager helm chart to be passed to the '
'deploy playbook.'
'Must be a local file path'
)
return parser
def _get_resources(self, parsed_args):
dcmanager_client = self.app.client_manager.subcloud_deploy_manager
kwargs = dict()
if not os.path.isfile(parsed_args.deploy_playbook):
error_msg = "deploy-playbook does not exist: %s" % \
parsed_args.deploy_playbook
raise exceptions.DCManagerClientException(error_msg)
kwargs['deploy_playbook'] = parsed_args.deploy_playbook
if not os.path.isfile(parsed_args.deploy_overrides):
error_msg = "deploy-overrides does not exist: %s" % \
parsed_args.deploy_overrides
raise exceptions.DCManagerClientException(error_msg)
kwargs['deploy_overrides'] = parsed_args.deploy_overrides
if not os.path.isfile(parsed_args.deploy_chart):
error_msg = "deploy-chart does not exist: %s" % \
parsed_args.deploy_chart
raise exceptions.DCManagerClientException(error_msg)
kwargs['deploy_chart'] = parsed_args.deploy_chart
try:
return dcmanager_client.subcloud_deploy_manager.\
subcloud_deploy_upload(**kwargs)
except Exception as e:
print(e)
error_msg = "Unable to upload subcloud deploy files"
raise exceptions.DCManagerClientException(error_msg)
class SubcloudDeployShow(base.DCManagerShowOne):
"""Show the uploaded deployment files."""
def _get_format_function(self):
return _format
def get_parser(self, prog_name):
parser = super(SubcloudDeployShow, self).get_parser(prog_name)
return parser
def _get_resources(self, parsed_args):
dcmanager_client = self.app.client_manager.subcloud_deploy_manager
return dcmanager_client.subcloud_deploy_manager.subcloud_deploy_show()

88
distributedcloud-client/dcmanagerclient/commands/v1/subcloud_manager.py
View File

@@ -19,15 +19,14 @@
# of an applicable Wind River license agreement.
#
import base64
import getpass
import os
import yaml
from osc_lib.command import command
from dcmanagerclient.commands.v1 import base
from dcmanagerclient import exceptions
from dcmanagerclient import utils
def format(subcloud=None):
@@ -136,17 +135,7 @@ class AddSubcloud(base.DCManagerShowOne):
)
parser.add_argument(
'--deploy-playbook',
required=False,
help='An optional ansible playbook to be run after the subcloud '
'has been successfully bootstrapped. It will be run with the '
'subcloud as the target and authentication is '
'handled automatically. '
'Can be either a local file path or a URL.'
)
parser.add_argument(
'--deploy-values',
'--deploy-config',
required=False,
help='YAML file containing subcloud variables to be passed to the '
'deploy playbook.'
@@ -182,52 +171,37 @@ class AddSubcloud(base.DCManagerShowOne):
def _get_resources(self, parsed_args):
dcmanager_client = self.app.client_manager.subcloud_manager
kwargs = dict()
kwargs['bootstrap-address'] = parsed_args.bootstrap_address
files = dict()
data = dict()
data['bootstrap-address'] = parsed_args.bootstrap_address
# Load the configuration from the install values yaml file
# Get the install values yaml file
if parsed_args.install_values is not None:
filename = parsed_args.install_values
stream = utils.get_contents_if_file(filename)
kwargs['install_values'] = yaml.safe_load(stream)
# Load the configuration from the bootstrap yaml file
filename = parsed_args.bootstrap_values
stream = utils.get_contents_if_file(filename)
kwargs.update(yaml.safe_load(stream))
# Load the the deploy playbook yaml file
if parsed_args.deploy_playbook is not None:
if parsed_args.deploy_values is None:
error_msg = "Error: Deploy playbook cannot be specified " \
"when the deploy values file has not been " \
"specified."
if not os.path.isfile(parsed_args.install_values):
error_msg = "install-values does not exist: %s" % \
parsed_args.install_values
raise exceptions.DCManagerClientException(error_msg)
filename = parsed_args.deploy_playbook
stream = utils.get_contents_if_file(filename)
kwargs['deploy_playbook'] = yaml.safe_load(stream)
files['install_values'] = parsed_args.install_values
# Load the configuration from the deploy values yaml file
if parsed_args.deploy_values is not None:
if parsed_args.deploy_playbook is None:
error_msg = "Error: Deploy values cannot be specified " \
"when a deploy playbook has not been specified."
raise exceptions.DCManagerClientException(error_msg)
# Get the bootstrap values yaml file
if not os.path.isfile(parsed_args.bootstrap_values):
error_msg = "bootstrap-values does not exist: %s" % \
parsed_args.bootstrap_values
raise exceptions.DCManagerClientException(error_msg)
files['bootstrap_values'] = parsed_args.bootstrap_values
filename = parsed_args.deploy_values
if os.path.isdir(filename):
error_msg = "Error: %s is a directory." % filename
raise exceptions.DCManagerClientException(error_msg)
try:
with open(filename, 'rb') as stream:
kwargs['deploy_values'] = yaml.safe_load(stream)
except Exception:
error_msg = "Error: Could not open file %s." % filename
# Get the deploy config yaml file
if parsed_args.deploy_config is not None:
if not os.path.isfile(parsed_args.deploy_config):
error_msg = "deploy-config does not exist: %s" % \
parsed_args.deploy_config
raise exceptions.DCManagerClientException(error_msg)
files['deploy_config'] = parsed_args.deploy_config
# Prompt the user for the subcloud's password if it isn't provided
if parsed_args.sysadmin_password is not None:
kwargs['sysadmin_password'] = parsed_args.sysadmin_password
data['sysadmin_password'] = base64.b64encode(
parsed_args.sysadmin_password.encode("utf-8"))
else:
while True:
password = getpass.getpass(
@@ -241,12 +215,14 @@ class AddSubcloud(base.DCManagerShowOne):
if password != confirm:
print("Passwords did not match")
continue
kwargs["sysadmin_password"] = password
data["sysadmin_password"] = base64.b64encode(
password.encode("utf-8"))
break
if parsed_args.install_values is not None:
if parsed_args.bmc_password is not None:
kwargs['bmc_password'] = parsed_args.bmc_password
data['bmc_password'] = base64.b64encode(
parsed_args.bmc_password.encode("utf-8"))
else:
while True:
password = getpass.getpass(
@@ -260,13 +236,15 @@ class AddSubcloud(base.DCManagerShowOne):
if password != confirm:
print("Passwords did not match")
continue
kwargs["bmc_password"] = password
data["bmc_password"] = base64.b64encode(
password.encode("utf-8"))
break
if parsed_args.group is not None:
kwargs['group_id'] = parsed_args.group
data['group_id'] = parsed_args.group
return dcmanager_client.subcloud_manager.add_subcloud(**kwargs)
return dcmanager_client.subcloud_manager.add_subcloud(files=files,
data=data)
class ListSubcloud(base.DCManagerLister):

4
distributedcloud-client/dcmanagerclient/shell.py
View File

@@ -37,6 +37,7 @@ from osc_lib.command import command
import argparse
from dcmanagerclient.commands.v1 import alarm_manager as am
from dcmanagerclient.commands.v1 import subcloud_deploy_manager as sdm
from dcmanagerclient.commands.v1 import subcloud_group_manager as gm
from dcmanagerclient.commands.v1 import subcloud_manager as sm
from dcmanagerclient.commands.v1 import sw_update_manager as sum
@@ -446,6 +447,7 @@ class DCManagerShell(app.App):
(object,),
dict(subcloud_manager=self.client,
subcloud_group_manager=self.client,
subcloud_deploy_manager=self.client,
alarm_manager=self.client,
sw_update_manager=self.client,
strategy_step_manager=self.client,
@@ -488,6 +490,8 @@ class DCManagerShell(app.App):
'subcloud-group list-subclouds': gm.ListSubcloudGroupSubclouds,
'subcloud-group show': gm.ShowSubcloudGroup,
'subcloud-group update': gm.UpdateSubcloudGroup,
'subcloud-deploy upload': sdm.SubcloudDeployUpload,
'subcloud-deploy show': sdm.SubcloudDeployShow,
'alarm summary': am.ListAlarmSummary,
'patch-strategy create': sum.CreatePatchStrategy,
'patch-strategy delete': sum.DeletePatchStrategy,

83
distributedcloud-client/dcmanagerclient/tests/v1/test_subcloud_deploy_manager.py Normal file
View File

@@ -0,0 +1,83 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
import os
import tempfile
from dcmanagerclient.api.v1 import subcloud_deploy_manager as sdm
from dcmanagerclient.commands.v1 \
import subcloud_deploy_manager as subcloud_deploy_cmd
from dcmanagerclient.tests import base
DEPLOY_PLAYBOOK = 'deployment-manager-playbook.yaml'
DEPLOY_OVERRIDES = 'deployment-manager-overrides-subcloud.yaml'
DEPLOY_CHART = 'deployment-manager.tgz'
SUBCLOUD_DEPLOY_DICT = {
'DEPLOY_PLAYBOOK': DEPLOY_PLAYBOOK,
'DEPLOY_OVERRIDES': DEPLOY_OVERRIDES,
'DEPLOY_CHART': DEPLOY_CHART
}
SUBCLOUD_DEPLOY = sdm.SubcloudDeploy(
deploy_playbook=SUBCLOUD_DEPLOY_DICT['DEPLOY_PLAYBOOK'],
deploy_overrides=SUBCLOUD_DEPLOY_DICT['DEPLOY_OVERRIDES'],
deploy_chart=SUBCLOUD_DEPLOY_DICT['DEPLOY_CHART']
)
class TestCLISubcloudDeployManagerV1(base.BaseCommandTest):
def setUp(self):
super(TestCLISubcloudDeployManagerV1, self).setUp()
# The client is the subcloud_deploy_manager
self.client = self.app.client_manager.subcloud_deploy_manager
def test_subcloud_deploy_show(self):
self.client.subcloud_deploy_manager.subcloud_deploy_show.\
return_value = [SUBCLOUD_DEPLOY]
actual_call = self.call(subcloud_deploy_cmd.SubcloudDeployShow)
self.assertEqual((DEPLOY_PLAYBOOK,
DEPLOY_OVERRIDES,
DEPLOY_CHART),
actual_call[1])
def test_subcloud_deploy_upload(self):
self.client.subcloud_deploy_manager.subcloud_deploy_upload.\
return_value = [SUBCLOUD_DEPLOY]
with tempfile.NamedTemporaryFile() as f1,\
tempfile.NamedTemporaryFile() as f2,\
tempfile.NamedTemporaryFile() as f3:
file_path_1 = os.path.abspath(f1.name)
file_path_2 = os.path.abspath(f2.name)
file_path_3 = os.path.abspath(f3.name)
actual_call = self.call(
subcloud_deploy_cmd.SubcloudDeployUpload,
app_args=[
'--deploy-playbook', file_path_1,
'--deploy-overrides', file_path_2,
'--deploy-chart', file_path_3])
self.assertEqual((DEPLOY_PLAYBOOK,
DEPLOY_OVERRIDES,
DEPLOY_CHART),
actual_call[1])

1
distributedcloud-client/requirements.txt
View File

@@ -10,3 +10,4 @@ PyYAML>=3.10.0 # MIT
requests!=2.12.2,!=2.13.0,>=2.10.0 # Apache-2.0
six>=1.9.0 # MIT
beautifulsoup4
requests-toolbelt