Refactor support for tar and git helm-charts

This commit aims to support helm-charts passed
as dir, git url or as a .tar package and unit
tests for these functionalities.
It also removes some of the remanescent Armada
sections from the app_manifest.yaml that i forgot
to remove in review 902843 and correct some
little flake8 warnings.

Test Plan:
PASS - Helm charts passed as dir in the
app_manifest path are working as expected
PASS - Helm charts passed as git url in the
app_manifest path are working as expected
PASS - Helm charts passed as tar packages in the
app_manifest path are working as expected

Story: 2010937
Task: 49130
Task: 49131

Change-Id: I1fc0e98f731c9a43f742b94d2044c57291876fc0
Signed-off-by: Tomás Barros <tomas.barros@encora.com>

.vscode/launch.json

@@ -13,7 +13,7 @@
             "console": "integratedTerminal",
             "justMyCode": true,
             "args": [
-                "--input=${workspaceFolder}/example/app-test-adminer-1.yaml",
+                "--input=${workspaceFolder}/example/app-test-example.yaml",
                 "--output=${workspaceFolder}/example/output",
                 "--overwrite"
             ]
@@ -29,7 +29,7 @@
             "console": "integratedTerminal",
             "justMyCode": true,
             "args": [
-                "--input=${workspaceFolder}/example/app-test-adminer-1.yaml",
+                "--input=${workspaceFolder}/example/app-test-example.yaml",
                 "--output=${workspaceFolder}/example/output",
                 "--overwrite"
             ]

app_manifest.yaml

@@ -8,21 +8,8 @@ appManifestFile-config:
     - name:  # required
       version:  # required
       path:  # required
-      chartGroup:  # required
+      chartGroup:  # required name
       # add more if you have more than one chart
-  chartGroup:
-    - name:  # required
-      description:  # required for Armada
-      sequenced:  # required for Armada <true/false>
-      chart_names:
-        -  # required
-        -  # optional
-        # add more if you have more than one chartgroup for your Armada app
-  manifest:
-    name:  # required for Armada
-    releasePrefix:  # required for Armada
-
-## For Armada packaging the sections bellow are not necessary.
 
 #################################################
 ## App Metadata Configuration
@@ -79,7 +66,7 @@ metadataFile-config:
 setupFile-config:
   metadata:
     author:  # required
-    author-email:  # required
+    author_email: # required
     url:  # required
     classifier:  # required
       -  # required

example/app-test-adminer-1.yaml

@@ -1,46 +0,0 @@
----
-## App Manifest Configuration
-appManifestFile-config:
-  appName: app-adminer
-  appVersion: 1.0-1
-  namespace: default
-  chart:
-    - name: adminer
-      version: 0.2.1
-      path: ./example/adminer
-  chartGroup:
-    - name:  adminer
-      chart_names:
-        -  adminer
-#################################################
-## App Metadata Configuration
-# for further details about possible configurations on this file, please
-# visit the link: https://wiki.openstack.org/wiki/StarlingX/Containers/StarlingXAppsInternals#metadata.yaml
-metadataFile-config:
-  # the following configurations are optional
-  # uncomment and configure properly the ones you need for your application metadata
-  upgrades:
-    auto_update: true
-
-  supported_k8s_version:
-    minimum: 1.21.8
-    maximum: 1.26.1
-
-  k8s_upgrades:
-    auto_update: true
-    timing: pre
-
-  maintain_user_overrides: true
-
-
-#################################################
-## App Setup Configuration
-# if you wish to see a setup.cfg example, please see the link
-# https://opendev.org/starlingx/app-dell-storage/src/branch/master/python3-k8sapp-dell-storage/k8sapp_dell_storage/setup.cfg
-setupFile-config:
-  metadata:
-    author: StarlingX
-    author_email: starlingx-discuss@lists.starlingx.io
-    url: https://www.starlingx.io/
-    classifier:
-      - "Environment :: OpenStack"

example/app-test-example.yaml

@@ -0,0 +1,40 @@
+---
+appManifestFile-config:
+  appName: app-test
+  appVersion: 1.0-1
+  namespace: default
+  chart:
+    - name: adminer
+      version: 0.2.1
+      path: https://github.com/cetic/helm-adminer.git # git example path
+      chartGroup: chartGroup-test
+    - name: tempo-vulture
+      version: 0.4.1
+      path: app-gen-tool/example/tempo-vulture/tempo-vulture-0.4.1.tgz # tar example path
+      chartGroup: chartGroup-test
+    - name: poc-starlingx
+      version: 1.5.2
+      path: app-gen-tool/example/poc-starlingx-messages # dir example path
+      chartGroup: chartGroup-test
+
+metadataFile-config:
+  upgrades:
+    auto_update: true
+
+  supported_k8s_version:
+    minimum: 1.21.8
+    maximum: 1.26.1
+
+  k8s_upgrades:
+    auto_update: true
+    timing: pre
+
+  maintain_user_overrides: true
+
+setupFile-config:
+  metadata:
+    author: StarlingX
+    author_email: starlingx-discuss@lists.starlingx.io
+    url: https://www.starlingx.io/
+    classifier:
+      - "Environment :: OpenStack"

example/poc-starlingx-messages/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

example/poc-starlingx-messages/Chart.yaml

@@ -0,0 +1,23 @@
+apiVersion: v2
+name: poc-starlingx
+description: A very very very basic messaging exchange app
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 1.5.2
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 1.5.2

example/poc-starlingx-messages/templates/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.kube.name }}
+  labels:
+    app: {{ .Values.kube.name }}
+spec:
+  replicas: {{ .Values.kube.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.kube.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.kube.name }}
+    spec:
+      containers:
+      - name: {{ .Values.kube.name }}
+        image: docker.io/brunomuniz/poc-starlingx:{{ .Values.image.tag }}
+        env:
+          {{- toYaml .Values.env | nindent 10 }}
+        ports:
+        - containerPort: {{ .Values.image.containerPort }}

example/poc-starlingx-messages/templates/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.kube.name }}
+  labels:
+    app: {{ .Values.kube.name }}
+spec:
+  type: NodePort
+  ports:
+  - port: 8100
+    targetPort: {{ .Values.image.containerPort }}
+    nodePort: {{ .Values.kube.port }}
+  selector:
+    app: {{ .Values.kube.name }}

example/poc-starlingx-messages/values.yaml

@@ -0,0 +1,22 @@
+# Default values for helm-chart.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+env:
+  - name: MODE
+    value: node
+  - name: SERVER
+    value: 127.0.0.1:8000
+  - name: PORT
+    value: "8000"
+
+image:
+  repository: docker.io/brunomuniz/poc-starlingx
+  tag: 1.5.2
+  containerPort: 8000
+
+kube:
+  port: 31234
+  replicas: 1
+  secret: my-docker-reg-secret
+  name: poc-starlingx

stx-app-generator/stx-app-generator/app_gen_tool/application.py

@@ -9,9 +9,11 @@ import yaml
 
 from abc import ABC
 from abc import abstractmethod
-from urllib import request
 
+from app_gen_tool.common import extract
+from app_gen_tool.common import get_chart_from_git
 from app_gen_tool.common import to_camel_case
+from app_gen_tool.common import transform_rel_path_into_abs_path
 from app_gen_tool import constants
 
 
@@ -28,7 +30,8 @@ class Application(ABC):
         # Initialize application config
         self._app: dict = {}
         self._app = app_data["appManifestFile-config"]
-        self._temp_app_dir = os.path.join(constants.TEMP_USER_DIR, self.get_app_name())
+        self._temp_app_dir = os.path.join(constants.TEMP_USER_DIR,
+                                          self.get_app_name())
         self.output_folder: str = output_folder
 
         self.APP_NAME = self._app["appName"]
@@ -42,15 +45,24 @@ class Application(ABC):
             = app_data["appManifestFile-config"]["chart"][0]["chartGroup"]
 
         # Create empty list that will contain all the charts
-        self._listcharts = {}
+        self._listcharts = {
-        self._listcharts["chart_names"] = []
+            "chart_names": [],
-        self._listcharts["namespace"] = self._app["namespace"]
+            "namespace": self._app["namespace"]
+        }
 
         # Initialize chart
         self._chart = app_data["appManifestFile-config"]["chart"]
         for i in range(len(self._chart)):
             self._chart[i]["namespace"] = self._app["namespace"]
             self._listcharts["chart_names"].append(self._chart[i]["name"])
+            self._chart[i]['pathType'] = self._define_chart_pathtype(i)
+            if self._chart[i]['pathType'] == 'git':
+                self._process_git_helm_charts(i)
+            else:
+                self._chart[i]['path'] = transform_rel_path_into_abs_path(os.getcwd(),
+                                                                          self._chart[i]['path'])
+            if self._chart[i]['pathType'] == 'tarball':
+                self._extract_helm_tars_to_temp_dir(chart_index=i)
             ret = self._get_values_file(self._chart[i]['path'])
             self._chart[i]['values'] = ret
 
@@ -77,41 +89,42 @@ class Application(ABC):
                 chart["version"],
             )
 
-            if chart["_pathType"] == "dir":
+            try:
-                try:
+                chart_metadata_f = open(chart_yaml_file, "r",
-                    chart_metadata_f = open(chart_yaml_file, "r", encoding="utf-8")
+                                        encoding="utf-8")
-                except Exception as e:
+            except Exception as e:
-                    print(f"ERROR: {e}")
+                print(f"ERROR: {e}")
-                    sys.exit(1)
+                sys.exit(1)
-                chart_file_lines = chart_metadata_f.readlines()
+            chart_file_lines = chart_metadata_f.readlines()
-                chart_file_lines = [
+            chart_file_lines = [
-                    cline
+                cline
-                    for cline in chart_file_lines
+                for cline in chart_file_lines
-                    if len(cline) > 0 and cline[0] != "#"
+                if len(cline) > 0 and cline[0] != "#"
-                ]
+            ]
-                chart_metadata_f.close()
+            chart_metadata_f.close()
-                for line in chart_file_lines:
+            for line in chart_file_lines:
-                    line = line.rstrip("\n")
+                line = line.rstrip("\n")
-                    line_data = line.split()
+                line_data = line.split()
-                    if not line_data:
+                if not line_data:
-                        continue
+                    continue
-                    if "name:" in line_data[0]:
+                if "name:" in line_data[0]:
-                        chart_file_data["name"] = line_data[-1]
+                    chart_file_data["name"] = line_data[-1]
-                    elif "version:" in line_data[0]:
+                elif "version:" in line_data[0]:
-                        chart_file_data["version"] = line_data[-1]
+                    chart_file_data["version"] = line_data[-1]
 
-                # To-do chart type different from dir
+            # To-do chart type different from dir
-                for key in manifest_data:
+            for key in manifest_data:
-                    err_str = ""
+                err_str = ""
-                    if key not in chart_file_data:
+                if key not in chart_file_data:
-                        err_str = \
+                    err_str = \
-                            f'"{key}" is present in app-manifest.yaml but not in {chart_yaml_file}'
+                        f'"{key}" is present in app-manifest.yaml but not in' \
-                        raise KeyError(err_str)
+                        f'{chart_yaml_file}'
-                    if manifest_data[key] != chart_file_data[key]:
+                    raise KeyError(err_str)
-                        err_str = \
+                if manifest_data[key] != chart_file_data[key]:
-                            f'"{key}" has different values ' \
+                    err_str = \
-                            f'in app-manifest.yaml and {chart_yaml_file}'
+                        f'"{key}" has different values ' \
-                        raise ValueError(err_str)
+                        f'in app-manifest.yaml and {chart_yaml_file}'
+                    raise ValueError(err_str)
 
     def get_app_name(self):
         """Return name of app.
@@ -121,7 +134,8 @@ class Application(ABC):
         """
         return self._app["appName"]
 
-    def _package_helm_chart(self, chart, chart_dir) -> bool:
+    @staticmethod
+    def _package_helm_chart(chart, chart_dir) -> bool:
         """Sub-process of app generation. Generate application helm-charts tarball.
 
         Args:
@@ -151,7 +165,8 @@ class Application(ABC):
             return False
 
         # package helm chart
-        cmd_package = ["helm", "package", chart["path"], "--destination=" + chart_dir]
+        cmd_package = ["helm", "package", chart["path"],
+                       "--destination=" + chart_dir]
         subproc = subprocess.run(
             cmd_package,
             env=os.environ.copy(),
@@ -172,7 +187,8 @@ class Application(ABC):
 
         return True
 
-    def _get_values_file(self, path):
+    @staticmethod
+    def _get_values_file(path):
         """Retrieve the content of a values file and saves them in a dictionary.
 
         Args:
@@ -181,6 +197,7 @@ class Application(ABC):
         Returns:
             dict: Data from the values.yaml found in the Helm chart path.
         """
+
         values_yaml_path = os.path.join(path, "values.yaml")
         try:
             with open(values_yaml_path, "r", encoding='utf-8') as f:
@@ -196,7 +213,8 @@ class Application(ABC):
 
     # pyyaml does not support writing yaml block with initial indent
     # add initial indent for yaml block substitution
-    def _write_yaml_to_manifest(self, key, src, init_indent):
+    @staticmethod
+    def _write_yaml_to_manifest(key, src, init_indent):
         """Return the values of a given key in a yaml file.
 
         Args:
@@ -207,9 +225,8 @@ class Application(ABC):
         Returns:
             str: Values with indentation and break lines.
         """
-        target = {}
+        target = {key: src}
         # add heading key
-        target[key] = src
         lines = yaml.safe_dump(target, sort_keys=False).split("\n")
         # remove ending space and first line
         lines.pop()
@@ -284,7 +301,8 @@ class Application(ABC):
             key = block_key[0].lower()
             indent = int(block_key[1])
             if key in dicts:
-                out_line = self._write_yaml_to_manifest(key, dicts[key], indent)
+                out_line = self._write_yaml_to_manifest(key, dicts[key],
+                                                        indent)
             else:
                 out_line = ""
         return out_line
@@ -303,102 +321,7 @@ class Application(ABC):
         ret = False
         chart_path = ""
         print(f'Processing chart {chart["name"]}...')
-        # check pathtype of the chart
+        chart_path = chart["path"]
-        if chart["_pathType"] == "git":
-            # download git
-
-            print("Processing chart _pathType git...")
-
-            if not os.path.exists(self._temp_app_dir):
-                os.makedirs(self._temp_app_dir)
-            # if the git folder exists, check git name and use that folder
-            # otherwise git clone from upstream
-            if not os.path.exists(self._temp_app_dir + chart["_gitname"]):
-                saved_pwd = os.getcwd()
-                os.chdir(self._temp_app_dir)
-                cmd = ["git", "clone", chart["path"]]
-
-                subproc = subprocess.run(
-                    cmd,
-                    env=os.environ.copy(),
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                )
-
-                if subproc.returncode != 0:
-                    output = str(subproc.stderr, encoding="utf-8")
-                    print(output)
-                    print(f'Error: git clone {chart["_gitname"]} failed')
-                    os.chdir(saved_pwd)
-                    return False
-                os.chdir(saved_pwd)
-            else:
-                # git pull to ensure folder up-to-date
-                saved_pwd = os.getcwd()
-                os.chdir(self._temp_app_dir + chart["_gitname"])
-                cmd = ["git", "pull"]
-
-                subproc = subprocess.run(
-                    cmd,
-                    env=os.environ.copy(),
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                )
-
-                if subproc.returncode != 0:
-                    output = str(subproc.stderr, encoding="utf-8")
-                    print(output)
-                    print(f'Error: git pull for {chart["_gitname"]} failed')
-                    os.chdir(saved_pwd)
-                    return False
-
-                os.chdir(saved_pwd)
-
-            chart_path = os.path.join(
-                self._temp_app_dir, chart["_gitname"], chart["subpath"]
-            )
-
-        elif chart["_pathType"] == "tarball":
-            print("Processing chart _pathType tarball...")
-
-            if not os.path.exists(self._temp_app_dir):
-                os.makedirs(self._temp_app_dir)
-            try:
-                # check whether it's a url or local tarball
-                if not os.path.exists(chart["path"]):
-                    # download tarball
-                    tarpath = self._temp_app_dir + chart["_tarname"] + ".tgz"
-                    if not os.path.exists(tarpath):
-                        res = request.urlopen(chart["path"])  # nosec
-                        with open(tarpath, "wb") as f:
-                            f.write(res.read())
-                else:
-                    tarpath = chart["path"]
-
-                # extract tarball
-                chart_tar = tarfile.open(tarpath, "r:gz")
-                chart_files = chart_tar.getnames()
-
-                # get tar arcname for packaging helm chart process
-                # TODO: compatible with the case that there is no arcname
-                chart["_tarArcname"] = chart_files[0].split("/")[0]
-                if not os.path.exists(chart["_tarArcname"]):
-                    for chart_file in chart_files:
-                        chart_tar.extract(chart_file, self._temp_app_dir)
-                chart_tar.close()
-            except Exception as e:
-                print(f"Error: {e}")
-                return False
-
-            chart_path = os.path.join(
-                self._temp_app_dir, chart["_tarArcname"], chart["subpath"]
-            )
-
-        elif chart["_pathType"] == "dir":
-            print("Processing chart _pathType dir...")
-
-            chart_path = chart["path"]
-
         # update chart path
         # remove ending '/', if it exists...
         chart["path"] = chart_path.rstrip("/")
@@ -433,7 +356,8 @@ class Application(ABC):
 
     # Sub-process of app generation
     # generate application sha256 file
-    def _gen_sha256(self, in_file):
+    @staticmethod
+    def _gen_sha256(in_file):
         """Sub-process of app generation. Generate hash function of a file.
 
         Args:
@@ -478,7 +402,8 @@ class Application(ABC):
         try:
             with open(checksum_file, "a", encoding="utf-8") as f:
                 for target_file in sorted(app_files):
-                    f.write(self._gen_sha256(target_file) + " *" + target_file + "\n")
+                    f.write(self._gen_sha256(
+                        target_file) + " *" + target_file + "\n")
         except Exception as e:
             print(f"Error: {e}")
 
@@ -496,7 +421,62 @@ class Application(ABC):
         os.chdir(store_cwd)
         return tarname
 
-    # For debug
+    def _extract_helm_tars_to_temp_dir(self, chart_index):
+        """Auxiliary function to extract helm charts passed as tar packages"""
+
+        try:
+            chart = self._chart[chart_index]
+            tarball_dir = chart['path']
+            tmp_rel_path = self._temp_app_dir
+            extract_path = os.path.join(tmp_rel_path, self.APP_NAME)
+            if not os.path.exists(extract_path):
+                os.makedirs(extract_path)
+            extract(tarball_dir, extract_path)
+            self._chart[chart_index]['tarball_path'] = \
+                chart['path']
+            self._chart[chart_index]['path'] = os.path.join(extract_path,
+                                                            chart['name'])
+        except Exception as e:
+            print(f"ERROR: {e}")
+            sys.exit(1)
+
+    def _define_chart_pathtype(self, i: int):
+        """Gets pathtype for each chart"""
+        chart = self._chart[i]
+        path = chart['path']
+        path_type = ''
+        if path.endswith('.tar') or path.endswith('.tar.gz') \
+                or path.endswith('.tgz'):
+            path_type = 'tarball'
+        elif path.endswith('.git'):
+            path_type = 'git'
+        else:
+            path_type = 'dir'
+        return path_type
+
+    def _process_git_helm_charts(self, chart_index: int):
+        self._chart[chart_index]['_gitname'] = re.search(r'[^/]+(?=\.git$)',
+                                                         self._chart[
+                                                             chart_index][
+                                                             'path']).group()
+        chart = self._chart[chart_index]
+        if not os.path.exists(self._temp_app_dir):
+            os.makedirs(self._temp_app_dir)
+        # if the git folder exists, check git name and use that folder
+        # otherwise git clone from upstream
+        new_path = os.path.join(self._temp_app_dir,
+                                chart['_gitname'])
+
+        _, saved_pwd = get_chart_from_git(
+            new_path, self._temp_app_dir,
+            chart['path'],
+            chart['_gitname']
+        )
+        self._chart[chart_index]['_gitUrl'] = chart['path']
+        self._chart[chart_index]['path'] = os.path.join(saved_pwd, new_path)
+        os.chdir(saved_pwd)
+        return True
+
     def print_app_data(self):
         """Debug Print App Data."""
         print(self._app)

stx-app-generator/stx-app-generator/app_gen_tool/common.py

@@ -1,4 +1,8 @@
 """Module for common methods accross entire codebase."""
+import os
+import subprocess
+import sys
+import tarfile
 
 
 def to_camel_case(s: str) -> str:
@@ -23,3 +27,81 @@ def uppercase_name(name: str) -> str:
         str: String converted to uppercase.
     """
     return name.upper().replace(" ", "_").replace("-", "_")
+
+
+def extract(tar_path='.', extract_path='.'):
+    """Extracts content from a .tar package stored locally or on the web"""
+
+    try:
+        if not os.path.exists(tar_path):
+            new_path = os.getcwd()
+            tar_path = transform_rel_path_into_abs_path(new_path, tar_path)
+        tar = tarfile.open(tar_path, 'r')
+        for item in tar:
+            tar.extract(item, extract_path)
+            if item.name.find(".tgz") != -1 or item.name.find(".tar") != -1:
+                extract(item.name, "./" + item.name[:item.name.rfind('/')])
+    except Exception as e:
+        print(f"Exception error {e} occurred during tar extraction!")
+        sys.exit(1)
+
+
+def get_chart_from_git(new_path: str, temp_dir: str, git_url: str, git_name: str):
+    """Auxiliary function to clone helm-charts from a git url
+
+    Returns a pair of type boolean, string.
+    The boolean indicates whether the git clone was successful or not.
+    The string indicates the saved pwd
+    """
+    if not os.path.exists(new_path):
+        saved_pwd = os.getcwd()
+        os.chdir(temp_dir)
+        cmd = ["git", "clone", git_url]
+
+        subproc = subprocess.run(
+            cmd,
+            env=os.environ.copy(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        if subproc.returncode != 0:
+            output = str(subproc.stderr, encoding="utf-8")
+            print(output)
+            print(f'Error: git clone {git_name} failed')
+            os.chdir(saved_pwd)
+            return False, saved_pwd
+        os.chdir(saved_pwd)
+    else:
+        # git pull to ensure folder up-to-date
+        saved_pwd = os.getcwd()
+        os.chdir(new_path)
+        cmd = ["git", "pull"]
+
+        subproc = subprocess.run(
+            cmd,
+            env=os.environ.copy(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        if subproc.returncode != 0:
+            output = str(subproc.stderr, encoding="utf-8")
+            print(output)
+            print(f'Error: git pull for {git_name} failed')
+            os.chdir(saved_pwd)
+            return False, saved_pwd
+    return True, saved_pwd
+
+
+def transform_rel_path_into_abs_path(abs_path: str, rel_path: str) -> str:
+    """It transforms the rel_path into an absolute path"""
+    if rel_path[:2] == './':
+        rel_path = rel_path.replace('./', 'app-gen-tool/')
+    longest_common_path = ""
+    for i in range(len(rel_path)):
+        common_path = rel_path[:i + 1]
+        if common_path in abs_path:
+            longest_common_path = common_path
+    replace_index = abs_path.find(longest_common_path)
+    return abs_path[:replace_index] + rel_path

stx-app-generator/stx-app-generator/app_gen_tool/constants.py

@@ -13,5 +13,5 @@ FLUXCD_LIFECYCLE_TEMPLATE = os.path.join('templates_plugins', 'lifecycle.templat
 
 # Other variables
 BIN_FETCH_CHART_INFO = os.path.join('scripts', 'fetch_chart_info.sh')
-TEMP_USER_DIR = os.path.join('tmp', getpass.getuser())
+TEMP_USER_DIR = os.path.join('/tmp', getpass.getuser())
 APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0]

stx-app-generator/stx-app-generator/app_gen_tool/fluxcd.py

@@ -618,8 +618,9 @@ class FluxCD(Application):
 
         for i, plug in enumerate(plugins_names):
             out += (
-                f'\t{i+1:03d}_{plug} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.'
+                f'\t{i + 1: 03d}_{plug} = k8sapp'
-                f'helm.{plug.replace("-","_")}'
+                f'_{self.APP_NAME_WITH_UNDERSCORE}.'
+                f'helm.{plug.replace("-", "_")}'
             )
 
             out += f':{plug.replace("-", " ").title().replace(" ", "")}Helm\n'

stx-app-generator/stx-app-generator/app_gen_tool/generator.py

@@ -1,7 +1,6 @@
 """Generator Main Module."""
 import argparse
 import os
-import re
 import shutil
 import sys
 import yaml
@@ -86,55 +85,6 @@ def check_manifest(manifest_data):  # pylint: disable=too-many-return-statements
                 f'Error: Chart attribute \'path\' is missing in chart {chart["name"]}.'
             )
             return False
-        else:
-            # TODO: To support branches/tags in git repo
-            if chart['path'].endswith('.git'):
-                if 'subpath' not in chart:
-                    print(
-                        'Error: Chart attribute "subpath" is missing in '
-                        f'chart {chart["name"]}.'
-                    )
-                    return False
-                chart['_pathType'] = 'git'
-                gitname = re.search(r'[^/]+(?=\.git$)', chart['path']).group()
-                if gitname:
-                    chart['_gitname'] = gitname
-                else:
-                    print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
-                    print(
-                        '       only "local dir", ".git", ".tar.gz", ".tgz" are supported'
-                    )
-                    return False
-            elif chart['path'].endswith('.tar.gz') or chart['path'].endswith('.tgz'):
-                if 'subpath' not in chart:
-                    print(
-                        'Error: Chart attribute "subpath" is missing in '
-                        f'chart {chart["name"]}.'
-                    )
-                    return False
-                chart['_pathType'] = 'tarball'
-                tarname = \
-                    re.search(
-                        r'[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)',
-                        chart['path']
-                    ).group()
-                if tarname:
-                    chart['_tarname'] = tarname
-                else:
-                    print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
-                    print(
-                        "       only 'local dir', '.git', '.tar.gz', '.tgz' are supported"
-                    )
-                    return False
-            else:
-                if not os.path.isdir(chart['path']):
-                    print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
-                    print(
-                        "       only 'local dir', '.git', '.tar.gz', '.tgz' are supported"
-                    )
-                    return False
-                chart['_pathType'] = 'dir'
-
     return True
 
 

stx-app-generator/stx-app-generator/tests/unit/resources/app-test.yaml

@@ -7,7 +7,7 @@ appManifestFile-config:
   chart:
     - name: adminer
       version: 0.2.1
-      path: ./stx-app-generator/stx-app-generator/tests/unit/resources/adminer/adminer
+      path: app-gen-tool/stx-app-generator/stx-app-generator/tests/unit/resources/adminer-0.2.1.tgz
       chartGroup: chartGroup-adminer
 #################################################
 ## App Metadata Configuration

stx-app-generator/stx-app-generator/tests/unit/test_application_class.py

@@ -1,33 +1,24 @@
+"""This testfile tests the Application class"""
+import git
 import os
 import shutil
-import tarfile
+from app_gen_tool.common import get_chart_from_git
-
 from app_gen_tool.fluxcd import FluxCD
 from app_gen_tool.generator import parse_yaml, create_app_directories, check_input_file
 
 
-def _extract(tar_url, extract_path='.'):
-    """Extracts content from a .tar package stored on the web"""
-    tar = tarfile.open(tar_url, 'r')
-    for item in tar:
-        tar.extract(item, extract_path)
-        if item.name.find(".tgz") != -1 or item.name.find(".tar") != -1:
-            _extract(item.name, "./" + item.name[:item.name.rfind('/')])
-
-
 class TestFluxCDAppGen:
     """Class to run unit tests on app-gen"""
 
     def setup_class(cls):
 
-        current_file = os.path.abspath(__file__)
+        cls.current_file = os.path.abspath(__file__)
-        current_folder = os.path.dirname(current_file)
+        cls.current_folder = os.path.dirname(cls.current_file)
-        resource_folder = os.path.join(current_folder, 'resources')
+        cls.resource_folder = os.path.join(cls.current_folder, 'resources')
-        input_file = os.path.join(resource_folder, 'app-test.yaml')
+        input_file = os.path.join(cls.resource_folder, 'app-test.yaml')
-        test_helm_chart = os.path.join(resource_folder, 'adminer-0.2.1.tgz')
 
-        cls.OUTPUT_FOLDER = os.path.join(current_folder, "TEST_OUTPUT")
+        cls.OUTPUT_FOLDER = os.path.join(cls.current_folder, "TEST_OUTPUT")
-        cls.helm_chart_folder = os.path.join(resource_folder, 'adminer')
+        cls.helm_chart_folder = os.path.join(cls.resource_folder, 'adminer')
         cls.app_data = parse_yaml(input_file)
 
         try:
@@ -42,8 +33,7 @@ class TestFluxCDAppGen:
         if not os.path.exists(cls.OUTPUT_FOLDER):
             os.makedirs(cls.OUTPUT_FOLDER, exist_ok=True)
 
-        if not os.path.exists(cls.helm_chart_folder):
+
-            _extract(test_helm_chart, cls.helm_chart_folder)
 
     def teardown_class(cls):
 
@@ -75,3 +65,27 @@ class TestFluxCDAppGen:
         file_exists = os.path.exists(output_file)
 
         assert file_exists
+        # assert True
+
+    def test_clone_helm_chart_from_git(self):
+        """Test cloning helm chart from a git url"""
+        helm_chart_git_url = "https://github.com/cetic/helm-adminer.git"
+        tmp_folder = os.path.join(self.current_folder, "tmp-git-test")
+        try:
+            if not os.path.exists(tmp_folder):
+                os.makedirs(tmp_folder, exist_ok=False)
+            git.Repo.clone_from(helm_chart_git_url, tmp_folder)
+            git_name = "helm-adminer"
+            new_path = os.path.join(tmp_folder, git_name)
+            get_chart_from_git(new_path, tmp_folder, helm_chart_git_url, git_name)
+            assert os.path.exists(new_path)
+        except git.exc.GitError:
+            print(f"git repository {helm_chart_git_url} does not exist or is not public!")
+            assert False
+        except Exception as e:
+            print(f"Exception error {e} occurred!")
+            assert False
+        finally:
+            if os.path.exists(tmp_folder):
+                shutil.rmtree(tmp_folder)
+            assert not os.path.exists(tmp_folder)

stx-app-generator/stx-app-generator/tests/unit/test_camel_case.py

@@ -1,17 +0,0 @@
-from app_gen_tool.common import to_camel_case
-
-
-class TestCameCase:
-
-    def setup_class(cls):
-        pass
-
-    def teardown_class(cls):
-        pass
-
-    def test_camel_case(self):
-
-        input_string = "ThisIsAString"
-        output_string = to_camel_case(input_string)
-
-        assert output_string == input_string.lower()

stx-app-generator/stx-app-generator/tests/unit/test_common_functions.py

@@ -0,0 +1,59 @@
+"""This testfile tests the common functions used across the repository"""
+import os
+import shutil
+from app_gen_tool.common import to_camel_case
+from app_gen_tool.common import extract, transform_rel_path_into_abs_path
+
+class CommonFunctions:
+
+    def __init__(self):
+        self.current_file = os.path.abspath(__file__)
+        self.current_folder = os.path.dirname(self.current_file)
+        self.resource_folder = os.path.join(self.current_folder, 'resources')
+
+    def setup_class(cls):
+        pass
+
+    def teardown_class(cls):
+        pass
+
+    def test_camel_case(self):
+
+        input_string = "ThisIsAString"
+        output_string = to_camel_case(input_string)
+
+        assert output_string == input_string.lower()
+
+    def test_tar_extraction(self):
+        """Test tar extraction function"""
+        tmp_folder = os.path.join('/tmp', "tmp-tar-test")
+        try:
+            if not os.path.exists(tmp_folder):
+                os.makedirs(tmp_folder, exist_ok=False)
+            app_name = 'tempo-vulture'
+            tarfile = 'tempo-vulture-0.4.1.tgz'
+            tarfile_path = os.path.join(self.resource_folder, tarfile)
+            extract(tarfile_path, tmp_folder)
+            assert os.path.exists(os.path.join(tmp_folder, app_name))
+        except Exception as e:
+            print(f"Exception error {e} occurred!")
+            assert False
+        finally:
+            if os.path.exists(tmp_folder):
+                shutil.rmtree(tmp_folder)
+            assert not os.path.exists(tmp_folder)
+
+    def test_transform_path_function(self):
+        rel_path_1 = '../../../../example/poc-starlingx-messages'
+        rel_path_2 = '../../../../example/tempo-vulture/tempo-vulture-0.4.1.tgz'
+        rel_path_3 = './resources/adminer-0.2.1.tgz'
+        tar_name_1 = '/tempo-vulture-0.4.1.tgz'
+        tar_name_2 = '/adminer-0.2.1.tgz'
+        abs_path_1 = transform_rel_path_into_abs_path(rel_path_1)
+        abs_path_2 = transform_rel_path_into_abs_path(rel_path_2)
+        abs_path_3 = transform_rel_path_into_abs_path(rel_path_3)
+        abs_path_2 = abs_path_2.replace(tar_name_1, '', 1)
+        abs_path_3 = abs_path_3.replace(tar_name_2, '', 1)
+        if os.path.exists(abs_path_1) and os.path.exists(abs_path_2) and os.path.exists(abs_path_3):
+            assert True
+        assert False

test-requirements.txt

@@ -6,4 +6,6 @@ yamllint<1.26.1;python_version>="3.0" # GPLv2
 pylint
 bandit
 pytest
-coverage
+coverage
+gitpython
+wheel