#!/bin/bash # # Copyright (c) 2022 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # set -e source $(dirname "$0")/lib/job_utils.sh require_job_env BUILD_HOME require_job_env DRY_RUN declare_job_env HELM_CHART_NAME declare_job_env HELM_CHART_PACKAGES declare_job_env EXTRA_HELM_CHART_PACKAGES declare_job_env IGNORE_HELM_CHARTS declare_job_env MANAGED_IMAGE_LIST_FILE load_build_env unset GREP_COLOR GREP_COLORS GREP_OPTIONS BUILD_STREAMS="stable dev" BUILD_TAGS="latest versioned" normalize_pattern_list() { echo "$1" | sed -r 's/[[:space:],]+/\n/g' | grep -v -E '^\s*$' || true } # Convert helm chart package name patterns to arrays HELM_CHART_PACKAGES="$(normalize_pattern_list "$HELM_CHART_PACKAGES")" declare -a HELM_CHART_PKG_PATTENRS if [[ -n "$HELM_CHART_PACKAGES" ]] ; then readarray -t HELM_CHART_PKG_PATTERNS <<<"$HELM_CHART_PACKAGES" fi EXTRA_HELM_CHART_PACKAGES="$(normalize_pattern_list "$EXTRA_HELM_CHART_PACKAGES")" declare -a EXTRA_HELM_CHART_PKG_PATTENRS if [[ -n "$EXTRA_HELM_CHART_PACKAGES" ]] ; then readarray -t EXTRA_HELM_CHART_PKG_PATTERNS <<<"$EXTRA_HELM_CHART_PACKAGES" fi # Usage: fnmatch PATTERN STRING... fnmatch() { local ptn="$1" ; shift while [[ "$#" -gt 0 ]] ; do case "$1" in $ptn) return 0 ;; esac shift done return 1 } # Usage: find_helm_apps PACKAGE_NAME_PATTERNS... # Prints "helm_tar_name:package_name,package_name,..." one per line find_helm_apps() { # find helm.inc files local inc_files_str inc_files_str=$(find "$BUILD_HOME/repo/cgcs-root" -name "${DOCKER_BASE_OS}_helm.inc") || return 1 [[ -n "$inc_files_str" ]] || return 0 local -a inc_files readarray -t inc_files <<<"$inc_files_str" || return 1 [[ "${#inc_files[@]}" -gt 0 ]] || return 0 # Each helm.inc file contais a list of DEB package names with helm charts local helm_packages helm_packages=$(cat "${inc_files[@]}" | \grep -v -E '^\s*(#.*)?$' ; [[ ${PIPESTATUS[0]} -eq 0 ]]) || return 1 # Hash: helm_tar_name => package_list local -A app_hash local ptn for ptn in "$@" ; do local pkg for pkg in $helm_packages ; do if fnmatch "$ptn" "$pkg" ; then local app # if main helm chart name is specified, group all matching helm chart # packages with the same helm char tar (app) name if [[ -n "$HELM_CHART_NAME" ]] ; then app="$HELM_CHART_NAME" # otherwise, create a separate helm chart tar whose name is derived # from the package else app="$(echo "$pkg" | sed -r 's/(-helm|-helm-fluxcd)$//g')" fi if [[ -z "${app_hash[$app]}" ]] ; then app_hash["$app"]="$pkg" else app_hash["$app"]="${app_hash[$app]},$pkg" fi fi done done for app in "${!app_hash[@]}" ; do echo "$app:${app_hash[$app]}" done | sort -u [[ "${PIPESTATUS[0]}" -eq 0 ]] } # find main helm charts MAIN_HELM_CHARTS=$(find_helm_apps "${HELM_CHART_PKG_PATTERNS[@]}") info "found primary helm charts: [$(echo ${MAIN_HELM_CHARTS})] pattern=[${HELM_CHART_PKG_PATTERNS[*]}]" # find extra helm charts EXTRA_HELM_CHARTS=$( set -e # temp file - all charts matching HELM_CHART_PACKAGES file1="$(mktemp -t "build-helm-charts.1.XXXXXX")" exec 3<>"$file1" rm "$file1" file1="/proc/self/fd/3" echo "$MAIN_HELM_CHARTS" >"$file1" # temp file all charts matching EXTRA_HELM_CHART_PACKAGES file2="$(mktemp -t "build-helm-charts.1.XXXXXX")" exec 4<>"$file2" rm "$file2" file2="/proc/self/fd/4" find_helm_apps "${EXTRA_HELM_CHART_PKG_PATTERNS[@]}" >"$file2" # lines unique to file2 comm -13 "$file1" "$file2" ) info "found extra helm charts: [$(echo ${EXTRA_HELM_CHARTS})] pattern=[${EXTRA_HELM_CHART_PKG_PATTERNS[*]}]" if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then info "IGNORE_HELM_CHARTS=$IGNORE_HELM_CHARTS" fi # find managed image list file use_managed_image_list=0 rm -rf "$WORKSPACE_ROOT/managed-image-lists" if [[ -n "$MANAGED_IMAGE_LIST_FILE" ]] ; then mkdir -p "$WORKSPACE_ROOT/managed-image-lists" file="$(cd "$BUILD_HOME/repo" && readlink -e "$MANAGED_IMAGE_LIST_FILE")" cp "$file" "$WORKSPACE_ROOT/managed-image-lists/" use_managed_image_list=1 fi # find image dirs relative to WORKSPACE_ROOT declare -a image_dirs declare -a extra_image_dirs if [[ $use_managed_image_list -eq 0 ]] ; then if [[ -d "$WORKSPACE_ROOT/std/build-images" ]] ; then image_dirs+=("std/build-images") fi if [[ -d "$WORKSPACE_ROOT/rt/build-images" ]] ; then image_dirs+=("rt/build-images") fi # copy any extra image-*.lst files to workspace so that # build containers can see them if [[ -n "$EXTRA_IMAGE_RECORD_DIR" ]] ; then if ! $DRY_RUN ; then rm -rf --one-file-system "$WORKSPACE_ROOT/extra-image-records"/* mkdir -p "$WORKSPACE_ROOT/extra-image-records" extra_image_dirs+=('extra-image-records') fi let extra_dir_index=0 || : # Look for extra image lists and copy them into extra-image-records directory. # Process them in reverse order, because the last list takes precedence # (see cgcs-root/build-tools/build-helm-charts.sh) # Also: rename while copying by adding a directory index (integer) to make # sure like-named files don't overwrite each other. for extra_dir in $(echo "$EXTRA_IMAGE_RECORD_DIR" | sed 's/,/ /g' | sed 's/ /\n/g' | tac | grep -v -E '^\s*$') ; do extra_dir_index_padded=$(printf '%03d' "$extra_dir_index") info "looking for extra image records in $extra_dir" ( cd "$extra_dir" ; ) || exit 1 if ! $DRY_RUN ; then for list_file in $(find -H "$extra_dir" -mindepth 1 -maxdepth 1 -name 'images-*.lst') ; do list_file_basename=$(basename "$list_file") \cp -v "${list_file}" "$WORKSPACE_ROOT/extra-image-records/${extra_dir_index_padded}-extra-${list_file_basename}" || exit 1 done fi let extra_dir_index++ || : done fi fi build_helm_charts() { local cmd="$1" stx_docker_cmd $DRY_RUN_ARG "set -e ; cd \"\$MY_REPO/build-tools\" ; export PATH=\"\$PWD:\$PATH\" ; $cmd" } copy_dir() { find "$1" -mindepth 1 -maxdepth 1 -exec cp -f -alr -t "$2" '{}' '+' } output_dir="$BUILD_HOME/workspace/helm-charts" tmp_output_dir=$BUILD_HOME/workspace/std/build-helm if [[ -d "$output_dir" ]] ; then rm -rf --one-file-system "$output_dir" || exit 1 fi mkdir -p "$output_dir" mkdir -p "$tmp_output_dir" # replace image tags in the main helm chart with the images generated # by this build if [[ ( "${#image_dirs[@]}" -gt 0 || "${#extra_image_dirs[@]}" -gt 0 || "$use_managed_image_list" -eq 1 ) && -n "${MAIN_HELM_CHARTS}" ]] ; then for build_stream in $BUILD_STREAMS ; do for build_tag in $BUILD_TAGS ; do for os in $DOCKER_BASE_OS ; do label="${os}-${build_stream}-${build_tag}" distroless_label="distroless-${build_stream}-${build_tag}" # look for image list files image_arg=$( sep= ( cd "$WORKSPACE_ROOT" if [[ $use_managed_image_list -eq 1 ]] ; then find "managed-image-lists" -type f else # Extra images first, because they have lower priority find "${extra_image_dirs[@]}" \ -mindepth 1 -maxdepth 1 \ -name "[0-9]*-extra-images-${label}.lst" \ -o -name "[0-9]*-extra-images-${distroless_label}.lst" \ | sort check_pipe_status || exit 1 # This build's images last find "${image_dirs[@]}" \ -mindepth 1 -maxdepth 1 \ -name "images-${label}.lst" \ -o -name "images-${distroless_label}.lst" check_pipe_status || exit 1 fi ) | while read image_list_file ; do echo -n "${sep}\$MY_WORKSPACE/${image_list_file}" sep="," done check_pipe_status || exit 1 ) check_pipe_status || exit 1 if [[ -z "$image_arg" ]] ; then continue fi for spec in ${MAIN_HELM_CHARTS} ; do app="${spec%%:*}" pkgs="${spec#*:}" cmd="build-helm-charts.sh" cmd+=" --verbose" cmd+=" --os ${os}" cmd+=" --label '${label}'" cmd+=" --image-record ${image_arg}" cmd+=" --app $app" cmd+=" --package $pkgs" if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then cmd+=" --ignore='$IGNORE_HELM_CHARTS'" fi cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/main-${label}.log\"" cmd+=" ; [[ \${PIPESTATUS[0]} -eq 0 ]]" notice "building primary helm chart $app ($label)" build_helm_charts "$cmd" || exit 1 copy_dir "$tmp_output_dir" "$output_dir" || exit 1 done done done done fi # all other helm charts: extract them from DEBs w/o replacing image tags for spec in $EXTRA_HELM_CHARTS ; do app="${spec%%:*}" pkgs="${spec#*:}" cmd="build-helm-charts.sh" cmd+=" --verbose" cmd+=" --os $DOCKER_BASE_OS" cmd+=" --app $app" cmd+=" --package $pkgs" if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then cmd+=" --ignore='$IGNORE_HELM_CHARTS'" fi cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/$app.log\"" notice "building extra helm chart $app" build_helm_charts "$cmd" || exit 1 copy_dir "$tmp_output_dir" "$output_dir" || exit 1 done if [[ -d "$output_dir/stx" && $(find "$output_dir/stx" -maxdepth 1 -type f -name '*.tgz' -print -quit | wc -l) -gt 0 ]] ; then notice "helm charts created in $output_dir/stx" ls -al "$output_dir/stx" else notice "no helm charts found" fi