fe1cf4ac78
Allow EXTRA_IMAGE_RECORD_DIR to include multiple directories, rather than one. TESTS ================= Set EXTRA_IMAGE_RECORD_DIR to multiple directories and make sure image lists from these directories are passed to the build-helm-charts.sh script in the correct order, ie reversed, and before the image lists generated by the current build. Story: 2010055 Task: 52266 Signed-off-by: Davlet Panech <davlet.panech@windriver.com> Change-Id: Ieb3f31bccd35d9fbf5f2a6cd619e2db5d610439d
293 lines
10 KiB
Bash
Executable File
293 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
#
|
|
# Copyright (c) 2022 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
set -e
|
|
source $(dirname "$0")/lib/job_utils.sh
|
|
|
|
require_job_env BUILD_HOME
|
|
require_job_env DRY_RUN
|
|
declare_job_env HELM_CHART_NAME
|
|
declare_job_env HELM_CHART_PACKAGES
|
|
declare_job_env EXTRA_HELM_CHART_PACKAGES
|
|
declare_job_env IGNORE_HELM_CHARTS
|
|
declare_job_env MANAGED_IMAGE_LIST_FILE
|
|
|
|
load_build_env
|
|
|
|
unset GREP_COLOR GREP_COLORS GREP_OPTIONS
|
|
|
|
BUILD_STREAMS="stable dev"
|
|
BUILD_TAGS="latest versioned"
|
|
|
|
normalize_pattern_list() {
|
|
echo "$1" | sed -r 's/[[:space:],]+/\n/g' | grep -v -E '^\s*$' || true
|
|
}
|
|
|
|
# Convert helm chart package name patterns to arrays
|
|
HELM_CHART_PACKAGES="$(normalize_pattern_list "$HELM_CHART_PACKAGES")"
|
|
declare -a HELM_CHART_PKG_PATTENRS
|
|
if [[ -n "$HELM_CHART_PACKAGES" ]] ; then
|
|
readarray -t HELM_CHART_PKG_PATTERNS <<<"$HELM_CHART_PACKAGES"
|
|
fi
|
|
|
|
EXTRA_HELM_CHART_PACKAGES="$(normalize_pattern_list "$EXTRA_HELM_CHART_PACKAGES")"
|
|
declare -a EXTRA_HELM_CHART_PKG_PATTENRS
|
|
if [[ -n "$EXTRA_HELM_CHART_PACKAGES" ]] ; then
|
|
readarray -t EXTRA_HELM_CHART_PKG_PATTERNS <<<"$EXTRA_HELM_CHART_PACKAGES"
|
|
fi
|
|
|
|
# Usage: fnmatch PATTERN STRING...
|
|
fnmatch() {
|
|
local ptn="$1" ; shift
|
|
while [[ "$#" -gt 0 ]] ; do
|
|
case "$1" in
|
|
$ptn) return 0 ;;
|
|
esac
|
|
shift
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Usage: find_helm_apps PACKAGE_NAME_PATTERNS...
|
|
# Prints "helm_tar_name:package_name,package_name,..." one per line
|
|
find_helm_apps() {
|
|
|
|
# find helm.inc files
|
|
local inc_files_str
|
|
inc_files_str=$(find "$BUILD_HOME/repo/cgcs-root" -name "${DOCKER_BASE_OS}_helm.inc") || return 1
|
|
[[ -n "$inc_files_str" ]] || return 0
|
|
local -a inc_files
|
|
readarray -t inc_files <<<"$inc_files_str" || return 1
|
|
[[ "${#inc_files[@]}" -gt 0 ]] || return 0
|
|
|
|
# Each helm.inc file contais a list of DEB package names with helm charts
|
|
local helm_packages
|
|
helm_packages=$(cat "${inc_files[@]}" | \grep -v -E '^\s*(#.*)?$' ; [[ ${PIPESTATUS[0]} -eq 0 ]]) || return 1
|
|
|
|
# Hash: helm_tar_name => package_list
|
|
local -A app_hash
|
|
local ptn
|
|
for ptn in "$@" ; do
|
|
local pkg
|
|
for pkg in $helm_packages ; do
|
|
if fnmatch "$ptn" "$pkg" ; then
|
|
local app
|
|
# if main helm chart name is specified, group all matching helm chart
|
|
# packages with the same helm char tar (app) name
|
|
if [[ -n "$HELM_CHART_NAME" ]] ; then
|
|
app="$HELM_CHART_NAME"
|
|
# otherwise, create a separate helm chart tar whose name is derived
|
|
# from the package
|
|
else
|
|
app="$(echo "$pkg" | sed -r 's/(-helm|-helm-fluxcd)$//g')"
|
|
fi
|
|
if [[ -z "${app_hash[$app]}" ]] ; then
|
|
app_hash["$app"]="$pkg"
|
|
else
|
|
app_hash["$app"]="${app_hash[$app]},$pkg"
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
for app in "${!app_hash[@]}" ; do
|
|
echo "$app:${app_hash[$app]}"
|
|
done | sort -u
|
|
[[ "${PIPESTATUS[0]}" -eq 0 ]]
|
|
}
|
|
|
|
# find main helm charts
|
|
MAIN_HELM_CHARTS=$(find_helm_apps "${HELM_CHART_PKG_PATTERNS[@]}")
|
|
info "found primary helm charts: [$(echo ${MAIN_HELM_CHARTS})] pattern=[${HELM_CHART_PKG_PATTERNS[*]}]"
|
|
|
|
# find extra helm charts
|
|
EXTRA_HELM_CHARTS=$(
|
|
set -e
|
|
|
|
# temp file - all charts matching HELM_CHART_PACKAGES
|
|
file1="$(mktemp -t "build-helm-charts.1.XXXXXX")"
|
|
exec 3<>"$file1"
|
|
rm "$file1"
|
|
file1="/proc/self/fd/3"
|
|
echo "$MAIN_HELM_CHARTS" >"$file1"
|
|
|
|
# temp file all charts matching EXTRA_HELM_CHART_PACKAGES
|
|
file2="$(mktemp -t "build-helm-charts.1.XXXXXX")"
|
|
exec 4<>"$file2"
|
|
rm "$file2"
|
|
file2="/proc/self/fd/4"
|
|
find_helm_apps "${EXTRA_HELM_CHART_PKG_PATTERNS[@]}" >"$file2"
|
|
|
|
# lines unique to file2
|
|
comm -13 "$file1" "$file2"
|
|
)
|
|
info "found extra helm charts: [$(echo ${EXTRA_HELM_CHARTS})] pattern=[${EXTRA_HELM_CHART_PKG_PATTERNS[*]}]"
|
|
|
|
if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then
|
|
info "IGNORE_HELM_CHARTS=$IGNORE_HELM_CHARTS"
|
|
fi
|
|
|
|
# find managed image list file
|
|
use_managed_image_list=0
|
|
rm -rf "$WORKSPACE_ROOT/managed-image-lists"
|
|
if [[ -n "$MANAGED_IMAGE_LIST_FILE" ]] ; then
|
|
mkdir -p "$WORKSPACE_ROOT/managed-image-lists"
|
|
file="$(cd "$BUILD_HOME/repo" && readlink -e "$MANAGED_IMAGE_LIST_FILE")"
|
|
cp "$file" "$WORKSPACE_ROOT/managed-image-lists/"
|
|
use_managed_image_list=1
|
|
fi
|
|
|
|
# find image dirs relative to WORKSPACE_ROOT
|
|
declare -a image_dirs
|
|
declare -a extra_image_dirs
|
|
if [[ $use_managed_image_list -eq 0 ]] ; then
|
|
if [[ -d "$WORKSPACE_ROOT/std/build-images" ]] ; then
|
|
image_dirs+=("std/build-images")
|
|
fi
|
|
if [[ -d "$WORKSPACE_ROOT/rt/build-images" ]] ; then
|
|
image_dirs+=("rt/build-images")
|
|
fi
|
|
# copy any extra image-*.lst files to workspace so that
|
|
# build containers can see them
|
|
if [[ -n "$EXTRA_IMAGE_RECORD_DIR" ]] ; then
|
|
if ! $DRY_RUN ; then
|
|
rm -rf --one-file-system "$WORKSPACE_ROOT/extra-image-records"/*
|
|
mkdir -p "$WORKSPACE_ROOT/extra-image-records"
|
|
extra_image_dirs+=('extra-image-records')
|
|
fi
|
|
let extra_dir_index=0 || :
|
|
|
|
# Look for extra image lists and copy them into extra-image-records directory.
|
|
# Process them in reverse order, because the last list takes precedence
|
|
# (see cgcs-root/build-tools/build-helm-charts.sh)
|
|
# Also: rename while copying by adding a directory index (integer) to make
|
|
# sure like-named files don't overwrite each other.
|
|
for extra_dir in $(echo "$EXTRA_IMAGE_RECORD_DIR" | sed 's/,/ /g' | sed 's/ /\n/g' | tac | grep -v -E '^\s*$') ; do
|
|
extra_dir_index_padded=$(printf '%03d' "$extra_dir_index")
|
|
info "looking for extra image records in $extra_dir"
|
|
( cd "$extra_dir" ; ) || exit 1
|
|
if ! $DRY_RUN ; then
|
|
for list_file in $(find -H "$extra_dir" -mindepth 1 -maxdepth 1 -name 'images-*.lst') ; do
|
|
list_file_basename=$(basename "$list_file")
|
|
\cp -v "${list_file}" "$WORKSPACE_ROOT/extra-image-records/${extra_dir_index_padded}-extra-${list_file_basename}" || exit 1
|
|
done
|
|
fi
|
|
let extra_dir_index++ || :
|
|
done
|
|
fi
|
|
fi
|
|
|
|
build_helm_charts() {
|
|
local cmd="$1"
|
|
stx_docker_cmd $DRY_RUN_ARG "set -e ; cd \"\$MY_REPO/build-tools\" ; export PATH=\"\$PWD:\$PATH\" ; $cmd"
|
|
}
|
|
|
|
copy_dir() {
|
|
find "$1" -mindepth 1 -maxdepth 1 -exec cp -f -alr -t "$2" '{}' '+'
|
|
}
|
|
|
|
output_dir="$BUILD_HOME/workspace/helm-charts"
|
|
tmp_output_dir=$BUILD_HOME/workspace/std/build-helm
|
|
|
|
if [[ -d "$output_dir" ]] ; then
|
|
rm -rf --one-file-system "$output_dir" || exit 1
|
|
fi
|
|
mkdir -p "$output_dir"
|
|
mkdir -p "$tmp_output_dir"
|
|
|
|
# replace image tags in the main helm chart with the images generated
|
|
# by this build
|
|
if [[ ( "${#image_dirs[@]}" -gt 0 || "${#extra_image_dirs[@]}" -gt 0 || "$use_managed_image_list" -eq 1 ) && -n "${MAIN_HELM_CHARTS}" ]] ; then
|
|
for build_stream in $BUILD_STREAMS ; do
|
|
for build_tag in $BUILD_TAGS ; do
|
|
for os in $DOCKER_BASE_OS ; do
|
|
label="${os}-${build_stream}-${build_tag}"
|
|
distroless_label="distroless-${build_stream}-${build_tag}"
|
|
# look for image list files
|
|
image_arg=$(
|
|
sep=
|
|
(
|
|
cd "$WORKSPACE_ROOT"
|
|
if [[ $use_managed_image_list -eq 1 ]] ; then
|
|
find "managed-image-lists" -type f
|
|
else
|
|
# Extra images first, because they have lower priority
|
|
find "${extra_image_dirs[@]}" \
|
|
-mindepth 1 -maxdepth 1 \
|
|
-name "[0-9]*-extra-images-${label}.lst" \
|
|
-o -name "[0-9]*-extra-images-${distroless_label}.lst" \
|
|
| sort
|
|
check_pipe_status || exit 1
|
|
# This build's images last
|
|
find "${image_dirs[@]}" \
|
|
-mindepth 1 -maxdepth 1 \
|
|
-name "images-${label}.lst" \
|
|
-o -name "images-${distroless_label}.lst"
|
|
check_pipe_status || exit 1
|
|
fi
|
|
) | while read image_list_file ; do
|
|
echo -n "${sep}\$MY_WORKSPACE/${image_list_file}"
|
|
sep=","
|
|
done
|
|
check_pipe_status || exit 1
|
|
)
|
|
check_pipe_status || exit 1
|
|
|
|
if [[ -z "$image_arg" ]] ; then
|
|
continue
|
|
fi
|
|
|
|
for spec in ${MAIN_HELM_CHARTS} ; do
|
|
app="${spec%%:*}"
|
|
pkgs="${spec#*:}"
|
|
|
|
cmd="build-helm-charts.sh"
|
|
cmd+=" --verbose"
|
|
cmd+=" --os ${os}"
|
|
cmd+=" --label '${label}'"
|
|
cmd+=" --image-record ${image_arg}"
|
|
cmd+=" --app $app"
|
|
cmd+=" --package $pkgs"
|
|
if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then
|
|
cmd+=" --ignore='$IGNORE_HELM_CHARTS'"
|
|
fi
|
|
cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/main-${label}.log\""
|
|
cmd+=" ; [[ \${PIPESTATUS[0]} -eq 0 ]]"
|
|
notice "building primary helm chart $app ($label)"
|
|
build_helm_charts "$cmd" || exit 1
|
|
copy_dir "$tmp_output_dir" "$output_dir" || exit 1
|
|
done
|
|
done
|
|
done
|
|
done
|
|
fi
|
|
|
|
# all other helm charts: extract them from DEBs w/o replacing image tags
|
|
for spec in $EXTRA_HELM_CHARTS ; do
|
|
app="${spec%%:*}"
|
|
pkgs="${spec#*:}"
|
|
cmd="build-helm-charts.sh"
|
|
cmd+=" --verbose"
|
|
cmd+=" --os $DOCKER_BASE_OS"
|
|
cmd+=" --app $app"
|
|
cmd+=" --package $pkgs"
|
|
if [[ -n "$IGNORE_HELM_CHARTS" ]] ; then
|
|
cmd+=" --ignore='$IGNORE_HELM_CHARTS'"
|
|
fi
|
|
cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/$app.log\""
|
|
notice "building extra helm chart $app"
|
|
build_helm_charts "$cmd" || exit 1
|
|
copy_dir "$tmp_output_dir" "$output_dir" || exit 1
|
|
done
|
|
|
|
if [[ -d "$output_dir/stx" && $(find "$output_dir/stx" -maxdepth 1 -type f -name '*.tgz' -print -quit | wc -l) -gt 0 ]] ; then
|
|
notice "helm charts created in $output_dir/stx"
|
|
ls -al "$output_dir/stx"
|
|
else
|
|
notice "no helm charts found"
|
|
fi
|