openstack/openstack-helm-images
Code Issues Proposed changes

Modify mini-mirror to build from yaml

Browse Source 
This modifies mini-mirror so that instead of parsing through .txt
documents to build the mirror, it instead has one yaml file consisting
of multiple yaml documents with a pre-defined structure that has all
of the same information. This will allow for easier modification and
visibility into the configuration of mini-mirror.

Along with that, this enables the manual setting of CodeName and Label
values of the mirror, as well as the ability to change the name of the
image that the build.sh script creates

Change-Id: I08646258492e8938b8892749172aea23e363af3d
This commit is contained in:
Michael Beaver
2019-11-13 16:33:06 -06:00
parent 101e8f8b33
commit d503870bfe
9 changed files with 146 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
mini-mirror/Dockerfile.ubuntu_bionic
View File

@@ -15,12 +15,13 @@
FROM ubuntu:18.04 as aptly
ARG APTLY_CONFIG_PATH=etc/aptly.conf
ARG MIRROR_SOURCE_DIR=sources
ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
ARG RELEASE_SIGN_KEY_PATH=etc
ARG RELEASE_SIGN_KEY_PASSPHRASE
RUN apt-get update
RUN apt-get install -y gnupg wget
RUN apt-get install -y gnupg wget jq python-pip
RUN pip install yq
COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
COPY tools/install_aptly.sh /opt/install_aptly.sh
@@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
RUN /opt/install_aptly.sh
COPY "${MIRROR_SOURCE_DIR}" /opt/sources
COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh

7
mini-mirror/Dockerfile.ubuntu_xenial
View File

@@ -15,12 +15,13 @@
FROM ubuntu:16.04 as aptly
ARG APTLY_CONFIG_PATH=etc/aptly.conf
ARG MIRROR_SOURCE_DIR=sources
ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
ARG RELEASE_SIGN_KEY_PATH=etc
ARG RELEASE_SIGN_KEY_PASSPHRASE
RUN apt-get update
RUN apt-get install -y wget
RUN apt-get install -y wget jq python-pip
RUN pip install yq
COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
COPY tools/install_aptly.sh /opt/install_aptly.sh
@@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
RUN /opt/install_aptly.sh
COPY "${MIRROR_SOURCE_DIR}" /opt/sources
COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh

65
mini-mirror/README.rst
View File

@@ -11,50 +11,39 @@ Build Requirements
Add mirror sources
~~~~~~~~~~~~~~~~~~
Mini-mirror requires a directory at build-time that contains the repositories
and packages that will be mirrored.
Mini-mirror requires a YAML file at build-time that contains the repositories
and packages that will be mirrored as different YAML documents.
.. code::
.. code:: yaml
---
name: <Repository name (i.e. the directory a source serves from)>
url: <URL link to the source repository>
key_url: <URL link to the key for the source repository>
codename: *<Override codename for the release file>
label: *<Override label for the release file>
aptly_config: | # *Inline aptly config JSON file to replace default
{ }
components: # List of Components
- <Component>
subrepos: # List of repositories within the source repository
- distribution: <Distribution name of the repository>
packages: # <List of all packages>
- name: <Package name>
version: *<Version of package to pin to>
...
---
# Additional repository document here
...
sources/
| -- source1-prefix/
|-- source-name/
|-- source.txt
|-- packages.txt
| -- source2-prefix/
|-- source-name/
|-- source.txt
|-- packages.txt
*Optional
Sources are defined as directories containing the files:
* source-prefix - a prefix to separate sources that have conflicting
distribution names (i.e. the directory a source serves from).
* source-name - the name of a source; used for record-keeping.
* source.txt - contains location and metadata information for a source.
* packages.txt - contains a list of packages, formatted as `package queries <https://www.aptly.info/doc/feature/query/>`_
for a source.
Example ``source.txt`` format:
.. code::
source_url source_key_url dist components
Example ``packages.txt`` format:
.. code::
package1
package2
package3 (>=3.6)
To specify the location of your sources directory, export the following
To specify the location of your sources YAML file, export the following
environment variable:
.. code:: bash
export MIRROR_SOURCE_DIR=/path/to/sources
export MIRROR_SOURCE_FILE=/path/to/sources.yaml
Generate a signing key
~~~~~~~~~~~~~~~~~~~~~~
@@ -92,8 +81,8 @@ environment variable:
.. NOTE::
Mini-mirror can be configured on a per-repo basis by adding an Aptly config
file to the root directory of a source. This overrides the Aptly config
file taken from ``APTLY_CONFIG_PATH``.
file to the .aptly_config key in the YAML document. This overrides
the Aptly config file taken from ``APTLY_CONFIG_PATH``.
Proxy
~~~~~

11
mini-mirror/build.sh
View File

@@ -20,7 +20,8 @@ SCRIPT_DIR=$(dirname "${SCRIPT}")
## Only build from main folder
cd "${SCRIPT_DIR}"/.. || exit
IMAGE="mini-mirror"
PROJECT_PATH="mini-mirror"
IMAGE=${IMAGE:-mini-mirror}
VERSION=${VERSION:-latest}
DISTRO=${DISTRO:-ubuntu_xenial}
REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"}
@@ -31,7 +32,7 @@ HTTPS_PROXY=${HTTPS_PROXY:-""}
NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"}
APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"}
MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"}
MIRROR_SOURCE_FILE=${MIRROR_SOURCE_FILE:-"mini-mirror-sources.yaml"}
RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"}
RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""}
@@ -42,7 +43,7 @@ APTLY_INSTALL_FROM=${APTLY_INSTALL_FROM:-"source"}
APTLY_REPO=${APTLY_REPO:-"https://github.com/smstone/aptly.git"}
APTLY_REFSPEC=${APTLY_REFSPEC:-"allow-custom-codename"}
docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
docker build -f "${PROJECT_PATH}"/Dockerfile."${DISTRO}" --network=host \
-t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \
--build-arg http_proxy="${HTTP_PROXY}" \
--build-arg https_proxy="${HTTPS_PROXY}" \
@@ -51,12 +52,12 @@ docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
--build-arg no_proxy="${NO_PROXY}" \
--build-arg NO_PROXY="${NO_PROXY}" \
--build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \
--build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \
--build-arg MIRROR_SOURCE_FILE="${MIRROR_SOURCE_FILE}" \
--build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \
--build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \
--build-arg APTLY_INSTALL_FROM="${APTLY_INSTALL_FROM}" \
--build-arg APTLY_REPO="${APTLY_REPO}" \
--build-arg APTLY_REFSPEC="${APTLY_REFSPEC}" \
${extra_build_args} "${IMAGE}"
${extra_build_args} "${PROJECT_PATH}"
cd - || exit

2
mini-mirror/etc/aptly.conf
View File

@@ -2,7 +2,7 @@
"rootDir": "/opt/.aptly",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"architectures": ["amd64"],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,

43
mini-mirror/mini-mirror-sources.yaml Normal file
View File

@@ -0,0 +1,43 @@
---
name: aptly-example
url: http://repo.aptly.info/
key_url: https://www.aptly.info/pubkey.txt
components:
- main
subrepos:
- distribution: squeeze
packages:
- name: aptly
...
# Example additional config for adding docker
#---
#name: docker
#url: https://download.docker.com/linux/ubuntu
#key_url: https://download.docker.com/linux/ubuntu/gpg
#aptly_config: |
# {
# "rootDir": "/opt/.aptly",
# "downloadConcurrency": 20,
# "downloadSpeedLimit": 0,
# "architectures": ["amd64"],
# "dependencyFollowSuggests": false,
# "dependencyFollowRecommends": false,
# "dependencyFollowAllVariants": false,
# "dependencyFollowSource": false,
# "dependencyVerboseResolve": true,
# "gpgDisableSign": true,
# "gpgDisableVerify": true,
# "gpgProvider": "gpg",
# "downloadSourcePackages": false,
# "skipLegacyPool": true,
# "ppaDistributorID": "ubuntu",
# "ppaCodename": ""
# }
#components:
# - stable
#subrepos:
# - distribution: xenial
# packages:
# - name: docker-ce
# version: 17.03.3~ce-0~ubuntu-xenial_amd64
#...

1
mini-mirror/sources/aptly-example/squeeze/packages.txt
View File

@@ -1 +0,0 @@
aptly

1
mini-mirror/sources/aptly-example/squeeze/source.txt
View File

@@ -1 +0,0 @@
http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main

85
mini-mirror/tools/publish_snapshots.sh
View File

@@ -14,23 +14,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
set -ex
if [ ! -z "$1" ]; then
if [[ ! -z "$1" ]]; then
gpg --import /opt/release.gpg
fi
for source_prefix in /opt/sources/*; do
for source in $source_prefix/*; do
read -r -a info < "${source}"/source.txt
repo=${info[0]}
key=${info[1]}
dist=${info[2]}
components=${info[*]:3}
sources=$(yq "." /opt/mini-mirror-sources.yaml | jq -s '.')
# Loop to iterate over each document in the YAML file.
# By base64 encoding and then decoding the output from jq we are able to
# cleanly iterate over the output even if it contains newlines, etc.
for source in $(echo "${sources}" | jq -r '.[] | @base64' ); do
_source() {
echo "${source}" | base64 --decode | jq -r "${*}"
}
source_name=$(_source '.name')
repo=$(_source '.url')
key=$(_source '.key_url')
components=$(_source '.components')
label=$(_source '.label')
codename=$(_source '.code_name')
# Loop to iterate over the `subrepo` list in the document
for subrepo in $(_source '.subrepos[] | @base64'); do
_subrepo() {
echo "${subrepo}" | base64 --decode | jq -r "${*}"
}
dist=$(_subrepo '.distribution')
# Use source specific aptly config when provided
if [ -f "${source}"/aptly.conf ]; then
conf="${source}"/aptly.conf
source_conf=$(_source '.aptly_config')
if [[ "$source_conf" != "null" ]]; then
echo "${source_conf}" > aptly.conf
conf=$(pwd)/aptly.conf
else
conf=/etc/aptly.conf
fi
@@ -38,11 +57,21 @@ for source_prefix in /opt/sources/*; do
# Create package query from well-defined package list.
#
# package1
# package2 ==> package1 | package2 | package3
# package2 ==> package1 | package2 (=1.0) | package3
# package3
#
packages=$(awk -v ORS=" | " '{ print $1 }' "${source}"/packages.txt)
packages="${packages::-3}"
# Grab packages from .subrepo
packages=$(_subrepo '.packages')
# Convert any found versions to strings
str_versions=$(echo "${packages}" | jq ' .[] | if .version != null then {name: .name, version: .version | tostring} else {name: .name} end')
# Format packages <pkg> and versions <ver> to "<pkg> (=@<ver>"
formatted_packages=$(echo "${str_versions}" | jq -r '. | join(" (=@")')
# Substitute "@<ver>" with "<ver>)" so the new format is "<pkg> (=<ver>)"
# and bring the packages on to one line separated by "@"
wrap_versions=$(echo "${formatted_packages}" | sed -r "s/@(.*)/\1\)/g" | tr "\n" "@")
# Substitute the "@" between packages with " | "
package_query=$(echo "${wrap_versions}" | sed -r "s/@/ \| /g" | sed -r "s/ \| $//g")
# Import source key
wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \
@@ -51,13 +80,14 @@ for source_prefix in /opt/sources/*; do
# Create a mirror of each component from a source's repository, update it,
# and publish a snapshot of it.
mirrors=()
for component in $components; do
name="${source}-${component}"
# Loop to iterate over the `component` list in the document
for component in $(echo "${components}" | jq -r '.[]' ); do
name="${source_name}-${dist}-${component}"
mirrors+=("$name")
aptly mirror create \
-config="${conf}" \
-filter="${packages}" \
-filter="${package_query}" \
-filter-with-deps \
"${name}" "${repo}" "${dist}" "${component}"
@@ -65,28 +95,35 @@ for source_prefix in /opt/sources/*; do
aptly snapshot create -config="${conf}" "${name}" from mirror "${name}"
done
# preserve the codename and label of the source repository
codename=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Codename: //p')
label=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Label: //p')
# If the codename or label have not been specified then acquire them from
# the mirror
if [[ "${codename}" == "null" ]]; then
codename=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Codename: //p')
fi
if [[ "${label}" == "null" ]]; then
label=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Label: //p')
fi
# Publish snapshot and sign if a key passphrase is provided.
com_list=$(echo "${components[@]}" | tr ' ' ',')
if [ ! -z "$1" ]; then
com_list=$(echo "${components}" | jq -r '. | join(",")')
if [[ ! -z "$1" ]]; then
aptly publish snapshot \
-batch=true \
-config="${conf}" \
-component="${com_list}" \
-distribution="${dist}" \
-passphrase="${1}" \
-codename="${codename}" \
-label="${label}" \
"${mirrors[@]}" "${source_prefix:13}"
"${mirrors[@]}" "${source_name}"
else
aptly publish snapshot \
-config="${conf}" \
-component="${com_list}" \
-distribution="${dist}" \
-codename="${codename}" \
-label="${label}" \
"${mirrors[@]}" "${source_prefix:13}"
"${mirrors[@]}" "${source_name}"
fi
done
done