diff --git a/mini-mirror/Dockerfile.ubuntu_bionic b/mini-mirror/Dockerfile.ubuntu_bionic index 8346f43f..66a22782 100644 --- a/mini-mirror/Dockerfile.ubuntu_bionic +++ b/mini-mirror/Dockerfile.ubuntu_bionic @@ -15,12 +15,13 @@ FROM ubuntu:18.04 as aptly ARG APTLY_CONFIG_PATH=etc/aptly.conf -ARG MIRROR_SOURCE_DIR=sources +ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml ARG RELEASE_SIGN_KEY_PATH=etc ARG RELEASE_SIGN_KEY_PASSPHRASE RUN apt-get update -RUN apt-get install -y gnupg wget +RUN apt-get install -y gnupg wget jq python-pip +RUN pip install yq COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf COPY tools/install_aptly.sh /opt/install_aptly.sh @@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename RUN /opt/install_aptly.sh -COPY "${MIRROR_SOURCE_DIR}" /opt/sources +COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh diff --git a/mini-mirror/Dockerfile.ubuntu_xenial b/mini-mirror/Dockerfile.ubuntu_xenial index 37cfda3f..ffc6e131 100644 --- a/mini-mirror/Dockerfile.ubuntu_xenial +++ b/mini-mirror/Dockerfile.ubuntu_xenial @@ -15,12 +15,13 @@ FROM ubuntu:16.04 as aptly ARG APTLY_CONFIG_PATH=etc/aptly.conf -ARG MIRROR_SOURCE_DIR=sources +ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml ARG RELEASE_SIGN_KEY_PATH=etc ARG RELEASE_SIGN_KEY_PASSPHRASE RUN apt-get update -RUN apt-get install -y wget +RUN apt-get install -y wget jq python-pip +RUN pip install yq COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf COPY tools/install_aptly.sh /opt/install_aptly.sh @@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename RUN /opt/install_aptly.sh -COPY "${MIRROR_SOURCE_DIR}" /opt/sources +COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh diff --git a/mini-mirror/README.rst b/mini-mirror/README.rst index d350bcd3..1ebdda2c 100644 --- a/mini-mirror/README.rst +++ b/mini-mirror/README.rst @@ -11,50 +11,39 @@ Build Requirements Add mirror sources ~~~~~~~~~~~~~~~~~~ -Mini-mirror requires a directory at build-time that contains the repositories -and packages that will be mirrored. +Mini-mirror requires a YAML file at build-time that contains the repositories +and packages that will be mirrored as different YAML documents. -.. code:: +.. code:: yaml + --- + name: + url: + key_url: + codename: * + label: * + aptly_config: | # *Inline aptly config JSON file to replace default + { } + components: # List of Components + - + subrepos: # List of repositories within the source repository + - distribution: + packages: # + - name: + version: * + ... + --- + # Additional repository document here + ... - sources/ - | -- source1-prefix/ - |-- source-name/ - |-- source.txt - |-- packages.txt - | -- source2-prefix/ - |-- source-name/ - |-- source.txt - |-- packages.txt +*Optional -Sources are defined as directories containing the files: -* source-prefix - a prefix to separate sources that have conflicting - distribution names (i.e. the directory a source serves from). -* source-name - the name of a source; used for record-keeping. -* source.txt - contains location and metadata information for a source. -* packages.txt - contains a list of packages, formatted as `package queries `_ - for a source. - -Example ``source.txt`` format: - - .. code:: - - source_url source_key_url dist components - -Example ``packages.txt`` format: - -.. code:: - - package1 - package2 - package3 (>=3.6) - -To specify the location of your sources directory, export the following +To specify the location of your sources YAML file, export the following environment variable: .. code:: bash - export MIRROR_SOURCE_DIR=/path/to/sources + export MIRROR_SOURCE_FILE=/path/to/sources.yaml Generate a signing key ~~~~~~~~~~~~~~~~~~~~~~ @@ -92,8 +81,8 @@ environment variable: .. NOTE:: Mini-mirror can be configured on a per-repo basis by adding an Aptly config - file to the root directory of a source. This overrides the Aptly config - file taken from ``APTLY_CONFIG_PATH``. + file to the .aptly_config key in the YAML document. This overrides + the Aptly config file taken from ``APTLY_CONFIG_PATH``. Proxy ~~~~~ diff --git a/mini-mirror/build.sh b/mini-mirror/build.sh index 209199cc..66ccd495 100755 --- a/mini-mirror/build.sh +++ b/mini-mirror/build.sh @@ -20,7 +20,8 @@ SCRIPT_DIR=$(dirname "${SCRIPT}") ## Only build from main folder cd "${SCRIPT_DIR}"/.. || exit -IMAGE="mini-mirror" +PROJECT_PATH="mini-mirror" +IMAGE=${IMAGE:-mini-mirror} VERSION=${VERSION:-latest} DISTRO=${DISTRO:-ubuntu_xenial} REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"} @@ -31,7 +32,7 @@ HTTPS_PROXY=${HTTPS_PROXY:-""} NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"} APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"} -MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"} +MIRROR_SOURCE_FILE=${MIRROR_SOURCE_FILE:-"mini-mirror-sources.yaml"} RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"} RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""} @@ -42,7 +43,7 @@ APTLY_INSTALL_FROM=${APTLY_INSTALL_FROM:-"source"} APTLY_REPO=${APTLY_REPO:-"https://github.com/smstone/aptly.git"} APTLY_REFSPEC=${APTLY_REFSPEC:-"allow-custom-codename"} -docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \ +docker build -f "${PROJECT_PATH}"/Dockerfile."${DISTRO}" --network=host \ -t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \ --build-arg http_proxy="${HTTP_PROXY}" \ --build-arg https_proxy="${HTTPS_PROXY}" \ @@ -51,12 +52,12 @@ docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \ --build-arg no_proxy="${NO_PROXY}" \ --build-arg NO_PROXY="${NO_PROXY}" \ --build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \ - --build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \ + --build-arg MIRROR_SOURCE_FILE="${MIRROR_SOURCE_FILE}" \ --build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \ --build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \ --build-arg APTLY_INSTALL_FROM="${APTLY_INSTALL_FROM}" \ --build-arg APTLY_REPO="${APTLY_REPO}" \ --build-arg APTLY_REFSPEC="${APTLY_REFSPEC}" \ - ${extra_build_args} "${IMAGE}" + ${extra_build_args} "${PROJECT_PATH}" cd - || exit diff --git a/mini-mirror/etc/aptly.conf b/mini-mirror/etc/aptly.conf index 78fc7f1f..7e0dcde3 100644 --- a/mini-mirror/etc/aptly.conf +++ b/mini-mirror/etc/aptly.conf @@ -2,7 +2,7 @@ "rootDir": "/opt/.aptly", "downloadConcurrency": 4, "downloadSpeedLimit": 0, - "architectures": [], + "architectures": ["amd64"], "dependencyFollowSuggests": false, "dependencyFollowRecommends": false, "dependencyFollowAllVariants": false, diff --git a/mini-mirror/mini-mirror-sources.yaml b/mini-mirror/mini-mirror-sources.yaml new file mode 100644 index 00000000..77c44c86 --- /dev/null +++ b/mini-mirror/mini-mirror-sources.yaml @@ -0,0 +1,43 @@ +--- +name: aptly-example +url: http://repo.aptly.info/ +key_url: https://www.aptly.info/pubkey.txt +components: + - main +subrepos: + - distribution: squeeze + packages: + - name: aptly +... +# Example additional config for adding docker +#--- +#name: docker +#url: https://download.docker.com/linux/ubuntu +#key_url: https://download.docker.com/linux/ubuntu/gpg +#aptly_config: | +# { +# "rootDir": "/opt/.aptly", +# "downloadConcurrency": 20, +# "downloadSpeedLimit": 0, +# "architectures": ["amd64"], +# "dependencyFollowSuggests": false, +# "dependencyFollowRecommends": false, +# "dependencyFollowAllVariants": false, +# "dependencyFollowSource": false, +# "dependencyVerboseResolve": true, +# "gpgDisableSign": true, +# "gpgDisableVerify": true, +# "gpgProvider": "gpg", +# "downloadSourcePackages": false, +# "skipLegacyPool": true, +# "ppaDistributorID": "ubuntu", +# "ppaCodename": "" +# } +#components: +# - stable +#subrepos: +# - distribution: xenial +# packages: +# - name: docker-ce +# version: 17.03.3~ce-0~ubuntu-xenial_amd64 +#... diff --git a/mini-mirror/sources/aptly-example/squeeze/packages.txt b/mini-mirror/sources/aptly-example/squeeze/packages.txt deleted file mode 100644 index a7620ba6..00000000 --- a/mini-mirror/sources/aptly-example/squeeze/packages.txt +++ /dev/null @@ -1 +0,0 @@ -aptly diff --git a/mini-mirror/sources/aptly-example/squeeze/source.txt b/mini-mirror/sources/aptly-example/squeeze/source.txt deleted file mode 100644 index 6eabe16a..00000000 --- a/mini-mirror/sources/aptly-example/squeeze/source.txt +++ /dev/null @@ -1 +0,0 @@ -http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main diff --git a/mini-mirror/tools/publish_snapshots.sh b/mini-mirror/tools/publish_snapshots.sh index fac62872..9336c25e 100755 --- a/mini-mirror/tools/publish_snapshots.sh +++ b/mini-mirror/tools/publish_snapshots.sh @@ -14,23 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -ex -if [ ! -z "$1" ]; then +if [[ ! -z "$1" ]]; then gpg --import /opt/release.gpg fi -for source_prefix in /opt/sources/*; do - for source in $source_prefix/*; do - read -r -a info < "${source}"/source.txt - repo=${info[0]} - key=${info[1]} - dist=${info[2]} - components=${info[*]:3} +sources=$(yq "." /opt/mini-mirror-sources.yaml | jq -s '.') + +# Loop to iterate over each document in the YAML file. +# By base64 encoding and then decoding the output from jq we are able to +# cleanly iterate over the output even if it contains newlines, etc. +for source in $(echo "${sources}" | jq -r '.[] | @base64' ); do + _source() { + echo "${source}" | base64 --decode | jq -r "${*}" + } + + source_name=$(_source '.name') + repo=$(_source '.url') + key=$(_source '.key_url') + components=$(_source '.components') + label=$(_source '.label') + codename=$(_source '.code_name') + + # Loop to iterate over the `subrepo` list in the document + for subrepo in $(_source '.subrepos[] | @base64'); do + _subrepo() { + echo "${subrepo}" | base64 --decode | jq -r "${*}" + } + + dist=$(_subrepo '.distribution') # Use source specific aptly config when provided - if [ -f "${source}"/aptly.conf ]; then - conf="${source}"/aptly.conf + source_conf=$(_source '.aptly_config') + if [[ "$source_conf" != "null" ]]; then + echo "${source_conf}" > aptly.conf + conf=$(pwd)/aptly.conf else conf=/etc/aptly.conf fi @@ -38,11 +57,21 @@ for source_prefix in /opt/sources/*; do # Create package query from well-defined package list. # # package1 - # package2 ==> package1 | package2 | package3 + # package2 ==> package1 | package2 (=1.0) | package3 # package3 # - packages=$(awk -v ORS=" | " '{ print $1 }' "${source}"/packages.txt) - packages="${packages::-3}" + + # Grab packages from .subrepo + packages=$(_subrepo '.packages') + # Convert any found versions to strings + str_versions=$(echo "${packages}" | jq ' .[] | if .version != null then {name: .name, version: .version | tostring} else {name: .name} end') + # Format packages and versions to " (=@" + formatted_packages=$(echo "${str_versions}" | jq -r '. | join(" (=@")') + # Substitute "@" with ")" so the new format is " (=)" + # and bring the packages on to one line separated by "@" + wrap_versions=$(echo "${formatted_packages}" | sed -r "s/@(.*)/\1\)/g" | tr "\n" "@") + # Substitute the "@" between packages with " | " + package_query=$(echo "${wrap_versions}" | sed -r "s/@/ \| /g" | sed -r "s/ \| $//g") # Import source key wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \ @@ -51,13 +80,14 @@ for source_prefix in /opt/sources/*; do # Create a mirror of each component from a source's repository, update it, # and publish a snapshot of it. mirrors=() - for component in $components; do - name="${source}-${component}" + # Loop to iterate over the `component` list in the document + for component in $(echo "${components}" | jq -r '.[]' ); do + name="${source_name}-${dist}-${component}" mirrors+=("$name") aptly mirror create \ -config="${conf}" \ - -filter="${packages}" \ + -filter="${package_query}" \ -filter-with-deps \ "${name}" "${repo}" "${dist}" "${component}" @@ -65,28 +95,35 @@ for source_prefix in /opt/sources/*; do aptly snapshot create -config="${conf}" "${name}" from mirror "${name}" done - # preserve the codename and label of the source repository - codename=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Codename: //p') - label=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Label: //p') + # If the codename or label have not been specified then acquire them from + # the mirror + if [[ "${codename}" == "null" ]]; then + codename=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Codename: //p') + fi + if [[ "${label}" == "null" ]]; then + label=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Label: //p') + fi # Publish snapshot and sign if a key passphrase is provided. - com_list=$(echo "${components[@]}" | tr ' ' ',') - if [ ! -z "$1" ]; then + com_list=$(echo "${components}" | jq -r '. | join(",")') + if [[ ! -z "$1" ]]; then aptly publish snapshot \ -batch=true \ + -config="${conf}" \ -component="${com_list}" \ -distribution="${dist}" \ -passphrase="${1}" \ -codename="${codename}" \ -label="${label}" \ - "${mirrors[@]}" "${source_prefix:13}" + "${mirrors[@]}" "${source_name}" else aptly publish snapshot \ + -config="${conf}" \ -component="${com_list}" \ -distribution="${dist}" \ -codename="${codename}" \ -label="${label}" \ - "${mirrors[@]}" "${source_prefix:13}" + "${mirrors[@]}" "${source_name}" fi done done