Use infra apt mirrors during image build

Image build often fail due to upstream
apt mirrors issues (unavailable due to maintenance,
traffic limits, etc). The change enables using
infra apt mirrors.

Change-Id: I5590507f09351709c00daafb47ecebdc9fdd536a

Dockerfile

@@ -30,6 +30,7 @@ ARG REGISTRY_INSECURE="False"
 ARG KEEP_ALL_WHEELS="False"
 ARG HORIZON_EXTRA_PANELS
 ARG PROJECT_PIP_EXTRAS
+ARG APT_MIRROR="https://archive.ubuntu.com/ubuntu/"
 
 # NOTE: This option is only applicable to apt/dpkg systems. The value is noop
 # for rpm based systems. This will not show up in the final image regardless.

playbooks/vars.yaml

@@ -5,6 +5,7 @@ image_registries:
   - host: "quay.io"
     org: "airshipit"
 container_command: docker
+apt_mirror: "[trusted=yes] https://{{ zuul_site_mirror_fqdn | default('archive.ubuntu.com') }}/{{ ansible_distribution | lower }}"
 
 base_image:
   context: "."
@@ -32,6 +33,7 @@ requirements_image:
     - "PROJECT='requirements'"
     - "PROJECT_REF={{ openstack_release }}"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 barbican_image:
   context: "."
@@ -50,6 +52,7 @@ barbican_image:
     - "KEEP_ALL_WHEELS=yes"
     - "DIST_PACKAGES='python3-dev gcc'"
     - "PIP_ARGS='--only-binary :none:'"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 cinder_image:
   context: "."
@@ -67,6 +70,7 @@ cinder_image:
     - "PIP_PACKAGES='python-swiftclient'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 cyborg_image:
   context: "."
@@ -83,6 +87,7 @@ cyborg_image:
     - "DIST_PACKAGES='pciutils'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 designate_image:
   context: "."
@@ -99,6 +104,7 @@ designate_image:
     - "PROFILES='fluent'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 glance_image:
   context: "."
@@ -116,6 +122,7 @@ glance_image:
     - "PIP_PACKAGES='python-swiftclient os-brick python-cinderclient oslo-rootwrap'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 heat_image:
   context: "."
@@ -133,6 +140,7 @@ heat_image:
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
     - "DIST_PACKAGES='curl'"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 horizon_image:
   context: "."
@@ -150,6 +158,7 @@ horizon_image:
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
     - "PIP_PACKAGES='pymemcache'"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 ironic_image:
   context: "."
@@ -167,6 +176,7 @@ ironic_image:
     - "DIST_PACKAGES='ethtool lshw iproute2'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 keystone_image:
   context: "."
@@ -184,6 +194,7 @@ keystone_image:
     - "PIP_PACKAGES='python-openstackclient'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 manila_image:
   context: "."
@@ -199,6 +210,7 @@ manila_image:
     - "PROJECT_REF={{ openstack_release }}"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 monasca_api_image:
   context: "."
@@ -216,6 +228,7 @@ monasca_api_image:
     - "PIP_PACKAGES='influxdb cassandra-driver sqlalchemy'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 neutron_image:
   context: "."
@@ -234,6 +247,7 @@ neutron_image:
     - "DIST_PACKAGES='jq ethtool lshw'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 nova_image:
   context: "."
@@ -251,6 +265,7 @@ nova_image:
     - "DIST_PACKAGES='net-tools openssh-server'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 octavia_image:
   context: "."
@@ -266,6 +281,7 @@ octavia_image:
     - "PROJECT_REF={{ openstack_release }}"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 openstack_client_pip_packages:
   - cmd2
@@ -306,6 +322,7 @@ openstack_client_image:
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
     - "PIP_PACKAGES='{{ openstack_client_pip_packages | join(' ') }}'"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 placement_image:
   context: "."
@@ -323,6 +340,7 @@ placement_image:
     - "PIP_PACKAGES='httplib2'"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 tacker_image:
   context: "."
@@ -338,6 +356,7 @@ tacker_image:
     - "PROJECT_REF={{ openstack_release }}"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"
 
 watcher_image:
   context: "."
@@ -353,3 +372,4 @@ watcher_image:
     - "PROJECT_REF={{ openstack_release }}"
     - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
     - "KEEP_ALL_WHEELS=yes"
+    - "APT_MIRROR='{{ apt_mirror }}'"

scripts/cleanup.sh

@@ -9,6 +9,9 @@ case ${distro} in
             patch \
             python3-virtualenv \
             virtualenv
+        if [ -f /etc/apt/sources.list.bak ]; then
+            mv /etc/apt/sources.list.bak /etc/apt/sources.list
+        fi
         rm -rf /var/lib/apt/lists/*
         ;;
     centos)

scripts/install.sh

@@ -16,6 +16,15 @@ rpm_python_packages=("python3")
 case ${distro} in
     ubuntu)
         export LC_CTYPE=C.UTF-8
+        # This overrides the base image configuration
+        echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated
+        mv /etc/apt/sources.list /etc/apt/sources.list.bak
+        cat > /etc/apt/sources.list <<EOF
+deb ${APT_MIRROR} ${distro_version} main universe
+deb ${APT_MIRROR} ${distro_version}-updates main universe
+deb ${APT_MIRROR} ${distro_version}-security main universe
+deb ${APT_MIRROR} ${distro_version}-backports main universe
+EOF
         apt-get update
         if [[ ! -z "$(apt-cache search ^python3-distutils$)" ]]; then
             dpkg_python_packages+=("python3-distutils")