diff --git a/Dockerfile b/Dockerfile index 9c9f1f71..dccac015 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,6 @@ ARG PROJECT_REPO=https://opendev.org/openstack/${PROJECT} ARG PROJECT_REF=master ARG PROJECT_RELEASE=master ARG EXTRA_PROJECTS="" -ARG DISTRO="" ARG PROFILES="" ARG PIP_PACKAGES="" ARG PIP_ARGS="" diff --git a/dockerfiles/ubuntu/Dockerfile b/Dockerfile.base similarity index 95% rename from dockerfiles/ubuntu/Dockerfile rename to Dockerfile.base index d65d8e4c..9e68c857 100644 --- a/dockerfiles/ubuntu/Dockerfile +++ b/Dockerfile.base @@ -16,11 +16,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-transport-https \ ca-certificates \ gnupg2 \ + lsb-release \ wget && \ wget -q -O- ${CEPH_KEY} | apt-key add - && \ if [ -n "${CEPH_REPO}" ]; then \ echo "${CEPH_REPO}" | tee /etc/apt/sources.list.d/ceph.list; \ fi && \ - apt-get remove -y wget gnupg2 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index a7844e43..fc2298be 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OpenStack LOCI OpenStack LOCI is a project designed to quickly build Lightweight OCI -compatible images of OpenStack services. +compatible images of OpenStack services based on Ubuntu. Additionally, we produce a "wheels" image for [requirements](https://github.com/openstack/requirements) containing all of the @@ -14,6 +14,7 @@ by LOCI. For simplicity, we will continue to use Keystone as an example. Note: To build locally, you will need a version of docker >= 17.05.0. +#### Base image You need to start by building a base image for your distribution that included the required build dependencies. Loci has included a collection of Dockerfiles to get you started with building a base image. These @@ -21,34 +22,41 @@ are located in the dockerfiles directory. It's easy to build a base image: ``` bash -$ docker build dockerfiles/ubuntu \ +$ docker build . \ + -f Dockerfile.base \ --build-arg FROM=ubuntu:jammy \ --build-arg CEPH_REPO='deb https://download.ceph.com/debian-reef/ jammy main' \ - --tag loci-base:ubuntu_jammy + --tag base:ubuntu_jammy ``` -Then you can build the rest of the service images locally: +#### Requirements image +The `requirements` image is where we put all the packages listed in the OpenStack +[upper constraints](https://opendev.org/openstack/requirements/src/branch/master/upper-constraints.txt) +together with their dependencies. This is a consistent set of packages so that if we install various +OpenStack components from this set of packages we can be sure they are compatible with each other. +In Loci we use multistage Dockerfile with the project image as a default target. +To build the `requirements` image use the following command ``` bash $ docker build . \ - --build-arg FROM=loci-base:ubuntu_jammy \ - --build-arg PROJECT=keystone \ - --tag loci-keystone:master-ubuntu_jammy + -f Dockerfile \ + --target requirements \ + --build-arg FROM=base:ubuntu_jammy \ + --build-arg PROJECT=requirements \ + --tag requirements:master-ubuntu_jammy ``` -The default base distro is Ubuntu Jammy, however, you can use the following form to build from a distro of your choice, in this case, CentOS: +#### Project image +Then you can build the rest of the service images using this requirements image: ``` bash -$ docker build dockerfiles/centos \ - --tag loci-base:centos - $ docker build . \ + --build-arg FROM=base:ubuntu_jammy \ + --build-arg WHEELS=requirements:master-ubuntu_jammy \ --build-arg PROJECT=keystone \ - --build-arg WHEELS="loci/requirements:master-centos" \ - --build-arg FROM=loci-base:centos \ - --tag loci-keystone:master-centos + --tag keystone:master-ubuntu_jammy ``` - -Loci will detect which base OS you're using, so if you need to add additional -features to your base image the Loci build will still run. +Here you can specify the `requirements` (WHEELS) image which is mounted during the build and is used +as a wheels repository. By default the `quay.io/airshipit/requirements:master-ubuntu_jammy` +is used. If building behind a proxy, remember to use build arguments to pass these through to the build: @@ -58,7 +66,7 @@ $ docker build . \ --build-arg https_proxy=$https_proxy \ --build-arg no_proxy=$no_proxy \ --build-arg PROJECT=keystone \ - --tag loci-keystone:master-ubuntu_jammy + --tag keystone:master-ubuntu_jammy ``` For more advanced building you can use docker build arguments to define: @@ -78,10 +86,6 @@ For more advanced building you can use docker build arguments to define: * `WHEELS` The location of the wheels Docker image. The image must contain wheels in the root directory. It is mounted while building other images. `[myregistry/]mydockernamespace/requirements[:tag]` - * `DISTRO` This is a helper variable used for scripts. It would primarily be - used in situations where the script would not detect the correct distro. - For example, you would set `DISTRO=centos` when running from an oraclelinux - base image. * `PROFILES` The bindep profiles to specify to configure which packages get installed. This is a space separated list. * `PIP_PACKAGES` Specify additional python packages you would like installed. @@ -89,17 +93,17 @@ For more advanced building you can use docker build arguments to define: you wanted to include rpdb, you would need to have built that into your WHEELS. * `KEEP_ALL_WHEELS` Set this to `True` if you want to keep all packages, even - not built ourselfs in the WHEEL image. Is useful for reproducible builts, - as 3rd party libraries will be keept in WHEEL image. + not built ourselfs in the WHEELS image. This is useful for reproducible builds, + as 3rd party libraries will be keept in the WHEELS image. * `PIP_ARGS` Specify additional pip parameters you would like. * `PIP_WHEEL_ARGS` Specify additional pip wheel parameters you would like. - Default is PIP_ARGS. + Default is PIP_ARGS. * `DIST_PACKAGES` Specify additional distribution packages you would like installed. * `EXTRA_BINDEP` Specify a bindep-* file to add in the container. It would - be considered next to the default bindep.txt. + be considered next to the default bindep.txt. * `EXTRA_PYDEP` Specify a pydep-* file to add in the container. It would - be considered next to the default pydep.txt. + be considered next to the default pydep.txt. * `REGISTRY_PROTOCOL` Set this to `https` if you are running your own registry on https, `http` if you are running on http, or leave it as `detect` if you want to re-use existing protocol detection. @@ -117,15 +121,7 @@ $ docker build . \ --build-arg PROJECT=keystone \ --build-arg PROJECT_REPO=https://review.opendev.org/openstack/keystone \ --build-arg PROJECT_REF=refs/changes/24/923324/10 \ - --tag loci-keystone:923324-10 -``` - -To build with the wheels from a private Docker registry rather than Docker Hub run: -``` bash -$ docker build . \ - --build-arg PROJECT=keystone \ - --build-arg WHEELS=172.17.0.1:5000/mydockernamespace/requirements:master-ubuntu_jammy \ - --tag loci-keystone:master-ubuntu_jammy + --tag keystone:923324-10 ``` To build cinder with lvm and ceph support you would run: @@ -157,7 +153,7 @@ do this we recommend that you perform any required customization in a child image using a pattern similar to: ``` Dockerfile -FROM loci/keystone:master-ubuntu_jammy +FROM quay.io/airshipit/keystone:master-ubuntu_jammy MAINTAINER you@example.com RUN set -x \ diff --git a/bindep.txt b/bindep.txt index 7c396ed3..1b684f23 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,47 +1,30 @@ apache2 [platform:dpkg platform:suse apache] -blas-devel [platform:rpm requirements] bridge-utils [(neutron linuxbridge) (nova linuxbridge)] build-essential [platform:dpkg requirements] -bzip2 [platform:rpm requirements] ceph-common [(ceph cinder) (ceph glance) (ceph nova) (ceph manila)] conntrack [platform:dpkg neutron] -conntrack-tools [platform:rpm neutron] -cyrus-sasl-devel [platform:rpm requirements] -device-mapper-multipath [platform:rpm (cinder purestorage) (nova purestorage)] -dhclient [platform:rpm octavia] dnsmasq [neutron] dnsmasq-utils [neutron] ebtables [neutron] -gcc [platform:rpm requirements] -gcc-c++ [platform:rpm requirements] genisoimage [(nova configdrive)] gettext [horizon] git [requirements] haproxy [neutron] -httpd [platform:rpm apache] ipmitool [(ironic ipmi)] iproute2 [platform:dpkg neutron nova] ipset [neutron] iptables [neutron (nova linuxbridge)] iputils-arping [platform:dpkg neutron] ipxe [platform:dpkg (ironic ipxe)] -ipxe-bootimgs [platform:rpm (ironic ipxe)] isc-dhcp-client [platform:dpkg octavia] -iscsi-initiator-utils [platform:rpm (cinder purestorage) (nova purestorage)] keepalived [neutron] -kvm [platform:rpm] -lapack-devel [platform:rpm requirements] libapache2-mod-auth-mellon [platform:dpkg (keystone apache mellon)] libapache2-mod-auth-openidc [platform:dpkg (keystone apache openidc)] libapache2-mod-wsgi-py3 [platform:dpkg apache] libblas-dev [platform:dpkg requirements] liberasurecode-dev [platform:dpkg requirements] -liberasurecode-devel [platform:rpm requirements] libffi-dev [platform:dpkg requirements] -libffi-devel [platform:rpm requirements] -libgcrypt [platform:rpm requirements] libjpeg-dev [platform:dpkg requirements] -libjpeg-devel [platform:rpm requirements] libkrb5-dev [platform:dpkg requirements] liblapack-dev [platform:dpkg requirements] libldap-2.4-2 [platform:dpkg (keystone ldap !jammy !noble)] @@ -51,7 +34,6 @@ libldap2-dev [platform:dpkg requirements] libmariadbclient-dev [platform:debian requirements] libmysqlclient-dev [platform:ubuntu requirements] libnss3-dev [platform:dpkg requirements] -libosinfo [platform:rpm nova] libosinfo-bin [platform:dpkg nova] libpcre3 [platform:ubuntu-noble] libpcre3-dev [platform:dpkg requirements] @@ -59,65 +41,33 @@ libpq-dev [platform:dpkg requirements] # libpythonx.y installation logic is moved to scripts to be able to dynamically # determine minor version from default python installed in a system. librdkafka-dev [platform:dpkg requirements] -libreswan [platform:rpm (neutron vpn)] libsasl2-dev [platform:dpkg requirements] libssl-dev [platform:dpkg requirements] libsystemd-dev [platform:dpkg requirements] libvirt-dev [platform:dpkg requirements] -libvirt-devel [platform:rpm requirements] libvirt0 [platform:dpkg nova] libxml2 [] libxml2-dev [platform:dpkg requirements] -libxml2-devel [platform:rpm requirements] -libxslt-devel [platform:rpm requirements] libxslt1-dev [platform:dpkg requirements] libyaml-dev [platform:dpkg requirements] -libyaml-devel [platform:rpm requirements] libz-dev [platform:dpkg requirements] lvm2 [(cinder lvm) (manila lvm)] -make [platform:rpm requirements] -mariadb-devel [platform:rpm requirements] -mod_auth_mellon [platform:rpm (keystone apache mellon)] -mod_auth_openidc [platform:rpm (keystone apache openidc)] -mod_ssl [platform:rpm apache] -mod_wsgi [platform:rpm apache] multipath-tools [platform:dpkg (cinder purestorage) (nova purestorage)] -mysql [platform:rpm placement] mysql-client [platform:dpkg placement] nginx [nginx] -# latest nss-devel breaks python-nss on centos:9 -nss-devel-3.53.1-17.el8_3 [platform:rpm requirements] -nss-util [platform:rpm requirements] open-iscsi [platform:dpkg (cinder purestorage) (nova purestorage)] -openldap-devel [platform:rpm requirements] openssh-client [platform:dpkg (nova migration)] -openssh-clients [platform:rpm (nova migration)] -openssl-devel [platform:rpm requirements] -openvswitch [platform:rpm (neutron openvswitch) (nova openvswitch)] openvswitch-switch [platform:dpkg (neutron openvswitch) (nova openvswitch)] -pcre-devel [platform:rpm requirements] pkg-config [platform:dpkg requirements] -pkgconfig [platform:rpm requirements] -postgresql-devel [platform:rpm requirements] postgresql-server-devel [platform:suse requirements] -python-mysql [platform:rpm (monasca api)] python3-dev [platform:dpkg requirements] -python3-devel [platform:rpm requirements] python3-mysqldb [platform:dpkg (monasca api)] python3-rados [platform:dpkg (glance ceph) (cinder ceph) (manila ceph) (nova ceph)] python3-rbd [(glance ceph) (cinder ceph) (manila ceph) (nova ceph)] -qemu-img [platform:rpm (nova qemu) (ironic qemu) (cinder qemu) (glance qemu)] qemu-utils [platform:dpkg (nova qemu) (ironic qemu) (cinder qemu) (glance qemu)] sg3-utils [platform:dpkg (cinder purestorage) (nova purestorage)] -sg3_utils [platform:rpm (cinder purestorage) (nova purestorage)] strongswan [platform:dpkg (neutron vpn)] -syslinux-tftpboot [platform:rpm (ironic tftp)] -systemd-devel [platform:rpm requirements] -targetcli [platform:rpm (cinder lvm)] -tftp-server [platform:rpm (ironic tftp)] tftpd-hpa [platform:dpkg (ironic tftp)] tgt [platform:dpkg (cinder lvm)] thin-provisioning-tools [platform:dpkg (cinder lvm)] -uuid [platform:rpm neutron] uuid-runtime [platform:dpkg neutron] -zlib-devel [platform:rpm requirements] diff --git a/dockerfiles/centos/CentOS.repo b/dockerfiles/centos/CentOS.repo deleted file mode 100644 index 69b79ce6..00000000 --- a/dockerfiles/centos/CentOS.repo +++ /dev/null @@ -1,36 +0,0 @@ -[appstream] -name=CentOS Linux $releasever - AppStream -baseurl=http://%%PACKAGE_MIRROR%%/$contentdir/$releasever/AppStream/$basearch/os/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[baseos] -name=CentOS Linux $releasever - BaseOS -baseurl=http://%%PACKAGE_MIRROR%%/$contentdir/$releasever/BaseOS/$basearch/os/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[extras] -name=CentOS Linux $releasever - Extras -baseurl=http://%%PACKAGE_MIRROR%%/$contentdir/$releasever/extras/$basearch/os/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[powertools] -name=CentOS Linux $releasever - PowerTools -baseurl=http://%%PACKAGE_MIRROR%%/$contentdir/$releasever/PowerTools/$basearch/os/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[openstack] -name=CentOS-$releasever - OpenStack -baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/cloud/$basearch/openstack-victoria/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud - -[centos-9-stream-appstream] -includepkgs=nss* -name=CentOS Linux $releasever stream - AppStream -baseurl=http://%%PACKAGE_MIRROR%%/$contentdir/$releasever-stream/AppStream/$basearch/os/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/dockerfiles/centos/Dockerfile b/dockerfiles/centos/Dockerfile deleted file mode 100644 index fb9fea4e..00000000 --- a/dockerfiles/centos/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG FROM=centos:9-stream -FROM ${FROM} - -ARG PACKAGE_MIRROR=mirror.centos.org -ARG PIP_INDEX_URL=https://pypi.python.org/simple/ -ARG PIP_TRUSTED_HOST=pypi.python.org -ENV PIP_INDEX_URL=${PIP_INDEX_URL} -ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST} - -RUN rm -rf /etc/yum.repos.d/* -COPY CentOS.repo /etc/yum.repos.d/ -COPY RPM-GPG-KEY-CentOS-SIG-Cloud /etc/pki/rpm-gpg/ -RUN sed -i "s|%%PACKAGE_MIRROR%%|${PACKAGE_MIRROR}|g" /etc/yum.repos.d/CentOS.repo \ - && yum update -y -# NOTE(aostapenko) repositories are restored on yum update, deleting them again -RUN find /etc/yum.repos.d/ -type f ! -name 'CentOS.repo' -delete diff --git a/dockerfiles/centos/RPM-GPG-KEY-CentOS-SIG-Cloud b/dockerfiles/centos/RPM-GPG-KEY-CentOS-SIG-Cloud deleted file mode 100644 index 91f8e1c5..00000000 --- a/dockerfiles/centos/RPM-GPG-KEY-CentOS-SIG-Cloud +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v2.0.22 (GNU/Linux) - -mQENBFVWcCcBCACfm3eQ0526/I0/p7HpR0NjK7K307XHhnbcbZv1sDUjQABDaqh0 -N4gnZcovf+3fj6pcdOmeOpGI0cKE7Fh68RbEIqyjB7l7+j1grjewR0oCFFZ38KGm -j+DWQrj1IJW7JU5fH/G0Cu66ix+dJPcuTB3PJTqXN3ce+4TuG09D+epgwfbHlqaT -pH2qHCu2uiGj/AaRSM/ZZzcInMaeleHSB+NChvaQ0W/m+kK5d/20d7sfkaTfI/pY -SrodCfVTYxfKAd0TLW03kimHs5/Rdz+iZWecVKv6aFxzaywbrOjmOsy2q0kEWIwX -MTZrq6cBRRuWyiXsI2zT2YHQ4UK44IxINiaJABEBAAG0WkNlbnRPUyBDbG91ZCBT -SUcgKGh0dHA6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lhbEludGVyZXN0R3JvdXAv -Q2xvdWQpIDxzZWN1cml0eUBjZW50b3Mub3JnPokBOQQTAQIAIwUCVVZwJwIbAwcL -CQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEPm5/ud2RCnmATUH/3HDtWxpFkmy -FiA3VGkMt5dp3bgCRSd84X6Orfx1LARowpI4LomCGglGBGXVJePBacwcclorbLaz -uWrW/wU0efz0aDB5c4NPg/yXfNvujvlda8ADJwZXVBQphzvaIKwl4PqBsEnxC10I -93T/0iyphAhfMRJ5R8AbEHMj7uF+TWTX/JoyQagllMqWTwoP4DFRutPdOmmjwvSV -kWItH7hq6z9+M4dhlqeoOvPbL5oCxX7TVmLck02Q5gI4syULOa7sqntzUQKFkhWp -9U0+5KrBQBKezrurrrkq/WZR3WNE1KQfNQ77f7S2JcXJdOaKgJ7xe7Y2flPq98Aq -wKXK7l1c3dc= -=W6yF ------END PGP PUBLIC KEY BLOCK----- diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml index 2ebf045d..27c556f8 100644 --- a/playbooks/vars.yaml +++ b/playbooks/vars.yaml @@ -9,8 +9,8 @@ apt_mirror: "[trusted=yes] https://{{ zuul_site_mirror_fqdn | default('archive.u base_image: context: "." - dockerfile: "dockerfiles/ubuntu/Dockerfile" - container_filename: "dockerfiles/ubuntu/Dockerfile" + dockerfile: "Dockerfile.base" + container_filename: "Dockerfile.base" repository: "base" tags: &tags - "{{ image_tag }}" diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 131a350a..a8a39d49 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -2,32 +2,15 @@ set -ex -case ${distro} in - ubuntu) - apt-get purge -y --auto-remove \ - git \ - patch \ - python3-virtualenv \ - virtualenv - if [ -f /etc/apt/sources.list.bak ]; then - mv /etc/apt/sources.list.bak /etc/apt/sources.list - fi - rm -rf /var/lib/apt/lists/* - ;; - centos) - # We should be removing 'patch' here, but that breaks - # centos as it tries to rip out systemd for some reason - yum -y autoremove \ - git \ - python-virtualenv \ - python3-virtualenv - yum clean all - ;; - *) - echo "Unknown distro: ${distro}" - exit 1 - ;; -esac +apt-get purge -y --auto-remove \ + git \ + patch \ + python3-virtualenv \ + virtualenv +if [ -f /etc/apt/sources.list.bak ]; then + mv /etc/apt/sources.list.bak /etc/apt/sources.list +fi +rm -rf /var/lib/apt/lists/* # Changing this option allows python to use libraries outside of the # virtualenv > 20 if they do not exist inside the venv. This is a requirement diff --git a/scripts/collect_info.sh b/scripts/collect_info.sh index 8e8a239b..ba9c97bf 100755 --- a/scripts/collect_info.sh +++ b/scripts/collect_info.sh @@ -8,18 +8,7 @@ PACKAGES_INFO="${INFO_DIR}/packages.txt" PIP_INFO="${INFO_DIR}/pip.txt" PROJECT_INFO="${INFO_DIR}/project.txt" -case ${distro} in - ubuntu) - dpkg -l > $PACKAGES_INFO - ;; - centos) - yum list installed > $PACKAGES_INFO - ;; - *) - echo "Unknown distro: ${distro}" - exit 1 - ;; -esac +dpkg -l > $PACKAGES_INFO pip freeze > $PIP_INFO cat > ${PROJECT_INFO} < /etc/apt/apt.conf.d/99allow-unauthenticated - mv /etc/apt/sources.list /etc/apt/sources.list.bak - cat > /etc/apt/sources.list < /etc/apt/apt.conf.d/99allow-unauthenticated +mv /etc/apt/sources.list /etc/apt/sources.list.bak +cat > /etc/apt/sources.list <