diff --git a/minikube-aio/Dockerfile b/minikube-aio/Dockerfile new file mode 100644 index 00000000..a6abf799 --- /dev/null +++ b/minikube-aio/Dockerfile @@ -0,0 +1,82 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM docker.io/ubuntu:bionic +MAINTAINER andrii.ostapenko@att.com + +ARG UBUNTU_URL=http://archive.ubuntu.com/ubuntu/ +ARG ALLOW_UNAUTHENTICATED=false + +COPY ./sources.list /etc/apt/ +RUN sed -i \ + -e "s|%%UBUNTU_URL%%|${UBUNTU_URL}|g" \ + /etc/apt/sources.list ;\ + echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";\n\ +Acquire::AllowInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";\n\ +Acquire::AllowDowngradeToInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";" \ + >> /etc/apt/apt.conf.d/allow-unathenticated + +ARG KUBE_VERSION="v1.16.2" +ARG MINIKUBE_VERSION="v1.3.1" +ARG CALICO_VERSION="v3.9" +ARG HELM_VERSION="v2.14.1" + +COPY images.txt install.sh /opt/ + +RUN set -ex; \ + apt-get update; \ + apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + ;\ + echo "export KUBE_VERSION=${KUBE_VERSION}" > /opt/versions.txt ;\ + echo "export MINIKUBE_VERSION=${MINIKUBE_VERSION}" >> /opt/versions.txt ;\ + echo "export CALICO_VERSION=${CALICO_VERSION}" >> /opt/versions.txt ;\ + echo "export HELM_VERSION=${HELM_VERSION}" >> /opt/versions.txt ;\ + GOOGLE_REPO_URL=https://storage.googleapis.com ;\ + MINIKUBE_REPO_URL=${GOOGLE_REPO_URL}/minikube/releases/${MINIKUBE_VERSION} ;\ + GOOGLE_KUBERNETES_REPO_URL=${GOOGLE_REPO_URL}/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64 ;\ + GOOGLE_HELM_REPO_URL=${GOOGLE_REPO_URL}/kubernetes-helm ;\ + CNI_REPO_URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION} ;\ + BINARY_DIR=/opt/binaries ;\ + mkdir ${BINARY_DIR}; \ + curl -sSLo ${BINARY_DIR}/minikube ${MINIKUBE_REPO_URL}/minikube-linux-amd64 ;\ + for BINARY in kubectl kubeadm kubelet; do \ + curl -sSLo ${BINARY_DIR}/${BINARY} ${GOOGLE_KUBERNETES_REPO_URL}/${BINARY} ;\ + done ;\ + TMP_DIR=$(mktemp -d) ;\ + curl -sSL ${GOOGLE_HELM_REPO_URL}/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} ;\ + mv ${TMP_DIR}/helm ${BINARY_DIR} ;\ + rm -rf ${TMP_DIR} ;\ + chmod +x ${BINARY_DIR}/* ;\ + curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o /opt/calico.yaml; \ + for CALICO_IMAGE in $(grep -oP '(?<=image:)\s*calico/.+$' /opt/calico.yaml); do \ + echo ${CALICO_IMAGE} >> /opt/images.txt ;\ + done ;\ + printf "kubernetesVersion: ${KUBE_VERSION}\napiVersion: kubeadm.k8s.io/v1beta1\nkind: ClusterConfiguration" >> /tmp/kubeadm.yaml ;\ + $BINARY_DIR/kubeadm config images list --config /tmp/kubeadm.yaml | tee -a /opt/images.txt ;\ + . /etc/os-release ;\ + echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/x${NAME}_${VERSION_ID}/ /" > \ + /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list ;\ + curl -sSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key | apt-key add - ;\ + apt-get update ;\ + apt-get install -y skopeo gettext ;\ + mkdir /opt/images ;\ + echo "$(envsubst < /opt/images.txt | sort | uniq)" > /opt/images.txt ;\ + for IMAGE in $(cat /opt/images.txt); do \ + path=/opt/images/$(echo ${IMAGE} | sed 's#[/:]#_#g').tar ;\ + skopeo copy docker://${IMAGE} docker-archive:${path}:${IMAGE} ;\ + done + +FROM scratch +COPY --from=0 /opt / diff --git a/minikube-aio/images.txt b/minikube-aio/images.txt new file mode 100644 index 00000000..822c551d --- /dev/null +++ b/minikube-aio/images.txt @@ -0,0 +1,3 @@ +gcr.io/k8s-minikube/storage-provisioner:v1.8.1 +gcr.io/kubernetes-helm/tiller:${HELM_VERSION} +k8s.gcr.io/kube-addon-manager:v9.0 diff --git a/minikube-aio/install.sh b/minikube-aio/install.sh new file mode 100755 index 00000000..5ca1c162 --- /dev/null +++ b/minikube-aio/install.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +SCRIPT_DIR=$(dirname $0) + +function extract { + cd "${SCRIPT_DIR}" + source versions.txt + MINIKUBE_CACHE_DIR=${HOME}/.minikube/cache/${KUBE_VERSION}/ + mkdir -p "${MINIKUBE_CACHE_DIR}" + chmod +x binaries/* + for binary in minikube kubectl helm; do + sudo mv binaries/${binary} /usr/local/bin/${binary} + done + for binary in kubeadm kubelet; do + mv binaries/${binary} "${MINIKUBE_CACHE_DIR}" + done + for image in images/*; do + sudo docker load < ${image} + done + cp calico.yaml /tmp/ + sudo docker images | sort | uniq | tee /tmp/loaded_images + cd - +} + +function configure_resolvconf { + # Setup resolv.conf to use the k8s api server, which is required for the + # kubelet to resolve cluster services. + sudo mv /etc/resolv.conf /etc/resolv.conf.backup + + # Create symbolic link to the resolv.conf file managed by systemd-resolved, as + # the kubelet.resolv-conf extra-config flag is automatically executed by the + # minikube start command, regardless of being passed in here + sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf + + sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf" + + # NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in + # the resolv.conf file unless using a proxy, then use the existing DNS servers, + # as custom DNS nameservers are commonly required when using a proxy server. + if [ -z "${HTTP_PROXY}" ]; then + sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf" + sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf" + else + sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do + sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf" + done + fi + + sudo bash -c "echo 'search svc.cluster.local cluster.local' >> /etc/resolv.conf" + sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf" + + sudo rm /etc/resolv.conf.backup +} + +# NOTE: Clean Up hosts file +sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts +sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts + +extract +configure_resolvconf + +# Prepare tmpfs for etcd +sudo mkdir -p /data +sudo mount -t tmpfs -o size=512m tmpfs /data + +# NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is +# required for validation; use calico for simplicity. +sudo -E minikube config set kubernetes-version "${KUBE_VERSION}" +sudo -E minikube config set vm-driver none +sudo -E minikube config set embed-certs true + +# NOTE(aostapenko) Minikube still tries to pull images with kubeadm config imagepull +# https://github.com/kubernetes/minikube/blob/v1.3.1/pkg/minikube/bootstrapper/kubeadm/kubeadm.go#L417 +# so we make it to fail fast and continue with existing images saving precious time +sudo sed -i 's/127.0.0.1.*/\0 k8s.gcr.io/g' /etc/hosts + +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_IN_STYLE=false +sudo -E minikube start \ + --docker-env HTTP_PROXY="${HTTP_PROXY}" \ + --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \ + --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \ + --network-plugin=cni \ + --extra-config=controller-manager.allocate-node-cidrs=true \ + --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 + +sudo sed -i 's/k8s.gcr.io//g' /etc/hosts + +kubectl apply -f /tmp/calico.yaml + +# Note: Patch calico daemonset to enable Prometheus metrics and annotations +tee /tmp/calico-node.yaml << EOF +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9091" + spec: + containers: + - name: calico-node + env: + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "true" + - name: FELIX_PROMETHEUSMETRICSPORT + value: "9091" +EOF +kubectl patch daemonset calico-node -n kube-system --patch "$(cat /tmp/calico-node.yaml)" + +# NOTE: Wait for dns to be running. +END=$(($(date +%s) + 240)) +until kubectl --namespace=kube-system \ + get pods -l k8s-app=kube-dns --no-headers -o name | grep -q "^pod/coredns"; do + NOW=$(date +%s) + [ "${NOW}" -gt "${END}" ] && exit 1 + echo "still waiting for dns" + sleep 10 +done +kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns + +# Deploy helm/tiller into the cluster +kubectl create -n kube-system serviceaccount helm-tiller +cat <&1) +Restart=always +ExecStart=/usr/local/bin/helm serve + +[Install] +WantedBy=multi-user.target +EOF + +sudo chmod 0640 /etc/systemd/system/helm-serve.service + +sudo systemctl daemon-reload +sudo systemctl restart helm-serve +sudo systemctl enable helm-serve + +# Remove stable repo, if present, to improve build time +helm repo remove stable || true + +# Set up local helm repo +helm repo add local http://localhost:8879/charts +helm repo update + +# Set required labels on host(s) +kubectl label nodes --all openstack-control-plane=enabled +kubectl label nodes --all openstack-compute-node=enabled +kubectl label nodes --all openvswitch=enabled +kubectl label nodes --all linuxbridge=enabled +kubectl label nodes --all ceph-mon=enabled +kubectl label nodes --all ceph-osd=enabled +kubectl label nodes --all ceph-mds=enabled +kubectl label nodes --all ceph-rgw=enabled +kubectl label nodes --all ceph-mgr=enabled + +# Add labels to the core namespaces +kubectl label --overwrite namespace default name=default +kubectl label --overwrite namespace kube-system name=kube-system +kubectl label --overwrite namespace kube-public name=kube-public +sudo docker images | sort | uniq | tee /tmp/images_after_installation diff --git a/minikube-aio/sources.list b/minikube-aio/sources.list new file mode 100644 index 00000000..ee1f9966 --- /dev/null +++ b/minikube-aio/sources.list @@ -0,0 +1,4 @@ +deb %%UBUNTU_URL%% bionic main universe +deb %%UBUNTU_URL%% bionic-updates main universe +deb %%UBUNTU_URL%% bionic-backports main universe +deb %%UBUNTU_URL%% bionic-security main universe diff --git a/zuul.d/minikube-aio.yaml b/zuul.d/minikube-aio.yaml new file mode 100644 index 00000000..d77f044b --- /dev/null +++ b/zuul.d/minikube-aio.yaml @@ -0,0 +1,58 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + check: + jobs: + - openstack-helm-images-build-minikube-aio + gate: + jobs: + - openstack-helm-images-upload-minikube-aio + promote: + jobs: + - openstack-helm-images-promote-minikube-aio + periodic: + jobs: + - openstack-helm-images-build-minikube-aio + +- job: + name: openstack-helm-images-build-minikube-aio + parent: openstack-helm-images-build + provides: common-images + description: Build minikube-aio image for OSH gates + vars: &minikube_aio_vars + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" + docker_images: + - context: minikube-aio + repository: openstackhelm/minikube-aio + tags: + - latest-ubuntu_bionic + - "ubuntu_bionic-{{ currentdate }}" + files: &minikube_aio_files + - minikube-aio/.* + - zuul.d/minikube-aio.yaml + +- job: + name: openstack-helm-images-upload-minikube-aio + parent: openstack-helm-images-upload + provides: common-images + description: Build and upload minikube-aio image + vars: *minikube_aio_vars + files: *minikube_aio_files + +- job: + name: openstack-helm-images-promote-minikube-aio + parent: openstack-helm-images-promote + description: Promote previously built minikube-aio image + vars: *minikube_aio_vars + files: *minikube_aio_files