From 42545f1c3c7b41ee4db8b55024cfa39a50eb06ab Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 6 Apr 2017 12:24:07 -0500 Subject: [PATCH] Use machinectl to manage image caches This change modifies the LXC image cache system to use machine control, which is part of systemd, to manage images for us. This will give us insight into the cached images which we had not had before all through the `machinectl` cli utility. This change also modifies the image fetch process allowing it to be faster and more transparent to the enduser. Part of the slowness in image fetching and caching is that it happens on every run even if it's not needed. This change will now check the cache expiry and state of the image within `machinectl` and only run the cache update when needed or instructed to do so. Documentation on what can be done with the `machinectl` CLI utility can be found here: * https://www.freedesktop.org/software/systemd/man/machinectl.html Change-Id: Ic7f8bf400ec5781b4be67539bc6c1523069d0ab2 Signed-off-by: Kevin Carter --- defaults/main.yml | 19 +- files/org.freedesktop.machine1.conf | 194 ++++++++++++++++++++ files/systemd-machined.service | 23 +++ files/var-lib-machines.mount | 16 ++ tasks/lxc_cache.yml | 71 +++---- tasks/lxc_cache_create.yml | 41 ++++- tasks/lxc_cache_preparation.yml | 56 +++--- tasks/lxc_cache_preparation_systemd_new.yml | 43 +++++ tasks/lxc_cache_preparation_systemd_old.yml | 141 ++++++++++++++ tasks/main.yml | 12 -- vars/redhat-7.yml | 3 + vars/ubuntu-16.04.yml | 4 + 12 files changed, 532 insertions(+), 91 deletions(-) create mode 100644 files/org.freedesktop.machine1.conf create mode 100644 files/systemd-machined.service create mode 100644 files/var-lib-machines.mount create mode 100644 tasks/lxc_cache_preparation_systemd_new.yml create mode 100644 tasks/lxc_cache_preparation_systemd_old.yml diff --git a/defaults/main.yml b/defaults/main.yml index 343845d8..1e0c6c5e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,10 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Timeout (in seconds) for the LXC cache to control how often the rootfs -# cache will be updated. By default this timeout is set to 1 day. -lxc_cache_timeout: 86400 - # Environment to use when building LXC caches # You can set http_proxy environment variables here for example # if you would like to proxy all image downloads @@ -115,7 +111,20 @@ lxc_cache_prep_pre_commands: '## pre command skipped ##' lxc_cache_prep_post_commands: '## post command skipped ##' # The DNS name of the LXD server to source the base container cache from -lxc_image_cache_server: images.linuxcontainers.org +lxc_image_cache_server: https://images.linuxcontainers.org + +# Path to images on server +lxc_image_cache_server_path: "{{ lxc_image_cache_server }}/images/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_map.variant }}/" + +# Local path to cached image +lxc_image_cache_path: "/var/lib/machines/{{ lxc_container_base_name }}" + +# Mode to pull image. This is used to pull the image from a remote source. +# Valid options are [pull-tar, pull-raw] +lxc_image_cache_pull_mode: pull-tar + +# Set this option to true to pull a new cached image. +lxc_image_cache_refresh: false # The keyservers to use when validating GPG keys for the downloaded cache lxc_image_cache_primary_keyserver: hkp://p80.pool.sks-keyservers.net:80 diff --git a/files/org.freedesktop.machine1.conf b/files/org.freedesktop.machine1.conf new file mode 100644 index 00000000..ed2d1cb8 --- /dev/null +++ b/files/org.freedesktop.machine1.conf @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/files/systemd-machined.service b/files/systemd-machined.service new file mode 100644 index 00000000..5477de85 --- /dev/null +++ b/files/systemd-machined.service @@ -0,0 +1,23 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Virtual Machine and Container Registration Service +Documentation=man:systemd-machined.service(8) +Documentation=http://www.freedesktop.org/wiki/Software/systemd/machined +Wants=machine.slice +After=machine.slice + +[Service] +ExecStart=/lib/systemd/systemd-machined +BusName=org.freedesktop.machine1 +CapabilityBoundingSet=CAP_KILL CAP_SYS_PTRACE CAP_SYS_ADMIN CAP_SETGID CAP_SYS_CHROOT CAP_DAC_READ_SEARCH CAP_DAC_OVERRIDE CAP_CHOWN CAP_FOWNER CAP_FSETID +WatchdogSec=3min + +# Note that machined cannot be placed in a mount namespace, since it +# needs access to the host's mount namespace in order to implement the +# "machinectl bind" operation. diff --git a/files/var-lib-machines.mount b/files/var-lib-machines.mount new file mode 100644 index 00000000..0d72c163 --- /dev/null +++ b/files/var-lib-machines.mount @@ -0,0 +1,16 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Virtual Machine and Container Storage +ConditionPathExists=/var/lib/machines.raw + +[Mount] +What=/var/lib/machines.raw +Where=/var/lib/machines +Type=btrfs +Options=loop \ No newline at end of file diff --git a/tasks/lxc_cache.yml b/tasks/lxc_cache.yml index dfc883df..b35c4d86 100644 --- a/tasks/lxc_cache.yml +++ b/tasks/lxc_cache.yml @@ -13,46 +13,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -- block: - - name: Create base container - lxc_container: - name: "LXC_NAME" - template: "download" - state: stopped - backing_store: "dir" - template_options: "{{ lxc_cache_download_template_options }} --keyserver {{ lxc_image_cache_primary_keyserver }}" - register: cache_download_primary - retries: 3 - delay: 10 - until: cache_download_primary | success - environment: "{{ lxc_cache_environment }}" - tags: - - lxc-cache - - lxc-cache-download - - lxc_hosts-install +- name: Set LXC cache path fact + set_fact: + cache_path_fact: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}" + cache_index_item: "{{ lxc_cache_map.distro }};{{ lxc_cache_map.release }};{{ lxc_cache_map.arch }};{{ lxc_cache_map.variant }}" + cache_time: "{{ ansible_date_time.epoch }}" + tags: + - always - rescue: - - name: Create base container - lxc_container: - name: "LXC_NAME" - template: "download" - state: stopped - backing_store: "dir" - template_options: "{{ lxc_cache_download_template_options }} --keyserver {{ lxc_image_cache_secondary_keyserver }}" - register: cache_download_secondary - retries: 3 - delay: 10 - until: cache_download_secondary | success - environment: "{{ lxc_cache_environment }}" - tags: - - lxc-cache - - lxc-cache-download - - lxc_hosts-install +- name : Check cached image status + command: "machinectl image-status {{ lxc_container_base_name }}" + register: cache_check + changed_when: false + failed_when: false + tags: + - always + +- name: Retrieve the expiry object + slurp: + src: "{{ cache_path_fact }}/expiry" + failed_when: false + register: expiry + when: + - not lxc_image_cache_refresh | bool + tags: + - always + +- name: Set cache refresh fact + set_fact: + lxc_image_cache_refresh: true + when: > + (cache_check.rc != 0) or + cache_time >= (expiry.content|default('MQo=') | b64decode) + tags: + - always - include: lxc_cache_preparation.yml + when: + - lxc_image_cache_refresh | bool tags: - lxc_hosts-config - include: lxc_cache_create.yml + when: + - lxc_image_cache_refresh | bool tags: - lxc_hosts-config diff --git a/tasks/lxc_cache_create.yml b/tasks/lxc_cache_create.yml index 6f6980e7..70fb0cd1 100644 --- a/tasks/lxc_cache_create.yml +++ b/tasks/lxc_cache_create.yml @@ -13,26 +13,51 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Create LXC cache dir + file: + path: "{{ cache_path_fact }}" + state: "directory" + recurse: true + - name: Remove existing cache archive file: - path: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}/rootfs.tar.xz" + path: "{{ cache_path_fact }}/rootfs.tar.xz" state: "absent" - tags: - - lxc-cache - - lxc-image-cache-create # This is using a shell command because the ansible archive module does not # provide for the options needed to properly create an LXC image archive. - name: Create lxc image shell: | - tar -Opc -C /var/lib/lxc/LXC_NAME/rootfs . | {{ lxc_xz_bin }} -{{ lxc_image_compression_ratio }} -c - > rootfs.tar.xz + tar -Opc -C {{ lxc_image_cache_path }} . | {{ lxc_xz_bin }} -{{ lxc_image_compression_ratio }} -c - > rootfs.tar.xz args: - chdir: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}/" + chdir: "{{ cache_path_fact }}/" notify: Destroy base container tags: - skip_ansible_lint - - lxc-cache - - lxc-image-cache-create + +- name: Update LXC image meta data + get_url: + url: "{{ lxc_image_cache_server }}/{{ item.split(';')[-1] }}/meta.tar.xz" + dest: "/tmp/meta.tar.xz" + with_items: "{{ lxc_images }}" + when: + - item | match("^{{ cache_index_item }}") + +- name: Place container metadata + unarchive: + src: "/tmp/meta.tar.xz" + dest: "{{ cache_path_fact }}" + remote_src: True + +- name: Set cache expiry + shell: "date -d @{{ (cache_time | int) + 31536000 }} > {{ cache_path_fact }}/expiry" + tags: + - skip_ansible_lint + +- name: Set build ID + shell: "echo {{ cache_time }} > {{ cache_path_fact }}/build_id" + tags: + - skip_ansible_lint - name: Create base container to use for overlayfs containers lxc_container: diff --git a/tasks/lxc_cache_preparation.yml b/tasks/lxc_cache_preparation.yml index 64d4f063..eaa0118d 100644 --- a/tasks/lxc_cache_preparation.yml +++ b/tasks/lxc_cache_preparation.yml @@ -13,40 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Pull SystemD Version + command: "systemctl --version" + changed_when: false + register: systemd_version + +- name: Retrieve Image Index + uri: + url: "{{ lxc_image_cache_server }}/meta/1.0/index-system" + return_content: yes + register: image_index + +- name: Set image index fact + set_fact: + lxc_images: "{{ image_index.content.splitlines() }}" + +- include: "lxc_cache_preparation_systemd_{{ (systemd_version.stdout_lines[0].split()[-1] | int > 219) | ternary('new', 'old') }}.yml" + - name: Generate apt keys from LXC host for the container cache shell: apt-key exportall > /root/repo.keys changed_when: False when: - ansible_pkg_mgr == 'apt' - tags: - - lxc-cache - - lxc-cache-apt-keys # TODO(evrardjp): replace this with a copy with remote_src: True # when ansible2.0 will be supported - name: Rsyncing files from the LXC host to the container cache shell: | if [[ -e "{{ item }}" ]]; then - rsync -av "{{ item }}" "/var/lib/lxc/LXC_NAME/rootfs{{ item }}" + rsync -av "{{ item }}" "{{ lxc_image_cache_path }}{{ item }}" fi args: executable: "/bin/bash" with_items: "{{ lxc_cache_map.copy_from_host }}" tags: - - lxc-cache - skip_ansible_lint - name: Copy files from deployment host to the container cache copy: src: "{{ item.src }}" - dest: "/var/lib/lxc/LXC_NAME/rootfs{{ item.dest | default(item.src) }}" + dest: "{{ lxc_image_cache_path }}{{ item.dest | default(item.src) }}" owner: "{{ item.owner | default('root') }}" group: "{{ item.group | default('root') }}" mode: "{{ item.mode | default('0644') }}" with_items: "{{ lxc_container_cache_files }}" - tags: - - lxc-cache - - lxc-cache-copy-files - name: Cached image preparation script copy: @@ -54,48 +64,33 @@ #!/usr/bin/env bash set -e -x {{ lxc_cache_map.cache_prep_commands }} - dest: "/var/lib/lxc/LXC_NAME/rootfs/usr/local/bin/cache-prep-commands.sh" + dest: "{{ lxc_image_cache_path }}/usr/local/bin/cache-prep-commands.sh" mode: "0755" - tags: - - lxc-cache - - lxc-cache-update # This task runs several commands against the cached image to speed up the # lxc_container_create playbook. - name: Prepare cached image setup commands - command: "chroot /var/lib/lxc/LXC_NAME/rootfs /usr/local/bin/cache-prep-commands.sh" + command: "chroot {{ lxc_image_cache_path }} /usr/local/bin/cache-prep-commands.sh" changed_when: false - tags: - - lxc-cache - - lxc-cache-update - name: Adjust sshd configuration in container lineinfile: - dest: "/var/lib/lxc/LXC_NAME/rootfs/etc/ssh/sshd_config" + dest: "{{ lxc_image_cache_path }}/etc/ssh/sshd_config" regexp: "{{ item.regexp }}" line: "{{ item.line }}" state: present with_items: "{{ lxc_cache_sshd_configuration }}" - tags: - - lxc-cache - - lxc-cache-update - name: Obtain the deploy system's ssh public key set_fact: lxc_container_ssh_key: "{{ lookup('file', '/root/.ssh/id_rsa.pub') }}" when: lxc_container_ssh_key is not defined - tags: - - lxc-cache - - lxc-cache-update - name: Deploy ssh public key into the cached image lineinfile: - dest: "/var/lib/lxc/LXC_NAME/rootfs/root/.ssh/authorized_keys" + dest: "{{ lxc_image_cache_path }}/.ssh/authorized_keys" line: "{{ lxc_container_ssh_key }}" create: true - tags: - - lxc-cache - - lxc-cache-update - name: Remove generated apt keys from LXC host file: @@ -104,13 +99,10 @@ when: - ansible_pkg_mgr == 'apt' changed_when: False - tags: - - lxc-cache - - lxc-cache-apt-keys - name: Remove requiretty for sudo on centos template: - dest: "/var/lib/lxc/LXC_NAME/rootfs/etc/sudoers.d/openstack-ansible" + dest: "{{ lxc_image_cache_path }}/etc/sudoers.d/openstack-ansible" owner: root group: root mode: "0440" diff --git a/tasks/lxc_cache_preparation_systemd_new.yml b/tasks/lxc_cache_preparation_systemd_new.yml new file mode 100644 index 00000000..cddfc572 --- /dev/null +++ b/tasks/lxc_cache_preparation_systemd_new.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(cloudnull): When modern SystemD is running everywhere this can be +# collapsed back into the base preparation task file. +- name : Remove old image cache + command: "machinectl remove {{ lxc_container_base_name }}" + register: cache_refresh + changed_when: cache_refresh.rc == 0 + failed_when: cache_refresh.rc not in [0, 1] + when: + - lxc_image_cache_refresh | bool + +- name: Retrieve base image + command: >- + machinectl + --verify=no + {{ lxc_image_cache_pull_mode }} + {{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz + {{ lxc_container_base_name }} + register: pull_image + until: pull_image | success + retries: 3 + delay: 1 + changed_when: pull_image.rc == 0 + failed_when: + - pull_image.rc != 0 + - "'failed' in pull_image.stderr | lower" + with_items: "{{ lxc_images }}" + when: + - item | match("^{{ cache_index_item }}") diff --git a/tasks/lxc_cache_preparation_systemd_old.yml b/tasks/lxc_cache_preparation_systemd_old.yml new file mode 100644 index 00000000..e61159d7 --- /dev/null +++ b/tasks/lxc_cache_preparation_systemd_old.yml @@ -0,0 +1,141 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# NOTE(cloudnull): This is only used when running SystemD <= 219 +# ============================================================== +# In later versions of SystemD this is automatically done for us +# by the machinectl cli on first run. +- name: Create sparse machines file + command: "truncate -s 11G /var/lib/machines.raw" + args: + creates: /var/lib/machines.raw + register: machines_create + +# In later versions of SystemD this is automatically done for us +# by the machinectl cli on first run. +- name: Format the machines file + filesystem: + fstype: btrfs + dev: /var/lib/machines.raw + when: + - machines_create | changed + +# In later versions of SystemD this is automatically done for us +# by the machinectl cli on first run. +- name: Create machines mount point + file: + path: "/var/lib/machines" + state: "directory" + recurse: true + +# In later versions of SystemD this unit file has been corrected +# and is packaged with systemd proper. +- name: Move machines mount into place + copy: + src: var-lib-machines.mount + dest: /lib/systemd/system/var-lib-machines.mount + register: mount_unit + when: + - machines_create | changed + +# In later versions of SystemD this is not needed. Referenced in +# the following ML post resolves the bug. +# * https://lists.freedesktop.org/archives/systemd-devel/2015-March/029151.html +- name: Move machined service into place + copy: + src: systemd-machined.service + dest: /lib/systemd/system/systemd-machined.service + register: machined_unit + when: + - machines_create | changed + +# In later versions of SystemD this is not needed. Referenced in +# the following commit resolves the bug. +# * https://cgit.freedesktop.org/systemd/systemd/commit/src/machine/org.freedesktop.machine1.conf?id=72c3897f77a7352618ea76b880a6764f52d6327b +- name: Move machine1 dbus config into place + copy: + src: org.freedesktop.machine1.conf + dest: /etc/dbus-1/system.d/org.freedesktop.machine1.conf + register: machine1_conf + when: + - machines_create | changed + +- name: Reload the System daemon + command: "systemctl daemon-reload" + when: > + mount_unit | changed or + machined_unit | changed or + machine1_conf | changed + +- name: Restart dbus + command: "systemctl reload dbus.service" + when: + - machine1_conf | changed + +- name: Mount all + shell: "mount | grep '/var/lib/machines' || (systemctl start var-lib-machines.mount && exit 3)" + register: mount_machines + changed_when: mount_machines.rc == 3 + failed_when: mount_machines.rc not in [0, 3] + tags: + - skip_ansible_lint + +- name: Restart machined + command: "systemctl restart systemd-machined.service" + when: + - machined_unit | changed + +# Because of this post and it's related bug(s) this is adding the container +# volumes the old way. The new way would simply be calling `machinectl`. +# * https://www.mail-archive.com/systemd-devel@lists.freedesktop.org/msg28255.html +- name : Remove old image cache + command: "btrfs subvolume delete /var/lib/machines/{{ lxc_container_base_name }}" + register: cache_refresh_del + changed_when: cache_refresh_del.rc == 0 + failed_when: cache_refresh_del.rc not in [0, 1] + when: + - lxc_image_cache_refresh | bool + +- name : Add image cache + command: "btrfs subvolume create /var/lib/machines/{{ lxc_container_base_name }}" + register: cache_refresh_add + changed_when: cache_refresh_add.rc == 0 + failed_when: cache_refresh_add.rc not in [0, 1] + when: + - lxc_image_cache_refresh | bool + +- name: Retrieve base image + get_url: + url: "{{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz" + dest: "/tmp/rootfs.tar.xz" + register: pull_image + until: pull_image | success + retries: 3 + delay: 1 + with_items: "{{ lxc_images }}" + when: + - item | match("^{{ cache_index_item }}") + +- name: Place container rootfs + unarchive: + src: "/tmp/rootfs.tar.xz" + dest: "/var/lib/machines/{{ lxc_container_base_name }}" + remote_src: True + +- name: Remove rootfs archive + file: + path: "/tmp/rootfs.tar.xz" + state: "absent" diff --git a/tasks/main.yml b/tasks/main.yml index ad0d8e6f..b119f86b 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -66,19 +66,7 @@ tags: - lxc_hosts-config -- name: Stat the prepared LXC cache - stat: - path: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}/rootfs.tar.xz" - register: lxc_cache_stat - tags: - - always - - include: lxc_cache.yml - static: no - when: - - "{{ not lxc_cache_stat.stat.exists - or (lxc_cache_stat.stat.exists - and lxc_cache_stat.stat.mtime > lxc_cache_timeout) }}" tags: - lxc_hosts-install - lxc_hosts-config diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index d66202fd..3d787e9a 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -18,6 +18,8 @@ system_config_dir: "/etc/sysconfig" # Required rpm packages. lxc_hosts_distro_packages: - bridge-utils + - btrfs-progs + - dbus - debootstrap - dnsmasq - git @@ -38,6 +40,7 @@ lxc_cache_map: distro: centos arch: amd64 release: 7 + variant: default copy_from_host: - /etc/yum.repos.d/ - /etc/yum/pluginconf.d/fastestmirror.conf diff --git a/vars/ubuntu-16.04.yml b/vars/ubuntu-16.04.yml index 07fbd593..d8809d60 100644 --- a/vars/ubuntu-16.04.yml +++ b/vars/ubuntu-16.04.yml @@ -21,8 +21,10 @@ lxc_hosts_distro_packages: - apparmor - apparmor-utils - bridge-utils + - btrfs-tools - cgmanager - cgroup-lite + - dbus - debootstrap - dnsmasq - git @@ -35,6 +37,7 @@ lxc_hosts_distro_packages: - lxc-templates - python-dev - python3-lxc + - systemd-container - pxz lxc_xz_bin: pxz @@ -44,6 +47,7 @@ lxc_cache_map: distro: ubuntu arch: "{{ lxc_architecture_mapping.get( ansible_architecture ) }}" release: xenial + variant: default copy_from_host: - /etc/apt/sources.list - /etc/apt/apt.conf.d/