Add Zuul job to test Loki backend with DevStack support

This patch introduces a new Zuul job that runs CI tests against the Loki backend. It also includes the required changes in the DevStack plugin to enable Loki support, ensuring that the driver can be deployed and verified through the CI pipeline.

Depends-On: https://review.opendev.org/c/openstack/cloudkitty/+/950868
Change-Id: I79026c8543f29fd6702954eb19b882ea213f92ff
Signed-off-by: Marihan Girgis <mgirgisf@redhat.com>

.zuul.yaml

@@ -144,6 +144,17 @@
         CLOUDKITTY_STORAGE_BACKEND: opensearch
         CLOUDKITTY_STORAGE_VERSION: 2
 
+- job:
+    name: cloudkitty-tempest-full-v2-storage-loki
+    parent: base-cloudkitty-v2-api-tempest-job
+    description: |
+      Job testing cloudkitty installation on devstack with python 3 and the
+      Loki v2 storage driver and running tempest tests
+    vars:
+      devstack_localrc:
+        CLOUDKITTY_STORAGE_BACKEND: loki
+        CLOUDKITTY_LOKI_URL: http://127.0.0.1:3100
+
 - job:
     name: cloudkitty-tox-bandit
     parent: openstack-tox
@@ -183,6 +194,7 @@
         - cloudkitty-tempest-full-v2-storage-elasticsearch
         - cloudkitty-tempest-full-v2-storage-opensearch
         - cloudkitty-tempest-full-v1-storage-sqlalchemy
+        - cloudkitty-tempest-full-v2-storage-loki
         - cloudkitty-tempest-full-ipv6-only
         - cloudkitty-tox-bandit:
             voting: false
@@ -194,5 +206,6 @@
         - cloudkitty-tempest-full-v2-storage-elasticsearch
         - cloudkitty-tempest-full-v2-storage-opensearch
         - cloudkitty-tempest-full-v1-storage-sqlalchemy
+        - cloudkitty-tempest-full-v2-storage-loki
         - cloudkitty-tempest-full-ipv6-only
         - cloudkitty-grenade-job

devstack/files/loki-config.yaml

@@ -0,0 +1,29 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 0.0.0.0
+  path_prefix: /opt/stack/data/loki
+  storage:
+    filesystem:
+      chunks_directory: /opt/stack/data/loki/chunks
+      rules_directory: /opt/stack/data/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  max_query_length: 745h

devstack/plugin.sh

@@ -167,6 +167,10 @@ function configure_cloudkitty {
         iniset $CLOUDKITTY_CONF storage_${CLOUDKITTY_STORAGE_BACKEND} index_name ${CLOUDKITTY_OPENSEARCH_INDEX}
     fi
 
+    if [ "$CLOUDKITTY_STORAGE_BACKEND" == "loki" ]; then
+        iniset $CLOUDKITTY_CONF storage_loki url ${CLOUDKITTY_LOKI_URL}
+    fi
+
     # collect
     iniset $CLOUDKITTY_CONF collect collector $CLOUDKITTY_COLLECTOR
     iniset $CLOUDKITTY_CONF "collector_${CLOUDKITTY_COLLECTOR}" auth_section authinfos
@@ -366,6 +370,53 @@ function install_opensearch {
     sudo systemctl start opensearch || sudo systemctl restart opensearch
 }
 
+function start_loki {
+    LOKI_SYSTEMD_SERVICE="devstack@loki.service"
+    loki_command="$CLOUDKITTY_BIN_DIR/loki"
+    loki_command+=" --config.file=${CLOUDKITTY_DIR}/devstack/files/loki-config.yaml"
+
+    write_user_unit_file $LOKI_SYSTEMD_SERVICE "$loki_command" "" "$STACK_USER"
+
+    enable_service $LOKI_SYSTEMD_SERVICE
+    start_service $LOKI_SYSTEMD_SERVICE
+}
+
+function install_loki_ubuntu {
+    local loki_url="https://github.com/grafana/loki/releases/download/v3.5.4/loki-linux-amd64.zip"
+    local loki_tmp="/tmp/loki-linux-amd64.zip"
+
+    sudo apt-get install -y unzip wget
+
+    wget -O ${loki_tmp} ${loki_url}
+    unzip -o ${loki_tmp} -d /tmp
+    sudo mv /tmp/loki-linux-amd64 $CLOUDKITTY_BIN_DIR/loki
+    sudo chmod +x $CLOUDKITTY_BIN_DIR/loki
+}
+
+function install_loki_fedora {
+    local loki_url="https://github.com/grafana/loki/releases/download/v3.5.4/loki-linux-amd64.zip"
+    local loki_tmp="/tmp/loki-linux-amd64.zip"
+
+    sudo dnf install -y unzip wget
+
+    wget -O ${loki_tmp} ${loki_url}
+    unzip -o ${loki_tmp} -d /tmp
+    sudo mv /tmp/loki-linux-amd64 $CLOUDKITTY_BIN_DIR/loki
+    sudo chmod +x $CLOUDKITTY_BIN_DIR/loki
+}
+
+function install_loki {
+    if is_ubuntu; then
+        install_loki_ubuntu
+    elif is_fedora; then
+        install_loki_fedora
+    else
+        die $LINENO "Distribution must be Debian or Fedora-based"
+    fi
+    # Start Loki service
+    start_loki
+}
+
 # install_cloudkitty() - Collect source and prepare
 function install_cloudkitty {
     git_clone $CLOUDKITTY_REPO $CLOUDKITTY_DIR $CLOUDKITTY_BRANCH
@@ -378,6 +429,8 @@ function install_cloudkitty {
         install_elasticsearch
     elif [ $CLOUDKITTY_STORAGE_BACKEND == 'opensearch' ]; then
         install_opensearch
+    elif [ $CLOUDKITTY_STORAGE_BACKEND == "loki" ]; then
+        install_loki
     fi
     if [ ${CLOUDKITTY_USE_UWSGI,,} == 'true' ]; then
         pip_install uwsgi

devstack/settings

@@ -84,3 +84,6 @@ CLOUDKITTY_ELASTICSEARCH_INDEX=${CLOUDKITTY_ELASTICSEARCH_INDEX:-"cloudkitty"}
 # Set opensearch info
 CLOUDKITTY_OPENSEARCH_HOST=${CLOUDKITTY_OPENSEARCH_HOST:-"http://localhost:9200"}
 CLOUDKITTY_OPENSEARCH_INDEX=${CLOUDKITTY_OPENSEARCH_INDEX:-"cloudkitty"}
+
+# Set loki info
+CLOUDKITTY_LOKI_URL=${CLOUDKITTY_LOKI_URL:-"http://localhost:3100"}