openstack/storlets
Code Issues Proposed changes

Rename STORLETS_CONTAINER_DEVICE

Browse Source 
This does not represent actual device, but the directory where relevant
data is stored.

Also rename the root directory in the container to avoid the term
"swift" within the container.

Change-Id: I128ac692ae33944ab95e251cc4cd545a3b8e6cd7
Signed-off-by: Takashi Kajinami <kajinamit@oss.nttdata.com>
This commit is contained in:
Takashi Kajinami
2025-08-13 00:50:21 +09:00
parent c911bc32fd
commit cfe221f529
8 changed files with 28 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
devstack/plugin.sh
View File

@@ -18,7 +18,6 @@
# - ``functions`` file
# - ``functions-common`` file
# - ``STACK_USER`` must be defined
# - ``SWIFT_DATA_DIR`` or ``DATA_DIR`` must be defined
# - ``lib/swift`` file
# - ``lib/keystone`` file
#
@@ -49,7 +48,9 @@ SWIFT_DEFAULT_PROJECT_DOMAIN_ID=${SWIFT_DEFAULT_PROJECT_DOMAIN_ID:-default}
SWIFT_CONF_DIR=${SWIFT_CONF_DIR:-/etc/swift}
# Storlets install tunables
STORLETS_CONTAINER_DEVICE=${STORLETS_CONTAINER_DEVICE:-/var/lib/storlets}
STORLETS_BIN_DIR=$(get_python_exec_prefix)
# TODO(tkajinam): Move this to DATA_DIR
STORLETS_DATA_DIR=${STORLETS_DATA_DIR:-/var/lib/storlets}
if is_fedora; then
STORLETS_CONTAINER_BASE_IMG=${STORLETS_CONTAINER_BASE_IMG:-quay.io/centos/centos:stream9}
else
@@ -64,19 +65,15 @@ STORLETS_LOG_CONTAIER_NAME=${STORLETS_LOG_CONTAIER_NAME:-log}
STORLETS_GATEWAY_MODULE=${STORLETS_GATEWAY_MODULE:-docker}
STORLETS_GATEWAY_CONF_FILE=${STORLETS_GATEWAY_CONF_FILE:-${SWIFT_CONF_DIR}/storlet_gateway.conf}
STORLETS_PROXY_EXECUTION_ONLY=${STORLETS_PROXY_EXECUTION_ONLY:-false}
STORLETS_SCRIPTS_DIR=${STORLETS_SCRIPTS_DIR:-"$STORLETS_CONTAINER_DEVICE"/scripts}
STORLETS_STORLETS_DIR=${STORLETS_STORLETS_DIR:-"$STORLETS_CONTAINER_DEVICE"/storlets/scopes}
STORLETS_LOGS_DIR=${STORLETS_LOGS_DIR:-"$STORLETS_CONTAINER_DEVICE"/logs/scopes}
STORLETS_CACHE_DIR=${STORLETS_CACHE_DIR:-"$STORLETS_CONTAINER_DEVICE"/cache/scopes}
STORLETS_PIPES_DIR=${STORLETS_PIPES_DIR:-"$STORLETS_CONTAINER_DEVICE"/pipes/scopes}
STORLETS_SCRIPTS_DIR=${STORLETS_SCRIPTS_DIR:-${STORLETS_DATA_DIR}/scripts}
STORLETS_STORLETS_DIR=${STORLETS_STORLETS_DIR:-${STORLETS_DATA_DIR}storlets/scopes}
STORLETS_LOGS_DIR=${STORLETS_LOGS_DIR:-${STORLETS_DATA_DIR}logs/scopes}
STORLETS_CACHE_DIR=${STORLETS_CACHE_DIR:-${STORLETS_DATA_DIR}cache/scopes}
STORLETS_PIPES_DIR=${STORLETS_PIPES_DIR:-${STORLETS_DATA_DIR}pipes/scopes}
STORLETS_RESTART_CONTAINER_TIMEOUT=${STORLETS_RESTART_CONTAINER_TIMEOUT:-3}
STORLETS_RUNTIME_TIMEOUT=${STORLETS_RUNTIME_TIMEOUT:-40}
STORLETS_JDK_VERSION=${STORLETS_JDK_VERSION:-11}
STORLETS_BIN_DIR=$(get_python_exec_prefix)
TMP_REGISTRY_PREFIX=/tmp/registry
if [ $STORLETS_GATEWAY_MODULE == 'podman' ]; then
CONTAINER_CMD=podman
elif [ $STORLETS_GATEWAY_MODULE == 'docker' ]; then
@@ -191,7 +188,7 @@ function prepare_storlets_install {
}
function _generate_jre_dockerfile {
JRE_DOCKERFILE=${TMP_REGISTRY_PREFIX}/repositories/storlet_engine_image/Dockerfile
JRE_DOCKERFILE=${STORLETS_DATA_DIR}/images/storlet_engine_image/Dockerfile
if is_fedora; then
JDK_PACKAGE="java-${STORLETS_JDK_VERSION}-openjdk-headless"
@@ -223,9 +220,9 @@ EOF
function create_base_jre_image {
echo "Create base jre image"
$CONTAINER_CMD pull $STORLETS_CONTAINER_BASE_IMG
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/storlet_engine_image
mkdir -p ${STORLETS_DATA_DIR}/images/storlet_engine_image
_generate_jre_dockerfile
cd ${TMP_REGISTRY_PREFIX}/repositories/storlet_engine_image
cd ${STORLETS_DATA_DIR}/images/storlet_engine_image
$CONTAINER_CMD build -t storlet_engine_image .
cd -
}
@@ -251,8 +248,8 @@ function install_storlets_code {
sudo cp ./bin/${bin_file} /usr/local/libexec/storlets/
done
sudo mkdir -p -m 0755 $STORLETS_CONTAINER_DEVICE
sudo chown -R "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" $STORLETS_CONTAINER_DEVICE
sudo mkdir -p -m 0755 $STORLETS_DATA_DIR
sudo chown -R "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" $STORLETS_DATA_DIR
# NOTE(takashi): We should cleanup egg-info directory here, otherwise it
# causes permission denined when installing package by tox.
@@ -305,10 +302,7 @@ function _modify_swift_conf {
function _generate_gateway_conf {
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT storlet_logcontainer $STORLETS_LOG_CONTAIER_NAME
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT cache_dir $STORLETS_CACHE_DIR
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT log_dir $STORLETS_LOGS_DIR
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT storlets_dir $STORLETS_STORLETS_DIR
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT pipes_dir $STORLETS_PIPES_DIR
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT host_root $STORLETS_DATA_DIR
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT restart_linux_container_timeout $STORLETS_RESTART_CONTAINER_TIMEOUT
iniset ${STORLETS_GATEWAY_CONF_FILE} DEFAULT storlet_timeout $STORLETS_RUNTIME_TIMEOUT
if [[ $STORLETS_GATEWAY_MODULE == 'podman' ]]; then
@@ -318,7 +312,7 @@ function _generate_gateway_conf {
}
function _generate_default_tenant_dockerfile {
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile
cat <<EOF > ${STORLETS_DATA_DIR}/images/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile
FROM storlet_engine_image
MAINTAINER root
EOF
@@ -326,9 +320,9 @@ EOF
function create_default_tenant_image {
SWIFT_DEFAULT_PROJECT_ID=`openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | awk '{ print $2 }'`
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
mkdir -p ${STORLETS_DATA_DIR}/images/$SWIFT_DEFAULT_PROJECT_ID
_generate_default_tenant_dockerfile
cd ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
cd ${STORLETS_DATA_DIR}/images/$SWIFT_DEFAULT_PROJECT_ID
$CONTAINER_CMD build -t ${SWIFT_DEFAULT_PROJECT_ID:0:13} .
cd -
}
@@ -383,7 +377,7 @@ function uninstall_storlets {
esac
echo "Cleaning all storlets runtime stuff..."
sudo rm -fr ${STORLETS_CONTAINER_DEVICE}
sudo rm -fr ${STORLETS_DATA_DIR}
# TODO(tkajinam): Remove config options
# TODO(tkajinam): Remove docker containers/images
}

2
doc/source/archive/invoking_storlets.rst
View File

@@ -114,7 +114,7 @@ Note that the log reflects the two invocations done above.
Storlet invocation done
About to invoke storlet
IdentityStorlet Invoked
Exec = /home/swift/identitystorlet/get42
Exec = /var/lib/storlets/identitystorlet/get42
Exit code = 42
Storlet invocation done

4
doc/source/engine_dev_installation.rst
View File

@@ -296,10 +296,6 @@ with the following content:
[DEFAULT]
storlet_logcontainer = storletlog
host_root = /var/lib/storlets
cache_dir = /var/lib/storlets/cache/scopes
log_dir = /var/lib/storlets/logs/scopes
storlets_dir = /var/lib/storlets/storlets/scopes
pipes_dir = /var/lib/storlets/pipes/scopes
storlet_timeout = 40
container_image_namespace =
restart_linux_container_timeout = 3

2
storlets/agent/daemon_factory/server.py
View File

@@ -102,7 +102,7 @@ class StorletDaemonFactory(SBusServer):
pargs = [python_interpreter, str_daemon_main_file, storlet_name,
uds_path, log_level, str(pool_size), self.container_id]
python_path = os.path.join('/home/swift/', storlet_name)
python_path = os.path.join('/var/lib/storlets', storlet_name)
if os.environ.get('PYTHONPATH'):
python_path = os.environ['PYTHONPATH'] + ':' + python_path
env = {'PYTHONPATH': python_path}

4
storlets/gateway/gateways/container/runtime.py
View File

@@ -86,7 +86,7 @@ class RunTimePaths(object):
directory.
This directory is called the storlet directories.
On the host side it is of the form <storlet_dir>/<scope>/<storlet_name>
On the sandbox side it is of the form /home/swift/<storlet_name>
On the sandbox side it is of the form /var/lib/storlets/<storlet_name>
<storlet_dir> comes from the configuration
<storlet_name> is the prefix of the jar.
@@ -107,7 +107,7 @@ class RunTimePaths(object):
self.factory_pipe_name = 'factory_pipe'
self.sandbox_pipe_dir = '/mnt/channels'
self.sandbox_storlet_base_dir = '/home/swift'
self.sandbox_storlet_base_dir = '/var/lib/storlets'
self.host_root_dir = conf.get('host_root', '/var/lib/storlets')
self.host_pipe_root_dir = \
conf.get('pipes_dir',

2
tests/unit/agent/daemon_factory/test_server.py
View File

@@ -104,7 +104,7 @@ class TestStorletDaemonFactory(unittest.TestCase):
pargs)
self.assertEqual(
{'PYTHONPATH': '/default/pythonpath:'
'/home/swift/test_storlet.TestStorlet'},
'/var/lib/storlets/test_storlet.TestStorlet'},
env)
def test_spawn_subprocess(self):

4
tests/unit/gateway/gateways/container/test_runtime.py
View File

@@ -146,7 +146,7 @@ class TestRuntimePaths(unittest.TestCase):
def test_get_sbox_storlet_dir(self):
self.assertEqual(
os.path.join('/home/swift', self.storlet_id),
os.path.join('/var/lib/storlets', self.storlet_id),
self.paths.get_sbox_storlet_dir(self.storlet_id))
def test_host_storlet_base_dir(self):
@@ -218,7 +218,7 @@ class TestRuntimePaths(unittest.TestCase):
'/var/lib/storlets/storlets/scopes/account/Storlet-1.0.jar',
runtime_paths.get_host_storlet_dir(storlet_id))
# And this one is a mount point in sand box?
self.assertEqual('/home/swift/Storlet-1.0.jar',
self.assertEqual('/var/lib/storlets/Storlet-1.0.jar',
runtime_paths.get_sbox_storlet_dir(storlet_id))
@with_tempdir

6
tests/unit/gateway/gateways/docker/test_runtime.py
View File

@@ -265,9 +265,9 @@ class TestDockerRunTimeSandbox(unittest.TestCase):
storlet_main = 'org.openstack.storlet.Storlet'
dependencies = ['dep1', 'dep2']
self.assertEqual(
'/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
'/home/swift/org.openstack.storlet.Storlet/dep1:'
'/home/swift/org.openstack.storlet.Storlet/dep2',
'/var/lib/storlets/org.openstack.storlet.Storlet/Storlet.jar:'
'/var/lib/storlets/org.openstack.storlet.Storlet/dep1:'
'/var/lib/storlets/org.openstack.storlet.Storlet/dep2',
self.sbox._get_storlet_classpath(storlet_main, storlet_id,
dependencies),)