Enable multiple threads, disabled muliple workers

This sets multiple threads in Deckhand's chart config (4)
and set workers to just 1.

Deckhand's database is not configured to work with multiprocessing.
Currently there is a data race on acquiring shared SQLAlchemy
engine pooled connection strings when workers > 1. As a
workaround, we use multiple threads but only 1 worker. For more
information, see:

https://github.com/att-comdev/deckhand/issues/20

Change-Id: I60adeffff5461fdda957124232bc5a606baae413
This commit is contained in:
Felipe Monteiro 2018-03-30 22:08:48 +01:00
parent 225638711b
commit 5c9efa9d74
6 changed files with 41 additions and 24 deletions

View File

@ -47,10 +47,15 @@ spec:
env: env:
- name: 'DECKHAND_API_TIMEOUT' - name: 'DECKHAND_API_TIMEOUT'
value: {{ .Values.conf.uwsgi.timeout | default 600 | quote }} value: {{ .Values.conf.uwsgi.timeout | default 600 | quote }}
# NOTE(fmontei): Deckhand's database is not configured to work with
# multiprocessing. Currently there is a data race on acquiring shared
# SQLAlchemy engine pooled connection strings when workers > 1. As a
# workaround, we use multiple threads but only 1 worker. For more
# information, see: https://github.com/att-comdev/deckhand/issues/20
- name: 'DECKHAND_API_WORKERS' - name: 'DECKHAND_API_WORKERS'
value: {{ .Values.conf.uwsgi.workers | default 4 | quote }} value: {{ .Values.conf.uwsgi.workers | default 1 | quote }}
- name: 'DECKHAND_API_THREADS' - name: 'DECKHAND_API_THREADS'
value: {{ .Values.conf.uwsgi.threads | default 1 | quote }} value: {{ .Values.conf.uwsgi.threads | default 4 | quote }}
image: {{ .Values.images.tags.deckhand }} image: {{ .Values.images.tags.deckhand }}
imagePullPolicy: {{ .Values.images.pull_policy }} imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}

View File

@ -179,8 +179,13 @@ secrets:
conf: conf:
uwsgi: uwsgi:
threads: 1 # NOTE(fmontei): Deckhand's database is not configured to work with
workers: 4 # multiprocessing. Currently there is a data race on acquiring shared
# SQLAlchemy engine pooled connection strings when workers > 1. As a
# workaround, we use multiple threads but only 1 worker. For more
# information, see: https://github.com/att-comdev/deckhand/issues/20
threads: 4
workers: 1
policy: policy:
admin_api: role:admin admin_api: role:admin
deckhand:create_cleartext_documents: rule:admin_api deckhand:create_cleartext_documents: rule:admin_api

View File

@ -33,7 +33,7 @@ CONFIG_FILES = ['deckhand.conf', 'deckhand-paste.ini']
def _get_config_files(env=None): def _get_config_files(env=None):
if env is None: if env is None:
env = os.environ env = os.environ
dirname = env.get('OS_DECKHAND_CONFIG_DIR', '/etc/deckhand').strip() dirname = env.get('DECKHAND_CONFIG_DIR', '/etc/deckhand').strip()
return [os.path.join(dirname, config_file) for config_file in CONFIG_FILES] return [os.path.join(dirname, config_file) for config_file in CONFIG_FILES]

View File

@ -139,8 +139,8 @@ Create the directory ``/etc/deckhand`` and copy the config file there::
To specify an alternative directory for the config file, run:: To specify an alternative directory for the config file, run::
$ export OS_DECKHAND_CONFIG_DIR=<PATH> $ export DECKHAND_CONFIG_DIR=<PATH>
$ [sudo] cp etc/deckhand/deckhand.conf.sample ${OS_DECKHAND_CONFIG_DIR}/deckhand.conf $ [sudo] cp etc/deckhand/deckhand.conf.sample ${DECKHAND_CONFIG_DIR}/deckhand.conf
To conveniently create an ephemeral PostgreSQL DB run:: To conveniently create an ephemeral PostgreSQL DB run::
@ -152,7 +152,7 @@ Retrieve the environment variable which contains connection information::
declare -x PIFPAF_POSTGRESQL_URL="postgresql://localhost/postgres?host=/tmp/tmpsg6tn3l9&port=9824" declare -x PIFPAF_POSTGRESQL_URL="postgresql://localhost/postgres?host=/tmp/tmpsg6tn3l9&port=9824"
Substitute the connection information into the config file in Substitute the connection information into the config file in
``${OS_DECKHAND_CONFIG_DIR}``:: ``${DECKHAND_CONFIG_DIR}``::
[database] [database]

View File

@ -20,10 +20,19 @@ set -ex
PORT=${PORT:-9000} PORT=${PORT:-9000}
# How long uWSGI should wait for each deckhand response # How long uWSGI should wait for each deckhand response
DECKHAND_API_TIMEOUT=${DECKHAND_API_TIMEOUT:-"600"} DECKHAND_API_TIMEOUT=${DECKHAND_API_TIMEOUT:-"600"}
# NOTE(fmontei): Deckhand's database is not configured to work with
# multiprocessing. Currently there is a data race on acquiring shared
# SQLAlchemy engine pooled connection strings when workers > 1. As a
# workaround, we use multiple threads but only 1 worker. For more
# information, see: https://github.com/att-comdev/deckhand/issues/20
# Number of uWSGI workers to handle API requests # Number of uWSGI workers to handle API requests
DECKHAND_API_WORKERS=${DECKHAND_API_WORKERS:-"4"} DECKHAND_API_WORKERS=${DECKHAND_API_WORKERS:-"1"}
# Threads per worker # Threads per worker
DECKHAND_API_THREADS=${DECKHAND_API_THREADS:-"1"} DECKHAND_API_THREADS=${DECKHAND_API_THREADS:-"4"}
# The Deckhand configuration directory containing deckhand.conf
DECKHAND_CONFIG_DIR=${DECKHAND_CONFIG_DIR:-"/etc/deckhand/deckhand.conf"}
# Start deckhand application # Start deckhand application
exec uwsgi \ exec uwsgi \

View File

@ -68,7 +68,7 @@ function gen_config {
export DECKHAND_TEST_URL=http://localhost:9000 export DECKHAND_TEST_URL=http://localhost:9000
export DATABASE_URL=postgresql+psycopg2://deckhand:password@$POSTGRES_IP:5432/deckhand export DATABASE_URL=postgresql+psycopg2://deckhand:password@$POSTGRES_IP:5432/deckhand
# Used by Deckhand's initialization script to search for config files. # Used by Deckhand's initialization script to search for config files.
export OS_DECKHAND_CONFIG_DIR=$CONF_DIR export DECKHAND_CONFIG_DIR=$CONF_DIR
cp etc/deckhand/logging.conf.sample $CONF_DIR/logging.conf cp etc/deckhand/logging.conf.sample $CONF_DIR/logging.conf
@ -198,19 +198,19 @@ gen_config
gen_paste gen_paste
gen_policy gen_policy
ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if [ -z "$DECKHAND_IMAGE" ]; then if [ -z "$DECKHAND_IMAGE" ]; then
log_section "Running Deckhand via uwsgi" log_section "Running Deckhand via uwsgi"
# NOTE(fmontei): Deckhand's database is not configured to work with
# Set --workers 2, so that concurrency is always tested. # multiprocessing. Currently there is a data race on acquiring shared
uwsgi \ # SQLAlchemy engine pooled connection strings when workers > 1. As a
--http :9000 \ # workaround, we use multiple threads but only 1 worker. For more
-w deckhand.cmd \ # information, see: https://github.com/att-comdev/deckhand/issues/20
--callable deckhand_callable \ export DECKHAND_API_WORKERS=1
--enable-threads \ export DECKHAND_API_THREADS=4
--workers 2 \ source $ROOTDIR/../entrypoint.sh &
--threads 1 \ sleep 5
-L \
--pyargv "--config-file $CONF_DIR/deckhand.conf" &
else else
log_section "Running Deckhand via Docker" log_section "Running Deckhand via Docker"
sudo docker run \ sudo docker run \
@ -229,8 +229,6 @@ echo $DECKHAND_ID
log_section Running tests log_section Running tests
ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Create folder for saving HTML test results. # Create folder for saving HTML test results.
if [ ! -d $ROOTDIR/results ]; then if [ ! -d $ROOTDIR/results ]; then
mkdir $ROOTDIR/results mkdir $ROOTDIR/results