From a5e57879ab318884493f9b0580c410adafc2e11a Mon Sep 17 00:00:00 2001 From: "Mahmoudi, Ahmad (am495p)" Date: Tue, 28 Jul 2020 19:38:59 +0000 Subject: [PATCH] Override uwsgi default config - Overrode uwsgi default configs to improve stability and performance. - Increased mas number of worker processes to increase capacity and performance. - Enabled uwsgi cheaper subsystem to scale worker processes dynamically. - Uplifted uwsgi to the latest release to bring bug fixes and improvements since 2018. Upgraded uwsgi to bring in bug fixes since 2018. For background information for this change please see: https://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html Change-Id: If067e9786e9dbbd39ef832dea6f51aa5523af4d7 --- .../templates/deployment-shipyard.yaml | 20 ++++++++ charts/shipyard/values.yaml | 16 ++++++- src/bin/shipyard_airflow/entrypoint.sh | 46 +++++++++++++++++-- src/bin/shipyard_airflow/requirements.txt | 2 +- 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/charts/shipyard/templates/deployment-shipyard.yaml b/charts/shipyard/templates/deployment-shipyard.yaml index 036cb667..bf6d93b1 100644 --- a/charts/shipyard/templates/deployment-shipyard.yaml +++ b/charts/shipyard/templates/deployment-shipyard.yaml @@ -62,6 +62,26 @@ spec: value: {{ .Values.conf.uwsgi.workers | quote }} - name: 'SHIPYARD_API_THREADS' value: {{ .Values.conf.uwsgi.threads | quote }} + - name: 'SHIPYARD_API_CHEAPER_ALGO' + value: {{ .Values.conf.uwsgi.cheaper_algo | quote }} + - name: 'SHIPYARD_API_CHEAPER' + value: {{ .Values.conf.uwsgi.cheaper | quote }} + - name: 'SHIPYARD_API_CHEAPER_INITIAL' + value: {{ .Values.conf.uwsgi.cheaper_initial | quote }} + - name: 'SHIPYARD_API_CHEAPER_STEP' + value: {{ .Values.conf.uwsgi.cheaper_step | quote }} + - name: 'SHIPYARD_API_CHEAPER_OVERLOAD' + value: {{ .Values.conf.uwsgi.cheaper_overload | quote }} + - name: 'SHIPYARD_API_CHEAPER_BUSYNESS_MULTIPLIER' + value: {{ .Values.conf.uwsgi.cheaper_busyness_multiplier | quote }} + - name: 'SHIPYARD_API_CHEAPER_BUSYNESS_MIN' + value: {{ .Values.conf.uwsgi.cheaper_busyness_min | quote }} + - name: 'SHIPYARD_API_CHEAPER_BUSYNESS_MAX' + value: {{ .Values.conf.uwsgi.cheaper_busyness_max | quote }} + - name: 'SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_ALERT' + value: {{ .Values.conf.uwsgi.cheaper_busyness_backlog_alert | quote }} + - name: 'SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_STEP' + value: {{ .Values.conf.uwsgi.cheaper_busyness_backlog_step | quote }} image: {{ .Values.images.tags.shipyard }} imagePullPolicy: {{ .Values.images.pull_policy }} {{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 1128df9c..7b73d8ce 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -328,8 +328,20 @@ secrets: conf: uwsgi: - threads: 1 - workers: 4 + # for more info about these default overrides please read + # https://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html + threads: 1 # Number of threads per worker/process + workers: 16 # maximum number of workers/processes + cheaper_algo: busyness # Use cheaper busyness plugin for dynamic scaling of workers + cheaper: 4 # Minimum number of workers allowed + cheaper_initial: 8 # Workers created at startup + cheaper_step: 4 # How many workers to spawn at a time + cheaper_overload: 1 # Length of a cycle in seconds + cheaper_busyness_multiplier: 60 # Cycles to wait before killing workers + cheaper_busyness_min: 20 # Below this threshold, kill workers + cheaper_busyness_max: 75 # Above this threshold, spawn new workers + cheaper_busyness_backlog_alert: 40 # Spawn emergency workers if more than this many requests are waiting in the queue + cheaper_busyness_backlog_step: 1 # Emergegency workers to create if there are too many requests in the queue policy: admin_create: role:admin or role:admin_ucp admin_read_access: rule:admin_create or role:admin_ucp_viewer diff --git a/src/bin/shipyard_airflow/entrypoint.sh b/src/bin/shipyard_airflow/entrypoint.sh index cd9faf89..08ed68c9 100644 --- a/src/bin/shipyard_airflow/entrypoint.sh +++ b/src/bin/shipyard_airflow/entrypoint.sh @@ -20,19 +20,57 @@ if [ "$1" = 'server' ]; then PORT=${PORT:-9000} HTTP_TIMEOUT=${HTTP_TIMEOUT:-600} # Number of uWSGI workers to handle API request - SHIPYARD_API_WORKERS=${SHIPYARD_API_WORKERS:-"4"} + SHIPYARD_API_WORKERS=${SHIPYARD_API_WORKERS:-"16"} #Threads per worker SHIPYARD_API_THREADS=${SHIPYARD_API_THREADS:-"1"} + # use uwsgi cheaper-busyness plugin for dynamic worker scaling + SHIPYARD_API_CHEAPER_ALGO=${SHIPYARD_API_CHEAPER_ALGO:-"busyness"} + # Minimum number of workers allowed + SHIPYARD_API_CHEAPER=${SHIPYARD_API_CHEAPER:-"4"} + # Number of workers created at startup + SHIPYARD_API_CHEAPER_INITIAL=${SHIPYARD_API_CHEAPER_INITIAL:-"8"} + # How many workers to spawn each time + SHIPYARD_API_CHEAPER_STEP=${SHIPYARD_API_CHEAPER_STEP:-"4"} + # Length of a busyness cycle in seconds + SHIPYARD_API_CHEAPER_OVERLOAD=${SHIPYARD_API_CHEAPER_OVERLOAD:-"1"} + # How many cycles to wait before killing workers due to low load + SHIPYARD_API_CHEAPER_BUSYNESS_MULTIPLIER=${SHIPYARD_API_CHEAPER_BUSYNESS_MULTIPLIER:-"60"} + # Below this threshold, kill workers (if stable for multiplier cycles) + SHIPYARD_API_CHEAPER_BUSYNESS_MIN=${SHIPYARD_API_CHEAPER_BUSYNESS_MIN:-"20"} + # Above this threshold, spawn new workers + SHIPYARD_API_CHEAPER_BUSYNESS_MAX=${SHIPYARD_API_CHEAPER_BUSYNESS_MAX:-"75"} + # Spawn emergency workers if more than this many requests are waiting in the queue + SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_ALERT=${SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_ALERT:-"40"} + # How many emergegency workers to create if there are too many requests in the queue + SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_STEP=${SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_STEP:-"1"} # Start shipyard application + # for more info about these default overrides please read + # https://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html exec uwsgi \ --http :${PORT} \ --paste config:/etc/shipyard/api-paste.ini \ --enable-threads \ -L \ --pyargv "--config-file /etc/shipyard/shipyard.conf" \ - --threads $SHIPYARD_API_THREADS \ - --workers $SHIPYARD_API_WORKERS \ - --http-timeout ${HTTP_TIMEOUT} + --threads ${SHIPYARD_API_THREADS} \ + --workers ${SHIPYARD_API_WORKERS} \ + --http-timeout ${HTTP_TIMEOUT} \ + --strict \ + --master \ + --vacuum \ + --single-interpreter \ + --die-on-term \ + --need-app \ + --cheaper-algo ${SHIPYARD_API_CHEAPER_ALGO} \ + --cheaper ${SHIPYARD_API_CHEAPER} \ + --cheaper-initial ${SHIPYARD_API_CHEAPER_INITIAL} \ + --cheaper-step ${SHIPYARD_API_CHEAPER_STEP} \ + --cheaper-overload ${SHIPYARD_API_CHEAPER_OVERLOAD} \ + --cheaper-busyness-multiplier ${SHIPYARD_API_CHEAPER_BUSYNESS_MULTIPLIER} \ + --cheaper-busyness-min ${SHIPYARD_API_CHEAPER_BUSYNESS_MIN} \ + --cheaper-busyness-max ${SHIPYARD_API_CHEAPER_BUSYNESS_MAX} \ + --cheaper-busyness-backlog-alert ${SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_ALERT} \ + --cheaper-busyness-backlog-step ${SHIPYARD_API_CHEAPER_BUSYNESS_BACKLOG_STEP} else CMD="shipyard" # Execute shipyard command diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index ee8c673d..f457fe0b 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -34,7 +34,7 @@ requests==2.20.0 setuptools==40.4.1 SQLAlchemy==1.3.15 ulid==1.1 -uwsgi==2.0.17 +uwsgi~=2.0.19.1 # To support profiling in non-prod Werkzeug==0.16.1