From e2b3e8ee8e09f7be59fc00ddfe480a757c807250 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 27 Jun 2018 16:57:11 -0500 Subject: [PATCH] Database tunability Add additional tunables to how SQLalchemy creates connection pools Also start including static doc assets to they can be published Change-Id: I268dc265a6b6cf1a200b235a5f99e65e89a95637 --- .gitignore | 1 - docs/source/_static/drydock.conf.sample | 402 ++++++++++++++++++++++++ docs/source/_static/policy.yaml.sample | 62 ++++ drydock_provisioner/config.py | 16 + drydock_provisioner/statemgmt/state.py | 6 +- etc/drydock/drydock.conf.sample | 14 + requirements-direct.txt | 2 +- requirements-lock.txt | 30 +- 8 files changed, 515 insertions(+), 18 deletions(-) create mode 100644 docs/source/_static/drydock.conf.sample create mode 100644 docs/source/_static/policy.yaml.sample diff --git a/.gitignore b/.gitignore index fa4fb3f6..f21b0208 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,6 @@ instance/ # Sphinx documentation docs/_build/ -docs/*/_static/ # PyBuilder target/ diff --git a/docs/source/_static/drydock.conf.sample b/docs/source/_static/drydock.conf.sample new file mode 100644 index 00000000..8c8e47e8 --- /dev/null +++ b/docs/source/_static/drydock.conf.sample @@ -0,0 +1,402 @@ +[DEFAULT] + +# +# From drydock_provisioner +# + +# Polling interval in seconds for checking subtask or downstream status (integer +# value) +#poll_interval = 10 + +# How long a leader has to check-in before leadership can be usurped, in seconds +# (integer value) +#leader_grace_period = 300 + +# How often will an instance attempt to claim leadership, in seconds (integer +# value) +#leadership_claim_interval = 30 + + +[database] + +# +# From drydock_provisioner +# + +# The URI database connect string. (string value) +#database_connect_string = + +# The SQLalchemy database connection pool size. (integer value) +#pool_size = 15 + +# Should DB connections be validated prior to use. (boolean value) +#pool_pre_ping = true + +# How long a request for a connection should wait before one becomes available. +# (integer value) +#pool_timeout = 30 + +# How many connections above pool_size are allowed to be open during high usage. +# (integer value) +#pool_overflow = 10 + +# Time, in seconds, when a connection should be closed and re-established. -1 +# for no recycling. (integer value) +#connection_recycle = -1 + + +[keystone_authtoken] + +# +# From drydock_provisioner +# + +# Authentication URL (string value) +#auth_url = + +# Domain ID to scope to (string value) +#domain_id = + +# Domain name to scope to (string value) +#domain_name = + +# Project ID to scope to (string value) +# Deprecated group/name - [keystone_authtoken]/tenant_id +#project_id = + +# Project name to scope to (string value) +# Deprecated group/name - [keystone_authtoken]/tenant_name +#project_name = + +# Domain ID containing project (string value) +#project_domain_id = + +# Domain name containing project (string value) +#project_domain_name = + +# Trust ID (string value) +#trust_id = + +# Optional domain ID to use with v3 and v2 parameters. It will be used for both +# the user and project domain in v3 and ignored in v2 authentication. (string +# value) +#default_domain_id = + +# Optional domain name to use with v3 API and v2 parameters. It will be used for +# both the user and project domain in v3 and ignored in v2 authentication. +# (string value) +#default_domain_name = + +# User id (string value) +#user_id = + +# Username (string value) +# Deprecated group/name - [keystone_authtoken]/user_name +#username = + +# User's domain id (string value) +#user_domain_id = + +# User's domain name (string value) +#user_domain_name = + +# User's password (string value) +#password = + +# +# From keystonemiddleware.auth_token +# + +# Complete "public" Identity API endpoint. This endpoint should not be an +# "admin" endpoint, as it should be accessible by all end users. Unauthenticated +# clients are redirected to this endpoint to authenticate. Although this +# endpoint should ideally be unversioned, client support in the wild varies. +# If you're using a versioned v2 endpoint here, then this should *not* be the +# same endpoint the service user utilizes for validating tokens, because normal +# end users may not be able to reach that endpoint. (string value) +#auth_uri = + +# API version of the admin Identity API endpoint. (string value) +#auth_version = + +# Do not handle authorization requests within the middleware, but delegate the +# authorization decision to downstream WSGI components. (boolean value) +#delay_auth_decision = false + +# Request timeout value for communicating with Identity API server. (integer +# value) +#http_connect_timeout = + +# How many times are we trying to reconnect when communicating with Identity API +# Server. (integer value) +#http_request_max_retries = 3 + +# Request environment key where the Swift cache object is stored. When +# auth_token middleware is deployed with a Swift cache, use this option to have +# the middleware share a caching backend with swift. Otherwise, use the +# ``memcached_servers`` option instead. (string value) +#cache = + +# Required if identity server requires client certificate (string value) +#certfile = + +# Required if identity server requires client certificate (string value) +#keyfile = + +# A PEM encoded Certificate Authority to use when verifying HTTPs connections. +# Defaults to system CAs. (string value) +#cafile = + +# Verify HTTPS connections. (boolean value) +#insecure = false + +# The region in which the identity server can be found. (string value) +#region_name = + +# Directory used to cache files related to PKI tokens. (string value) +#signing_dir = + +# Optionally specify a list of memcached server(s) to use for caching. If left +# undefined, tokens will instead be cached in-process. (list value) +# Deprecated group/name - [keystone_authtoken]/memcache_servers +#memcached_servers = + +# In order to prevent excessive effort spent validating tokens, the middleware +# caches previously-seen tokens for a configurable duration (in seconds). Set to +# -1 to disable caching completely. (integer value) +#token_cache_time = 300 + +# Determines the frequency at which the list of revoked tokens is retrieved from +# the Identity service (in seconds). A high number of revocation events combined +# with a low cache duration may significantly reduce performance. Only valid for +# PKI tokens. (integer value) +#revocation_cache_time = 10 + +# (Optional) If defined, indicate whether token data should be authenticated or +# authenticated and encrypted. If MAC, token data is authenticated (with HMAC) +# in the cache. If ENCRYPT, token data is encrypted and authenticated in the +# cache. If the value is not one of these options or empty, auth_token will +# raise an exception on initialization. (string value) +# Possible values: +# None - +# MAC - +# ENCRYPT - +#memcache_security_strategy = None + +# (Optional, mandatory if memcache_security_strategy is defined) This string is +# used for key derivation. (string value) +#memcache_secret_key = + +# (Optional) Number of seconds memcached server is considered dead before it is +# tried again. (integer value) +#memcache_pool_dead_retry = 300 + +# (Optional) Maximum total number of open connections to every memcached server. +# (integer value) +#memcache_pool_maxsize = 10 + +# (Optional) Socket timeout in seconds for communicating with a memcached +# server. (integer value) +#memcache_pool_socket_timeout = 3 + +# (Optional) Number of seconds a connection to memcached is held unused in the +# pool before it is closed. (integer value) +#memcache_pool_unused_timeout = 60 + +# (Optional) Number of seconds that an operation will wait to get a memcached +# client connection from the pool. (integer value) +#memcache_pool_conn_get_timeout = 10 + +# (Optional) Use the advanced (eventlet safe) memcached client pool. The +# advanced pool will only work under python 2.x. (boolean value) +#memcache_use_advanced_pool = false + +# (Optional) Indicate whether to set the X-Service-Catalog header. If False, +# middleware will not ask for service catalog on token validation and will not +# set the X-Service-Catalog header. (boolean value) +#include_service_catalog = true + +# Used to control the use and type of token binding. Can be set to: "disabled" +# to not check token binding. "permissive" (default) to validate binding +# information if the bind type is of a form known to the server and ignore it if +# not. "strict" like "permissive" but if the bind type is unknown the token will +# be rejected. "required" any form of token binding is needed to be allowed. +# Finally the name of a binding method that must be present in tokens. (string +# value) +#enforce_token_bind = permissive + +# If true, the revocation list will be checked for cached tokens. This requires +# that PKI tokens are configured on the identity server. (boolean value) +#check_revocations_for_cached = false + +# Hash algorithms to use for hashing PKI tokens. This may be a single algorithm +# or multiple. The algorithms are those supported by Python standard +# hashlib.new(). The hashes will be tried in the order given, so put the +# preferred one first for performance. The result of the first hash will be +# stored in the cache. This will typically be set to multiple values only while +# migrating from a less secure algorithm to a more secure one. Once all the old +# tokens are expired this option should be set to a single value for better +# performance. (list value) +#hash_algorithms = md5 + +# Authentication type to load (string value) +# Deprecated group/name - [keystone_authtoken]/auth_plugin +#auth_type = + +# Config Section from which to load plugin specific options (string value) +#auth_section = + + +[libvirt_driver] + +# +# From drydock_provisioner +# + +# Polling interval in seconds for querying libvirt status (integer value) +#poll_interval = 10 + + +[logging] + +# +# From drydock_provisioner +# + +# Global log level for Drydock (string value) +#log_level = INFO + +# Logger name for the top-level logger (string value) +#global_logger_name = drydock_provisioner + +# Logger name for OOB driver logging (string value) +#oobdriver_logger_name = ${global_logger_name}.oobdriver + +# Logger name for Node driver logging (string value) +#nodedriver_logger_name = ${global_logger_name}.nodedriver + +# Logger name for API server logging (string value) +#control_logger_name = ${global_logger_name}.control + + +[maasdriver] + +# +# From drydock_provisioner +# + +# The API key for accessing MaaS (string value) +#maas_api_key = + +# The URL for accessing MaaS API (string value) +#maas_api_url = + +# Polling interval for querying MaaS status in seconds (integer value) +#poll_interval = 10 + + +[network] + +# +# From drydock_provisioner +# + +# Timeout for initial read of outgoing HTTP calls from Drydock in seconds. +# (integer value) +#http_client_connect_timeout = 16 + +# Timeout for initial read of outgoing HTTP calls from Drydock in seconds. +# (integer value) +#http_client_read_timeout = 300 + +# Number of retries for transient errors of outgoing HTTP calls from Drydock. +# (integer value) +#http_client_retries = 3 + + +[oslo_policy] + +# +# From oslo.policy +# + +# The file that defines policies. (string value) +#policy_file = policy.json + +# Default rule. Enforced when a requested rule is not found. (string value) +#policy_default_rule = default + +# Directories where policy configuration files are stored. They can be relative +# to any directory in the search path defined by the config_dir option, or +# absolute paths. The file defined by policy_file must exist for these +# directories to be searched. Missing or empty directories are ignored. (multi +# valued) +#policy_dirs = policy.d + + +[plugins] + +# +# From drydock_provisioner +# + +# Module path string of a input ingester to enable (string value) +#ingester = drydock_provisioner.ingester.plugins.yaml.YamlIngester + +# List of module path strings of OOB drivers to enable (list value) +#oob_driver = drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver + +# Module path string of the Node driver to enable (string value) +#node_driver = drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver + +# Module path string of the Network driver enable (string value) +#network_driver = + + +[pyghmi_driver] + +# +# From drydock_provisioner +# + +# Polling interval in seconds for querying IPMI status (integer value) +#poll_interval = 10 + + +[timeouts] + +# +# From drydock_provisioner +# + +# Fallback timeout when a specific one is not configured (integer value) +#drydock_timeout = 5 + +# Timeout in minutes for creating site network templates (integer value) +#create_network_template = 2 + +# Timeout in minutes for creating user credentials (integer value) +#configure_user_credentials = 2 + +# Timeout in minutes for initial node identification (integer value) +#identify_node = 10 + +# Timeout in minutes for node commissioning and hardware configuration (integer +# value) +#configure_hardware = 30 + +# Timeout in minutes for configuring node networking (integer value) +#apply_node_networking = 5 + +# Timeout in minutes for configuring node storage (integer value) +#apply_node_storage = 5 + +# Timeout in minutes for configuring node platform (integer value) +#apply_node_platform = 5 + +# Timeout in minutes for deploying a node (integer value) +#deploy_node = 45 + +# Timeout in minutes between deployment completion and the all boot actions +# reporting status (integer value) +#bootaction_final_status = 15 diff --git a/docs/source/_static/policy.yaml.sample b/docs/source/_static/policy.yaml.sample new file mode 100644 index 00000000..65706bf5 --- /dev/null +++ b/docs/source/_static/policy.yaml.sample @@ -0,0 +1,62 @@ +# Actions requiring admin authority +#"admin_required": "role:admin or is_admin:1" + +# Get task status +# GET /api/v1.0/tasks +# GET /api/v1.0/tasks/{task_id} +#"physical_provisioner:read_task": "role:admin" + +# Create a task +# POST /api/v1.0/tasks +#"physical_provisioner:create_task": "role:admin" + +# Create validate_design task +# POST /api/v1.0/tasks +#"physical_provisioner:validate_design": "role:admin" + +# Create verify_site task +# POST /api/v1.0/tasks +#"physical_provisioner:verify_site": "role:admin" + +# Create prepare_site task +# POST /api/v1.0/tasks +#"physical_provisioner:prepare_site": "role:admin" + +# Create verify_nodes task +# POST /api/v1.0/tasks +#"physical_provisioner:verify_nodes": "role:admin" + +# Create prepare_nodes task +# POST /api/v1.0/tasks +#"physical_provisioner:prepare_nodes": "role:admin" + +# Create deploy_nodes task +# POST /api/v1.0/tasks +#"physical_provisioner:deploy_nodes": "role:admin" + +# Create destroy_nodes task +# POST /api/v1.0/tasks +#"physical_provisioner:destroy_nodes": "role:admin" + +# Read build data for a node +# GET /api/v1.0/nodes/{nodename}/builddata +#"physical_provisioner:read_build_data": "role:admin" + +# Read loaded design data +# GET /api/v1.0/designs +# GET /api/v1.0/designs/{design_id} +#"physical_provisioner:read_data": "role:admin" + +# Load design data +# POST /api/v1.0/designs +# POST /api/v1.0/designs/{design_id}/parts +#"physical_provisioner:ingest_data": "role:admin" + +# et health status +# GET /api/v1.0/health/extended +#"physical_provisioner:health_data": "role:admin" + +# Validate site design +# POST /api/v1.0/validatedesign +#"physical_provisioner:validate_site_design": "role:admin" + diff --git a/drydock_provisioner/config.py b/drydock_provisioner/config.py index 03036ace..de1ff2c2 100644 --- a/drydock_provisioner/config.py +++ b/drydock_provisioner/config.py @@ -96,6 +96,22 @@ class DrydockConfig(object): 'pool_size', default=15, help='The SQLalchemy database connection pool size.'), + cfg.BoolOpt( + 'pool_pre_ping', + default=True, + help='Should DB connections be validated prior to use.'), + cfg.IntOpt( + 'pool_timeout', + default=30, + help='How long a request for a connection should wait before one becomes available.'), + cfg.IntOpt( + 'pool_overflow', + default=10, + help='How many connections above pool_size are allowed to be open during high usage.'), + cfg.IntOpt( + 'connection_recycle', + default=-1, + help='Time, in seconds, when a connection should be closed and re-established. -1 for no recycling.'), ] # Options for the boot action framework diff --git a/drydock_provisioner/statemgmt/state.py b/drydock_provisioner/statemgmt/state.py index 1177883e..2166b167 100644 --- a/drydock_provisioner/statemgmt/state.py +++ b/drydock_provisioner/statemgmt/state.py @@ -43,7 +43,11 @@ class DrydockState(object): """Connect the state manager to the persistent DB.""" self.db_engine = create_engine( config.config_mgr.conf.database.database_connect_string, - pool_size=config.config_mgr.conf.database.pool_size) + pool_size=config.config_mgr.conf.database.pool_size, + pool_pre_ping=config.config_mgr.conf.database.pool_pre_ping, + max_overflow=config.config_mgr.conf.database.pool_overflow, + pool_timeout=config.config_mgr.conf.database.pool_timeout, + pool_recycle=config.config_mgr.conf.database.connection_recycle) self.db_metadata = MetaData(bind=self.db_engine) self.tasks_tbl = tables.Tasks(self.db_metadata) diff --git a/etc/drydock/drydock.conf.sample b/etc/drydock/drydock.conf.sample index 09eb4b9f..b5a21873 100644 --- a/etc/drydock/drydock.conf.sample +++ b/etc/drydock/drydock.conf.sample @@ -29,6 +29,20 @@ # The SQLalchemy database connection pool size. (integer value) #pool_size = 15 +# Should DB connections be validated prior to use. (boolean value) +#pool_pre_ping = true + +# How long a request for a connection should wait before one becomes available. +# (integer value) +#pool_timeout = 30 + +# How many connections above pool_size are allowed to be open during high usage. +# (integer value) +#pool_overflow = 10 + +# Time, in seconds, when a connection should be closed and re-established. -1 +# for no recycling. (integer value) +#connection_recycle = -1 [keystone_authtoken] diff --git a/requirements-direct.txt b/requirements-direct.txt index 11daf662..974097c4 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -16,7 +16,7 @@ oslo.policy==1.22.1 iso8601==0.1.11 keystoneauth1==3.3.0 alembic==0.8.2 -sqlalchemy==1.1.14 +sqlalchemy==1.2.8 psycopg2==2.7.3.1 jsonschema==2.6.0 jinja2==2.9.6 diff --git a/requirements-lock.txt b/requirements-lock.txt index efdc2ae1..e127932b 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -1,6 +1,6 @@ alembic==0.8.2 -amqp==2.2.2 -Babel==2.5.3 +amqp==2.3.2 +Babel==2.6.0 Beaker==1.9.1 cachetools==2.1.0 certifi==2018.4.16 @@ -14,13 +14,13 @@ falcon==1.4.1 fasteners==0.14.1 futurist==1.7.0 greenlet==0.4.13 -idna==2.6 +idna==2.7 iso8601==0.1.11 Jinja2==2.9.6 jsonschema==2.6.0 keystoneauth1==3.3.0 keystonemiddleware==4.9.1 -kombu==4.2.0 +kombu==4.2.1 libvirt-python==3.10.0 Mako==1.0.7 MarkupSafe==1.0 @@ -31,19 +31,19 @@ netifaces==0.10.7 oauthlib==2.1.0 oslo.concurrency==3.27.0 oslo.config==5.2.0 -oslo.context==2.20.0 +oslo.context==2.21.0 oslo.i18n==3.20.0 -oslo.log==3.38.1 -oslo.messaging==6.2.0 +oslo.log==3.39.0 +oslo.messaging==6.5.0 oslo.middleware==3.35.0 oslo.policy==1.22.1 -oslo.serialization==2.25.0 +oslo.serialization==2.27.0 oslo.service==1.31.2 -oslo.utils==3.36.2 +oslo.utils==3.36.3 oslo.versionedobjects==1.23.0 Paste==2.0.3 PasteDeploy==1.5.2 -pbr==4.0.3 +pbr==4.0.4 pip==10.0.1 positional==1.2.1 prettytable==0.7.2 @@ -56,24 +56,24 @@ pymongo==3.6.1 pyparsing==2.2.0 python-dateutil==2.7.3 python-editor==1.0.3 -python-keystoneclient==3.16.0 +python-keystoneclient==3.17.0 python-mimeparse==1.6.0 pytz==2018.4 PyYAML==3.12 repoze.lru==0.7 -requests==2.18.4 +requests==2.19.1 rfc3986==1.1.0 Routes==2.4.1 setuptools==39.2.0 six==1.11.0 -SQLAlchemy==1.1.14 +SQLAlchemy==1.2.8 statsd==3.2.2 stevedore==1.28.0 tenacity==4.12.0 ulid2==0.1.1 -urllib3==1.22 +urllib3==1.23 uWSGI==2.0.15 vine==1.1.4 -WebOb==1.8.1 +WebOb==1.8.2 wheel==0.31.1 wrapt==1.10.11