Add better caching to jsonpath-ng wrapper functions
This patchset uses beaker (used by Promenade and Drydock) to achieve better caching around jsonpath-ng wrapper functions (jsonpath_replace and jsonpath_parse). Change-Id: Ifae24775b4741ade7673dc91776c35d2de5b9065
This commit is contained in:
parent
dbcc03776d
commit
d55ee9fb6e
|
@ -17,18 +17,26 @@ import copy
|
||||||
import re
|
import re
|
||||||
import string
|
import string
|
||||||
|
|
||||||
|
from beaker.cache import CacheManager
|
||||||
|
from beaker.util import parse_cache_config_options
|
||||||
import jsonpath_ng
|
import jsonpath_ng
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
from oslo_utils import excutils
|
from oslo_utils import excutils
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
from deckhand.conf import config
|
||||||
from deckhand import errors
|
from deckhand import errors
|
||||||
|
|
||||||
|
CONF = config.CONF
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Cache for JSON paths computed from path strings because jsonpath_ng
|
# Cache for JSON paths computed from path strings because jsonpath_ng
|
||||||
# is computationally expensive.
|
# is computationally expensive.
|
||||||
_PATH_CACHE = dict()
|
_CACHE_OPTS = {
|
||||||
|
'cache.type': 'memory',
|
||||||
|
'expire': CONF.jsonpath.cache_timeout,
|
||||||
|
}
|
||||||
|
_CACHE = CacheManager(**parse_cache_config_options(_CACHE_OPTS))
|
||||||
|
|
||||||
_ARRAY_RE = re.compile(r'.*\[\d+\].*')
|
_ARRAY_RE = re.compile(r'.*\[\d+\].*')
|
||||||
|
|
||||||
|
@ -54,17 +62,13 @@ def _normalize_jsonpath(jsonpath):
|
||||||
return jsonpath
|
return jsonpath
|
||||||
|
|
||||||
|
|
||||||
def _jsonpath_parse_cache(jsonpath):
|
@_CACHE.cache()
|
||||||
|
def _jsonpath_parse(jsonpath):
|
||||||
"""Retrieve the parsed jsonpath path
|
"""Retrieve the parsed jsonpath path
|
||||||
|
|
||||||
Utilizes a cache of parsed values to eliminate re-parsing
|
Utilizes a cache of parsed values to eliminate re-parsing
|
||||||
"""
|
"""
|
||||||
if jsonpath not in _PATH_CACHE:
|
return jsonpath_ng.parse(jsonpath)
|
||||||
p = jsonpath_ng.parse(jsonpath)
|
|
||||||
_PATH_CACHE[jsonpath] = p
|
|
||||||
else:
|
|
||||||
p = _PATH_CACHE[jsonpath]
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def jsonpath_parse(data, jsonpath, match_all=False):
|
def jsonpath_parse(data, jsonpath, match_all=False):
|
||||||
|
@ -96,7 +100,7 @@ def jsonpath_parse(data, jsonpath, match_all=False):
|
||||||
# Do something with the extracted secret from the source document.
|
# Do something with the extracted secret from the source document.
|
||||||
"""
|
"""
|
||||||
jsonpath = _normalize_jsonpath(jsonpath)
|
jsonpath = _normalize_jsonpath(jsonpath)
|
||||||
p = _jsonpath_parse_cache(jsonpath)
|
p = _jsonpath_parse(jsonpath)
|
||||||
|
|
||||||
matches = p.find(data)
|
matches = p.find(data)
|
||||||
if matches:
|
if matches:
|
||||||
|
@ -179,7 +183,7 @@ def jsonpath_replace(data, value, jsonpath, pattern=None):
|
||||||
'or "$"' % jsonpath)
|
'or "$"' % jsonpath)
|
||||||
|
|
||||||
def _do_replace():
|
def _do_replace():
|
||||||
p = _jsonpath_parse_cache(jsonpath)
|
p = _jsonpath_parse(jsonpath)
|
||||||
p_to_change = p.find(data)
|
p_to_change = p.find(data)
|
||||||
|
|
||||||
if p_to_change:
|
if p_to_change:
|
||||||
|
|
|
@ -21,9 +21,8 @@ CONF = cfg.CONF
|
||||||
barbican_group = cfg.OptGroup(
|
barbican_group = cfg.OptGroup(
|
||||||
name='barbican',
|
name='barbican',
|
||||||
title='Barbican Options',
|
title='Barbican Options',
|
||||||
help="""
|
help="Barbican options for allowing Deckhand to communicate with "
|
||||||
Barbican options for allowing Deckhand to communicate with Barbican.
|
"Barbican.")
|
||||||
""")
|
|
||||||
|
|
||||||
barbican_opts = [
|
barbican_opts = [
|
||||||
# TODO(fmontei): Drop these options and related group once Keystone
|
# TODO(fmontei): Drop these options and related group once Keystone
|
||||||
|
@ -35,6 +34,19 @@ barbican_opts = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
jsonpath_group = cfg.OptGroup(
|
||||||
|
name='jsonpath',
|
||||||
|
title='JSONPath Options',
|
||||||
|
help="JSONPath options for allowing JSONPath logic to be configured.")
|
||||||
|
|
||||||
|
|
||||||
|
jsonpath_opts = [
|
||||||
|
cfg.IntOpt('cache_timeout', default=3600,
|
||||||
|
help="How long JSONPath lookup results should remain cached "
|
||||||
|
"in memory.")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
default_opts = [
|
default_opts = [
|
||||||
cfg.BoolOpt('profiler', default=False,
|
cfg.BoolOpt('profiler', default=False,
|
||||||
help="Enables profiling of API requests. Do NOT use in "
|
help="Enables profiling of API requests. Do NOT use in "
|
||||||
|
@ -48,6 +60,7 @@ default_opts = [
|
||||||
def register_opts(conf):
|
def register_opts(conf):
|
||||||
conf.register_group(barbican_group)
|
conf.register_group(barbican_group)
|
||||||
conf.register_opts(barbican_opts, group=barbican_group)
|
conf.register_opts(barbican_opts, group=barbican_group)
|
||||||
|
conf.register_opts(jsonpath_opts, group=jsonpath_group)
|
||||||
conf.register_opts(default_opts)
|
conf.register_opts(default_opts)
|
||||||
ks_loading.register_auth_conf_options(conf, group='keystone_authtoken')
|
ks_loading.register_auth_conf_options(conf, group='keystone_authtoken')
|
||||||
ks_loading.register_auth_conf_options(conf, group=barbican_group.name)
|
ks_loading.register_auth_conf_options(conf, group=barbican_group.name)
|
||||||
|
@ -68,7 +81,8 @@ def list_opts():
|
||||||
ks_loading.get_session_conf_options() +
|
ks_loading.get_session_conf_options() +
|
||||||
ks_loading.get_auth_common_conf_options() +
|
ks_loading.get_auth_common_conf_options() +
|
||||||
ks_loading.get_auth_plugin_conf_options('v3password')
|
ks_loading.get_auth_plugin_conf_options('v3password')
|
||||||
)
|
),
|
||||||
|
jsonpath_group: jsonpath_opts
|
||||||
}
|
}
|
||||||
return opts
|
return opts
|
||||||
|
|
||||||
|
|
|
@ -12,11 +12,18 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import jsonpath_ng
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from testtools.matchers import Equals
|
||||||
|
from testtools.matchers import MatchesAny
|
||||||
|
|
||||||
from deckhand.common import utils
|
from deckhand.common import utils
|
||||||
from deckhand.tests.unit import base as test_base
|
from deckhand.tests.unit import base as test_base
|
||||||
|
|
||||||
|
|
||||||
class TestUtils(test_base.DeckhandTestCase):
|
class TestJSONPathReplace(test_base.DeckhandTestCase):
|
||||||
|
"""Validate that JSONPath replace function works."""
|
||||||
|
|
||||||
def test_jsonpath_replace_creates_object(self):
|
def test_jsonpath_replace_creates_object(self):
|
||||||
path = ".values.endpoints.admin"
|
path = ".values.endpoints.admin"
|
||||||
|
@ -40,3 +47,43 @@ class TestUtils(test_base.DeckhandTestCase):
|
||||||
expected = {'values': {'endpoints0': {'admin': 'foo'}}}
|
expected = {'values': {'endpoints0': {'admin': 'foo'}}}
|
||||||
result = utils.jsonpath_replace({}, 'foo', path)
|
result = utils.jsonpath_replace({}, 'foo', path)
|
||||||
self.assertEqual(expected, result)
|
self.assertEqual(expected, result)
|
||||||
|
|
||||||
|
|
||||||
|
class TestJSONPathUtilsCaching(test_base.DeckhandTestCase):
|
||||||
|
"""Validate that JSONPath caching works."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestJSONPathUtilsCaching, self).setUp()
|
||||||
|
self.jsonpath_call_count = 0
|
||||||
|
|
||||||
|
def fake_parse(value):
|
||||||
|
self.jsonpath_call_count += 1
|
||||||
|
return jsonpath_ng.parse(value)
|
||||||
|
|
||||||
|
self.fake_jsonpath_ng = fake_parse
|
||||||
|
|
||||||
|
def test_jsonpath_parse_replace_cache(self):
|
||||||
|
"""Validate caching for both parsing and replacing functions."""
|
||||||
|
path = ".values.endpoints.admin"
|
||||||
|
expected = {'values': {'endpoints': {'admin': 'foo'}}}
|
||||||
|
|
||||||
|
# Mock jsonpath_ng to return a monkey-patched parse function that
|
||||||
|
# keeps track of call count and yet calls the actual function.
|
||||||
|
with mock.patch.object(utils, 'jsonpath_ng', # noqa: H210
|
||||||
|
parse=self.fake_jsonpath_ng):
|
||||||
|
# Though this is called 3 times, the cached function should only
|
||||||
|
# be called once, with the cache returning the cached value early.
|
||||||
|
for _ in range(3):
|
||||||
|
result = utils.jsonpath_replace({}, 'foo', path)
|
||||||
|
self.assertEqual(expected, result)
|
||||||
|
|
||||||
|
# Though this is called 3 times, the cached function should only
|
||||||
|
# be called once, with the cache returning the cached value early.
|
||||||
|
for _ in range(3):
|
||||||
|
result = utils.jsonpath_parse(expected, path)
|
||||||
|
self.assertEqual('foo', result)
|
||||||
|
|
||||||
|
# Assert that the actual function was called <= 1 times. (Allow for 0
|
||||||
|
# in case CI jobs clash.)
|
||||||
|
self.assertThat(
|
||||||
|
self.jsonpath_call_count, MatchesAny(Equals(0), Equals(1)))
|
||||||
|
|
|
@ -23,6 +23,7 @@ psycopg2==2.7.4
|
||||||
uwsgi==2.0.17
|
uwsgi==2.0.17
|
||||||
jsonpath-ng==1.4.3
|
jsonpath-ng==1.4.3
|
||||||
jsonschema==2.6.0
|
jsonschema==2.6.0
|
||||||
|
beaker==1.9.1
|
||||||
|
|
||||||
oslo.cache>=1.30.1 # Apache-2.0
|
oslo.cache>=1.30.1 # Apache-2.0
|
||||||
oslo.concurrency>=3.27.0 # Apache-2.0
|
oslo.concurrency>=3.27.0 # Apache-2.0
|
||||||
|
|
Loading…
Reference in New Issue