ValidationPolicy integration with Validations API
This PS integrates ValidationPolicy logic with the Deckhand Validations API. Support for multiple ValidationPolicy documents is included. If a ValidationPolicy is found, then the validations contained therein are used to determine whether a revision is successful or not. For example, if a VP contains 'promenade-schema-validation' then DH will return success if the externally registered validation result for that validation is success. However, if the result was never registered in DH then the returned result is 'failure'. In addition, if "extra" validations are registered (that is validations not present in any VP) then they are effectively ignored. An error message is added with enough details to indicate why the validation is ignored. This PS adds unit tests to verify the correct behavior for the above scenarios. Functional tests and documentation changes will be added in a follow up once design is ironed out. Change-Id: I44c657974589ea3563e0a23ad667894329048b46
This commit is contained in:
Felipe Monteiro
Scott Hussey
parent
65c459d1f9
commit
d82d0cfaf7
|
|
@ -1008,6 +1008,23 @@ def revision_rollback(revision_id, latest_revision, session=None):
|
|||
####################
|
||||
|
||||
|
||||
def _get_validation_policies_for_revision(revision_id, session=None):
|
||||
session = session or get_session()
|
||||
|
||||
# Check if a ValidationPolicy for the revision exists.
|
||||
validation_policies = document_get_all(
|
||||
session, revision_id=revision_id, deleted=False,
|
||||
schema=types.VALIDATION_POLICY_SCHEMA)
|
||||
if not validation_policies:
|
||||
# Otherwise return early.
|
||||
LOG.info('Failed to find a ValidationPolicy for revision ID %s.'
|
||||
'Only the "%s" results will be included in the response.',
|
||||
revision_id, types.DECKHAND_SCHEMA_VALIDATION)
|
||||
validation_policies = []
|
||||
|
||||
return validation_policies
|
||||
|
||||
|
||||
@require_revision_exists
|
||||
def validation_create(revision_id, val_name, val_data, session=None):
|
||||
session = session or get_session()
|
||||
|
|
@ -1037,6 +1054,7 @@ def validation_get_all(revision_id, session=None):
|
|||
# has its own validation but for this query we want to return the result
|
||||
# of the overall validation for the revision. If just 1 document failed
|
||||
# validation, we regard the validation for the whole revision as 'failure'.
|
||||
session = session or get_session()
|
||||
|
||||
query = raw_query("""
|
||||
SELECT DISTINCT name, status FROM validations as v1
|
||||
|
|
@ -1050,8 +1068,35 @@ def validation_get_all(revision_id, session=None):
|
|||
ORDER BY name, status;
|
||||
""", revision_id=revision_id)
|
||||
|
||||
result = query.fetchall()
|
||||
return result
|
||||
result = {v[0]: v for v in query.fetchall()}
|
||||
actual_validations = set(v[0] for v in result.values())
|
||||
|
||||
validation_policies = _get_validation_policies_for_revision(revision_id)
|
||||
if not validation_policies:
|
||||
return result.values()
|
||||
|
||||
# TODO(fmontei): Raise error for expiresAfter conflicts for duplicate
|
||||
# validations across ValidationPolicy documents.
|
||||
expected_validations = set()
|
||||
for vp in validation_policies:
|
||||
expected_validations = expected_validations.union(
|
||||
list(v['name'] for v in vp['data'].get('validations', [])))
|
||||
|
||||
missing_validations = expected_validations - actual_validations
|
||||
extra_validations = actual_validations - expected_validations
|
||||
|
||||
# If an entry in the ValidationPolicy was never POSTed, set its status
|
||||
# to failure.
|
||||
for missing_validation in missing_validations:
|
||||
result[missing_validation] = (missing_validation, 'failure')
|
||||
|
||||
# If an entry is not in the ValidationPolicy but was externally registered,
|
||||
# then override its status to "ignored [{original_status}]".
|
||||
for extra_validation in extra_validations:
|
||||
result[extra_validation] = (
|
||||
extra_validation, 'ignored [%s]' % result[extra_validation][1])
|
||||
|
||||
return result.values()
|
||||
|
||||
|
||||
@require_revision_exists
|
||||
|
|
@ -1062,15 +1107,75 @@ def validation_get_all_entries(revision_id, val_name, session=None):
|
|||
.filter_by(**{'revision_id': revision_id, 'name': val_name})\
|
||||
.order_by(models.Validation.created_at.asc())\
|
||||
.all()
|
||||
result = [e.to_dict() for e in entries]
|
||||
result_map = {}
|
||||
for r in result:
|
||||
result_map.setdefault(r['name'], [])
|
||||
result_map[r['name']].append(r)
|
||||
actual_validations = set(v['name'] for v in result)
|
||||
|
||||
return [e.to_dict() for e in entries]
|
||||
validation_policies = _get_validation_policies_for_revision(revision_id)
|
||||
if not validation_policies:
|
||||
return result
|
||||
|
||||
# TODO(fmontei): Raise error for expiresAfter conflicts for duplicate
|
||||
# validations across ValidationPolicy documents.
|
||||
expected_validations = set()
|
||||
for vp in validation_policies:
|
||||
expected_validations |= set(
|
||||
v['name'] for v in vp['data'].get('validations', []))
|
||||
|
||||
missing_validations = expected_validations - actual_validations
|
||||
extra_validations = actual_validations - expected_validations
|
||||
|
||||
# If an entry in the ValidationPolicy was never POSTed, set its status
|
||||
# to failure.
|
||||
for missing_name in missing_validations:
|
||||
if missing_name == val_name:
|
||||
result.append({
|
||||
'id': len(result),
|
||||
'name': val_name,
|
||||
'status': 'failure',
|
||||
'errors': [{
|
||||
'message': 'The result for this validation was never '
|
||||
'externally registered so its status defaulted '
|
||||
'to "failure".'
|
||||
}]
|
||||
})
|
||||
break
|
||||
|
||||
# If an entry is not in the ValidationPolicy but was externally registered,
|
||||
# then override its status to "ignored [{original_status}]".
|
||||
for extra_name in extra_validations:
|
||||
for entry in result_map[extra_name]:
|
||||
original_status = entry['status']
|
||||
entry['status'] = 'ignored [%s]' % original_status
|
||||
|
||||
entry.setdefault('errors', [])
|
||||
for vp in validation_policies:
|
||||
entry['errors'].append({
|
||||
'message': (
|
||||
'The result for this validation was externally '
|
||||
'registered but has been ignored because it is not '
|
||||
'found in the validations for ValidationPolicy [%s]'
|
||||
' %s: %s.' % (
|
||||
vp['schema'], vp['metadata']['name'],
|
||||
', '.join(v['name'] for v in vp['data'].get(
|
||||
'validations', []))
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_revision_exists
|
||||
def validation_get_entry(revision_id, val_name, entry_id, session=None):
|
||||
session = session or get_session()
|
||||
|
||||
entries = validation_get_all_entries(
|
||||
revision_id, val_name, session=session)
|
||||
|
||||
try:
|
||||
return entries[entry_id]
|
||||
except IndexError:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import os
|
||||
import yaml
|
||||
|
||||
import mock
|
||||
|
|
@ -47,11 +46,7 @@ validator:
|
|||
VALIDATION_RESULT_ALT = """
|
||||
---
|
||||
status: success
|
||||
errors:
|
||||
- documents:
|
||||
- schema: promenade/Slaves/v1
|
||||
name: kubernetes-slaves
|
||||
message: No slave nodes found.
|
||||
errors: []
|
||||
validator:
|
||||
name: promenade
|
||||
version: 1.1.2
|
||||
|
|
@ -85,21 +80,6 @@ class ValidationsControllerBaseTest(test_base.BaseControllerTest):
|
|||
headers={'Content-Type': 'application/x-yaml'}, body=policy)
|
||||
return resp
|
||||
|
||||
|
||||
class TestValidationsControllerPostValidate(ValidationsControllerBaseTest):
|
||||
"""Test suite for validating positive scenarios for post-validations with
|
||||
Validations controller.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestValidationsControllerPostValidate, self).setUp()
|
||||
dataschema_schema = os.path.join(
|
||||
os.getcwd(), 'deckhand', 'engine', 'schemas',
|
||||
'dataschema_schema.yaml')
|
||||
with open(dataschema_schema, 'r') as f:
|
||||
self.dataschema_schema = yaml.safe_load(f.read())
|
||||
self._monkey_patch_document_validation()
|
||||
|
||||
def _monkey_patch_document_validation(self):
|
||||
"""Workaround for testing complex validation scenarios by forcibly
|
||||
passing in `pre_validate=False`.
|
||||
|
|
@ -116,6 +96,16 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest):
|
|||
side_effect=monkey_patch, autospec=True).start()
|
||||
self.addCleanup(mock.patch.stopall)
|
||||
|
||||
|
||||
class TestValidationsControllerPostValidate(ValidationsControllerBaseTest):
|
||||
"""Test suite for validating positive scenarios for post-validations with
|
||||
Validations controller.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestValidationsControllerPostValidate, self).setUp()
|
||||
self._monkey_patch_document_validation()
|
||||
|
||||
def test_create_validation(self):
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_validation': '@'}
|
||||
|
|
@ -192,6 +182,7 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest):
|
|||
}
|
||||
]
|
||||
}
|
||||
body['results'] = sorted(body['results'], key=lambda x: x['name'])
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
def test_list_validation_entries(self):
|
||||
|
|
@ -829,6 +820,325 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest):
|
|||
self.assertEqual(expected_body, body)
|
||||
|
||||
|
||||
class TestValidationsControllerWithValidationPolicy(
|
||||
ValidationsControllerBaseTest):
|
||||
|
||||
def setUp(self):
|
||||
super(TestValidationsControllerWithValidationPolicy, self).setUp()
|
||||
self._monkey_patch_document_validation()
|
||||
|
||||
def test_validation_with_validation_policy_success(self):
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:list_validations': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a `ValidationPolicy` which is used to check whether a revision
|
||||
# passed all the validations.
|
||||
validation_policy = yaml.safe_load("""
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: site-deploy-ready
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: deckhand-schema-validation
|
||||
...
|
||||
""")
|
||||
revision_id = self._create_revision(payload=[validation_policy])
|
||||
|
||||
# Validate that the validation was created and reports success.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 1,
|
||||
'results': [
|
||||
{'name': 'deckhand-schema-validation', 'status': 'success'}
|
||||
]
|
||||
}
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
def test_with_validation_policy_external_validation(self):
|
||||
"""Validate that a ValidationPolicy with an externally registered
|
||||
validation that is successful passes.
|
||||
"""
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_validation': '@',
|
||||
'deckhand:list_validations': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a `ValidationPolicy` which expects two validations.
|
||||
validation_policy = yaml.safe_load("""
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: site-deploy-ready
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: deckhand-schema-validation
|
||||
- name: promenade-schema-validation
|
||||
...
|
||||
""")
|
||||
revision_id = self._create_revision(payload=[validation_policy])
|
||||
|
||||
# Create the external validation for "promenade-schema-validation".
|
||||
resp = self._create_validation(
|
||||
revision_id, 'promenade-schema-validation', VALIDATION_RESULT_ALT)
|
||||
self.assertEqual(201, resp.status_code)
|
||||
|
||||
# Validate that the validation was created and reports success.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 2,
|
||||
'results': [
|
||||
{'name': 'deckhand-schema-validation', 'status': 'success'},
|
||||
{'name': 'promenade-schema-validation', 'status': 'success'}
|
||||
]
|
||||
}
|
||||
body['results'] = sorted(body['results'], key=lambda x: x['name'])
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
def test_with_multiple_validation_policy_external_validation(self):
|
||||
"""Validate that two ValidationPolicy documents, one that references
|
||||
the internal deckhand-schema-validation, and the other which requires
|
||||
an externally registered validation, produces a successful validation
|
||||
result.
|
||||
"""
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_validation': '@',
|
||||
'deckhand:list_validations': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create two `ValidationPolicy` documents.
|
||||
validation_policies = yaml.safe_load_all("""
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: vp-1
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: deckhand-schema-validation
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: vp-2
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: promenade-schema-validation
|
||||
...
|
||||
""")
|
||||
revision_id = self._create_revision(payload=validation_policies)
|
||||
|
||||
# Create the external validation for "promenade-schema-validation".
|
||||
resp = self._create_validation(
|
||||
revision_id, 'promenade-schema-validation', VALIDATION_RESULT_ALT)
|
||||
self.assertEqual(201, resp.status_code)
|
||||
|
||||
# Validate that the validation was created and reports success.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 2,
|
||||
'results': [
|
||||
{'name': 'deckhand-schema-validation', 'status': 'success'},
|
||||
{'name': 'promenade-schema-validation', 'status': 'success'}
|
||||
]
|
||||
}
|
||||
body['results'] = sorted(body['results'], key=lambda x: x['name'])
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
def test_with_validation_policy_missing_external_validation(self):
|
||||
"""Validate that a ValidationPolicy with a missing externally
|
||||
registered validation that is listed under the validations for the
|
||||
ValidationPolicy defaults to "failure".
|
||||
"""
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:list_validations': '@',
|
||||
'deckhand:show_validation': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a `ValidationPolicy` which expects two validations but do not
|
||||
# create the validation for "promenade-schema-validation".
|
||||
validation_policy = yaml.safe_load("""
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: site-deploy-ready
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: deckhand-schema-validation
|
||||
- name: promenade-schema-validation
|
||||
...
|
||||
""")
|
||||
revision_id = self._create_revision(payload=[validation_policy])
|
||||
|
||||
# Validate that the validation was created and that the missing one
|
||||
# defaults to "failure".
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 2,
|
||||
'results': [
|
||||
{'name': 'deckhand-schema-validation', 'status': 'success'},
|
||||
{'name': 'promenade-schema-validation', 'status': 'failure'}
|
||||
]
|
||||
}
|
||||
body['results'] = sorted(body['results'], key=lambda x: x['name'])
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
# Validate that 'promenade-schema-validation' is 'failure' even though
|
||||
# it was never externally registered.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations/%s' % (
|
||||
revision_id, 'promenade-schema-validation'),
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 1,
|
||||
'results': [{'id': 0, 'status': 'failure'}]
|
||||
}
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
# Validate information explaining why 'promenade-schema-validation'
|
||||
# failed is returned. Note that DH should be smart enough to say that
|
||||
# it was never registered externally, which is why it's 'failure'.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations/%s/entries/0' % (
|
||||
revision_id, 'promenade-schema-validation'),
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
|
||||
expected_msg = ('The result for this validation was never externally '
|
||||
'registered so its status defaulted to "failure".')
|
||||
expected_body = {
|
||||
'name': 'promenade-schema-validation',
|
||||
'status': 'failure',
|
||||
'createdAt': None,
|
||||
'expiresAfter': None,
|
||||
'errors': [{'message': expected_msg}]
|
||||
}
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
def test_with_validation_policy_extra_external_validation(self):
|
||||
"""Validate that a ValidationPolicy with extra externally registered
|
||||
validations that aren't listed under the validations for the
|
||||
ValidationPolicy defaults to "ignored [{original_status}]".
|
||||
"""
|
||||
rules = {'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_validation': '@',
|
||||
'deckhand:list_validations': '@',
|
||||
'deckhand:show_validation': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a `ValidationPolicy` with only 1 validation.
|
||||
validation_policy = yaml.safe_load("""
|
||||
---
|
||||
schema: deckhand/ValidationPolicy/v1
|
||||
metadata:
|
||||
schema: metadata/Control/v1
|
||||
name: site-deploy-ready
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
data:
|
||||
validations:
|
||||
- name: deckhand-schema-validation
|
||||
...
|
||||
""")
|
||||
revision_id = self._create_revision(payload=[validation_policy])
|
||||
|
||||
# Register an extra validation not in the ValidationPolicy.
|
||||
resp = self._create_validation(
|
||||
revision_id, 'promenade-schema-validation', VALIDATION_RESULT)
|
||||
self.assertEqual(201, resp.status_code)
|
||||
|
||||
# Validate that the extra validation is ignored.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 2,
|
||||
'results': [
|
||||
{'name': 'deckhand-schema-validation', 'status': 'success'},
|
||||
{'name': 'promenade-schema-validation',
|
||||
'status': 'ignored [failure]'}
|
||||
]
|
||||
}
|
||||
body['results'] = sorted(body['results'], key=lambda x: x['name'])
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
# Validate that 'promenade-schema-validation' is 'ignored [failure]'
|
||||
# even though it was externally registered.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations/%s' % (
|
||||
revision_id, 'promenade-schema-validation'),
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
expected_body = {
|
||||
'count': 1,
|
||||
'results': [{'id': 0, 'status': 'ignored [failure]'}]
|
||||
}
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
# Validate information explaining why 'promenade-schema-validation'
|
||||
# is ignored is returned.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/validations/%s/entries/0' % (
|
||||
revision_id, 'promenade-schema-validation'),
|
||||
headers={'Content-Type': 'application/x-yaml'})
|
||||
self.assertEqual(200, resp.status_code)
|
||||
body = yaml.safe_load(resp.text)
|
||||
|
||||
expected_msg = ('The result for this validation was externally '
|
||||
'registered but has been ignored because it is not '
|
||||
'found in the validations for ValidationPolicy [%s] '
|
||||
'%s: %s.' % (validation_policy['schema'],
|
||||
validation_policy['metadata']['name'],
|
||||
types.DECKHAND_SCHEMA_VALIDATION))
|
||||
expected_errors = yaml.safe_load(VALIDATION_RESULT)['errors']
|
||||
expected_errors.append({'message': expected_msg})
|
||||
|
||||
expected_body = {
|
||||
'name': 'promenade-schema-validation',
|
||||
'status': 'ignored [failure]',
|
||||
'createdAt': None,
|
||||
'expiresAfter': None,
|
||||
'errors': expected_errors
|
||||
}
|
||||
self.assertEqual(expected_body, body)
|
||||
|
||||
|
||||
class TestValidationsControllerPreValidate(ValidationsControllerBaseTest):
|
||||
"""Test suite for validating positive scenarios for pre-validations with
|
||||
Validations controller.
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ variable via ``PIFPAF_URL`` which is referenced by Deckhand's unit test suite.
|
|||
Overview
|
||||
--------
|
||||
|
||||
Unit testing currently uses an in-memory sqlite SQLite. Since Deckhand's
|
||||
Unit testing currently uses an in-memory SQLite database. Since Deckhand's
|
||||
primary function is to serve as the back-end storage for UCP, the majority
|
||||
of unit tests perform actual database operations. Mocking is used sparingly
|
||||
because Deckhand is a fairly insular application that lives at the bottom
|
||||
|
|
|
|||
Loading…
Reference in New Issue