From c788621486e545910075e4d5f29c5207d68c95f2 Mon Sep 17 00:00:00 2001 From: Alexander Hughes Date: Wed, 24 Apr 2019 10:52:06 -0500 Subject: [PATCH] Allow buffer mode to be configured for upload Shipyard helper's upload documents method supports two default modes of buffer. None, and append. We want to allow the user to dictate which mode is used to support the other methods Shipyard itself has. To accomplish this we add a new command line argument, a new variable and leave existing behavior intact via the new default 'auto' mode Change-Id: I7a252efa7fe7a766152c42d9398c3290d7e52a13 --- doc/source/cli/cli.rst | 15 ++++++++++++++- pegleg/cli.py | 18 ++++++++++++++++-- pegleg/engine/exceptions.py | 10 ++++++++++ pegleg/engine/util/shipyard_helper.py | 20 +++++++++++++------- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/doc/source/cli/cli.rst b/doc/source/cli/cli.rst index 9e753930..0393af88 100644 --- a/doc/source/cli/cli.rst +++ b/doc/source/cli/cli.rst @@ -414,11 +414,24 @@ Please reference Shipyard's `CLI documentation`_ for information related to thes Specifies a UUID (8-4-4-4-12 format) that will be used to correlate logs, transactions, etc. in downstream activities triggered by this interaction. +**-b / --buffer-mode** (Optional). Default is auto. + +Set the buffer mode when uploading documents. Supported buffer modes +include append, replace, auto. + +append: Add the collection to the Shipyard Buffer, only if that +collection does not already exist in the Shipyard buffer. + +replace: Clear the Shipyard Buffer before adding the specified +collection. + +auto: Let Pegleg determine the appropriate buffer mode to use. + Usage: :: - ./pegleg.sh site upload --context-marker= + ./pegleg.sh site upload --context-marker= --buffer= Site Secrets Group ------------------ diff --git a/pegleg/cli.py b/pegleg/cli.py index 4fdd7661..28c61f8e 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -354,11 +354,25 @@ def lint_site(*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name): 'interaction ', required=False, type=click.UUID) +@click.option( + '-b', + '--buffer-mode', + 'buffer_mode', + required=False, + default='auto', + show_default=True, + help='Set the buffer mode when uploading documents. Supported buffer ' + 'modes include append, replace, auto.\n' + 'append: Add the collection to the Shipyard Buffer, only if that ' + 'collection does not already exist in the Shipyard buffer.\n' + 'replace: Clear the Shipyard Buffer before adding the specified ' + 'collection.\n' + 'auto: Let Pegleg determine the appropriate buffer mode to use.') @SITE_REPOSITORY_ARGUMENT @click.pass_context def upload(ctx, *, os_project_domain_name, os_user_domain_name, os_project_name, os_username, - os_password, os_auth_url, context_marker, site_name): + os_password, os_auth_url, context_marker, site_name, buffer_mode): if not ctx.obj: ctx.obj = {} @@ -378,7 +392,7 @@ def upload(ctx, *, os_project_domain_name, ctx.obj['context_marker'] = str(context_marker) ctx.obj['site_name'] = site_name - click.echo(ShipyardHelper(ctx).upload_documents()) + click.echo(ShipyardHelper(ctx, buffer_mode).upload_documents()) @site.group( diff --git a/pegleg/engine/exceptions.py b/pegleg/engine/exceptions.py index f424858c..6be2622b 100644 --- a/pegleg/engine/exceptions.py +++ b/pegleg/engine/exceptions.py @@ -132,3 +132,13 @@ class SaltInsufficientLengthException(PeglegBaseException): """Exception raised when salt is too short.""" message = 'PEGLEG_SALT must be at least 24 characters long.' + + +# +# Shipyard Helper Exceptions +# + +class InvalidBufferModeException(PeglegBaseException): + """Exception raised when invalid buffer mode specified""" + + message = 'BUFFER MODE must be one of: append, auto, replace' diff --git a/pegleg/engine/util/shipyard_helper.py b/pegleg/engine/util/shipyard_helper.py index 39f98b6f..f7537956 100644 --- a/pegleg/engine/util/shipyard_helper.py +++ b/pegleg/engine/util/shipyard_helper.py @@ -18,7 +18,7 @@ import uuid import yaml -from pegleg.engine.exceptions import PeglegBaseException +from pegleg.engine import exceptions from pegleg.engine.util import files from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement @@ -29,14 +29,14 @@ from shipyard_client.api_client.shipyardclient_context import \ LOG = logging.getLogger(__name__) -class AuthValuesError(PeglegBaseException): +class AuthValuesError(exceptions.PeglegBaseException): """Shipyard authentication failed. """ def __init__(self, *, diagnostic): self.diagnostic = diagnostic -class DocumentUploadError(PeglegBaseException): +class DocumentUploadError(exceptions.PeglegBaseException): """Exception occurs while uploading documents""" def __init__(self, message): @@ -52,7 +52,7 @@ class ShipyardHelper(object): 4. Formats response from Shipyard api_client """ - def __init__(self, context): + def __init__(self, context, buffer_mode='auto'): """ Initializes params to be used by Shipyard @@ -71,6 +71,7 @@ class ShipyardHelper(object): self.client_context = ShipyardClientContext( self.auth_vars, self.context_marker) self.api_client = ShipyardClient(self.client_context) + self.buffer_mode = buffer_mode def upload_documents(self): """Uploads documents to Shipyard """ @@ -82,10 +83,15 @@ class ShipyardHelper(object): # Append flag is not required for the first # collection being uploaded to Shipyard. It # is needed for subsequent collections. - if idx == 0: - buffer_mode = None + if self.buffer_mode == 'auto': + if idx == 0: + buffer_mode = None + else: + buffer_mode = 'append' + elif self.buffer_mode == 'append' or self.buffer_mode == 'replace': + buffer_mode = self.buffer_mode else: - buffer_mode = 'append' + raise exceptions.InvalidBufferModeException() # Decrypt the documents if encrypted pegleg_secret_mgmt = PeglegSecretManagement(