Shipyard API Client/CLI - Airflow Logs Retrieval

Change-Id: I7e93c6cd5a4922cb33eaf16d32b0af579d3b9e1c

shipyard_client/api_client/shipyard_api_client.py

@@ -1,17 +1,16 @@
-# -*- coding: utf-8 -*-
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import enum
 import json
 
@@ -33,6 +32,7 @@ class ApiPaths(enum.Enum):
     GET_ACTION_DETAIL = _BASE_URL + 'actions/{}'
     GET_VALIDATION_DETAIL = _BASE_URL + 'actions/{}/validationdetails/{}'
     GET_STEP_DETAIL = _BASE_URL + 'actions/{}/steps/{}'
+    GET_STEP_LOG = _BASE_URL + 'actions/{}/steps/{}/logs'
     POST_CONTROL_ACTION = _BASE_URL + 'actions/{}/control/{}'
     GET_WORKFLOWS = _BASE_URL + 'workflows'
     GET_DAG_DETAIL = _BASE_URL + 'workflows/{}'
@@ -177,6 +177,31 @@ class ShipyardClient(BaseClient):
         )
         return self.get_resp(url)
 
+    def get_step_log(self, action_id=None, step_id=None, try_number=None):
+        """
+        Retrieve logs for a particular step
+        :param str action_id: Unique action id
+        :param str step_id: step id
+        :param int try_number: The logs that user wants to retrieve. Note that
+                               a workflow can retry multiple times with the
+                               names of the logs as 1.log, 2.log, 3.log, etc.
+                               Logs from the last attempt will be returned if
+                               'try' is not specified.
+        :returns: Logs for the step
+        :rtype: Response object
+        """
+        if try_number:
+            query_params = {'try': try_number}
+        else:
+            query_params = {}
+
+        url = ApiPaths.GET_STEP_LOG.value.format(
+            self.get_endpoint(),
+            action_id,
+            step_id
+        )
+        return self.get_resp(url, query_params)
+
     def post_control_action(self, action_id=None, control_verb=None):
         """
         Allows for issuing DAG controls against an action.

shipyard_client/cli/commands.py

@@ -23,6 +23,7 @@ from .create import commands as create
 from .describe import commands as describe
 from .get import commands as get
 from .help import commands as help
+from .logs import commands as logs
 from shipyard_client.cli.input_checks import check_control_action, check_id
 
 
@@ -110,6 +111,7 @@ shipyard.add_command(create.create)
 shipyard.add_command(describe.describe)
 shipyard.add_command(get.get)
 shipyard.add_command(help.help)
+shipyard.add_command(logs.logs)
 
 
 # To Invoke Control Commands

shipyard_client/cli/help/commands.py

@@ -16,7 +16,7 @@
 
 import click
 
-from shipyard_client.cli.help.output import default, actions, configdocs
+from shipyard_client.cli.help.output import actions, configdocs, default, logs
 
 
 @click.group()
@@ -44,6 +44,8 @@ def help(ctx, topic=None):
         click.echo(actions())
     elif topic == 'configdocs':
         click.echo(configdocs())
+    elif topic == 'logs':
+        click.echo(logs())
     else:
         ctx.fail("Invalid topic. Run command 'shipyard help' for a list of "
                  " available topics.")

shipyard_client/cli/help/output.py

@@ -54,3 +54,11 @@ Supported Commands:
     shipyard commit configdocs
     shipyard create configdocs
     shipyard get configdocs'''
+
+
+def logs():
+    return '''LOGS
+Allows users to query and view logs using Shipyard
+
+Supported Commands:
+    shipyard logs step'''

shipyard_client/cli/logs/actions.py

@@ -0,0 +1,54 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from shipyard_client.cli.action import CliAction
+from shipyard_client.cli import format_utils
+
+
+class LogsStep(CliAction):
+    """Action to Retrieve Logs for a particular Step"""
+
+    def __init__(self, ctx, action_id, step_id, try_number=None):
+        """Sets parameters."""
+        super().__init__(ctx)
+        self.logger.debug(
+            "LogsStep action initialized with action_id=%s and step_id=%s",
+            action_id, step_id)
+        self.action_id = action_id
+        self.step_id = step_id
+        self.try_number = try_number
+
+    def invoke(self):
+        """Calls API Client and formats response from API Client"""
+        self.logger.debug("Calling API Client get_step_log.")
+        return self.get_api_client().get_step_log(
+            action_id=self.action_id,
+            step_id=self.step_id,
+            try_number=self.try_number)
+
+    # Handle 404 with default error handler for cli.
+    cli_handled_err_resp_codes = [404]
+
+    # Handle 200 responses using the cli_format_response_handler
+    cli_handled_succ_resp_codes = [200]
+
+    def cli_format_response_handler(self, response):
+        """CLI output handler
+
+        Effectively passes through the logs received.
+        :param response: a requests response object
+        :returns: a string representing a CLI appropriate response
+            Handles 200 responses
+        """
+        return format_utils.raw_format_response_handler(response)

shipyard_client/cli/logs/commands.py

@@ -0,0 +1,96 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Logs command
+
+import click
+
+from click_default_group import DefaultGroup
+
+from shipyard_client.cli.logs.actions import LogsStep
+
+
+@click.group(cls=DefaultGroup, default='logs_default_command')
+@click.pass_context
+def logs(ctx):
+    """
+    Get the logs of a particular step.
+    For more information on logs commands
+    please enter the logs command followed by '--help'
+    Example: shipyard logs step --help
+
+    FOR NAMESPACE ENTRIES:
+        COMMAND: (no sub command)
+        DESCRIPTION: Retrieves the logs of the supplied namespaced item.
+        FORMAT: shipyard logs <namespace item>
+        EXAMPLE: shipyard logs step/01CASSSZT7CP1F0NKHCAJBCJGR/action_xcom
+    """
+
+
+@logs.command('logs_default_command', short_help="")
+@click.argument('namespace_item')
+@click.pass_context
+def logs_default_command(ctx, namespace_item):
+    try:
+        namespace = namespace_item.split("/")
+        if namespace[0] == 'step':
+            if len(namespace) == 4:
+                ctx.invoke(logs_step,
+                           action=namespace[1],
+                           step_id=namespace[2],
+                           try_number=namespace[3])
+            else:
+                ctx.invoke(logs_step,
+                           action=namespace[1],
+                           step_id=namespace[2],
+                           try_number=None)
+        else:
+            raise Exception('Invalid namespaced logs action')
+    except Exception:
+        ctx.fail("Invalid namespace item. Please utilize the following "
+                 "format for the namespace item."
+                 "step: step/action_id/step_id")
+
+
+LOGS_STEP = """
+COMMAND: logs step
+DESCRIPTION: Retrieves logs for a particular step.
+FORMAT: shipyard logs step <step_id> --action=<action_id> --try=<try>
+EXAMPLE:
+shipyard logs step drydock_validate_site_design
+ --action=01C7ECDZF7MC8JEVE8NA8PS764 --try=2
+"""
+
+SHORT_LOGS_STEP = ("Retrieve logs for a particular step.")
+
+
+@logs.command('step', help=LOGS_STEP, short_help=SHORT_LOGS_STEP)
+@click.argument('step_id', nargs=1)
+@click.option(
+    '--action',
+    '-a',
+    required=True,
+    help='The action name for this step')
+@click.option(
+    '--try',
+    '-t',
+    'try_number',
+    help='The try number that provides the context for this step')
+@click.pass_context
+def logs_step(ctx, action, step_id, try_number=None):
+
+    click.echo(LogsStep(ctx,
+                        action,
+                        step_id,
+                        try_number).invoke_and_return_resp())

shipyard_client/tests/unit/apiclient_test/test_shipyard_api_client.py

@@ -219,3 +219,18 @@ def test_get_dag_details(*args):
     result = shipyard_client.get_dag_detail(workflow_id)
     assert result['url'] == '{}/workflows/{}'.format(
         shipyard_client.get_endpoint(), workflow_id)
+
+
+@mock.patch.object(BaseClient, 'post_resp', replace_post_rep)
+@mock.patch.object(BaseClient, 'get_resp', replace_get_resp)
+@mock.patch.object(BaseClient, 'get_endpoint', replace_get_endpoint)
+def test_get_step_log(*args):
+    shipyard_client = get_api_client()
+    action_id = '01C9VVQSCFS7V9QB5GBS3WFVSE'
+    step_id = 'action_xcom'
+    try_number = '2'
+    result = shipyard_client.get_step_log(action_id, step_id, try_number)
+    params = result['params']
+    assert result['url'] == '{}/actions/{}/steps/{}/logs'.format(
+        shipyard_client.get_endpoint(), action_id, step_id)
+    assert params['try'] == try_number