diff --git a/docs/source/API.rst b/docs/source/API.rst index 5c6c6d6c..7d823b8f 100644 --- a/docs/source/API.rst +++ b/docs/source/API.rst @@ -20,6 +20,34 @@ tasks API The Tasks API is used for creating and listing asynchronous tasks to be executed by the Drydock orchestrator. See :ref:`task` for details on creating tasks and field information. +nodes API +--------- + +GET nodes +^^^^^^^^^ + +The Nodes API will provide a report of current nodes as known by the node provisioner +and their status with a few hardware details. + +GET nodes/hostname/builddata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Get all the build data record for node ``hostname``. The response will be a list of +objects in the below form.:: + + { + "node_name": "hostname", + "generator": "description of how data was generated", + "collected_date": ios8601 UTC datestamp, + "task_id": "UUID of task initiating collection", + "data_format": "MIME-type of data_element", + "data_element": "Collected data" + } + +If the query parameter ``latest`` is passed with a value of ``true``, then only +the most recently collected data for each ``generator`` will be included in the +response. + bootdata -------- diff --git a/docs/source/task.rst b/docs/source/task.rst index 37c50cf4..1b1f0175 100644 --- a/docs/source/task.rst +++ b/docs/source/task.rst @@ -88,7 +88,7 @@ When querying the state of an existing task, the below document will be returned { "Kind": "Task", - "apiVersion": "v1", + "apiVersion": "v1.0", "task_id": "uuid", "action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node", "design_ref": "http_uri|deckhand_uri|file_uri", @@ -144,4 +144,24 @@ consist of the below:: "ts": iso8601 UTC timestamp, } +Task Build Data +~~~~~~~~~~~~~~~ +When querying the detail state of an existing task, adding the parameter ``builddata=true`` +in the query string will add one additional field with a list of build data elements +collected by this task.:: + + { + "Kind": "Task", + "apiVersion": "v1", + .... + "build_data": [ + { + "node_name": "foo", + "task_id": "uuid", + "collected_data": iso8601 UTC timestamp, + "generator": "lshw", + "data_format": "application/json", + "data_element": "{ \"id\": \"foo\", \"class\": \"system\" ...}" + } + ] diff --git a/drydock_provisioner/control/api.py b/drydock_provisioner/control/api.py index b9505990..bed5b45f 100644 --- a/drydock_provisioner/control/api.py +++ b/drydock_provisioner/control/api.py @@ -21,6 +21,7 @@ from .designs import DesignsPartResource from .tasks import TasksResource from .tasks import TaskResource from .nodes import NodesResource +from .nodes import NodeBuildDataResource from .health import HealthResource from .bootaction import BootactionUnitsResource from .bootaction import BootactionFilesResource @@ -75,6 +76,9 @@ def start_api(state_manager=None, ingester=None, orchestrator=None): # API to list current MaaS nodes ('/nodes', NodesResource()), + # API to get build data for a node + ('/nodes/{hostname}/builddata', + NodeBuildDataResource(state_manager=state_manager)), # API for nodes to discover their boot actions during curtin install ('/bootactions/nodes/{hostname}/units', BootactionUnitsResource( diff --git a/drydock_provisioner/control/nodes.py b/drydock_provisioner/control/nodes.py index 60724b02..2c9b7cbb 100644 --- a/drydock_provisioner/control/nodes.py +++ b/drydock_provisioner/control/nodes.py @@ -20,7 +20,7 @@ from drydock_provisioner import config from drydock_provisioner.drivers.node.maasdriver.api_client import MaasRequestFactory from drydock_provisioner.drivers.node.maasdriver.models.machine import Machines -from .base import BaseResource +from .base import BaseResource, StatefulResource class NodesResource(BaseResource): @@ -57,3 +57,32 @@ class NodesResource(BaseResource): self.error(req.context, "Unknown error: %s" % str(ex), exc_info=ex) self.return_error( resp, falcon.HTTP_500, message="Unknown error", retry=False) + + +class NodeBuildDataResource(StatefulResource): + """Resource for returning build data for a node.""" + + @policy.ApiEnforcer('physical_provisioner:read_build_data') + def on_get(self, req, resp, hostname): + try: + latest = req.params.get('latest', 'false').upper() + latest = True if latest == 'TRUE' else False + + node_bd = self.state_manager.get_build_data( + node_name=hostname, latest=latest) + + if not node_bd: + self.return_error( + resp, + falcon.HTTP_404, + message="No build data found", + retry=False) + else: + node_bd = [bd.to_dict() for bd in node_bd] + resp.status = falcon.HTTP_200 + resp.body = json.dumps(node_bd) + resp.content_type = falcon.MEDIA_JSON + except Exception as ex: + self.error(req.context, "Unknown error: %s" % str(ex), exc_info=ex) + self.return_error( + resp, falcon.HTTP_500, message="Unknown error", retry=False) diff --git a/drydock_provisioner/control/tasks.py b/drydock_provisioner/control/tasks.py index 37889096..e475024d 100644 --- a/drydock_provisioner/control/tasks.py +++ b/drydock_provisioner/control/tasks.py @@ -300,7 +300,6 @@ class TaskResource(StatefulResource): """Handler for GET method.""" try: task = self.state_manager.get_task(uuid.UUID(task_id)) - if task is None: self.info(req.context, "Task %s does not exist" % task_id) self.return_error( @@ -310,7 +309,15 @@ class TaskResource(StatefulResource): retry=False) return - resp.body = json.dumps(task.to_dict()) + resp_data = task.to_dict() + builddata = req.params.get('builddata', 'false').upper() + + if builddata == "TRUE": + task_bd = self.state_manager.get_build_data( + task_id=task.get_id()) + resp_data['build_data'] = [bd.to_dict() for bd in task_bd] + + resp.body = json.dumps(resp_data) resp.status = falcon.HTTP_200 except Exception as ex: self.error(req.context, "Unknown error: %s" % (str(ex))) diff --git a/drydock_provisioner/objects/task.py b/drydock_provisioner/objects/task.py index ba30fea1..2ca4a779 100644 --- a/drydock_provisioner/objects/task.py +++ b/drydock_provisioner/objects/task.py @@ -558,7 +558,7 @@ class TaskStatus(object): def to_dict(self): return { 'kind': 'Status', - 'apiVersion': 'v1', + 'apiVersion': 'v1.0', 'metadata': {}, 'message': self.message, 'reason': self.reason, diff --git a/drydock_provisioner/policy.py b/drydock_provisioner/policy.py index d994a203..9fefd182 100644 --- a/drydock_provisioner/policy.py +++ b/drydock_provisioner/policy.py @@ -95,6 +95,13 @@ class DrydockPolicy(object): 'path': '/api/v1.0/tasks', 'method': 'POST' }]), + policy.DocumentedRuleDefault( + 'physical_provisioner:read_build_data', 'role:admin', + 'Read build data for a node', + [{ + 'path': '/api/v1.0/nodes/{nodename}/builddata', + 'method': 'GET', + }]), ] # Data Management Policy diff --git a/tests/integration/postgres/test_api_builddata.py b/tests/integration/postgres/test_api_builddata.py new file mode 100644 index 00000000..a84d3e67 --- /dev/null +++ b/tests/integration/postgres/test_api_builddata.py @@ -0,0 +1,152 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test nodes build data API with Postgres backend.""" +import pytest +from falcon import testing + +import uuid +import datetime +import random + +import drydock_provisioner.objects as objects + +from drydock_provisioner import policy +from drydock_provisioner.control.api import start_api + +import falcon + + +class TestNodeBuildDataApi(): + def test_read_builddata_all(self, falcontest, seeded_builddata): + """Test that by default the API returns all build data for a node.""" + url = '/api/v1.0/nodes/foo/builddata' + + # Seed the database with build data + nodelist = ['foo'] + count = 3 + seeded_builddata(nodelist=nodelist, count=count) + + # TODO(sh8121att) Make fixture for request header forging + hdr = { + 'Content-Type': 'application/json', + 'X-IDENTITY-STATUS': 'Confirmed', + 'X-USER-NAME': 'Test', + 'X-ROLES': 'admin' + } + + resp = falcontest.simulate_get(url, headers=hdr) + + assert resp.status == falcon.HTTP_200 + + resp_body = resp.json + + assert len(resp_body) == count + + def test_read_builddata_latest(self, falcontest, seeded_builddata): + """Test that the ``latest`` parameter works.""" + url = '/api/v1.0/nodes/foo/builddata' + + req_hdr = { + 'Content-Type': 'application/json', + 'X-IDENTITY-STATUS': 'Confirmed', + 'X-USER-NAME': 'Test', + 'X-ROLES': 'admin', + } + + # Seed the database with build data + nodelist = ['foo'] + generatorlist = ['hello', 'hello', 'bye'] + count = 3 + seeded_builddata( + nodelist=nodelist, + generatorlist=generatorlist, + count=count, + random_dates=True) + + resp = falcontest.simulate_get( + url, headers=req_hdr, query_string="latest=true") + + assert resp.status == falcon.HTTP_200 + + resp_body = resp.json + + # Should only be a single instance for each unique generator + assert len(resp_body) == len(set(generatorlist)) + + @pytest.fixture() + def seeded_builddata(self, blank_state): + """Provide function to seed the database with build data.""" + + def seed_build_data(nodelist=['foo'], + tasklist=None, + generatorlist=None, + count=1, + random_dates=True): + """Seed the database with ``count`` build data elements for each node. + + If tasklist is specified, it should be a list of length ``count`` such that + as build_data are added for a node, each task_id will be used one for each node + + :param nodelist: list of string nodenames for build data + :param tasklist: list of uuid.UUID IDs for task. If omitted, uuids will be generated + :param gneratorlist: list of string generatos to assign to build data. If omitted, 'hello_world' + is used. + :param count: number build data elements to create for each node + :param random_dates: whether to generate random dates in the past 180 days or create each + build data element with utcnow() + """ + for n in nodelist: + for i in range(count): + if random_dates: + collected_date = datetime.datetime.utcnow( + ) - datetime.timedelta(days=random.randint(1, 180)) + else: + collected_date = None + if tasklist: + task_id = tasklist[i] + else: + task_id = uuid.uuid4() + if generatorlist: + generator = generatorlist[i] + else: + generator = 'hello_world' + bd = objects.BuildData( + node_name=n, + task_id=task_id, + generator=generator, + data_format='text/plain', + collected_date=collected_date, + data_element='Hello World!') + blank_state.post_build_data(bd) + i = i + 1 + + return seed_build_data + + # TODO(sh8121att) Make this a general fixture in conftest.py + @pytest.fixture() + def falcontest(self, drydock_state, deckhand_ingester, + deckhand_orchestrator): + """Create a test harness for the the Falcon API framework.""" + policy.policy_engine = policy.DrydockPolicy() + policy.policy_engine.register_policy() + + return testing.TestClient( + start_api( + state_manager=drydock_state, + ingester=deckhand_ingester, + orchestrator=deckhand_orchestrator)) + + @policy.ApiEnforcer('physical_provisioner:read_task') + def target_function(self, req, resp): + return True diff --git a/tests/integration/postgres/test_api_tasks.py b/tests/integration/postgres/test_api_tasks.py index a7e70183..0923aaed 100644 --- a/tests/integration/postgres/test_api_tasks.py +++ b/tests/integration/postgres/test_api_tasks.py @@ -12,96 +12,106 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test tasks API with Postgres backend.""" +import pytest +from falcon import testing -import uuid import json -from drydock_provisioner import policy -from drydock_provisioner.orchestrator.orchestrator import Orchestrator +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.objects as objects +from drydock_provisioner import policy +from drydock_provisioner.control.api import start_api from drydock_provisioner.control.base import DrydockRequestContext -from drydock_provisioner.control.tasks import TasksResource import falcon class TestTasksApi(): - def test_read_tasks(self, mocker, blank_state): - ''' DrydockPolicy.authorized() should correctly use oslo_policy to enforce - RBAC policy based on a DrydockRequestContext instance - ''' + def test_read_tasks(self, falcontest, blank_state): + """Test that the tasks API responds with list of tasks.""" + url = '/api/v1.0/tasks' - mocker.patch('oslo_policy.policy.Enforcer') + # TODO(sh8121att) Make fixture for request header forging + hdr = { + 'Content-Type': 'application/json', + 'X-IDENTITY-STATUS': 'Confirmed', + 'X-USER-NAME': 'Test', + 'X-ROLES': 'admin' + } - ctx = DrydockRequestContext() - policy_engine = policy.DrydockPolicy() - - # Mock policy enforcement - policy_mock_config = {'authorize.return_value': True} - policy_engine.enforcer.configre_mock(**policy_mock_config) - - api = TasksResource(state_manager=blank_state) - - # Configure context - project_id = str(uuid.uuid4().hex) - ctx.project_id = project_id - user_id = str(uuid.uuid4().hex) - ctx.user_id = user_id - ctx.roles = ['admin'] - ctx.set_policy_engine(policy_engine) - - # Configure mocked request and response - req = mocker.MagicMock() - resp = mocker.MagicMock() - req.context = ctx - - api.on_get(req, resp) + resp = falcontest.simulate_get(url, headers=hdr) assert resp.status == falcon.HTTP_200 - def test_create_task(self, mocker, blank_state): - mocker.patch('oslo_policy.policy.Enforcer') - - ingester = mocker.MagicMock() - orch = mocker.MagicMock( - spec=Orchestrator, - wraps=Orchestrator(state_manager=blank_state, ingester=ingester)) - + def test_read_tasks_builddata(self, falcontest, blank_state, + deckhand_orchestrator): + """Test that the tasks API includes build data when prompted.""" + req_hdr = { + 'Content-Type': 'application/json', + 'X-IDENTITY-STATUS': 'Confirmed', + 'X-USER-NAME': 'Test', + 'X-ROLES': 'admin', + } + # Seed DB with a task ctx = DrydockRequestContext() - policy_engine = policy.DrydockPolicy() + task = deckhand_orchestrator.create_task( + action=hd_fields.OrchestratorAction.PrepareNodes, + design_ref='http://foo.com', + context=ctx) + + # Seed DB with build data for task + build_data = objects.BuildData( + node_name='foo', + task_id=task.get_id(), + generator='hello_world', + data_format='text/plain', + data_element='Hello World!') + blank_state.post_build_data(build_data) + + url = '/api/v1.0/tasks/%s' % str(task.get_id()) + + resp = falcontest.simulate_get( + url, headers=req_hdr, query_string="builddata=true") + + assert resp.status == falcon.HTTP_200 + + resp_body = resp.json + + assert isinstance(resp_body.get('build_data'), list) + + assert len(resp_body.get('build_data')) == 1 + + def test_create_task(self, falcontest, blank_state): + url = '/api/v1.0/tasks' + + req_hdr = { + 'Content-Type': 'application/json', + 'X-IDENTITY-STATUS': 'Confirmed', + 'X-USER-NAME': 'Test', + 'X-ROLES': 'admin', + } json_body = json.dumps({ 'action': 'verify_site', 'design_ref': 'http://foo.com', - }).encode('utf-8') + }) - # Mock policy enforcement - policy_mock_config = {'authorize.return_value': True} - policy_engine.enforcer.configure_mock(**policy_mock_config) - - api = TasksResource(orchestrator=orch, state_manager=blank_state) - - # Configure context - project_id = str(uuid.uuid4().hex) - ctx.project_id = project_id - user_id = str(uuid.uuid4().hex) - ctx.user_id = user_id - ctx.roles = ['admin'] - ctx.set_policy_engine(policy_engine) - - # Configure mocked request and response - req = mocker.MagicMock(spec=falcon.Request) - req.content_type = 'application/json' - req.stream.read.return_value = json_body - resp = mocker.MagicMock(spec=falcon.Response) - - req.context = ctx - - api.on_post(req, resp) + resp = falcontest.simulate_post(url, headers=req_hdr, body=json_body) assert resp.status == falcon.HTTP_201 - assert resp.get_header('Location') is not None + assert resp.headers.get('Location') is not None - @policy.ApiEnforcer('physical_provisioner:read_task') - def target_function(self, req, resp): - return True + # TODO(sh8121att) Make this a general fixture in conftest.py + @pytest.fixture() + def falcontest(self, drydock_state, deckhand_ingester, + deckhand_orchestrator): + """Create a test harness for the the Falcon API framework.""" + policy.policy_engine = policy.DrydockPolicy() + policy.policy_engine.register_policy() + + return testing.TestClient( + start_api( + state_manager=drydock_state, + ingester=deckhand_ingester, + orchestrator=deckhand_orchestrator))