From cc77125953cd25eacfb2da39f8370f5f875aea5c Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 1 May 2018 08:24:12 -0500 Subject: [PATCH] [411387] Schema update to support repo - Support one or more repo specifications for a site - Add object model for repository - Add testing for repository parsing Update freeze job with make target - Update the requirements freeze job to have a mark target that rebuilds the tox virtualenv each run - Update Dockerfile to create a valid /etc/protocols file Change-Id: I9d09b7dd7226827995e23756ff968b36eaa4d16c --- Makefile | 5 +++ .../ingester/plugins/deckhand.py | 20 ++++++++++ drydock_provisioner/objects/site.py | 12 ++++-- drydock_provisioner/schemas/region.yaml | 37 +++++++++++++++++++ requirements-direct.txt | 4 +- requirements-lock.txt | 4 +- tests/unit/test_ingester.py | 20 ++++++++++ tests/yaml_samples/deckhand_fullsite.yaml | 11 ++++++ tox.ini | 4 +- 9 files changed, 109 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 5e57f331..d45bbda8 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,11 @@ coverage_test: build_drydock external_dep unit_tests: external_dep tox -re py35 +# Freeze full set of Python requirements +.PHONY: req_freeze +req_freeze: + tox -re freeze + # Run the drydock container and exercise simple tests .PHONY: run_drydock run_drydock: build_drydock diff --git a/drydock_provisioner/ingester/plugins/deckhand.py b/drydock_provisioner/ingester/plugins/deckhand.py index 6148e7ed..d5cd5043 100644 --- a/drydock_provisioner/ingester/plugins/deckhand.py +++ b/drydock_provisioner/ingester/plugins/deckhand.py @@ -217,6 +217,26 @@ class DeckhandIngester(IngesterPlugin): model.authorized_keys = [k for k in auth_keys] + repos = data.get('repositories', None) + + if repos: + model.repositories = self.process_drydock_region_repo_list(repos) + + return model + + def process_drydock_region_repo_list(self, data): + """Process a package repository list. + + :param data: The data from the ``repositories`` key in a Region document + """ + model = objects.RepositoryList() + + for k, v in data.items(): + if k == 'remove_unlisted': + model.remove_unlisted = v + else: + model.append(objects.Repository(name=k, **v)) + return model def process_drydock_rack(self, name, data): diff --git a/drydock_provisioner/objects/site.py b/drydock_provisioner/objects/site.py index 999354cb..8e8b1e0a 100644 --- a/drydock_provisioner/objects/site.py +++ b/drydock_provisioner/objects/site.py @@ -90,8 +90,6 @@ class NodeTagDefinitionList(base.DrydockObjectListBase, base.DrydockObject): } -# Need to determine how best to define a repository that can encompass -# all repositories needed @base.DrydockObjectRegistry.register class Repository(base.DrydockObject): @@ -99,12 +97,19 @@ class Repository(base.DrydockObject): fields = { 'name': ovo_fields.StringField(), + 'url': ovo_fields.StringField(), + 'repo_type': ovo_fields.StringField(), + 'gpgkey': ovo_fields.StringField(nullable=True), + 'distributions': ovo_fields.ListOfStringsField(nullable=True), + 'components': ovo_fields.ListOfStringsField(nullable=True), + 'arches': ovo_fields.ListOfStringsField(default=['amd64']), + 'options': ovo_fields.DictOfStringsField(nullable=True) } def __init__(self, **kwargs): super(Repository, self).__init__(**kwargs) - # TagDefinition keyed by tag + # Repository keyed by tag def get_id(self): return self.name @@ -116,6 +121,7 @@ class RepositoryList(base.DrydockObjectListBase, base.DrydockObject): fields = { 'objects': ovo_fields.ListOfObjectsField('Repository'), + 'remove_unlisted': ovo_fields.BooleanField(default=False), } diff --git a/drydock_provisioner/schemas/region.yaml b/drydock_provisioner/schemas/region.yaml index 37e4da8c..05d16a6f 100644 --- a/drydock_provisioner/schemas/region.yaml +++ b/drydock_provisioner/schemas/region.yaml @@ -28,4 +28,41 @@ data: type: 'array' items: type: 'string' + repositories: + # top level is class (e.g. apt, rpm) + type: 'object' + properties: + remove_unlisted: + type: 'boolean' + additionalPropties: + type: 'object' + properties: + repo_type: + type: 'string' + pattern: 'apt|rpm' + url: + type: 'string' + distributions: + type: 'array' + items: + type: 'string' + components: + type: 'array' + items: + type: 'string' + gpgkey: + type: 'string' + arches: + type: 'array' + items: + type: 'string' + options: + type: 'object' + additionalProperties: + type: 'string' + additionalProperties: false + required: + - 'repo_type' + - 'url' + - 'arches' additionalProperties: false diff --git a/requirements-direct.txt b/requirements-direct.txt index 3a422a7f..b54e1306 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -7,14 +7,14 @@ requests oauthlib uwsgi==2.0.15 pymongo==3.6.1 -oslo.config==3.16.0 +oslo.config==5.2.0 click==6.7 PasteDeploy==1.5.2 PTable==0.9.2 keystonemiddleware==4.9.1 oslo.policy==1.22.1 iso8601==0.1.11 -keystoneauth1==2.13.0 +keystoneauth1==3.3.0 alembic==0.8.2 sqlalchemy==1.1.14 psycopg2==2.7.3.1 diff --git a/requirements-lock.txt b/requirements-lock.txt index 2b81c1c8..bee5be9d 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -17,7 +17,7 @@ idna==2.6 iso8601==0.1.11 Jinja2==2.9.6 jsonschema==2.6.0 -keystoneauth1==2.13.0 +keystoneauth1==3.3.0 keystonemiddleware==4.9.1 kombu==4.1.0 libvirt-python==3.10.0 @@ -29,7 +29,7 @@ netaddr==0.7.19 netifaces==0.10.7 oauthlib==2.0.7 oslo.concurrency==3.27.0 -oslo.config==3.16.0 +oslo.config==5.2.0 oslo.context==2.20.0 oslo.i18n==3.20.0 oslo.log==3.38.1 diff --git a/tests/unit/test_ingester.py b/tests/unit/test_ingester.py index 3f86a348..613c80c1 100644 --- a/tests/unit/test_ingester.py +++ b/tests/unit/test_ingester.py @@ -31,6 +31,26 @@ class TestClass(object): assert len(design_data.host_profiles) == 2 assert len(design_data.baremetal_nodes) == 3 + def test_ingest_deckhand_repos(self, input_files, setup, deckhand_ingester): + """Test that the ingester properly parses repo definitions.""" + input_file = input_files.join("deckhand_fullsite.yaml") + + design_state = DrydockState() + design_ref = "file://%s" % str(input_file) + + design_status, design_data = deckhand_ingester.ingest_data( + design_state=design_state, design_ref=design_ref) + + assert design_status.status == objects.fields.ValidationResult.Success + + region_def = design_data.get_site() + + assert len(region_def.repositories) == 1 + assert region_def.repositories.remove_unlisted + + for r in region_def.repositories: + assert 'docker' in r.url + def test_ingest_deckhand_docref_exists(self, input_files, setup, deckhand_ingester): """Test that each processed document has a doc_ref.""" diff --git a/tests/yaml_samples/deckhand_fullsite.yaml b/tests/yaml_samples/deckhand_fullsite.yaml index ae190b57..366e8bf4 100644 --- a/tests/yaml_samples/deckhand_fullsite.yaml +++ b/tests/yaml_samples/deckhand_fullsite.yaml @@ -30,6 +30,17 @@ data: - tag: 'test' definition_type: 'lshw_xpath' definition: "//node[@id=\"display\"]/'clock units=\"Hz\"' > 1000000000" + repositories: + remove_unlisted: true + docker: + repo_type: apt + url: https://docker.io/repo + distributions: + - ubuntu-xenial + gpgkey: |+ + -----BLAH BLAH----- + STUFF + -----BLAH BLAH----- authorized_keys: - | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDENeyO5hLPbLLQRZ0oafTYWs1ieo5Q+XgyZQs51Ju diff --git a/tox.ini b/tox.ini index b235e95e..1f5b246b 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,9 @@ deps= -rrequirements-test.txt [testenv:freeze] -whitelist_externals=rm +whitelist_externals= + rm + sh deps= -rrequirements-direct.txt commands=