summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-10-29 16:53:58 +0000
committerGerrit Code Review <review@openstack.org>2018-10-29 16:53:58 +0000
commit313c4ed86c9cd2d6432d0f8674711f567af68987 (patch)
treefd4f02457a395c57297a6d01cc758ed817413c26
parent25dfde96e59f9eff2bb0aa66d244dd307c4b622f (diff)
parent76f12648f69b9786a90fac7debc9041eb27d1faf (diff)
Merge "refactor: Allow site_by_params to take in list of fields"
-rw-r--r--pegleg/engine/lint.py4
-rw-r--r--pegleg/engine/site.py18
-rw-r--r--pegleg/engine/util/definition.py38
3 files changed, 27 insertions, 33 deletions
diff --git a/pegleg/engine/lint.py b/pegleg/engine/lint.py
index c678ad0..550c065 100644
--- a/pegleg/engine/lint.py
+++ b/pegleg/engine/lint.py
@@ -183,9 +183,7 @@ def _verify_no_unexpected_files(*, sitenames=None):
183 expected_directories = set() 183 expected_directories = set()
184 for site_name in sitenames: 184 for site_name in sitenames:
185 params = util.definition.load_as_params(site_name) 185 params = util.definition.load_as_params(site_name)
186 expected_directories.update( 186 expected_directories.update(util.files.directories_for(**params))
187 util.files.directories_for(
188 site_name=params['site_name'], site_type=params['site_type']))
189 LOG.debug('expected_directories: %s', expected_directories) 187 LOG.debug('expected_directories: %s', expected_directories)
190 found_directories = util.files.existing_directories() 188 found_directories = util.files.existing_directories()
191 LOG.debug('found_directories: %s', found_directories) 189 LOG.debug('found_directories: %s', found_directories)
diff --git a/pegleg/engine/site.py b/pegleg/engine/site.py
index 2f043a5..902e1aa 100644
--- a/pegleg/engine/site.py
+++ b/pegleg/engine/site.py
@@ -117,20 +117,12 @@ def list_(output_stream):
117 117
118 # Create a table to output site information for all sites for a given repo 118 # Create a table to output site information for all sites for a given repo
119 site_table = PrettyTable() 119 site_table = PrettyTable()
120 site_table.field_names = ['site_name', 'site_type'] 120 field_names = ['site_name', 'site_type']
121 site_table.field_names = field_names
121 122
122 for site_name in util.files.list_sites(): 123 for site_name in util.files.list_sites():
123 params = util.definition.load_as_params(site_name) 124 params = util.definition.load_as_params(site_name, *field_names)
124 # TODO(felipemonteiro): This is a temporary hack around legacy manifest 125 site_table.add_row(list(map(lambda k: params[k], field_names)))
125 # repositories containing the name of a directory that symbolizes a
126 # repository. Once all these manifest repositories migrate over to Git
127 # references instead, remove this hack.
128 # NOTE(felipemonteiro): The 'revision' information can instead be
129 # computed using :func:`process_site_repository` and storing into
130 # a configuration via a "set_site_revision" function, for example.
131 if 'revision' in params:
132 params.pop('revision')
133 site_table.add_row([params['site_name'], params['site_type']])
134 # Write table to specified output_stream 126 # Write table to specified output_stream
135 output_stream.write(site_table.get_string() + "\n") 127 output_stream.write(site_table.get_string() + "\n")
136 128
@@ -141,6 +133,8 @@ def show(site_name, output_stream):
141 # Create a table to output site information for specific site 133 # Create a table to output site information for specific site
142 site_table = PrettyTable() 134 site_table = PrettyTable()
143 site_table.field_names = ['revision', 'site_name', 'site_type', 'files'] 135 site_table.field_names = ['revision', 'site_name', 'site_type', 'files']
136 # TODO(felipemonteiro): Drop support for 'revision' once manifest
137 # repositories have removed it altogether.
144 if 'revision' in data.keys(): 138 if 'revision' in data.keys():
145 for file in data['files']: 139 for file in data['files']:
146 site_table.add_row( 140 site_table.add_row(
diff --git a/pegleg/engine/util/definition.py b/pegleg/engine/util/definition.py
index a551014..117d2a3 100644
--- a/pegleg/engine/util/definition.py
+++ b/pegleg/engine/util/definition.py
@@ -30,18 +30,25 @@ def load(site, primary_repo_base=None):
30 return files.slurp(path(site, primary_repo_base)) 30 return files.slurp(path(site, primary_repo_base))
31 31
32 32
33def load_as_params(site_name, primary_repo_base=None): 33def load_as_params(site_name, *fields, primary_repo_base=None):
34 """Load site definition for given ``site_name`` and return data as params.
35
36 :param str site_name: Name of the site.
37 :param iterable fields: List of parameter fields to return. Defaults to
38 ``('site_name', 'site_type')``.
39 :param str primary_repo_base: Path to primary repository.
40 :returns: key-value pairs of parameters, whose keys are a subset of those
41 specified by ``fields``.
42 :rtype: dict
43 """
44 if not fields:
45 # Default legacy fields.
46 fields = ('site_name', 'site_type')
47
34 definition = load(site_name, primary_repo_base) 48 definition = load(site_name, primary_repo_base)
35 # TODO(felipemonteiro): Currently we are filtering out "revision" from
36 # the params that are returned by this function because it is no longer
37 # supported. This is a workaround. As soon as the site definition repos
38 # switch to real repository format, then we can drop that workaround.
39 # Ideally, we should:
40 # 1) validate the site-definition.yaml format using lint module
41 # 2) extract only the required params here
42 params = definition.get('data', {}) 49 params = definition.get('data', {})
43 params['site_name'] = site_name 50 params['site_name'] = site_name
44 return params 51 return {k: v for k, v in params.items() if k in fields}
45 52
46 53
47def path(site_name, primary_repo_base=None): 54def path(site_name, primary_repo_base=None):
@@ -63,17 +70,14 @@ def pluck(site_definition, key):
63 70
64def site_files(site_name): 71def site_files(site_name):
65 params = load_as_params(site_name) 72 params = load_as_params(site_name)
66 for filename in files.search( 73 for filename in files.search(files.directories_for(**params)):
67 files.directories_for(
68 site_name=params['site_name'], site_type=params['site_type'])):
69 yield filename 74 yield filename
70 75
71 76
72def site_files_by_repo(site_name): 77def site_files_by_repo(site_name):
73 """Yield tuples of repo_base, file_name.""" 78 """Yield tuples of repo_base, file_name."""
74 params = load_as_params(site_name) 79 params = load_as_params(site_name)
75 dir_map = files.directories_for_each_repo( 80 dir_map = files.directories_for_each_repo(**params)
76 site_name=params['site_name'], site_type=params['site_type'])
77 for repo, dl in dir_map.items(): 81 for repo, dl in dir_map.items():
78 for filename in files.search(dl): 82 for filename in files.search(dl):
79 yield (repo, filename) 83 yield (repo, filename)
@@ -93,8 +97,7 @@ def documents_for_each_site():
93 97
94 for sitename in sitenames: 98 for sitename in sitenames:
95 params = load_as_params(sitename) 99 params = load_as_params(sitename)
96 paths = files.directories_for( 100 paths = files.directories_for(**params)
97 site_name=params['site_name'], site_type=params['site_type'])
98 filenames = set(files.search(paths)) 101 filenames = set(files.search(paths))
99 for filename in filenames: 102 for filename in filenames:
100 with open(filename) as f: 103 with open(filename) as f:
@@ -116,8 +119,7 @@ def documents_for_site(sitename):
116 documents = [] 119 documents = []
117 120
118 params = load_as_params(sitename) 121 params = load_as_params(sitename)
119 paths = files.directories_for( 122 paths = files.directories_for(**params)
120 site_name=params['site_name'], site_type=params['site_type'])
121 filenames = set(files.search(paths)) 123 filenames = set(files.search(paths))
122 for filename in filenames: 124 for filename in filenames:
123 with open(filename) as f: 125 with open(filename) as f: