Implement Pipenv dependency management

Pipenv is a tool that brings better package dependency management to
python. It can automatically create and manage virtualenv as well as
managing package dependencies using Pipfile and Pipfile.lock. Adding
this dependency manager into Airship projects will decrease package
version conflicts between projects and help increase security through
hash validation of packages and vulnerability scans.

Changes:
- Imports requirements.txt type files into Pipfile
- Pipenv dependency management in tox
- Switches Safety package for "pipenv check", an implementation of
Safety
- Adds `-e` flag on VCS dependencies to resolve good versions on all
packages
- Unpins or loosens pins on "dev"-type packages

Pipenv Docs: https://docs.pipenv.org/en/latest/

Helpful Pipenv Guide: https://realpython.com/pipenv-guide/

Change-Id: I47e7e7b440d76103b4984499e6ffce4482a59353

.zuul.yaml

@@ -49,12 +49,13 @@
 
 - job:
     name: pegleg-dependency-vulnerability-check
-    parent: openstack-tox
+    parent: tox-py36
     voting: false
     timeout: 600
-    nodeset: spyglass-single-node
+    nodeset: ubuntu-bionic
     vars:
       tox_envlist: safety
+      bindep_profile: test py36
 
 - job:
     name: airship-pegleg-docker-build-gate-ubuntu_xenial

Pipfile

@@ -0,0 +1,34 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pytest = "*"
+pytest-cov = "*"
+testfixtures = "*"
+pytest-xdist = "*"
+requests = ">=2.20.0"
+urllib3 = ">=1.24.3"
+chardet = ">=3.0.4"
+flake8 = "*"
+yapf = ">=0.27.0"
+hacking = ">=1.1.0"
+flake8-import-order = ">=0.18.1"
+bandit = ">=1.6.0"
+docker = ">=3.7.2"
+
+[packages]
+click = ">=6.7"
+jsonschema = "~=2.6.0"
+cryptography = "~=2.3.1"
+python-dateutil = "~=2.7.3"
+GitPython = "~=2.1.11"
+PyYAML = "~=5.1"
+deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "a4850f8c4fc02dd1971282c6d9e4beb9cd5be320",editable = true}
+shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "1472a22dce698ea77f15b007906b263eef470f52",subdirectory = "src/bin/shipyard_client",editable = true}
+promenade = {git = "https://opendev.org/airship/promenade.git",ref = "89f9842b2ec428709a05fd0a8d4cd1796043375d",editable = true}
+oslo-utils = "~=3.41.0"
+
+[requires]
+python_version = "3.6"

requirements.txt

@@ -1,16 +1,118 @@
-gitpython==2.1.11
+-i https://pypi.org/simple
+-e git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320#egg=deckhand
+-e git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d#egg=promenade
+-e git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard-client&subdirectory=src/bin/shipyard_client
+alembic==1.0.1
+amqp==2.5.0
+argparse==1.4.0
+arrow==0.12.1
+asn1crypto==0.24.0
+attrs==19.1.0
+babel==2.7.0
+beaker==1.10.0
+cachetools==3.1.1
+certifi==2019.6.16
+cffi==1.12.3
+chardet==3.0.4
+click-default-group==1.2
 click==6.7
-jsonschema==2.6.0
+cliff==2.15.0
-pyyaml==5.1
+cmd2==0.9.16 ; python_version >= '3.0'
+colorama==0.4.1
 cryptography==2.3.1
-python-dateutil==2.7.3
+debtcollector==1.21.0
-docker==3.7.2
+decorator==4.4.0
+deepdiff==3.3.0
+dnspython==1.16.0
+dogpile.cache==0.7.1
+eventlet==0.25.0
+extras==1.0.0
+falcon==1.4.1
+fasteners==0.15
+fixtures==3.0.0
+flake8==2.6.2
+futurist==1.8.1
+gitdb2==2.0.5
+gitpython==2.1.13
+greenlet==0.4.15
+hacking==1.1.0
+idna==2.7
+iso8601==0.1.12
+jinja2==2.10.1
+jsonpath-ng==1.4.3
+jsonpickle==1.2
+jsonschema==2.6.0
+keystoneauth1==3.11.1
+keystonemiddleware==5.3.0
+kombu==4.6.3
+linecache2==1.0.0
+mako==1.1.0
+markupsafe==1.1.1
+mccabe==0.5.3
+monotonic==1.5
+msgpack==0.6.1
+netaddr==0.7.19
+netifaces==0.10.9
+networkx==2.2
+os-service-types==1.7.0
+oslo-utils==3.41.0
+oslo.cache==1.31.1
+oslo.concurrency==3.28.1
+oslo.config==6.6.2
+oslo.context==2.22.1
+oslo.db==4.41.1
+oslo.i18n==3.23.1
+oslo.log==3.40.1
+oslo.messaging==9.1.1
+oslo.middleware==3.36.0
+oslo.policy==1.40.1
+oslo.serialization==2.28.1
+oslo.service==1.40.0
+oslo.utils==3.40.2
+paste==3.0.1
+pastedeploy==1.5.2
+pbr==5.1.0
+ply==3.11
+prettytable==0.7.2
+psycopg2==2.7.5
+ptable==0.9.2
+pycadf==2.9.0
+pycodestyle==2.0.0
+pycparser==2.19
+pyflakes==1.2.3
+pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'
+pyparsing==2.4.2
+pyperclip==1.7.0
+python-barbicanclient==4.7.0
+python-dateutil==2.7.5
+python-editor==1.0.4
+python-keystoneclient==3.18.0
+python-memcached==1.59
+python-mimeparse==1.6.0
+pytz==2019.2
+pyyaml==5.1
+repoze.lru==0.7
 requests==2.20.0
+rfc3986==1.3.2
+routes==2.4.1
+six==1.11.0
+smmap2==2.0.5
+sqlalchemy-migrate==0.12.0
+sqlalchemy==1.3.6
+sqlparse==0.3.0
+statsd==3.3.0
+stevedore==1.30.0
+tempita==0.5.2
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.3.0
+traceback2==1.4.0
+unittest2==1.1.0
 urllib3==1.24.3
-chardet==3.0.4
+uwsgi==2.0.17.1
-oslo.utils==3.41.0
+vine==1.3.0
-
+wcwidth==0.1.7
-# External dependencies
+webob==1.8.5
-git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320
+werkzeug==0.14.1
-git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard_client&subdirectory=src/bin/shipyard_client
+wrapt==1.11.2
-git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d
+yappi==1.0

test-requirements.txt

@@ -1,16 +0,0 @@
-# Testing
-pytest>=5.0.1
-pytest-cov>=2.5.1
-testfixtures>=6.8.2
-pytest-xdist>=1.23.2
-
-# Formatting
-yapf>=0.27.0
-
-# Linting
-hacking>=1.1.0
-flake8-import-order>=0.18.1
-
-# Security
-bandit>=1.6.0
-safety>=1.8.5

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist = py36,py37,pep8,docs,cover
-minversion = 2.3.1
+minversion = 3.6
 skipsdist = True
 
 [testenv]
@@ -9,14 +9,16 @@ setenv =
   VIRTUAL_ENV={envdir}
   LANGUAGE=en_US
   LC_ALL=en_US.utf-8
+  PIPENV_VERBOSITY=-1
+  PIPENV_IGNORE_PIPFILE=1
 deps =
-  -r{toxinidir}/requirements.txt
+  pipenv
-  -r{toxinidir}/test-requirements.txt
 passenv = http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
 whitelist_externals =
   bash
   find
 commands =
+  pipenv install --dev
   find . -type f -name "*.pyc" -delete
   bash -c "{toxinidir}/tools/install-cfssl.sh"
   {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'
@@ -24,15 +26,18 @@ commands =
 [testenv:fmt]
 basepython = python3
 deps =
-  -r{toxinidir}/test-requirements.txt
+  pipenv
 commands =
+  pipenv install --dev
   yapf -ir {toxinidir}/pegleg {toxinidir}/tests
 
 [testenv:pep8]
 basepython = python3
 deps =
-  -r{toxinidir}/test-requirements.txt
+  pipenv
+  safety
 commands =
+  pipenv install --dev
   bash -c "{toxinidir}/tools/gate/whitespace-linter.sh"
   bandit -r pegleg -n 5
   flake8 {toxinidir}/pegleg
@@ -43,9 +48,10 @@ whitelist_externals =
 [testenv:docs]
 basepython = python3
 deps =
-  -r{toxinidir}/requirements.txt
+  pipenv
   -r{toxinidir}/doc/requirements.txt
 commands =
+  pipenv install --dev
   bash -c "{toxinidir}/tools/gate/build-docs.sh"
 whitelist_externals =
   bash
@@ -57,18 +63,18 @@ commands = bandit -r pegleg -n 5
 [testenv:safety]
 basepython = python3
 deps =
+  pipenv
   safety
 commands =
-  safety check -r {toxinidir}/requirements.txt --full-report
+  pipenv check
-  safety check -r {toxinidir}/test-requirements.txt --full-report
   safety check -r {toxinidir}/doc/requirements.txt --full-report
 
 [testenv:cover]
 basepython = python3
 deps =
-  -r{toxinidir}/requirements.txt
+  pipenv
-  -r{toxinidir}/test-requirements.txt
 commands =
+  pipenv install --dev
   {toxinidir}/tools/install-cfssl.sh
   bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=pegleg --cov-report \
       html:cover --cov-report xml:cover/coverage.xml --cov-report term \
@@ -76,6 +82,15 @@ commands =
 whitelist_externals =
   bash
 
+[testenv:update-requirements]
+deps =
+    pipenv
+commands =
+  pipenv lock --clear
+  bash -c "pipenv lock -r > {toxinidir}/requirements.txt"
+whitelist_externals =
+  bash
+
 [testenv:releasenotes]
 basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt