From 70410cc478e6aec4abad88c84d68684edfedb376 Mon Sep 17 00:00:00 2001
From: Ahmad Mahmoudi <ahmad.mahmoudi@att.com>
Date: Fri, 21 Feb 2020 06:27:28 +0000
Subject: [PATCH] (fix) Address image build issues, bionic

- With bionic image based shipyard docker images, uwsgi crashes
  with segmentation fault, when it tries to load the psycopg2 library,
  causing the api become unreachable on both shipyard docker images.
  This happens because psycopg2 2.7.x and uwsgi binary wheels are built
  with incompatible ssl libraries. This patch upgrades psycopg2 to the
  latest release to address this issue.

- The existing image build script cannot run in a docker or a pod,
  based pipeline because of two reasons:
  - The build script runs a docker (docker-in-docker) and mounts a
    volume.
    In a dind case, volume bind mounts will not work, because the nested
    container will need the host file system's path for the source path.
  - The shipyard service listens to its exposed service port in the
    nested docker network namespace, which is not reachable from the host
    pod/container.
This patch address both of the above issues. It first creates the
container, copies needed config files to the container and then starts
it. Also it execs into the nested docker to access the shipyard services
in a dind (docker-in-dcoker) case.

Change-Id: Ifdfed539babab01608bfaef37001bb79cd3a080d
---
 images/airflow/requirements.txt           |  6 +++---
 src/bin/shipyard_airflow/requirements.txt |  6 +++---
 tools/shipyard_image_run.sh               | 23 +++++++++++++++--------
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/images/airflow/requirements.txt b/images/airflow/requirements.txt
index 8c04604b..3516aa82 100644
--- a/images/airflow/requirements.txt
+++ b/images/airflow/requirements.txt
@@ -16,7 +16,7 @@ pytz==2018.5
 pyOpenSSL==18.0.0
 ndg-httpsclient==0.5.1
 pyasn1==0.4.4
-psycopg2==2.7.5
+psycopg2-binary==2.8.4
 docker==3.5.0
 # Airflow is now installed in the Dockerfile directory to allow for
 # overriding where it is sourced from
@@ -27,6 +27,6 @@ marshmallow-sqlalchemy==0.18.0
 tabulate==0.8.03
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@7e5d81f50f1f8c0d58d0973ae0b3065fd5b62451#egg=deckhand
-git+https://opendev.org/airship/drydock.git@22a4f01cb7880828f7b955c56d53603b6170415a#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
+git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt
index f34d7aa3..cb4156a7 100644
--- a/src/bin/shipyard_airflow/requirements.txt
+++ b/src/bin/shipyard_airflow/requirements.txt
@@ -26,7 +26,7 @@ networkx==2.1 # common/deployment_group
 oslo.config==6.4.0
 oslo.policy==1.38.1
 PasteDeploy==1.5.2
-psycopg2==2.7.4
+psycopg2-binary==2.8.4
 python-dateutil==2.7.3
 python-memcached==1.59
 requests==2.20.0
@@ -42,6 +42,6 @@ marshmallow-sqlalchemy==0.18.0
 tabulate==0.8.03
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@7e5d81f50f1f8c0d58d0973ae0b3065fd5b62451#egg=deckhand
-git+https://opendev.org/airship/drydock.git@22a4f01cb7880828f7b955c56d53603b6170415a#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
+git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
diff --git a/tools/shipyard_image_run.sh b/tools/shipyard_image_run.sh
index f52e437b..ac670dde 100755
--- a/tools/shipyard_image_run.sh
+++ b/tools/shipyard_image_run.sh
@@ -18,19 +18,27 @@ set -x
 IMAGE=$1
 USE_PROXY=${USE_PROXY:-false}
 
-# Collect necessary files and run shipyard image in docker
 mkdir -p build/.tmprun/etc
-cp $PWD/etc/shipyard/api-paste.ini build/.tmprun/etc
-cp $PWD/tools/resources/shipyard.conf build/.tmprun/etc
-docker run \
+docker create \
     -v $PWD/build/.tmprun/etc:/etc/shipyard \
     -p 9000:9000 \
-    --name shipyard_test ${IMAGE} \
-    &
+    --name shipyard_test ${IMAGE}
 
+docker cp $PWD/etc/shipyard/api-paste.ini shipyard_test:/etc/shipyard
+docker cp $PWD/tools/resources/shipyard.conf shipyard_test:/etc/shipyard
+docker start shipyard_test &
 sleep 5
 
+# If the image build pipeline is running in a pod/docker (docker-in-docker),
+# we'll need to exec into the nested container's network namespace to acces the
+# shipyard api.
+GOOD="HTTP/1.1 200 OK"
 RESULT="$(curl -i 'http://127.0.0.1:9000/versions' --noproxy '*' | tr '\r' '\n' | head -1)"
+if [[ "${RESULT}" != "${GOOD}" ]]; then
+  if docker exec -t shipyard_test /bin/bash -c "curl -i 'http://127.0.0.1:9000/versions' --noproxy '*' | tr '\r' '\n' | head -1 | grep 'HTTP/1.1 200 OK'"; then
+    RESULT="${GOOD}"
+  fi
+fi
 
 if [ "${USE_PROXY}" == "true" ]; then
   CLI_RESULT="$(docker run -t --rm --net=host --env HTTP_PROXY="${PROXY}" --env HTTPS_PROXY="${PROXY}" ${IMAGE} help | tr '\r' '\n' | head -1)"
@@ -40,8 +48,7 @@ fi
 
 docker stop shipyard_test
 docker rm shipyard_test
-rm -r build/.tmprun
-GOOD="HTTP/1.1 200 OK"
+rm -rf $PWD/build/.tmprun
 CLI_GOOD="THE SHIPYARD COMMAND"
 if [[ ${RESULT} == ${GOOD} && ${CLI_RESULT} == ${CLI_GOOD} ]]; then
     exit 0