From 1d5792df65c1f9202c8c7785834e6af7cac7408f Mon Sep 17 00:00:00 2001
From: Bryan <bryan@turbojets.net>
Date: Tue, 2 Feb 2016 02:40:01 -0800
Subject: [PATCH] initial commit

---
 .gitignore                    |  1 +
 Dockerfile                    | 60 ++++++++++++++++++++++++++
 NOTICE                        |  5 +++
 README.md                     | 81 ++++++++++++++++++++++++++++++++++-
 assets/aptly.conf             | 17 ++++++++
 assets/gpg_batch.sh           | 17 ++++++++
 assets/nginx.conf.sh          | 12 ++++++
 assets/startup.sh             | 25 +++++++++++
 assets/supervisord.nginx.conf |  5 +++
 assets/update_mirror.sh       | 61 ++++++++++++++++++++++++++
 build.sh                      | 21 +++++++++
 push.sh                       | 51 ++++++++++++++++++++++
 run.sh                        | 33 ++++++++++++++
 shell.sh                      | 12 ++++++
 vars                          | 38 ++++++++++++++++
 15 files changed, 438 insertions(+), 1 deletion(-)
 create mode 100644 .gitignore
 create mode 100644 Dockerfile
 create mode 100644 NOTICE
 create mode 100644 assets/aptly.conf
 create mode 100755 assets/gpg_batch.sh
 create mode 100755 assets/nginx.conf.sh
 create mode 100755 assets/startup.sh
 create mode 100644 assets/supervisord.nginx.conf
 create mode 100755 assets/update_mirror.sh
 create mode 100755 build.sh
 create mode 100755 push.sh
 create mode 100755 run.sh
 create mode 100755 shell.sh
 create mode 100644 vars

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d5905e3
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+builds
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..5c0f44b
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,60 @@
+# Copyright 2016 Bryan J. Hong
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:14.04
+
+MAINTAINER bryan@turbojets.net
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Add Aptly repository
+RUN echo "deb http://repo.aptly.info/ squeeze main" > /etc/apt/sources.list.d/aptly.list
+RUN apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
+
+# Add Nginx repository
+RUN echo "deb http://nginx.org/packages/ubuntu/ trusty nginx" > /etc/apt/sources.list.d/nginx.list
+RUN echo "deb-src http://nginx.org/packages/ubuntu/ trusty nginx" >> /etc/apt/sources.list.d/nginx.list
+RUN apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62
+
+# Update APT repository and install packages
+RUN apt-get -q update        \
+ && apt-get -y install aptly \
+      bzip2                  \
+      gnupg                  \
+      gpgv                   \
+      supervisor             \
+      nginx
+
+# Install GPG Generator 
+COPY assets/gpg_batch.sh /opt/gpg_batch.sh
+
+# Install Aptly Configuration
+COPY assets/aptly.conf /etc/aptly.conf
+
+# Install Mirror Update Script
+COPY assets/update_mirror.sh /opt/update_mirror.sh
+
+# Install Nginx Config
+COPY assets/nginx.conf.sh /opt/nginx.conf.sh
+COPY assets/supervisord.nginx.conf /etc/supervisor/conf.d/nginx.conf
+RUN echo "daemon off;" >> /etc/nginx/nginx.conf
+
+# Install Startup script
+COPY assets/startup.sh /opt/startup.sh
+
+# Bind mount location
+VOLUME [ "/opt/aptly" ]
+
+# Execute Startup script when container starts
+ENTRYPOINT [ "/opt/startup.sh" ]
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..98aab4c
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,5 @@
+docker-aptly
+Copyright 2016 Bryan J. Hong
+
+This product contains software (https://github.com/bryanhong/docker-aptly) developed
+by Bryan Hong (http://github.com/bryanhong) licensed under the Apache License.
diff --git a/README.md b/README.md
index 1318da9..5fad4a4 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,81 @@
-# docker-aptly
+#docker-aptly
+
 Dockerfile and support scripts to run aptly in a container backed by nginx
+
+from [aptly.info](http://aptly.info): 
+>aptly is a swiss army knife for Debian repository management: it allows you to mirror remote repositories, manage local package repositories, take snapshots, pull new versions of packages along with dependencies, publish as Debian repository.
+
+##Requirements / Dependencies
+
+* Docker 1.6 or higher, we are using the Docker syslog driver in this container and this feature made its debut in 1.6
+* ```vars``` needs to be populated with the appropriate variables.
+
+##Commands and variables
+
+* ```vars```: Variables for Docker registry, the application, and aptly repository data location
+* ```build.sh```: Build the Docker image locally
+* ```run.sh```: Starts the Docker container, it the image hasn't been built locally, it is fetched from the repository set in vars
+* ```push.sh```: Pushes the latest locally built image to the repository set in vars
+* ```shell.sh```: get a shell within the container
+
+##How this image/container works
+
+####Data
+All of aptly's data (including PGP keys and GPG keyrings) is bind mounted outside of the container to preserve it if the container is removed or rebuilt. Set the location for the bind mount in ```vars``` before starting the container. If you're going to host a mirror of Ubuntu's main repository, you'll need upwards of 35GB of free space as of Feb 2016, plan for growth.
+####Networking
+By default, Docker will map port 80 on the Docker host to port 80 within the container where nginx is configured to listen. You can change the external listening port in ```vars``` to map to any port you like.
+####Security
+The GPG password you set in ```vars``` is stored in plain text and is visible as an environment variable inside the container. It is set as an enviornment variable to allow for automation of repository updates without user interaction. The GPG password can be removed completely but it is safer to encrypt the GPG keyrings because they are bind mounted outside the container to avoid the necessity of regenerating/redistributing keys if the container is removed or rebuilt.
+
+##Usage
+
+####Configure the container
+
+1. Configure application specific variables in ```vars```
+
+####Build the image
+
+1. Run ```./build.sh```
+
+####Start the container
+
+1. Run ```./run.sh```
+2. Wait until the GPG keyrings are created (not 0 bytes) before proceeding (it can take a few minutes). They will be in the bind mount location you chose in ```vars```
+
+####Create a mirror of Ubuntu's main repository
+1. The initial download of the repository may take quite some time depending on your bandwidth limits, it may be in your best interest to open a tmux or screen session before proceeding.
+2. Attach to the container ```./shell.sh```
+3. By default, ```/opt/update_mirror.sh``` will automate the creation of an Ubuntu 14.04 Trusty repository, if you want a different release, modify the variables in the script.
+4. Run ```/opt/update_mirror.sh```
+5. If the script fails due to network disconnects etc, just re-run it.
+
+When the script completes, you should have a functional mirror that you can point a client to.
+
+####Point a host at the mirror
+
+1. Fetch the public PGP key from your aptly repository and add it to your trusted repositories
+
+ ```
+ wget http://FQDN.OF.APTLY/aptly_repo_key.pub
+ apt-key add aptly_repo_key.pub
+ ```
+
+2. Backup then replace /etc/apt/sources.list
+
+ ```
+ cp /etc/apt/sources.list /etc/apt/sources.list.bak
+ echo "deb http://FQDN.OF.APTLY/ ubuntu main" > /etc/apt/sources.list
+ apt-get update
+ ```
+ 
+ You should be able to install packages at this point!
+ 
+Checkout the excellent aptly documentation [here](http://www.aptly.info/doc/overview/)
+ 
+####Pushing your image to the registry
+
+If you're happy with your container and ready to share with others, push your image up to a [Docker registry](https://docs.docker.com/docker-hub/) and save any other changes you've made so the image can be easily changed or rebuilt in the future.
+
+1. Run ```./push.sh```
+
+> NOTE: If your image will be used FROM other containers you might want to use ```./push.sh flatten``` to consolidate the AUFS layers into a single layer. Keep in mind, you may lose Dockerfile attributes when your image is flattened.
diff --git a/assets/aptly.conf b/assets/aptly.conf
new file mode 100644
index 0000000..2ab6f51
--- /dev/null
+++ b/assets/aptly.conf
@@ -0,0 +1,17 @@
+{
+  "rootDir": "/opt/aptly",
+  "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
+  "architectures": [],
+  "dependencyFollowSuggests": false,
+  "dependencyFollowRecommends": false,
+  "dependencyFollowAllVariants": false,
+  "dependencyFollowSource": false,
+  "gpgDisableSign": false,
+  "gpgDisableVerify": false,
+  "downloadSourcePackages": false,
+  "ppaDistributorID": "ubuntu",
+  "ppaCodename": "",
+  "S3PublishEndpoints": {},
+  "SwiftPublishEndpoints": {}
+}
diff --git a/assets/gpg_batch.sh b/assets/gpg_batch.sh
new file mode 100755
index 0000000..79448c2
--- /dev/null
+++ b/assets/gpg_batch.sh
@@ -0,0 +1,17 @@
+#! /bin/bash
+cat << EOF > /opt/gpg_batch
+%echo Generating a GPG key, might take a while
+Key-Type: RSA
+Key-Length: 2048
+Subkey-Type: ELG-E
+Subkey-Length: 1024
+Name-Real: ${FULL_NAME}
+Name-Comment: Aptly Repo Signing
+Name-Email: ${EMAIL_ADDRESS}
+Expire-Date: 0
+Passphrase: ${GPG_PASSWORD}
+%pubring /opt/aptly/aptly.pub
+%secring /opt/aptly/aptly.sec 
+%commit
+%echo done
+EOF
diff --git a/assets/nginx.conf.sh b/assets/nginx.conf.sh
new file mode 100755
index 0000000..80886a9
--- /dev/null
+++ b/assets/nginx.conf.sh
@@ -0,0 +1,12 @@
+#! /bin/bash
+
+cat << EOF > /etc/nginx/conf.d/default.conf
+server {
+  root /opt/aptly/public;
+  server_name ${HOSTNAME};
+
+  location / {
+    autoindex on;
+  }
+}
+EOF
diff --git a/assets/startup.sh b/assets/startup.sh
new file mode 100755
index 0000000..e01f038
--- /dev/null
+++ b/assets/startup.sh
@@ -0,0 +1,25 @@
+#! /bin/bash
+
+# If the repository GPG keypair doesn't exist, create it.
+if [[ ! -f /opt/aptly/aptly.sec ]] || [[ ! -f /opt/aptly/aptly.pub ]]; then
+  /opt/gpg_batch.sh
+  gpg --batch --gen-key /opt/gpg_batch
+fi
+
+# Import Ubuntu keyrings
+gpg --no-default-keyring \
+    --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \
+    --export | \
+gpg --no-default-keyring \
+    --keyring trustedkeys.gpg \
+    --import
+
+# Aptly looks in /root/.gnupg for default keyrings
+ln -sf /opt/aptly/aptly.sec /root/.gnupg/secring.gpg
+ln -sf /opt/aptly/aptly.pub /root/.gnupg/pubring.gpg
+
+# Generate Nginx Config
+/opt/nginx.conf.sh
+
+# Start Supervisor
+/usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf
diff --git a/assets/supervisord.nginx.conf b/assets/supervisord.nginx.conf
new file mode 100644
index 0000000..bfae082
--- /dev/null
+++ b/assets/supervisord.nginx.conf
@@ -0,0 +1,5 @@
+[program:nginx]
+command=/usr/sbin/nginx
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
diff --git a/assets/update_mirror.sh b/assets/update_mirror.sh
new file mode 100755
index 0000000..16fab84
--- /dev/null
+++ b/assets/update_mirror.sh
@@ -0,0 +1,61 @@
+#! /bin/bash
+set -e
+
+UBUNTU_RELEASE=trusty
+UPSTREAM_URL="http://archive.ubuntu.com/ubuntu/"
+REPOS=( ${UBUNTU_RELEASE} ${UBUNTU_RELEASE}-updates ${UBUNTU_RELEASE}-security )
+
+# Export the GPG Public key
+if [[ ! -f /opt/aptly/aptly_repo_key.pub ]]; then
+  gpg --export --armor > /opt/aptly/${HOSTNAME}_signing.key
+fi
+
+# Create repository mirrors if they don't exist
+set +e
+for repo in ${REPOS[@]}; do
+  aptly mirror list -raw | grep "^${repo}$"
+  if [[ $? -ne 0 ]]; then
+    echo "Creating mirror of ${repo} repository."
+    aptly mirror create \
+      -architectures=amd64 ${repo} ${UPSTREAM_URL} ${repo} main
+  fi
+done
+set -e
+
+# Update all repository mirrors
+for repo in ${REPOS[@]}; do
+  echo "Updating ${repo} repository mirror.."
+  aptly mirror update ${repo}
+done
+
+# Create snapshots of updated repositories
+for repo in ${REPOS[@]}; do
+  echo "Creating snapshot of ${repo} repository mirror.."
+  aptly snapshot create ${repo}-`date +%Y%m%d%H` from mirror ${repo}
+done
+
+# Merge snapshots into a single snapshot with updates applied
+echo "Merging snapshots into one.." 
+aptly snapshot merge -latest                 \
+  ${UBUNTU_RELEASE}-merged-`date +%Y%m%d%H`  \
+  ${UBUNTU_RELEASE}-`date +%Y%m%d%H`         \
+  ${UBUNTU_RELEASE}-updates-`date +%Y%m%d%H` \
+  ${UBUNTU_RELEASE}-security-`date +%Y%m%d%H`
+
+# Publish the latest merged snapshot
+set +e
+aptly publish list -raw | awk '{print $2}' | grep "^${UBUNTU_RELEASE}$"
+if [[ $? -eq 0 ]]; then
+  aptly publish switch            \
+    -passphrase="${GPG_PASSWORD}" \
+    ${UBUNTU_RELEASE} ${UBUNTU_RELEASE}-merged-`date +%Y%m%d%H`
+else
+  aptly publish snapshot \
+    -passphrase="${GPG_PASSWORD}" \
+    -distribution=${UBUNTU_RELEASE} ${UBUNTU_RELEASE}-merged-`date +%Y%m%d%H`
+fi
+set -e
+
+# Generate Aptly Graph
+aptly graph
+cp `ls -rt /tmp/aptly-graph*.png | tail -n1` /opt/aptly/public/aptly_graph.png
diff --git a/build.sh b/build.sh
new file mode 100755
index 0000000..71db177
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source vars
+
+docker build -t "${REPO_NAME}/${APP_NAME}:${TAG}" .
+
+# If the build was successful (0 exit code)...
+if [ $? -eq 0 ]; then
+  echo
+  echo "Build of ${REPO_NAME}/${APP_NAME}:${TAG} completed OK"
+  echo
+
+  # log build details to builds file
+  echo "`date` => ${REPO_NAME}/${APP_NAME}:${TAG}" >> builds
+
+# The build exited with an error.
+else
+  echo "Build failed!"
+  exit 1
+
+fi
diff --git a/push.sh b/push.sh
new file mode 100755
index 0000000..a104e3d
--- /dev/null
+++ b/push.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+source vars
+
+#This will take the latest locally built image and push it to the repository as
+#configured in vars and tag it as latest.
+
+if [[ ! -f builds ]]; then
+  echo
+  echo "It appears that the Docker image hasn't been built yet, run build.sh first"
+  echo
+  exit 1
+fi
+
+LATESTIMAGE=`tail -1 builds | awk '{print $8}'`
+
+# Flatten is here as an option and not the default because with the export/import
+# process we lose Dockerfile attributes like PORT and VOLUMES. Flattening helps if
+# we are concerned about hitting the AUFS 42 layer limit or creating an image that
+# other containers source FROM
+
+DockerExport () {
+  docker export ${APP_NAME} | docker import - ${REPO_NAME}/${APP_NAME}:latest
+}
+
+DockerPush () {
+  docker push ${REPO_NAME}/${APP_NAME}:latest
+}
+
+case "$1" in
+  flatten)
+    docker inspect ${APP_NAME} > /dev/null 2>&1
+    if [[ $? -ne 0 ]]; then
+      echo "The ${APP_NAME} container doesn't appear to exist, exiting"
+      exit 1
+    fi
+    RUNNING=`docker inspect ${APP_NAME} | python -c 'import sys, json; print json.load(sys.stdin)[0]["State"]["Running"]'`
+    if [[ "${RUNNING}" = "True" ]]; then
+      echo "Stopping ${APP_NAME} container for export"
+      docker stop ${APP_NAME}
+      DockerExport
+      DockerPush
+    else
+      DockerExport
+      DockerPush
+    fi
+    ;;
+  *)
+    docker tag -f ${LATESTIMAGE} ${REPO_NAME}/${APP_NAME}:latest
+    DockerPush
+esac
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..5e8b7f5
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+source vars
+
+#If there is a locally built image present, prefer that over the
+#one in the registry, we're going to assume you're working on changes
+#to the image.
+
+if [[ ! -f builds ]]; then
+  LATESTIMAGE=${REPO_NAME}/${APP_NAME}:latest
+else
+  LATESTIMAGE=`tail -1 builds | awk '{print $8}'`
+fi
+echo
+echo "Starting $APP_NAME..."
+echo
+echo -n "Container ID: "
+docker run \
+--detach=true \
+--log-driver=syslog \
+--name="${APP_NAME}" \
+--restart=always \
+-e FULL_NAME="${FULL_NAME}" \
+-e EMAIL_ADDRESS="${EMAIL_ADDRESS}" \
+-e GPG_PASSWORD="${GPG_PASSWORD}" \
+-e HOSTNAME=${HOSTNAME} \
+-v ${APTLY_DATADIR}:/opt/aptly \
+-p ${DOCKER_HOST_PORT}:80 \
+${LATESTIMAGE}
+# Other useful options
+# -p DOCKERHOST_PORT:CONTAINER_PORT \
+# -e "ENVIRONMENT_VARIABLE_NAME=VALUE" \
+# -v /DOCKERHOST/PATH:/CONTAINER/PATH \
diff --git a/shell.sh b/shell.sh
new file mode 100755
index 0000000..6c4fc78
--- /dev/null
+++ b/shell.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+source vars
+
+docker inspect ${APP_NAME} > /dev/null 2>&1
+if [[ $? -ne 0 ]]; then
+  echo "The ${APP_NAME} container doesn't appear to exist, exiting"
+fi
+
+CONTAINER_ID=`docker inspect ${APP_NAME} | python -c 'import sys, json; print json.load(sys.stdin)[0]["Id"]'`
+
+docker exec -it ${CONTAINER_ID} /bin/bash
diff --git a/vars b/vars
new file mode 100644
index 0000000..6bddc99
--- /dev/null
+++ b/vars
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+#### BEGIN APP SPECIFIC VARIABLES
+
+# Name of the container
+APP_NAME=aptly
+# Docker Hub Username or internal registry (e.g. docker-registry.example.com:5000)
+REPO_NAME="myusername"
+
+# Name used on repository signing key
+FULL_NAME="First Last"
+# Email address of the repository key signer
+EMAIL_ADDRESS=user@example.com
+# Password used to encrypt the signing key
+GPG_PASSWORD=repo1234
+# The directory on the Docker host to store repository data
+APTLY_DATADIR=/path/to/lots/ofspace
+# FQDN of the Docker host that the aptly container will run on 
+HOSTNAME=aptly.example.com
+# TCP port that aptly will be reachable on, set to something else if you already
+# have a web server running on your Docker host
+DOCKER_HOST_PORT=80
+
+#### END APP SPECIFIC VARIABLES
+#### BEGIN GENERIC VARIABLES
+  
+# Get an SHA sum of all files involved in building the image so the image can be tagged
+# this will provide assurance that any image with the same tag was built the same way. 
+SHASUM=`find . -type f \
+        -not -path "*/.git/*" \
+        -not -path "*.gitignore*" \
+        -not -path "*builds*" \
+        -not -path "*run.sh*" \
+        -exec shasum {} + | awk '{print $1}' | sort | shasum | cut -c1-4`
+  
+TAG="`date +%Y%m%d`-${SHASUM}"
+
+#### END GENERIC VARIABLES