Development: Update Dockerfile and create possibility to run local cluster

To help in developing features, the Dockerfile and its entrypoint have been extended.
The README.md explains stuff in detail, in short:

- you can now run a Patroni cluster with a single command

Dockerfile

@@ -13,23 +13,23 @@ RUN apt-get update -y
 RUN apt-get upgrade -y
 
 ENV PGVERSION 9.4
-RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-pip -y
+RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-kazoo -y
-RUN pip install zake
 
 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
 
 RUN mkdir -p /patroni/helpers
 ADD patroni.py /patroni/patroni.py
 ADD helpers /patroni/helpers
-ADD postgres0.yml /patroni/
 
-ENV ETCDVERSION 2.0.12
+ENV ETCDVERSION 2.0.13
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
 ## Setting up a simple script that will serve as an entrypoint
-RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err && chown postgres:postgres /var/log/etcd.*
+RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml
-RUN chown postgres:postgres -R /patroni/ /data/
+RUN chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml
-ADD entrypoint.sh /entrypoint.sh
+ADD docker/entrypoint.sh /entrypoint.sh
+
+EXPOSE 4001 5432 2380
 
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres

docker/README.md

@@ -0,0 +1,46 @@
+# Patroni Dockerfile
+You can run Patroni in a docker container using this Dockerfile, or by using the Docker image at
+    https://os-registry.stups.zalan.do/acid/patroni-1.0
+
+This Dockerfile is meant in aiding development of Patroni and quick testing of features. It is not a production-worthy
+Dockerfile
+
+# Examples
+
+## Standalone Patroni
+
+    docker run -d os-registry.stups.zalan.do/acid/patroni:1.0
+
+## Multiple Patroni's communicating with a standalone etcd inside Docker
+
+Basically what you would do would be: 
+
+* Run 1 container which provides etcd
+
+        docker run -d <IMAGE> --etcd-only
+
+* Run n containers running Patroni, passing the `--etcd` option to the `docker run` command
+
+        docker run -d <IMAGE> --etcd=<IP FROM etcd CONTAINER:PORT>
+
+To automate this you can run the following script:
+
+    dev_patroni_cluster.sh [OPTIONS]
+
+    Options:
+
+        --image     IMAGE    The Docker image to use for the cluster
+        --members   INT      The number of members for the cluster
+        --name      NAME     The name of the new cluster
+
+Example session:
+
+    $ ./dev_patroni_cluster.sh --image os-registry.stups.zalan.do/acid/patroni:1.0 --members=2 --name=bravo
+    The etcd container is 6be871a11cb373406ca5ea1c6b39e140fdde9fb1d6177212d6ad0c0d1bd9b563, ip=172.17.1.24
+    Started Patroni container 67e611f2eca7c40f9e6e0e24a4a8f2cba7e3e56d22a420e15ab9240a37a9d7a4, ip=172.17.1.25
+    Started Patroni container 47dd12ae635ab83b039f5889e250048b606ed5e48e3650b69e365e7e1d4acbcf, ip=172.17.1.26
+    $ docker ps
+    CONTAINER ID        IMAGE                                         COMMAND                CREATED             STATUS              PORTS                          NAMES
+    47dd12ae635a        os-registry.stups.zalan.do/acid/patroni:1.0   "/bin/bash /entrypoi   10 seconds ago      Up 8 seconds        4001/tcp, 5432/tcp, 2380/tcp   bravo_OR64g8bx
+    67e611f2eca7        os-registry.stups.zalan.do/acid/patroni:1.0   "/bin/bash /entrypoi   11 seconds ago      Up 10 seconds       2380/tcp, 4001/tcp, 5432/tcp   bravo_si9no8iz
+    6be871a11cb3        os-registry.stups.zalan.do/acid/patroni:1.0   "/bin/bash /entrypoi   12 seconds ago      Up 10 seconds       4001/tcp, 5432/tcp, 2380/tcp   bravo_etcd

docker/dev_patroni_cluster.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+DOCKER_IMAGE="os-registry.stups.zalan.do/acid/patroni:1.0"
+MEMBERS=3
+
+
+function usage()
+{
+    cat <<__EOF__
+Usage: $0 
+
+Options:
+
+    --image     IMAGE    The Docker image to use for the cluster
+    --members   INT      The number of members for the cluster
+    --name      NAME     The name of the new cluster
+
+Examples:
+
+    $0 --image ${DOCKER_IMAGE}
+    $0
+    $0 --image ${DOCKER_IMAGE} --members=2
+__EOF__
+}
+
+
+optspec=":-:"
+while getopts "$optspec" optchar; do
+    case "${optchar}" in
+        -)
+            case "${OPTARG}" in
+                help)
+                    usage
+                    exit 0
+                    ;;
+                name)
+                    PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+                    ;;
+                name=*)
+                    PATRONI_SCOPE="${OPTARG#*=}"
+                    ;;
+                image)
+                    DOCKER_IMAGE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+                    ;;
+                image=*)
+                    DOCKER_IMAGE="${OPTARG#*=}"
+                    ;;
+                members)
+                    MEMBERS="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+                    ;;
+                members=*)
+                    MEMBERS="${OPTARG#*=}"
+                    ;;
+                *)
+                    if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then
+                        echo "Unknown option --${OPTARG}" >&2
+                    fi
+                    ;;
+            esac;;
+        *)
+            if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then
+                echo "Non-option argument: '-${OPTARG}'" >&2
+                usage
+                exit 1
+            fi
+            ;;
+    esac
+done
+
+function random_name()
+{
+    cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | head -c 8
+}
+
+if [ -z ${PATRONI_SCOPE} ]
+then
+    PATRONI_SCOPE=$(random_name)
+fi
+
+etcd_container=$(docker run -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only)
+etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container})
+echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}"
+
+for i in $(seq 1 "${MEMBERS}")
+do
+    container_name=$(random_name)
+    patroni_container=$(docker run -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}")
+    patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container})
+    echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}"
+done

docker/entrypoint.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+function usage()
+{
+    cat <<__EOF__
+Usage: $0 
+
+Options:
+
+    --etcd      ETCD    Provide an external etcd to connect to
+    --name      NAME    Give the cluster a specific name 
+    --etcd-only         Do not run Patroni, run a standalone etcd
+
+Examples:
+
+    $0 --etcd=127.17.0.84:4001
+    $0 --etcd-only
+    $0 
+    $0 --name=true_scotsman
+__EOF__
+}
+
+DOCKER_IP=$(hostname --ip-address)
+PATRONI_SCOPE=batman
+
+optspec=":vh-:"
+while getopts "$optspec" optchar; do
+    case "${optchar}" in
+        -)
+            case "${OPTARG}" in
+                etcd-only)
+                    exec etcd --data-dir /tmp/etcd.data \
+                        -advertise-client-urls=http://${DOCKER_IP}:4001 \
+                        -listen-client-urls=http://0.0.0.0:4001 \
+                        -listen-peer-urls=http://0.0.0.0:2380 
+                    exit 0
+                    ;;
+                name)
+                    PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+                    ;;
+                name=*)
+                    PATRONI_SCOPE=${OPTARG#*=}
+                    ;;
+                etcd)
+                    ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+                    ;;
+                etcd=*)
+                    ETCD_CLUSTER=${OPTARG#*=}
+                    ;;
+                help)
+                    usage
+                    exit 0
+                    ;;
+                *)
+                    if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then
+                        echo "Unknown option --${OPTARG}" >&2
+                    fi
+                    ;;
+            esac;;
+        *)
+            if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then
+                echo "Non-option argument: '-${OPTARG}'" >&2
+                usage
+                exit 1
+            fi
+            ;;
+    esac
+done
+
+if [ -z ${ETCD_CLUSTER} ]
+then
+    etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err &
+    ETCD_CLUSTER="127.0.0.1:4001"
+fi
+
+cat > /patroni/postgres.yml <<__EOF__
+
+ttl: &ttl 30
+loop_wait: &loop_wait 10
+scope: &scope ${PATRONI_SCOPE}
+restapi:
+  listen: 127.0.0.1:8008
+  connect_address: 127.0.0.1:8008
+etcd:
+  scope: *scope
+  ttl: *ttl
+  host: ${ETCD_CLUSTER}
+postgresql:
+  name: postgresql_${DOCKER_IP//./_} ## Replication slots do not allow dots in their name
+  scope: *scope
+  listen: 0.0.0.0:5432
+  connect_address: ${DOCKER_IP}:5432
+  data_dir: data/postgresql0
+  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - hostssl all all 0.0.0.0/0 md5
+  - host replication replicator ${DOCKER_IP}/16    md5
+  replication:
+    username: replicator
+    password: rep-pass
+    network:  127.0.0.1/32
+  superuser:
+    password: zalando
+  admin:
+    username: admin
+    password: admin
+  parameters:
+    archive_mode: "on"
+    wal_level: hot_standby
+    archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f
+    max_wal_senders: 20 
+    listen_addresses: 0.0.0.0
+    wal_keep_segments: 8
+    archive_timeout: 1800s
+    max_replication_slots: 20
+    hot_standby: "on"
+__EOF__
+
+cat /patroni/postgres.yml
+
+exec /patroni/patroni.py /patroni/postgres.yml

entrypoint.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err &
-exec /patroni/patroni.py /patroni/postgres0.yml "$@"