From c30d8dbd1ae278795e36d680999e8f8d0b1f4e4a Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Wed, 5 Aug 2015 14:12:45 +0200 Subject: [PATCH] Development: Update Dockerfile and create possibility to run local cluster To help in developing features, the Dockerfile and its entrypoint have been extended. The README.md explains stuff in detail, in short: - you can now run a Patroni cluster with a single command --- Dockerfile | 14 ++-- docker/README.md | 46 +++++++++++++ docker/dev_patroni_cluster.sh | 90 +++++++++++++++++++++++++ docker/entrypoint.sh | 122 ++++++++++++++++++++++++++++++++++ entrypoint.sh | 3 - 5 files changed, 265 insertions(+), 10 deletions(-) create mode 100644 docker/README.md create mode 100755 docker/dev_patroni_cluster.sh create mode 100755 docker/entrypoint.sh delete mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index d8e27c2d..147cad38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,23 +13,23 @@ RUN apt-get update -y RUN apt-get upgrade -y ENV PGVERSION 9.4 -RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-pip -y -RUN pip install zake +RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-kazoo -y ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH RUN mkdir -p /patroni/helpers ADD patroni.py /patroni/patroni.py ADD helpers /patroni/helpers -ADD postgres0.yml /patroni/ -ENV ETCDVERSION 2.0.12 +ENV ETCDVERSION 2.0.13 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl ## Setting up a simple script that will serve as an entrypoint -RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err && chown postgres:postgres /var/log/etcd.* -RUN chown postgres:postgres -R /patroni/ /data/ -ADD entrypoint.sh /entrypoint.sh +RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml +RUN chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml +ADD docker/entrypoint.sh /entrypoint.sh + +EXPOSE 4001 5432 2380 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] USER postgres diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..e6245adc --- /dev/null +++ b/docker/README.md @@ -0,0 +1,46 @@ +# Patroni Dockerfile +You can run Patroni in a docker container using this Dockerfile, or by using the Docker image at + https://os-registry.stups.zalan.do/acid/patroni-1.0 + +This Dockerfile is meant in aiding development of Patroni and quick testing of features. It is not a production-worthy +Dockerfile + +# Examples + +## Standalone Patroni + + docker run -d os-registry.stups.zalan.do/acid/patroni:1.0 + +## Multiple Patroni's communicating with a standalone etcd inside Docker + +Basically what you would do would be: + +* Run 1 container which provides etcd + + docker run -d --etcd-only + +* Run n containers running Patroni, passing the `--etcd` option to the `docker run` command + + docker run -d --etcd= + +To automate this you can run the following script: + + dev_patroni_cluster.sh [OPTIONS] + + Options: + + --image IMAGE The Docker image to use for the cluster + --members INT The number of members for the cluster + --name NAME The name of the new cluster + +Example session: + + $ ./dev_patroni_cluster.sh --image os-registry.stups.zalan.do/acid/patroni:1.0 --members=2 --name=bravo + The etcd container is 6be871a11cb373406ca5ea1c6b39e140fdde9fb1d6177212d6ad0c0d1bd9b563, ip=172.17.1.24 + Started Patroni container 67e611f2eca7c40f9e6e0e24a4a8f2cba7e3e56d22a420e15ab9240a37a9d7a4, ip=172.17.1.25 + Started Patroni container 47dd12ae635ab83b039f5889e250048b606ed5e48e3650b69e365e7e1d4acbcf, ip=172.17.1.26 + $ docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 47dd12ae635a os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 10 seconds ago Up 8 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_OR64g8bx + 67e611f2eca7 os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 11 seconds ago Up 10 seconds 2380/tcp, 4001/tcp, 5432/tcp bravo_si9no8iz + 6be871a11cb3 os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 12 seconds ago Up 10 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_etcd diff --git a/docker/dev_patroni_cluster.sh b/docker/dev_patroni_cluster.sh new file mode 100755 index 00000000..00a8264a --- /dev/null +++ b/docker/dev_patroni_cluster.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +DOCKER_IMAGE="os-registry.stups.zalan.do/acid/patroni:1.0" +MEMBERS=3 + + +function usage() +{ + cat <<__EOF__ +Usage: $0 + +Options: + + --image IMAGE The Docker image to use for the cluster + --members INT The number of members for the cluster + --name NAME The name of the new cluster + +Examples: + + $0 --image ${DOCKER_IMAGE} + $0 + $0 --image ${DOCKER_IMAGE} --members=2 +__EOF__ +} + + +optspec=":-:" +while getopts "$optspec" optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help) + usage + exit 0 + ;; + name) + PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + name=*) + PATRONI_SCOPE="${OPTARG#*=}" + ;; + image) + DOCKER_IMAGE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + image=*) + DOCKER_IMAGE="${OPTARG#*=}" + ;; + members) + MEMBERS="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + members=*) + MEMBERS="${OPTARG#*=}" + ;; + *) + if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + esac;; + *) + if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then + echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 1 + fi + ;; + esac +done + +function random_name() +{ + cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | head -c 8 +} + +if [ -z ${PATRONI_SCOPE} ] +then + PATRONI_SCOPE=$(random_name) +fi + +etcd_container=$(docker run -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only) +etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container}) +echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}" + +for i in $(seq 1 "${MEMBERS}") +do + container_name=$(random_name) + patroni_container=$(docker run -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}") + patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container}) + echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}" +done diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..697bab66 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +function usage() +{ + cat <<__EOF__ +Usage: $0 + +Options: + + --etcd ETCD Provide an external etcd to connect to + --name NAME Give the cluster a specific name + --etcd-only Do not run Patroni, run a standalone etcd + +Examples: + + $0 --etcd=127.17.0.84:4001 + $0 --etcd-only + $0 + $0 --name=true_scotsman +__EOF__ +} + +DOCKER_IP=$(hostname --ip-address) +PATRONI_SCOPE=batman + +optspec=":vh-:" +while getopts "$optspec" optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + etcd-only) + exec etcd --data-dir /tmp/etcd.data \ + -advertise-client-urls=http://${DOCKER_IP}:4001 \ + -listen-client-urls=http://0.0.0.0:4001 \ + -listen-peer-urls=http://0.0.0.0:2380 + exit 0 + ;; + name) + PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + name=*) + PATRONI_SCOPE=${OPTARG#*=} + ;; + etcd) + ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + etcd=*) + ETCD_CLUSTER=${OPTARG#*=} + ;; + help) + usage + exit 0 + ;; + *) + if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + esac;; + *) + if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then + echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 1 + fi + ;; + esac +done + +if [ -z ${ETCD_CLUSTER} ] +then + etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err & + ETCD_CLUSTER="127.0.0.1:4001" +fi + +cat > /patroni/postgres.yml <<__EOF__ + +ttl: &ttl 30 +loop_wait: &loop_wait 10 +scope: &scope ${PATRONI_SCOPE} +restapi: + listen: 127.0.0.1:8008 + connect_address: 127.0.0.1:8008 +etcd: + scope: *scope + ttl: *ttl + host: ${ETCD_CLUSTER} +postgresql: + name: postgresql_${DOCKER_IP//./_} ## Replication slots do not allow dots in their name + scope: *scope + listen: 0.0.0.0:5432 + connect_address: ${DOCKER_IP}:5432 + data_dir: data/postgresql0 + maximum_lag_on_failover: 1048576 # 1 megabyte in bytes + pg_hba: + - host all all 0.0.0.0/0 md5 + - hostssl all all 0.0.0.0/0 md5 + - host replication replicator ${DOCKER_IP}/16 md5 + replication: + username: replicator + password: rep-pass + network: 127.0.0.1/32 + superuser: + password: zalando + admin: + username: admin + password: admin + parameters: + archive_mode: "on" + wal_level: hot_standby + archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f + max_wal_senders: 20 + listen_addresses: 0.0.0.0 + wal_keep_segments: 8 + archive_timeout: 1800s + max_replication_slots: 20 + hot_standby: "on" +__EOF__ + +cat /patroni/postgres.yml + +exec /patroni/patroni.py /patroni/postgres.yml diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index ab76a0dd..00000000 --- a/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err & -exec /patroni/patroni.py /patroni/postgres0.yml "$@"