Merge pull request #225 from zalando/feature/confd

sample haproxy templates for confd
2026-08-25 14:53:37 +00:00 · 2016-06-28 15:23:54 +02:00
parent fe0bc2f3eb d65d1028a7
commit 6fdd01a8eb
8 changed files with 207 additions and 82 deletions
@@ -3,36 +3,47 @@
 FROM ubuntu:16.04
 MAINTAINER Feike Steenbergen <[email protected]>

-RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend
-RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
+RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend

 ENV PGVERSION 9.5
+ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
 RUN apt-get update -y \
    && apt-get upgrade -y \
-    && apt-get install -y curl postgresql-${PGVERSION} python-psycopg2 python-yaml python-requests python-six python-click \
-         python-dateutil python-tzlocal python-urllib3 python-dnspython python-pip python-setuptools python-kazoo python \
-    && pip install python-etcd==0.4.3 python-consul \
+    && apt-get install -y curl jq haproxy postgresql-${PGVERSION} python-psycopg2 python-yaml \
+        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
+        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
+    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
    && apt-get remove -y python-pip python-setuptools \
    && apt-get autoremove -y \
        # Clean up
    && apt-get clean -y \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* /root/.cache

-ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
+ENV ETCDVERSION 2.3.6
+RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+    | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
+
+ENV CONFDVERSION 0.11.0
+RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
+    && chmod +x /usr/local/bin/confd

 ADD patronictl.py patroni.py docker/entrypoint.sh /
 ADD patroni /patroni/
-RUN    ln -s /patroni/patroni.py /usr/local/bin/patroni \
-    && ln -s /patroni/patronictl.py /usr/local/bin/patronictl
-
-ENV ETCDVERSION 2.3.6
-RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
+ADD extras/confd /etc/confd
+RUN ln -s /patronictl.py /usr/local/bin/patronictl

 ### Setting up a simple script that will serve as an entrypoint
-RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml \
-    && chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml
+RUN mkdir /data/ && touch /pgpass /patroni.yml /var/run/haproxy.pid \
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/haproxy.pid \
+    && for name in etcd haproxy; do \
+        for ext in log err; do \
+            touch /var/log/$name.$ext \
+            && chown postgres:postgres /var/log/$name.$ext; \
+        done; \
+    done

-EXPOSE 4001 5432 2380
+EXPOSE 2379 5432 8008

 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres
@@ -67,24 +67,37 @@ while getopts "$optspec" optchar; do
    esac
 done

-function random_name()
-{
-    cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8
-}
-
-if [ -z ${PATRONI_SCOPE} ]
-then
-    PATRONI_SCOPE=$(random_name)
+if [ -z ${PATRONI_SCOPE} ]; then
+    PATRONI_SCOPE=$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 8)
 fi

-etcd_container=$(docker run -P -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only)
-etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container})
-echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}"
+function docker_run()
+{
+    local name=$1
+    shift
+    container=$(docker run -d --name=$name $*)
+    container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${container})
+    echo "Started container ${name}, ip=${container_ip}"
+}

-for i in $(seq 1 "${MEMBERS}")
-do
-    container_name=$(random_name)
-    patroni_container=$(docker run -P -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}")
-    patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container})
-    echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}"
+
+ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
+docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
+
+DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
+PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
+
+for i in $(seq 1 "${MEMBERS}"); do
+    container_name=postgres${i}
+    docker_run "${PATRONI_SCOPE}_${container_name}" \
+        -v patroni:/patroni \
+        $DOCKER_ARGS \
+        $PATRONI_ENV \
+        -e PATRONI_NAME=${container_name} \
+        ${DOCKER_IMAGE}
 done
+
+docker_run "${PATRONI_SCOPE}_haproxy" \
+    -p=5000 -p=5001 \
+    $DOCKER_ARGS \
+    ${DOCKER_IMAGE} --confd
@@ -7,49 +7,39 @@ Usage: $0

 Options:

-    --etcd      ETCD    Provide an external etcd to connect to
-    --name      NAME    Give the cluster a specific name
-    --etcd-only         Do not run Patroni, run a standalone etcd
+    --etcd      Do not run Patroni, run a standalone etcd
+    --confd     Do not run Patroni, run a standalone confd

 Examples:

-    $0 --etcd=127.17.0.84:4001
-    $0 --etcd-only
+    $0 --etcd
+    $0 --confd
    $0
-    $0 --name=true_scotsman
 __EOF__
 }

 DOCKER_IP=$(hostname --ip-address)
 PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
+ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380"

 optspec=":vh-:"
 while getopts "$optspec" optchar; do
    case "${optchar}" in
        -)
            case "${OPTARG}" in
-                etcd-only)
-                    exec etcd --data-dir /tmp/etcd.data \
-                        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-                        -listen-client-urls=http://0.0.0.0:4001 \
-                        -listen-peer-urls=http://0.0.0.0:2380
-                    exit 0
+                confd)
+                    while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
+                        sleep 1
+                    done
+                    haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
+                    exec confd -prefix=${PATRONI_NAMESPACE:-/service}/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
+                    ;;
+                etcd)
+                    exec etcd $ETCD_ARGS
                    ;;
                cheat)
                    CHEAT=1
                    ;;
-                name)
-                    PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                name=*)
-                    PATRONI_SCOPE=${OPTARG#*=}
-                    ;;
-                etcd)
-                    ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                etcd=*)
-                    ETCD_CLUSTER=${OPTARG#*=}
-                    ;;
                help)
                    usage
                    exit 0
@@ -71,32 +61,27 @@ while getopts "$optspec" optchar; do
 done

 ## We start an etcd
-if [ -z ${ETCD_CLUSTER} ]
-then
-    etcd --data-dir /tmp/etcd.data \
-        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-        -listen-client-urls=http://0.0.0.0:4001 \
-        -listen-peer-urls=http://0.0.0.0:2380 > /var/log/etcd.log 2> /var/log/etcd.err &
-    ETCD_CLUSTER="127.0.0.1:4001"
+if [ -z ${PATRONI_ETCD_HOST} ]; then
+    etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
+    export PATRONI_ETCD_HOST="127.0.0.1:2379"
 fi

 export PATRONI_SCOPE
-export PATRONI_NAME="${HOSTNAME}"
-export PATRONI_ETCD_HOST="$ETCD_CLUSTER"
+export PATRONI_NAME="${PATRONI_NAME:-${HOSTNAME}}"
 export PATRONI_RESTAPI_CONNECT_ADDRESS="${DOCKER_IP}:8008"
 export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008"
-export PATRONI_admin_PASSWORD="admin"
-export PATRONI_admin_OPTIONS="createdb, createrole"
+export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:=admin}"
+export PATRONI_admin_OPTIONS="$PATRONI_admin_OPTIONS:-createdb, createrole}"
 export PATRONI_POSTGRESQL_CONNECT_ADDRESS="${DOCKER_IP}:5432"
 export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432"
 export PATRONI_POSTGRESQL_DATA_DIR="data/${PATRONI_SCOPE}"
-export PATRONI_REPLICATION_USERNAME="replicator"
-export PATRONI_REPLICATION_PASSWORD="abcd"
-export PATRONI_SUPERUSER_USERNAME="postgres"
-export PATRONI_SUPERUSER_PASSWORD="postgres"
+export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}"
+export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-abcd}"
+export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
+export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
 export PATRONI_POSTGRESQL_PGPASS="$HOME/.pgpass"

-cat > /patroni/postgres.yaml <<__EOF__
+cat > /patroni.yml <<__EOF__
 bootstrap:
  dcs:
    postgresql:
@@ -108,14 +93,10 @@ bootstrap:
 __EOF__

 mkdir -p "$HOME/.config/patroni"
-ln -s /patroni/postgres.yaml "$HOME/.config/patroni/patronictl.yaml"
+ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"

-if [ ! -z $CHEAT ]
-then
-    while :
-    do
-        sleep 60
-    done
-else
-    exec python /patroni.py /patroni/postgres.yaml
-fi
+[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
+
+while true; do
+    sleep 60
+done
@@ -0,0 +1,8 @@
+PATRONI_RESTAPI_USERNAME=admin
+PATRONI_RESTAPI_PASSWORD=admin
+PATRONI_SUPERUSER_USERNAME=postgres
+PATRONI_SUPERUSER_PASSWORD=postgres
+PATRONI_REPLICATION_USERNAME=replicator
+PATRONI_REPLICATION_PASSWORD=replicate
+PATRONI_admin_PASSWORD=admin
+PATRONI_admin_OPTIONS=createdb,createrole
@@ -0,0 +1,13 @@
+### confd
+
+`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
+You need to copy content of `confd` directory into /etcd/confd and run confd service:
+```bash
+$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
+```
+It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
+
+
+### startup-scripts
+
+`startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni.
@@ -0,0 +1,13 @@
+[template]
+#prefix = "/service/batman"
+#owner = "haproxy"
+#mode = "0644"
+src = "haproxy.tmpl"
+dest = "/etc/haproxy/haproxy.cfg"
+
+check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
+reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
+
+keys = [
+    "/members/",
+]
@@ -0,0 +1,28 @@
+global
+	maxconn 100
+
+defaults
+	log	global
+	mode	tcp
+	retries 2
+	timeout client 30m
+	timeout connect 4s
+	timeout server 30m
+	timeout check 5s
+
+frontend master_postgresql
+	bind *:5000
+	default_backend backend_master
+
+frontend replicas_postgresql
+	bind *:5001
+	default_backend backend_replicas
+
+backend backend_master
+	option httpchk OPTIONS /master
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}
+backend backend_replicas
+	option httpchk OPTIONS /replica
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}
@@ -0,0 +1,58 @@
+# docker compose file for running a 3-node PostgreSQL cluster
+# with etcd as the SIS
+
+patroni_etcd:
+    container_name: patroni_etcd
+    image: patroni
+    command: --etcd
+
+dbnode1:
+    image: patroni
+    hostname: dbnode1
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode1
+        PATRONI_SCOPE: testcluster
+
+dbnode2:
+    image: patroni
+    hostname: dbnode2
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode2
+        PATRONI_SCOPE: testcluster
+
+dbnode3:
+    image: patroni
+    hostname: dbnode3
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode3
+        PATRONI_SCOPE: testcluster
+
+haproxy:
+    image: patroni
+    links:
+        - patroni_etcd:patroni_etcd
+    ports:
+        - "5000"
+        - "5001"
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_SCOPE: testcluster
+    command: --confd