diff --git a/Dockerfile b/Dockerfile index 421e775f..43f6f12f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,36 +3,47 @@ FROM ubuntu:16.04 MAINTAINER Feike Steenbergen -RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend -RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend +RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \ + && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend ENV PGVERSION 9.5 +ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH RUN apt-get update -y \ && apt-get upgrade -y \ - && apt-get install -y curl postgresql-${PGVERSION} python-psycopg2 python-yaml python-requests python-six python-click \ - python-dateutil python-tzlocal python-urllib3 python-dnspython python-pip python-setuptools python-kazoo python \ - && pip install python-etcd==0.4.3 python-consul \ + && apt-get install -y curl jq haproxy postgresql-${PGVERSION} python-psycopg2 python-yaml \ + python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \ + python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \ + && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \ && apt-get remove -y python-pip python-setuptools \ && apt-get autoremove -y \ # Clean up && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* /root/.cache -ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH +ENV ETCDVERSION 2.3.6 +RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \ + | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl + +ENV CONFDVERSION 0.11.0 +RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \ + && chmod +x /usr/local/bin/confd ADD patronictl.py patroni.py docker/entrypoint.sh / ADD patroni /patroni/ -RUN ln -s /patroni/patroni.py /usr/local/bin/patroni \ - && ln -s /patroni/patronictl.py /usr/local/bin/patronictl - -ENV ETCDVERSION 2.3.6 -RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl +ADD extras/confd /etc/confd +RUN ln -s /patronictl.py /usr/local/bin/patronictl ### Setting up a simple script that will serve as an entrypoint -RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml \ - && chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml +RUN mkdir /data/ && touch /pgpass /patroni.yml /var/run/haproxy.pid \ + && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/haproxy.pid \ + && for name in etcd haproxy; do \ + for ext in log err; do \ + touch /var/log/$name.$ext \ + && chown postgres:postgres /var/log/$name.$ext; \ + done; \ + done -EXPOSE 4001 5432 2380 +EXPOSE 2379 5432 8008 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] USER postgres diff --git a/docker/dev_patroni_cluster.sh b/docker/dev_patroni_cluster.sh index dcc18f87..2260133e 100755 --- a/docker/dev_patroni_cluster.sh +++ b/docker/dev_patroni_cluster.sh @@ -67,24 +67,37 @@ while getopts "$optspec" optchar; do esac done -function random_name() -{ - cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8 -} - -if [ -z ${PATRONI_SCOPE} ] -then - PATRONI_SCOPE=$(random_name) +if [ -z ${PATRONI_SCOPE} ]; then + PATRONI_SCOPE=$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 8) fi -etcd_container=$(docker run -P -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only) -etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container}) -echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}" +function docker_run() +{ + local name=$1 + shift + container=$(docker run -d --name=$name $*) + container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${container}) + echo "Started container ${name}, ip=${container_ip}" +} -for i in $(seq 1 "${MEMBERS}") -do - container_name=$(random_name) - patroni_container=$(docker run -P -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}") - patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container}) - echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}" + +ETCD_CONTAINER="${PATRONI_SCOPE}_etcd" +docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd + +DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379" +PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ') + +for i in $(seq 1 "${MEMBERS}"); do + container_name=postgres${i} + docker_run "${PATRONI_SCOPE}_${container_name}" \ + -v patroni:/patroni \ + $DOCKER_ARGS \ + $PATRONI_ENV \ + -e PATRONI_NAME=${container_name} \ + ${DOCKER_IMAGE} done + +docker_run "${PATRONI_SCOPE}_haproxy" \ + -p=5000 -p=5001 \ + $DOCKER_ARGS \ + ${DOCKER_IMAGE} --confd diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index c329d2bb..48f82460 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -7,49 +7,39 @@ Usage: $0 Options: - --etcd ETCD Provide an external etcd to connect to - --name NAME Give the cluster a specific name - --etcd-only Do not run Patroni, run a standalone etcd + --etcd Do not run Patroni, run a standalone etcd + --confd Do not run Patroni, run a standalone confd Examples: - $0 --etcd=127.17.0.84:4001 - $0 --etcd-only + $0 --etcd + $0 --confd $0 - $0 --name=true_scotsman __EOF__ } DOCKER_IP=$(hostname --ip-address) PATRONI_SCOPE=${PATRONI_SCOPE:-batman} +ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380" optspec=":vh-:" while getopts "$optspec" optchar; do case "${optchar}" in -) case "${OPTARG}" in - etcd-only) - exec etcd --data-dir /tmp/etcd.data \ - -advertise-client-urls=http://${DOCKER_IP}:4001 \ - -listen-client-urls=http://0.0.0.0:4001 \ - -listen-peer-urls=http://0.0.0.0:2380 - exit 0 + confd) + while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do + sleep 1 + done + haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D + exec confd -prefix=${PATRONI_NAMESPACE:-/service}/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10 + ;; + etcd) + exec etcd $ETCD_ARGS ;; cheat) CHEAT=1 ;; - name) - PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) - ;; - name=*) - PATRONI_SCOPE=${OPTARG#*=} - ;; - etcd) - ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) - ;; - etcd=*) - ETCD_CLUSTER=${OPTARG#*=} - ;; help) usage exit 0 @@ -71,32 +61,27 @@ while getopts "$optspec" optchar; do done ## We start an etcd -if [ -z ${ETCD_CLUSTER} ] -then - etcd --data-dir /tmp/etcd.data \ - -advertise-client-urls=http://${DOCKER_IP}:4001 \ - -listen-client-urls=http://0.0.0.0:4001 \ - -listen-peer-urls=http://0.0.0.0:2380 > /var/log/etcd.log 2> /var/log/etcd.err & - ETCD_CLUSTER="127.0.0.1:4001" +if [ -z ${PATRONI_ETCD_HOST} ]; then + etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err & + export PATRONI_ETCD_HOST="127.0.0.1:2379" fi export PATRONI_SCOPE -export PATRONI_NAME="${HOSTNAME}" -export PATRONI_ETCD_HOST="$ETCD_CLUSTER" +export PATRONI_NAME="${PATRONI_NAME:-${HOSTNAME}}" export PATRONI_RESTAPI_CONNECT_ADDRESS="${DOCKER_IP}:8008" export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008" -export PATRONI_admin_PASSWORD="admin" -export PATRONI_admin_OPTIONS="createdb, createrole" +export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:=admin}" +export PATRONI_admin_OPTIONS="$PATRONI_admin_OPTIONS:-createdb, createrole}" export PATRONI_POSTGRESQL_CONNECT_ADDRESS="${DOCKER_IP}:5432" export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432" export PATRONI_POSTGRESQL_DATA_DIR="data/${PATRONI_SCOPE}" -export PATRONI_REPLICATION_USERNAME="replicator" -export PATRONI_REPLICATION_PASSWORD="abcd" -export PATRONI_SUPERUSER_USERNAME="postgres" -export PATRONI_SUPERUSER_PASSWORD="postgres" +export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}" +export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-abcd}" +export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}" +export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}" export PATRONI_POSTGRESQL_PGPASS="$HOME/.pgpass" -cat > /patroni/postgres.yaml <<__EOF__ +cat > /patroni.yml <<__EOF__ bootstrap: dcs: postgresql: @@ -108,14 +93,10 @@ bootstrap: __EOF__ mkdir -p "$HOME/.config/patroni" -ln -s /patroni/postgres.yaml "$HOME/.config/patroni/patronictl.yaml" +ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml" -if [ ! -z $CHEAT ] -then - while : - do - sleep 60 - done -else - exec python /patroni.py /patroni/postgres.yaml -fi +[ -z $CHEAT ] && exec python /patroni.py /patroni.yml + +while true; do + sleep 60 +done diff --git a/docker/patroni-secrets.env b/docker/patroni-secrets.env new file mode 100644 index 00000000..7c0f840e --- /dev/null +++ b/docker/patroni-secrets.env @@ -0,0 +1,8 @@ +PATRONI_RESTAPI_USERNAME=admin +PATRONI_RESTAPI_PASSWORD=admin +PATRONI_SUPERUSER_USERNAME=postgres +PATRONI_SUPERUSER_PASSWORD=postgres +PATRONI_REPLICATION_USERNAME=replicator +PATRONI_REPLICATION_PASSWORD=replicate +PATRONI_admin_PASSWORD=admin +PATRONI_admin_OPTIONS=createdb,createrole diff --git a/extras/README.md b/extras/README.md new file mode 100644 index 00000000..a4c3d59a --- /dev/null +++ b/extras/README.md @@ -0,0 +1,13 @@ +### confd + +`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool +You need to copy content of `confd` directory into /etcd/confd and run confd service: +```bash +$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10 +``` +It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary. + + +### startup-scripts + +`startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni. diff --git a/extras/confd/conf.d/haproxy.toml b/extras/confd/conf.d/haproxy.toml new file mode 100644 index 00000000..f9f04fbc --- /dev/null +++ b/extras/confd/conf.d/haproxy.toml @@ -0,0 +1,13 @@ +[template] +#prefix = "/service/batman" +#owner = "haproxy" +#mode = "0644" +src = "haproxy.tmpl" +dest = "/etc/haproxy/haproxy.cfg" + +check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}" +reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)" + +keys = [ + "/members/", +] diff --git a/extras/confd/templates/haproxy.tmpl b/extras/confd/templates/haproxy.tmpl new file mode 100644 index 00000000..4309ed0c --- /dev/null +++ b/extras/confd/templates/haproxy.tmpl @@ -0,0 +1,28 @@ +global + maxconn 100 + +defaults + log global + mode tcp + retries 2 + timeout client 30m + timeout connect 4s + timeout server 30m + timeout check 5s + +frontend master_postgresql + bind *:5000 + default_backend backend_master + +frontend replicas_postgresql + bind *:5001 + default_backend backend_replicas + +backend backend_master + option httpchk OPTIONS /master +{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}} +{{end}} +backend backend_replicas + option httpchk OPTIONS /replica +{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}} +{{end}} diff --git a/patroni-compose-etcd-3.yml b/patroni-compose-etcd-3.yml new file mode 100644 index 00000000..580feabd --- /dev/null +++ b/patroni-compose-etcd-3.yml @@ -0,0 +1,58 @@ +# docker compose file for running a 3-node PostgreSQL cluster +# with etcd as the SIS + +patroni_etcd: + container_name: patroni_etcd + image: patroni + command: --etcd + +dbnode1: + image: patroni + hostname: dbnode1 + links: + - patroni_etcd:patroni_etcd + volumes: + - patroni:/patroni + env_file: docker/patroni-secrets.env + environment: + PATRONI_ETCD_HOST: patroni_etcd:2379 + PATRONI_NAME: dbnode1 + PATRONI_SCOPE: testcluster + +dbnode2: + image: patroni + hostname: dbnode2 + links: + - patroni_etcd:patroni_etcd + volumes: + - patroni:/patroni + env_file: docker/patroni-secrets.env + environment: + PATRONI_ETCD_HOST: patroni_etcd:2379 + PATRONI_NAME: dbnode2 + PATRONI_SCOPE: testcluster + +dbnode3: + image: patroni + hostname: dbnode3 + links: + - patroni_etcd:patroni_etcd + volumes: + - patroni:/patroni + env_file: docker/patroni-secrets.env + environment: + PATRONI_ETCD_HOST: patroni_etcd:2379 + PATRONI_NAME: dbnode3 + PATRONI_SCOPE: testcluster + +haproxy: + image: patroni + links: + - patroni_etcd:patroni_etcd + ports: + - "5000" + - "5001" + environment: + PATRONI_ETCD_HOST: patroni_etcd:2379 + PATRONI_SCOPE: testcluster + command: --confd