Dev docker images improvements (#2677)

- configurable image
- ETCD_UNSUPPORTED_ARCH env in docker-compose
- Build confd and citus for arm64 images

Dockerfile

@@ -53,14 +53,26 @@ RUN set -ex \
     && curl -sL "https://github.com/coreos/etcd/releases/download/v$ETCDVERSION/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
             | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
 \
-    # Download confd
-    && curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
-            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
+    && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
+        # Build confd
+        apt-get install -y git make \
+        && curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \
+        && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \
+        && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \
+        && make -C confd \
+        && cp confd/bin/confd /usr/local/bin/confd \
+        && rm -rf /confd /usr/local/go; \
+    else \
+        # Download confd
+        curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
+            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \
+    fi \
 \
     # Clean up all useless packages and some files
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
                 exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                git make \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* \

Dockerfile.citus

@@ -20,14 +20,26 @@ RUN set -ex \
     && export DEBIAN_FRONTEND=noninteractive \
     && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
     && apt-get update -y \
-    # postgres:10 is based on debian, which has the patroni package. We will install all required dependencies
+    # postgres:PG_MAJOR is based on debian, which has the patroni package. We will install all required dependencies
     && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
                             net-tools iputils-ping --fix-missing \
-    && curl https://install.citusdata.com/community/deb.sh | bash \
-    && apt-get -y install postgresql-$PG_MAJOR-citus-11.2 \
+    && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
+        apt-get install -y postgresql-server-dev-$PG_MAJOR \
+                           git gcc make autoconf \
+                           libc6-dev flex libcurl4-gnutls-dev \
+                           libicu-dev libkrb5-dev liblz4-dev \
+                           libpam0g-dev libreadline-dev libselinux1-dev\
+                           libssl-dev libxslt1-dev libzstd-dev uuid-dev \
+        && git clone -b "main" https://github.com/citusdata/citus.git \
+        && MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \
+        && cd citus && ./configure && make install && cd ../ && rm -rf /citus; \
+    else \
+        curl https://install.citusdata.com/community/deb.sh | bash \
+        && apt-get -y install postgresql-$PG_MAJOR-citus-11.2; \
+    fi \
     && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
@@ -55,9 +67,19 @@ RUN set -ex \
     && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \
             | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
 \
-    # Download confd
-    && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-$(dpkg --print-architecture) \
-            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
+    && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
+        # Build confd
+        curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \
+        && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \
+        && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \
+        && make -C confd \
+        && cp confd/bin/confd /usr/local/bin/confd \
+        && rm -rf /confd /usr/local/go; \
+    else \
+        # Download confd
+        curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
+            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \
+    fi \
     # Prepare client cert for HAProxy
     && cat /etc/ssl/private/ssl-cert-snakeoil.key  /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \
 \
@@ -65,6 +87,9 @@ RUN set -ex \
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
                 exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                postgresql-server-dev-$PG_MAJOR git gcc make autoconf \
+                libc6-dev flex libcurl4-gnutls-dev libicu-dev libkrb5-dev liblz4-dev \
+                libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* \

docker-compose-citus.yml

@@ -16,7 +16,7 @@ networks:
 
 services:
     etcd1: &etcd
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         environment:
             ETCDCTL_API: 3
@@ -25,6 +25,7 @@ services:
             ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
             ETCD_INITIAL_CLUSTER_STATE: new
             ETCD_INITIAL_CLUSTER_TOKEN: tutorial
+            ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
         command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
@@ -42,7 +43,7 @@ services:
         command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: haproxy
@@ -64,7 +65,7 @@ services:
             PGSSLROOTCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem
 
     coord1:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: coord1
@@ -75,7 +76,7 @@ services:
             PATRONI_CITUS_GROUP: 0
 
     coord2:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: coord2
@@ -85,7 +86,7 @@ services:
             PATRONI_NAME: coord2
 
     coord3:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: coord3
@@ -96,7 +97,7 @@ services:
 
 
     work1-1:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: work1-1
@@ -107,7 +108,7 @@ services:
             PATRONI_CITUS_GROUP: 1
 
     work1-2:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: work1-2
@@ -118,7 +119,7 @@ services:
 
 
     work2-1:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: work2-1
@@ -129,7 +130,7 @@ services:
             PATRONI_CITUS_GROUP: 2
 
     work2-2:
-        image: patroni-citus
+        image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: work2-2

docker-compose.yml

@@ -14,7 +14,7 @@ networks:
 
 services:
     etcd1: &etcd
-        image: patroni
+        image: ${PATRONI_TEST_IMAGE:-patroni}
         networks: [ demo ]
         environment:
             ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
@@ -22,6 +22,7 @@ services:
             ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
             ETCD_INITIAL_CLUSTER_STATE: new
             ETCD_INITIAL_CLUSTER_TOKEN: tutorial
+            ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
         command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
@@ -39,7 +40,7 @@ services:
         command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
-        image: patroni
+        image: ${PATRONI_TEST_IMAGE:-patroni}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: haproxy
@@ -54,7 +55,7 @@ services:
             PATRONI_SCOPE: demo
 
     patroni1:
-        image: patroni
+        image: ${PATRONI_TEST_IMAGE:-patroni}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: patroni1
@@ -64,7 +65,7 @@ services:
             PATRONI_NAME: patroni1
 
     patroni2:
-        image: patroni
+        image: ${PATRONI_TEST_IMAGE:-patroni}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: patroni2
@@ -74,7 +75,7 @@ services:
             PATRONI_NAME: patroni2
 
     patroni3:
-        image: patroni
+        image: ${PATRONI_TEST_IMAGE:-patroni}
         networks: [ demo ]
         env_file: docker/patroni.env
         hostname: patroni3