From 44e58a1ba11bb47e3ea186797bbe5c477f5a4d9e Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Mon, 15 May 2023 11:40:35 +0200 Subject: [PATCH] Dev docker images improvements (#2677) - configurable image - ETCD_UNSUPPORTED_ARCH env in docker-compose - Build confd and citus for arm64 images --- Dockerfile | 18 +++++++++++++++--- Dockerfile.citus | 37 +++++++++++++++++++++++++++++++------ docker-compose-citus.yml | 19 ++++++++++--------- docker-compose.yml | 11 ++++++----- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index e424f588..5892e1ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,14 +53,26 @@ RUN set -ex \ && curl -sL "https://github.com/coreos/etcd/releases/download/v$ETCDVERSION/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \ | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \ \ - # Download confd - && curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \ - > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \ + && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ + # Build confd + apt-get install -y git make \ + && curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \ + && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \ + && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \ + && make -C confd \ + && cp confd/bin/confd /usr/local/bin/confd \ + && rm -rf /confd /usr/local/go; \ + else \ + # Download confd + curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \ + > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \ + fi \ \ # Clean up all useless packages and some files && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \ libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \ exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \ + git make \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ diff --git a/Dockerfile.citus b/Dockerfile.citus index d5432b15..06a84683 100644 --- a/Dockerfile.citus +++ b/Dockerfile.citus @@ -20,14 +20,26 @@ RUN set -ex \ && export DEBIAN_FRONTEND=noninteractive \ && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \ && apt-get update -y \ - # postgres:10 is based on debian, which has the patroni package. We will install all required dependencies + # postgres:PG_MAJOR is based on debian, which has the patroni package. We will install all required dependencies && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y vim curl less jq locales haproxy sudo \ python3-etcd python3-kazoo python3-pip busybox \ net-tools iputils-ping --fix-missing \ - && curl https://install.citusdata.com/community/deb.sh | bash \ - && apt-get -y install postgresql-$PG_MAJOR-citus-11.2 \ + && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ + apt-get install -y postgresql-server-dev-$PG_MAJOR \ + git gcc make autoconf \ + libc6-dev flex libcurl4-gnutls-dev \ + libicu-dev libkrb5-dev liblz4-dev \ + libpam0g-dev libreadline-dev libselinux1-dev\ + libssl-dev libxslt1-dev libzstd-dev uuid-dev \ + && git clone -b "main" https://github.com/citusdata/citus.git \ + && MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \ + && cd citus && ./configure && make install && cd ../ && rm -rf /citus; \ + else \ + curl https://install.citusdata.com/community/deb.sh | bash \ + && apt-get -y install postgresql-$PG_MAJOR-citus-11.2; \ + fi \ && pip3 install dumb-init \ \ # Cleanup all locales but en_US.UTF-8 @@ -55,9 +67,19 @@ RUN set -ex \ && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \ \ - # Download confd - && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-$(dpkg --print-architecture) \ - > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \ + && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ + # Build confd + curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \ + && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \ + && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \ + && make -C confd \ + && cp confd/bin/confd /usr/local/bin/confd \ + && rm -rf /confd /usr/local/go; \ + else \ + # Download confd + curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \ + > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \ + fi \ # Prepare client cert for HAProxy && cat /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \ \ @@ -65,6 +87,9 @@ RUN set -ex \ && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \ libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \ exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \ + postgresql-server-dev-$PG_MAJOR git gcc make autoconf \ + libc6-dev flex libcurl4-gnutls-dev libicu-dev libkrb5-dev liblz4-dev \ + libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ diff --git a/docker-compose-citus.yml b/docker-compose-citus.yml index f141bc3b..7ff2a2c5 100644 --- a/docker-compose-citus.yml +++ b/docker-compose-citus.yml @@ -16,7 +16,7 @@ networks: services: etcd1: &etcd - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] environment: ETCDCTL_API: 3 @@ -25,6 +25,7 @@ services: ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 ETCD_INITIAL_CLUSTER_STATE: new ETCD_INITIAL_CLUSTER_TOKEN: tutorial + ETCD_UNSUPPORTED_ARCH: arm64 container_name: demo-etcd1 hostname: etcd1 command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380 @@ -42,7 +43,7 @@ services: command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380 haproxy: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: haproxy @@ -64,7 +65,7 @@ services: PGSSLROOTCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem coord1: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord1 @@ -75,7 +76,7 @@ services: PATRONI_CITUS_GROUP: 0 coord2: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord2 @@ -85,7 +86,7 @@ services: PATRONI_NAME: coord2 coord3: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord3 @@ -96,7 +97,7 @@ services: work1-1: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work1-1 @@ -107,7 +108,7 @@ services: PATRONI_CITUS_GROUP: 1 work1-2: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work1-2 @@ -118,7 +119,7 @@ services: work2-1: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work2-1 @@ -129,7 +130,7 @@ services: PATRONI_CITUS_GROUP: 2 work2-2: - image: patroni-citus + image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work2-2 diff --git a/docker-compose.yml b/docker-compose.yml index 626391de..996c2c82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ networks: services: etcd1: &etcd - image: patroni + image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] environment: ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 @@ -22,6 +22,7 @@ services: ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 ETCD_INITIAL_CLUSTER_STATE: new ETCD_INITIAL_CLUSTER_TOKEN: tutorial + ETCD_UNSUPPORTED_ARCH: arm64 container_name: demo-etcd1 hostname: etcd1 command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380 @@ -39,7 +40,7 @@ services: command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380 haproxy: - image: patroni + image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: haproxy @@ -54,7 +55,7 @@ services: PATRONI_SCOPE: demo patroni1: - image: patroni + image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni1 @@ -64,7 +65,7 @@ services: PATRONI_NAME: patroni1 patroni2: - image: patroni + image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni2 @@ -74,7 +75,7 @@ services: PATRONI_NAME: patroni2 patroni3: - image: patroni + image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni3