From 08865512f82a1f0c3dc38a0d2e9bece1c776117a Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 12:06:04 +0200 Subject: [PATCH 1/4] Differentiate between connection address and listen address For example you can listen on 0.0.0.0 but provide service on some didicated ip. This should cover all possible use-cases when you are running behind firewall, load balancer or inside for example docker. --- helpers/postgresql.py | 4 ++-- postgres0.yml | 1 + postgres1.yml | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index a061a52c..c3870cb6 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -38,7 +38,7 @@ class Postgresql: self.config = config self.connection_string = 'postgres://{username}:{password}@{listen}/postgres'.format( - listen=self.config['listen'], **self.replication) + listen=self.config['connection_address'], **self.replication) self.conn = None self.cursor_holder = None @@ -46,7 +46,7 @@ class Postgresql: def cursor(self): if not self.cursor_holder: - self.conn = psycopg2.connect('postgres://{}/postgres'.format(self.config['listen'])) + self.conn = psycopg2.connect('postgres://{}/postgres'.format(self.config['connection_address'])) self.conn.autocommit = True self.cursor_holder = self.conn.cursor() diff --git a/postgres0.yml b/postgres0.yml index 163fd3b1..01116059 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -6,6 +6,7 @@ etcd: postgresql: name: postgresql0 listen: 127.0.0.1:5432 + connection_address: 127.0.0.1:5432 data_dir: data/postgresql0 maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: diff --git a/postgres1.yml b/postgres1.yml index 3a5bf93c..d6296845 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -6,6 +6,7 @@ etcd: postgresql: name: postgresql1 listen: 127.0.0.1:5433 + connection_address: 127.0.0.1:5433 data_dir: data/postgresql1 maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: From 75ff91d01702921c57f6b7df6f700428969715b8 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 12:13:05 +0200 Subject: [PATCH 2/4] rename connection_address to connect_address --- helpers/postgresql.py | 4 ++-- postgres0.yml | 2 +- postgres1.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index c3870cb6..b0d2b8fd 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -38,7 +38,7 @@ class Postgresql: self.config = config self.connection_string = 'postgres://{username}:{password}@{listen}/postgres'.format( - listen=self.config['connection_address'], **self.replication) + listen=self.config['connect_address'], **self.replication) self.conn = None self.cursor_holder = None @@ -46,7 +46,7 @@ class Postgresql: def cursor(self): if not self.cursor_holder: - self.conn = psycopg2.connect('postgres://{}/postgres'.format(self.config['connection_address'])) + self.conn = psycopg2.connect('postgres://{}/postgres'.format(self.config['connect_address'])) self.conn.autocommit = True self.cursor_holder = self.conn.cursor() diff --git a/postgres0.yml b/postgres0.yml index 01116059..5f4262f9 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -6,7 +6,7 @@ etcd: postgresql: name: postgresql0 listen: 127.0.0.1:5432 - connection_address: 127.0.0.1:5432 + connect_address: 127.0.0.1:5432 data_dir: data/postgresql0 maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: diff --git a/postgres1.yml b/postgres1.yml index d6296845..31a5dca3 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -6,7 +6,7 @@ etcd: postgresql: name: postgresql1 listen: 127.0.0.1:5433 - connection_address: 127.0.0.1:5433 + connect_address: 127.0.0.1:5433 data_dir: data/postgresql1 maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: From af9ebfa3b9d5d3b62a86e10266f7c9ed67cca03f Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 12:20:24 +0200 Subject: [PATCH 3/4] Use connect_address instead of listen when building connection_string --- helpers/postgresql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index b0d2b8fd..3c669ab5 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -37,8 +37,8 @@ class Postgresql: self.config = config - self.connection_string = 'postgres://{username}:{password}@{listen}/postgres'.format( - listen=self.config['connect_address'], **self.replication) + self.connection_string = 'postgres://{username}:{password}@{connect_address}/postgres'.format( + connect_address=self.config['connect_address'], **self.replication) self.conn = None self.cursor_holder = None From 0b669e0d20126462cf23cd1bf0ef943b6f00025f Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Fri, 15 May 2015 13:52:20 +0200 Subject: [PATCH 4/4] Cleanup Removed aws reference from Dockerfile, remove unnecessary steps. Connect using libpq parameters, therefore providing nice application name for Governor. --- Dockerfile | 22 +++++++--------------- helpers/postgresql.py | 2 +- requirements.txt | 1 + 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53d56965..42a90964 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,31 @@ ## This Dockerfile is meant to aid in the building and debugging governor whilst developing on your local machine ## It has all the necessary components to play/debug with a single node appliance, running etcd -FROM ubuntu:14.04 +FROM zalando/ubuntu:14.04.1-1 MAINTAINER Feike Steenbergen # Add PGDG repositories RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list -RUN apt-get install wget ca-certificates -y -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - RUN apt-get update -y RUN apt-get upgrade -y ENV PGVERSION 9.4 -RUN apt-get install curl python python-pip python-psycopg2 python-yaml postgresql-${PGVERSION} -y +RUN apt-get install python python-psycopg2 python-yaml python-requests postgresql-${PGVERSION} -y -RUN ln -s /usr/lib/postgresql/* /usr/lib/postgresql/current -ENV PATH /usr/lib/postgresql/current/bin:$PATH +ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH RUN mkdir -p /governor/helpers ADD governor.py /governor/governor.py -ADD requirements.txt /governor/requirements.txt ADD helpers /governor/helpers ADD postgres0.yml /governor/ -## As we are standalone, remove any reference to AWS -RUN sed -i '/aws_use_host_address/d' /governor/postgres0.yml -ENV ETCDVERSION 2.0.9 -RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz -o etcd-v${ETCDVERSION}-linux-amd64.tar.gz && tar vzxf etcd-v${ETCDVERSION}-linux-amd64.tar.gz && cp etcd-v${ETCDVERSION}-linux-amd64/etcd* /bin/ - -## Most requirements should already have been met, only as an extra precaution -RUN pip install -r /governor/requirements.txt +ENV ETCDVERSION 2.0.10 +RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl ## Setting up a simple script that will serve as an entrypoint RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err && chown postgres:postgres /var/log/etcd.* RUN chown postgres:postgres -R /governor/ /data/ -RUN /bin/echo -e "etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err &\n/governor/governor.py /governor/postgres0.yml \"$@\"" >> /entrypoint.sh && chmod +x /entrypoint.sh +RUN /bin/echo -e "etcd --data-dir /tmp/etcd.data > /var/log/etcd.log 2> /var/log/etcd.err &\n/governor/governor.py /governor/postgres0.yml \"\$@\"" >> /entrypoint.sh && chmod +x /entrypoint.sh ENTRYPOINT /entrypoint.sh USER postgres diff --git a/helpers/postgresql.py b/helpers/postgresql.py index ce1955f4..7eb884c7 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -56,7 +56,7 @@ class Postgresql: def cursor(self): if not self.cursor_holder: - self.conn = psycopg2.connect('postgres://{}/postgres'.format(self.config['connect_address'])) + self.conn = psycopg2.connect(**self.libpq_parameters) self.conn.autocommit = True self.cursor_holder = self.conn.cursor() diff --git a/requirements.txt b/requirements.txt index 377437fe..2185e5c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ PyYAML psycopg2 +requests