Replacing SNS with Argus (#38)

* use argus for tr1d1um as webhook store

* update config that includes admin token

* save changes

* small bash file linting fixes

* consul latest works

* Remove SNS as a dep

deploy/docker-compose/config_dynamodb.sh

@@ -0,0 +1,15 @@
+AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb  --endpoint-url http://localhost:8000 create-table \
+    --table-name gifnoc \
+    --attribute-definitions \
+        AttributeName=bucket,AttributeType=S \
+        AttributeName=uuid,AttributeType=S \
+    --key-schema \
+        AttributeName=bucket,KeyType=HASH \
+        AttributeName=uuid,KeyType=RANGE \
+    --provisioned-throughput \
+        ReadCapacityUnits=10,WriteCapacityUnits=5 \
+    --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \
+    --region us-east-2
+
+
+AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb  --endpoint-url http://localhost:8000 --region us-east-2 update-time-to-live --table-name gifnoc --time-to-live-specification "Enabled=true, AttributeName=expires"

deploy/docker-compose/deploy.sh

@@ -1,27 +1,30 @@
 #!/bin/bash
 
-DIR=$( cd $(dirname $0) ; pwd -P )
+DIR=$( cd "$(dirname "$0")" || exit; pwd -P )
 ROOT_DIR=$DIR/../../
 
-pushd $ROOT_DIR
+pushd "$ROOT_DIR" || exit
 echo "Building Simulator..."
-docker build -t simulator:local $ROOT_DIR/simulator
+docker build -t simulator:local "$ROOT_DIR/simulator"
-popd
+popd || exit
 
 echo "Building goaws..."
 git clone [email protected]:kcajmagic/goaws.git /tmp/goaws
-pushd /tmp/goaws
+pushd /tmp/goaws || exit
 git checkout adding_http_support
 docker build -t goaws:local .
-popd
+popd || exit
 
 echo "Running services..."
-CADUCEUS_VERSION=${CADUCEUS_VERSION:-0.2.7} \
+CADUCEUS_VERSION=${CADUCEUS_VERSION:-0.4.2} \
-TR1D1UM_VERSION=${TR1D1UM_VERSION:-0.5.0} \
+ARGUS_VERSION=${ARGUS_VERSION:-0.3.9} \
+TR1D1UM_VERSION=${TR1D1UM_VERSION:-0.5.3} \
 SCYTALE_VERSION=${SCYTALE_VERSION:-0.1.5} \
 PETASOS_VERSION=${PETASOS_VERSION:-0.1.4} \
 TALARIA_VERSION=${TALARIA_VERSION:-0.5.9} \
 THEMIS_VERSION=${THEMIS_VERSION:-0.4.3} \
 SIMULATOR_VERSION=${SIMULATOR_VERSION:-local} \
-docker-compose -f $ROOT_DIR/deploy/docker-compose/docker-compose.yml up -d $@
+docker-compose -f "$ROOT_DIR/deploy/docker-compose/docker-compose.yml" up -d "$@"
+
+bash config_dynamodb.sh
 

deploy/docker-compose/docFiles/argus.yaml

@@ -0,0 +1,165 @@
+---
+prometheus:
+    defaultNamespace: xmidt
+    defaultSubsystem: argus
+    constLabels:
+      development: "true"
+  
+log:
+  file: stdout
+  level: DEBUG
+
+health:
+  disableLogging: false
+  custom:
+    server: development
+
+servers:
+  primary:
+    address: :6600
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+  metrics:
+    address: :6601
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+
+  health:
+    address: :6602
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+
+#yugabyte:
+#  # hosts is and array of address and port used to connect to the cluster.
+#  hosts:
+#    - "localhost:9042"
+#  # database is the name of the database being connected to.
+#  database: "argus"
+#  # opTimeout is the timeout for database calls after argus is connected.
+#  # If the opTimeout is set to 0, it defaults to 10s.
+#  # (Optional) defaults to 10s
+#  opTimeout: 100ms
+
+#  # username is the username to use when connecting to the database.
+#  # (Optional)
+#  username: "cassandra"
+#
+#  # password is the password to use when connecting to the database.
+#  # (Optional)
+#  password: "cassandra"
+#
+#  # SSLRootCert is the root cert to use when connecting to the database.
+#  # The SSLKey and SSLCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslRootCert: "/etc/argus/ca.crt"
+#
+#  # SSLKey is the SSL key to use when connecting to the database.  The
+#  # SSLRootCert and SSLCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslKey: "/etc/argus/node.0.0.0.0.key"
+#
+#  # SSLCert is the SSL cert to use when connecting to the database.  The SSLKey
+#  # and SSLRootCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslCert: "/etc/argus/node.0.0.0.0.crt"
+#
+#  # If you want to verify the hostname and server cert (like a wildcard for cass cluster) then you should turn this on
+#  # This option is basically the inverse of InSecureSkipVerify
+#  # See InSecureSkipVerify in http://golang.org/pkg/crypto/tls/ for more info
+#  # (Optional) defaults to false
+#  #enableHostVerification: false
+
+# dyanmo is the configuration block to communicate with dynamoDB.
+dynamo:
+
+  # endpoint is used to set a custom aws endpoint.
+  # (Optional)
+  endpoint: http://dynamodb:8000
+
+  # table is the name of the table that is already configured with bucket and id as the key.
+  table: "gifnoc"
+
+  # region is where request should go to.
+  region: "us-east-2"
+
+  # maxRetires is the maximum times the application will retry the request to the db.
+  # (Optional) default: 3
+  maxRetries: 3
+
+  # accessKey is the AWS accessKey to access dynamodb.
+  accessKey: "accessKey"
+
+  # secretKey is the AWS secretKey to go with the accessKey to access dynamodb.
+  secretKey: "secretKey"
+
+# request is a config section related to operation authorization
+# and request validation.
+request:
+  authorization:
+    # adminToken serves as a master key which allows performing operations on any
+    # item regardless of their ownership status.
+    adminToken: "Hzu1WpIe7S8G"
+
+  validation:
+    # maxTTL specifies the cap for the TTL of items when values are specified.
+    maxTTL: "24h"
+
+##############################################################################
+# Authorization Credentials
+##############################################################################
+
+# authHeader is a list of Basic Auth credentials intended to be used for local testing purposes
+# WARNING! Be sure to remove this from your production config
+authHeader: ["dXNlcjpwYXNz"]
+
+# jwtValidator provides Bearer auth configuration
+jwtValidator:
+  keys:
+    Factory:
+        uri: "http://themis:6500/keys/local"
+    purpose: 0
+    updateInterval: 24h
+
+# capabilityCheck provides the details needed for checking an incoming JWT's
+# capabilities.  If the type of check isn't provided, no checking is done.  The
+# type can be "monitor" or "enforce".  If it is empty or a different value, no
+# checking is done.  If "monitor" is provided, the capabilities are checked but
+# the request isn't rejected when there isn't a valid capability for the
+# request. Instead, a message is logged.  When "enforce" is provided, a request
+# that doesn't have the needed capability is rejected.
+#
+# The capability is expected to have the format:
+#
+# {prefix}{endpoint}:{method}
+#
+# The prefix can be a regular expression.  If it's empty, no capability check
+# is done.  The endpoint is a regular expression that should match the endpoint
+# the request was sent to. The method is usually the method of the request, such as
+# GET.  The accept all method is a catchall string that indicates the capability
+# is approved for all methods.
+# (Optional)
+#capabilityCheck:
+# # type provides the mode for capability checking.
+# type: "monitor"
+# # prefix provides the regex to match the capability before the endpoint.
+# prefix: "xmidt"
+# # acceptAllMethod provides a way to have a capability that allows all
+# # methods for a specific endpoint.
+# acceptAllMethod: "all"
+# # endpointBuckets provides regular expressions to use against the request
+# # endpoint in order to group requests for a metric label.
+# endpointBuckets:
+#   - "store\\b"
+#   - "store/.*\\b"

deploy/docker-compose/docFiles/caduceus.yaml

@@ -46,21 +46,16 @@
     - 1000
     - 10000
 
-  aws:
-    accessKey: "supbro"
-    secretKey: "nahbro"
-    env: local-dev
-    sns:
-      awsEndpoint: http://goaws:4100
-      region: "us-east-1"
-      topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus
-      urlPath: "/api/v2/aws/sns"
-  waitForDns: 0
   authHeader: ["dXNlcjpwYXNz"]
-  start:
+
-    duration: 1
+  webhook:
-    apiPath: http://caduceus:6000/hooks
+    argus:
-    authHeader: dXNlcjpwYXNz
+      bucket: webhooks
+      address: http://argus:6600
+      pullInterval: 5s
+      adminToken: Hzu1WpIe7S8G
+      auth:
+        Basic: Basic dXNlcjpwYXNz
 
   service:
     defaultScheme: http

deploy/docker-compose/docFiles/prometheus.yml

@@ -68,6 +68,13 @@ scrape_configs:
     static_configs:
       - targets: ['tr1d1um:6103']
 
+  - job_name: 'argus'
+         # metrics_path defaults to '/metrics'
+         # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['argus:6601']
+  
   - job_name: 'themis'
     static_configs:
       - targets: ['themis:6502']

deploy/docker-compose/docFiles/sns.yaml

@@ -1,35 +0,0 @@
-Local:                              # Environment name that can be passed on the command line
-                                    #     (i.e.: ./goaws [Local | Dev]  -- defaults to 'Local')
-  Host: goaws                   # hostname of the goaws system  (for docker-compose this is the tag name of the container)
-# you can now use either 1 port for both sns and sqs or alternatively you can comment out Port and use SqsPort + SnsPort for compatibilyt with
-# yopa and (fage-sns + face-sqs).  If both ways are in the config file on the one "Port" will be used by GoAws
-  Port: 4100                        # port to listen on.
-# SqsPort: 9324                     # alterante Sqs Port
-# SnsPort: 9292                     # alternate Sns Port
-  Region: us-east-1
-  LogMessages: true                 # Log messages (true/false)
-  LogFile: ./goaws_messages.log  # Log filename (for message logging
-  QueueAttributeDefaults:           # default attributes for all queues
-    VisibilityTimeout: 30              # message visibility timeout
-    ReceiveMessageWaitTimeSeconds: 0   # receive message max wait time
-  Topics:                           # List of topic to create at startup
-    - Name: xmidt-local-caduceus            # Topic name - no Subscriptions
-
-Dev:                                # Another environment
-  Host: goaws
-  Port: 4100
-# SqsPort: 9324
-# SnsPort: 9292
-  LogMessages: true
-  LogFile: ./goaws_messages.log
-  Queues:
-    - Name: dev-queue1
-    - Name: dev-queue2
-  Topics:
-    - Name: dev-topic1
-      Subscriptions:
-        - QueueName: dev-queue3
-          Raw: false
-        - QueueName: dev-queue4
-          Raw: true
-    - Name: dev-topic2

deploy/docker-compose/docFiles/talaria-0.yaml

@@ -62,7 +62,7 @@
   jwtValidator:
     keys:
       Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
       purpose: 0
       updateInterval: 24h 
   

deploy/docker-compose/docFiles/talaria-1.yaml

@@ -64,7 +64,7 @@
   jwtValidator:
     keys:
       Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
       purpose: 0
       updateInterval: 24h
   

deploy/docker-compose/docFiles/talaria-2.yaml

@@ -64,7 +64,7 @@
   jwtValidator:
     keys:
       Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
       purpose: 0
       updateInterval: 24h
 

deploy/docker-compose/docFiles/tr1d1um.yaml

@@ -26,29 +26,21 @@
       subsystem: "tr1d1um"
     readTimeout: "15s"
     idleTimeout: "15s"
+
   log:
     file: "stdout"
     level: "INFO"
     json: false
     reducedLoggingResponseCodes: [200, 504]
     
-  webhooksEnabled: true
+  webhook:
-
+    argus:
-  start:
+      bucket: webhooks
-    duration: 1
+      address: http://argus:6600
-    apiPath: tr1d1um:6100/hooks
+      pullInterval: 5s
-
+      adminToken: Hzu1WpIe7S8G
-  waitForDns: 0
+      auth:
-
+        Basic: Basic dXNlcjpwYXNz
-  aws:
-    accessKey: "supbro"
-    secretKey: "nahbro"
-    env: local-dev
-    sns:
-      awsEndpoint: http://goaws:4100
-      region: "us-east-1"
-      topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus
-      urlPath: "/api/v2/aws/sns"
 
   authHeader: ["dXNlcjpwYXNz"]
   targetURL: http://scytale:6300

deploy/docker-compose/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - 6002:6002
       - 6003:6003
     depends_on:
-      - goaws
+      - argus
     volumes:
       - ./docFiles/caduceus.yaml:/etc/caduceus/caduceus.yaml
     networks:
@@ -21,7 +21,7 @@ services:
       - 6101:6101
       - 6102:6102
     depends_on:
-      - goaws
+      - argus
       - scytale
     volumes:
       - ./docFiles/tr1d1um.yaml:/etc/tr1d1um/tr1d1um.yaml
@@ -73,7 +73,6 @@ services:
       - 6302:6302
       - 6303:6303
     depends_on:
-      - goaws
       - petasos
     volumes:
       - ./docFiles/scytale.yaml:/etc/scytale/scytale.yaml
@@ -108,16 +107,6 @@ services:
     networks:
       - xmidt
 
-  goaws:
-    image: goaws:local
-    container_name: goaws
-    ports:
-      - 4100:4100
-    networks:
-      - xmidt
-    volumes:
-      - ./docFiles/sns.yaml:/conf/goaws.yaml
-
   prometheus:
     image: prom/prometheus
     networks:
@@ -176,5 +165,26 @@ services:
       - ./docFiles/consul-1.json:/consul.json
     command: "agent -server -bootstrap-expect 1 -ui -client 0.0.0.0 -config-file consul.json"
   
+  argus:
+    image: xmidt/argus:${ARGUS_VERSION}
+    ports:
+      - 6600:6600
+      - 6601:6601
+      - 6602:6602
+    depends_on:
+      - dynamodb
+    volumes:
+      - ./docFiles/argus.yaml:/etc/argus/argus.yaml
+    networks:
+      - xmidt
+
+  dynamodb:
+    image: amazon/dynamodb-local
+    hostname: dynamodb-local
+    ports:
+      - "8000:8000"
+    networks:
+      - xmidt
+
 networks:
   xmidt: