Replacing SNS with Argus (#38)

* use argus for tr1d1um as webhook store * update config that includes admin token * save changes * small bash file linting fixes * consul latest works * Remove SNS as a dep
2026-09-03 02:09:19 +00:00 · 2020-12-18 13:38:13 -08:00
parent c2da56c111
commit 96e7b4764f
11 changed files with 243 additions and 91 deletions
@@ -0,0 +1,165 @@
+---
+prometheus:
+    defaultNamespace: xmidt
+    defaultSubsystem: argus
+    constLabels:
+      development: "true"
+  
+log:
+  file: stdout
+  level: DEBUG
+
+health:
+  disableLogging: false
+  custom:
+    server: development
+
+servers:
+  primary:
+    address: :6600
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+  metrics:
+    address: :6601
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+
+  health:
+    address: :6602
+    disableHTTPKeepAlives: true
+    header:
+      X-Midt-Server:
+        - argus
+      X-Midt-Version:
+        - development
+
+#yugabyte:
+#  # hosts is and array of address and port used to connect to the cluster.
+#  hosts:
+#    - "localhost:9042"
+#  # database is the name of the database being connected to.
+#  database: "argus"
+#  # opTimeout is the timeout for database calls after argus is connected.
+#  # If the opTimeout is set to 0, it defaults to 10s.
+#  # (Optional) defaults to 10s
+#  opTimeout: 100ms
+
+#  # username is the username to use when connecting to the database.
+#  # (Optional)
+#  username: "cassandra"
+#
+#  # password is the password to use when connecting to the database.
+#  # (Optional)
+#  password: "cassandra"
+#
+#  # SSLRootCert is the root cert to use when connecting to the database.
+#  # The SSLKey and SSLCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslRootCert: "/etc/argus/ca.crt"
+#
+#  # SSLKey is the SSL key to use when connecting to the database.  The
+#  # SSLRootCert and SSLCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslKey: "/etc/argus/node.0.0.0.0.key"
+#
+#  # SSLCert is the SSL cert to use when connecting to the database.  The SSLKey
+#  # and SSLRootCert must also be provided in order to connect securely.
+#  # (Optional)
+#  #sslCert: "/etc/argus/node.0.0.0.0.crt"
+#
+#  # If you want to verify the hostname and server cert (like a wildcard for cass cluster) then you should turn this on
+#  # This option is basically the inverse of InSecureSkipVerify
+#  # See InSecureSkipVerify in http://golang.org/pkg/crypto/tls/ for more info
+#  # (Optional) defaults to false
+#  #enableHostVerification: false
+
+# dyanmo is the configuration block to communicate with dynamoDB.
+dynamo:
+
+  # endpoint is used to set a custom aws endpoint.
+  # (Optional)
+  endpoint: http://dynamodb:8000
+
+  # table is the name of the table that is already configured with bucket and id as the key.
+  table: "gifnoc"
+
+  # region is where request should go to.
+  region: "us-east-2"
+
+  # maxRetires is the maximum times the application will retry the request to the db.
+  # (Optional) default: 3
+  maxRetries: 3
+
+  # accessKey is the AWS accessKey to access dynamodb.
+  accessKey: "accessKey"
+
+  # secretKey is the AWS secretKey to go with the accessKey to access dynamodb.
+  secretKey: "secretKey"
+
+# request is a config section related to operation authorization
+# and request validation.
+request:
+  authorization:
+    # adminToken serves as a master key which allows performing operations on any
+    # item regardless of their ownership status.
+    adminToken: "Hzu1WpIe7S8G"
+
+  validation:
+    # maxTTL specifies the cap for the TTL of items when values are specified.
+    maxTTL: "24h"
+
+##############################################################################
+# Authorization Credentials
+##############################################################################
+
+# authHeader is a list of Basic Auth credentials intended to be used for local testing purposes
+# WARNING! Be sure to remove this from your production config
+authHeader: ["dXNlcjpwYXNz"]
+
+# jwtValidator provides Bearer auth configuration
+jwtValidator:
+  keys:
+    Factory:
+        uri: "http://themis:6500/keys/local"
+    purpose: 0
+    updateInterval: 24h
+
+# capabilityCheck provides the details needed for checking an incoming JWT's
+# capabilities.  If the type of check isn't provided, no checking is done.  The
+# type can be "monitor" or "enforce".  If it is empty or a different value, no
+# checking is done.  If "monitor" is provided, the capabilities are checked but
+# the request isn't rejected when there isn't a valid capability for the
+# request. Instead, a message is logged.  When "enforce" is provided, a request
+# that doesn't have the needed capability is rejected.
+#
+# The capability is expected to have the format:
+#
+# {prefix}{endpoint}:{method}
+#
+# The prefix can be a regular expression.  If it's empty, no capability check
+# is done.  The endpoint is a regular expression that should match the endpoint
+# the request was sent to. The method is usually the method of the request, such as
+# GET.  The accept all method is a catchall string that indicates the capability
+# is approved for all methods.
+# (Optional)
+#capabilityCheck:
+# # type provides the mode for capability checking.
+# type: "monitor"
+# # prefix provides the regex to match the capability before the endpoint.
+# prefix: "xmidt"
+# # acceptAllMethod provides a way to have a capability that allows all
+# # methods for a specific endpoint.
+# acceptAllMethod: "all"
+# # endpointBuckets provides regular expressions to use against the request
+# # endpoint in order to group requests for a metric label.
+# endpointBuckets:
+#   - "store\\b"
+#   - "store/.*\\b"
@@ -46,21 +46,16 @@
    - 1000
    - 10000

-  aws:
-    accessKey: "supbro"
-    secretKey: "nahbro"
-    env: local-dev
-    sns:
-      awsEndpoint: http://goaws:4100
-      region: "us-east-1"
-      topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus
-      urlPath: "/api/v2/aws/sns"
-  waitForDns: 0
  authHeader: ["dXNlcjpwYXNz"]
-  start:
-    duration: 1
-    apiPath: http://caduceus:6000/hooks
-    authHeader: dXNlcjpwYXNz
+
+  webhook:
+    argus:
+      bucket: webhooks
+      address: http://argus:6600
+      pullInterval: 5s
+      adminToken: Hzu1WpIe7S8G
+      auth:
+        Basic: Basic dXNlcjpwYXNz

  service:
    defaultScheme: http
@@ -68,6 +68,13 @@ scrape_configs:
    static_configs:
      - targets: ['tr1d1um:6103']

+  - job_name: 'argus'
+         # metrics_path defaults to '/metrics'
+         # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['argus:6601']
+  
  - job_name: 'themis'
    static_configs:
      - targets: ['themis:6502']
@@ -1,35 +0,0 @@
-Local:                              # Environment name that can be passed on the command line
-                                    #     (i.e.: ./goaws [Local | Dev]  -- defaults to 'Local')
-  Host: goaws                   # hostname of the goaws system  (for docker-compose this is the tag name of the container)
-# you can now use either 1 port for both sns and sqs or alternatively you can comment out Port and use SqsPort + SnsPort for compatibilyt with
-# yopa and (fage-sns + face-sqs).  If both ways are in the config file on the one "Port" will be used by GoAws
-  Port: 4100                        # port to listen on.
-# SqsPort: 9324                     # alterante Sqs Port
-# SnsPort: 9292                     # alternate Sns Port
-  Region: us-east-1
-  LogMessages: true                 # Log messages (true/false)
-  LogFile: ./goaws_messages.log  # Log filename (for message logging
-  QueueAttributeDefaults:           # default attributes for all queues
-    VisibilityTimeout: 30              # message visibility timeout
-    ReceiveMessageWaitTimeSeconds: 0   # receive message max wait time
-  Topics:                           # List of topic to create at startup
-    - Name: xmidt-local-caduceus            # Topic name - no Subscriptions
-
-Dev:                                # Another environment
-  Host: goaws
-  Port: 4100
-# SqsPort: 9324
-# SnsPort: 9292
-  LogMessages: true
-  LogFile: ./goaws_messages.log
-  Queues:
-    - Name: dev-queue1
-    - Name: dev-queue2
-  Topics:
-    - Name: dev-topic1
-      Subscriptions:
-        - QueueName: dev-queue3
-          Raw: false
-        - QueueName: dev-queue4
-          Raw: true
-    - Name: dev-topic2
@@ -62,7 +62,7 @@
  jwtValidator:
    keys:
      Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
      purpose: 0
      updateInterval: 24h 
  
@@ -64,7 +64,7 @@
  jwtValidator:
    keys:
      Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
      purpose: 0
      updateInterval: 24h
  
@@ -64,7 +64,7 @@
  jwtValidator:
    keys:
      Factory:
-          uri: "http://themis:6500/keys/{keyId}"
+          uri: "http://themis:6500/keys/local"
      purpose: 0
      updateInterval: 24h

@@ -26,29 +26,21 @@
      subsystem: "tr1d1um"
    readTimeout: "15s"
    idleTimeout: "15s"
+
  log:
    file: "stdout"
    level: "INFO"
    json: false
    reducedLoggingResponseCodes: [200, 504]
    
-  webhooksEnabled: true
-
-  start:
-    duration: 1
-    apiPath: tr1d1um:6100/hooks
-
-  waitForDns: 0
-
-  aws:
-    accessKey: "supbro"
-    secretKey: "nahbro"
-    env: local-dev
-    sns:
-      awsEndpoint: http://goaws:4100
-      region: "us-east-1"
-      topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus
-      urlPath: "/api/v2/aws/sns"
+  webhook:
+    argus:
+      bucket: webhooks
+      address: http://argus:6600
+      pullInterval: 5s
+      adminToken: Hzu1WpIe7S8G
+      auth:
+        Basic: Basic dXNlcjpwYXNz

  authHeader: ["dXNlcjpwYXNz"]
  targetURL: http://scytale:6300