diff --git a/deploy/docker-compose/config_dynamodb.sh b/deploy/docker-compose/config_dynamodb.sh new file mode 100755 index 0000000..6a5652a --- /dev/null +++ b/deploy/docker-compose/config_dynamodb.sh @@ -0,0 +1,15 @@ +AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 create-table \ + --table-name gifnoc \ + --attribute-definitions \ + AttributeName=bucket,AttributeType=S \ + AttributeName=uuid,AttributeType=S \ + --key-schema \ + AttributeName=bucket,KeyType=HASH \ + AttributeName=uuid,KeyType=RANGE \ + --provisioned-throughput \ + ReadCapacityUnits=10,WriteCapacityUnits=5 \ + --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \ + --region us-east-2 + + +AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 --region us-east-2 update-time-to-live --table-name gifnoc --time-to-live-specification "Enabled=true, AttributeName=expires" diff --git a/deploy/docker-compose/deploy.sh b/deploy/docker-compose/deploy.sh index cd0fb3f..2faa1d2 100755 --- a/deploy/docker-compose/deploy.sh +++ b/deploy/docker-compose/deploy.sh @@ -1,27 +1,30 @@ #!/bin/bash -DIR=$( cd $(dirname $0) ; pwd -P ) +DIR=$( cd "$(dirname "$0")" || exit; pwd -P ) ROOT_DIR=$DIR/../../ -pushd $ROOT_DIR +pushd "$ROOT_DIR" || exit echo "Building Simulator..." -docker build -t simulator:local $ROOT_DIR/simulator -popd +docker build -t simulator:local "$ROOT_DIR/simulator" +popd || exit echo "Building goaws..." git clone git@github.com:kcajmagic/goaws.git /tmp/goaws -pushd /tmp/goaws +pushd /tmp/goaws || exit git checkout adding_http_support docker build -t goaws:local . -popd +popd || exit echo "Running services..." -CADUCEUS_VERSION=${CADUCEUS_VERSION:-0.2.7} \ -TR1D1UM_VERSION=${TR1D1UM_VERSION:-0.5.0} \ +CADUCEUS_VERSION=${CADUCEUS_VERSION:-0.4.2} \ +ARGUS_VERSION=${ARGUS_VERSION:-0.3.9} \ +TR1D1UM_VERSION=${TR1D1UM_VERSION:-0.5.3} \ SCYTALE_VERSION=${SCYTALE_VERSION:-0.1.5} \ PETASOS_VERSION=${PETASOS_VERSION:-0.1.4} \ TALARIA_VERSION=${TALARIA_VERSION:-0.5.9} \ THEMIS_VERSION=${THEMIS_VERSION:-0.4.3} \ SIMULATOR_VERSION=${SIMULATOR_VERSION:-local} \ -docker-compose -f $ROOT_DIR/deploy/docker-compose/docker-compose.yml up -d $@ +docker-compose -f "$ROOT_DIR/deploy/docker-compose/docker-compose.yml" up -d "$@" + +bash config_dynamodb.sh diff --git a/deploy/docker-compose/docFiles/argus.yaml b/deploy/docker-compose/docFiles/argus.yaml new file mode 100644 index 0000000..4094652 --- /dev/null +++ b/deploy/docker-compose/docFiles/argus.yaml @@ -0,0 +1,165 @@ +--- +prometheus: + defaultNamespace: xmidt + defaultSubsystem: argus + constLabels: + development: "true" + +log: + file: stdout + level: DEBUG + +health: + disableLogging: false + custom: + server: development + +servers: + primary: + address: :6600 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - argus + X-Midt-Version: + - development + metrics: + address: :6601 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - argus + X-Midt-Version: + - development + + health: + address: :6602 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - argus + X-Midt-Version: + - development + +#yugabyte: +# # hosts is and array of address and port used to connect to the cluster. +# hosts: +# - "localhost:9042" +# # database is the name of the database being connected to. +# database: "argus" +# # opTimeout is the timeout for database calls after argus is connected. +# # If the opTimeout is set to 0, it defaults to 10s. +# # (Optional) defaults to 10s +# opTimeout: 100ms + +# # username is the username to use when connecting to the database. +# # (Optional) +# username: "cassandra" +# +# # password is the password to use when connecting to the database. +# # (Optional) +# password: "cassandra" +# +# # SSLRootCert is the root cert to use when connecting to the database. +# # The SSLKey and SSLCert must also be provided in order to connect securely. +# # (Optional) +# #sslRootCert: "/etc/argus/ca.crt" +# +# # SSLKey is the SSL key to use when connecting to the database. The +# # SSLRootCert and SSLCert must also be provided in order to connect securely. +# # (Optional) +# #sslKey: "/etc/argus/node.0.0.0.0.key" +# +# # SSLCert is the SSL cert to use when connecting to the database. The SSLKey +# # and SSLRootCert must also be provided in order to connect securely. +# # (Optional) +# #sslCert: "/etc/argus/node.0.0.0.0.crt" +# +# # If you want to verify the hostname and server cert (like a wildcard for cass cluster) then you should turn this on +# # This option is basically the inverse of InSecureSkipVerify +# # See InSecureSkipVerify in http://golang.org/pkg/crypto/tls/ for more info +# # (Optional) defaults to false +# #enableHostVerification: false + +# dyanmo is the configuration block to communicate with dynamoDB. +dynamo: + + # endpoint is used to set a custom aws endpoint. + # (Optional) + endpoint: http://dynamodb:8000 + + # table is the name of the table that is already configured with bucket and id as the key. + table: "gifnoc" + + # region is where request should go to. + region: "us-east-2" + + # maxRetires is the maximum times the application will retry the request to the db. + # (Optional) default: 3 + maxRetries: 3 + + # accessKey is the AWS accessKey to access dynamodb. + accessKey: "accessKey" + + # secretKey is the AWS secretKey to go with the accessKey to access dynamodb. + secretKey: "secretKey" + +# request is a config section related to operation authorization +# and request validation. +request: + authorization: + # adminToken serves as a master key which allows performing operations on any + # item regardless of their ownership status. + adminToken: "Hzu1WpIe7S8G" + + validation: + # maxTTL specifies the cap for the TTL of items when values are specified. + maxTTL: "24h" + +############################################################################## +# Authorization Credentials +############################################################################## + +# authHeader is a list of Basic Auth credentials intended to be used for local testing purposes +# WARNING! Be sure to remove this from your production config +authHeader: ["dXNlcjpwYXNz"] + +# jwtValidator provides Bearer auth configuration +jwtValidator: + keys: + Factory: + uri: "http://themis:6500/keys/local" + purpose: 0 + updateInterval: 24h + +# capabilityCheck provides the details needed for checking an incoming JWT's +# capabilities. If the type of check isn't provided, no checking is done. The +# type can be "monitor" or "enforce". If it is empty or a different value, no +# checking is done. If "monitor" is provided, the capabilities are checked but +# the request isn't rejected when there isn't a valid capability for the +# request. Instead, a message is logged. When "enforce" is provided, a request +# that doesn't have the needed capability is rejected. +# +# The capability is expected to have the format: +# +# {prefix}{endpoint}:{method} +# +# The prefix can be a regular expression. If it's empty, no capability check +# is done. The endpoint is a regular expression that should match the endpoint +# the request was sent to. The method is usually the method of the request, such as +# GET. The accept all method is a catchall string that indicates the capability +# is approved for all methods. +# (Optional) +#capabilityCheck: +# # type provides the mode for capability checking. +# type: "monitor" +# # prefix provides the regex to match the capability before the endpoint. +# prefix: "xmidt" +# # acceptAllMethod provides a way to have a capability that allows all +# # methods for a specific endpoint. +# acceptAllMethod: "all" +# # endpointBuckets provides regular expressions to use against the request +# # endpoint in order to group requests for a metric label. +# endpointBuckets: +# - "store\\b" +# - "store/.*\\b" \ No newline at end of file diff --git a/deploy/docker-compose/docFiles/caduceus.yaml b/deploy/docker-compose/docFiles/caduceus.yaml index 177b9cc..7f8bda9 100644 --- a/deploy/docker-compose/docFiles/caduceus.yaml +++ b/deploy/docker-compose/docFiles/caduceus.yaml @@ -46,21 +46,16 @@ - 1000 - 10000 - aws: - accessKey: "supbro" - secretKey: "nahbro" - env: local-dev - sns: - awsEndpoint: http://goaws:4100 - region: "us-east-1" - topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus - urlPath: "/api/v2/aws/sns" - waitForDns: 0 authHeader: ["dXNlcjpwYXNz"] - start: - duration: 1 - apiPath: http://caduceus:6000/hooks - authHeader: dXNlcjpwYXNz + + webhook: + argus: + bucket: webhooks + address: http://argus:6600 + pullInterval: 5s + adminToken: Hzu1WpIe7S8G + auth: + Basic: Basic dXNlcjpwYXNz service: defaultScheme: http diff --git a/deploy/docker-compose/docFiles/prometheus.yml b/deploy/docker-compose/docFiles/prometheus.yml index db2f383..416b3b4 100644 --- a/deploy/docker-compose/docFiles/prometheus.yml +++ b/deploy/docker-compose/docFiles/prometheus.yml @@ -68,6 +68,13 @@ scrape_configs: static_configs: - targets: ['tr1d1um:6103'] + - job_name: 'argus' + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['argus:6601'] + - job_name: 'themis' static_configs: - targets: ['themis:6502'] diff --git a/deploy/docker-compose/docFiles/sns.yaml b/deploy/docker-compose/docFiles/sns.yaml deleted file mode 100644 index 382573c..0000000 --- a/deploy/docker-compose/docFiles/sns.yaml +++ /dev/null @@ -1,35 +0,0 @@ -Local: # Environment name that can be passed on the command line - # (i.e.: ./goaws [Local | Dev] -- defaults to 'Local') - Host: goaws # hostname of the goaws system (for docker-compose this is the tag name of the container) -# you can now use either 1 port for both sns and sqs or alternatively you can comment out Port and use SqsPort + SnsPort for compatibilyt with -# yopa and (fage-sns + face-sqs). If both ways are in the config file on the one "Port" will be used by GoAws - Port: 4100 # port to listen on. -# SqsPort: 9324 # alterante Sqs Port -# SnsPort: 9292 # alternate Sns Port - Region: us-east-1 - LogMessages: true # Log messages (true/false) - LogFile: ./goaws_messages.log # Log filename (for message logging - QueueAttributeDefaults: # default attributes for all queues - VisibilityTimeout: 30 # message visibility timeout - ReceiveMessageWaitTimeSeconds: 0 # receive message max wait time - Topics: # List of topic to create at startup - - Name: xmidt-local-caduceus # Topic name - no Subscriptions - -Dev: # Another environment - Host: goaws - Port: 4100 -# SqsPort: 9324 -# SnsPort: 9292 - LogMessages: true - LogFile: ./goaws_messages.log - Queues: - - Name: dev-queue1 - - Name: dev-queue2 - Topics: - - Name: dev-topic1 - Subscriptions: - - QueueName: dev-queue3 - Raw: false - - QueueName: dev-queue4 - Raw: true - - Name: dev-topic2 diff --git a/deploy/docker-compose/docFiles/talaria-0.yaml b/deploy/docker-compose/docFiles/talaria-0.yaml index 2784ab4..1feb6b8 100644 --- a/deploy/docker-compose/docFiles/talaria-0.yaml +++ b/deploy/docker-compose/docFiles/talaria-0.yaml @@ -62,7 +62,7 @@ jwtValidator: keys: Factory: - uri: "http://themis:6500/keys/{keyId}" + uri: "http://themis:6500/keys/local" purpose: 0 updateInterval: 24h diff --git a/deploy/docker-compose/docFiles/talaria-1.yaml b/deploy/docker-compose/docFiles/talaria-1.yaml index 74093a7..a0a33f6 100644 --- a/deploy/docker-compose/docFiles/talaria-1.yaml +++ b/deploy/docker-compose/docFiles/talaria-1.yaml @@ -64,7 +64,7 @@ jwtValidator: keys: Factory: - uri: "http://themis:6500/keys/{keyId}" + uri: "http://themis:6500/keys/local" purpose: 0 updateInterval: 24h diff --git a/deploy/docker-compose/docFiles/talaria-2.yaml b/deploy/docker-compose/docFiles/talaria-2.yaml index 443fe31..eb351cc 100644 --- a/deploy/docker-compose/docFiles/talaria-2.yaml +++ b/deploy/docker-compose/docFiles/talaria-2.yaml @@ -64,7 +64,7 @@ jwtValidator: keys: Factory: - uri: "http://themis:6500/keys/{keyId}" + uri: "http://themis:6500/keys/local" purpose: 0 updateInterval: 24h diff --git a/deploy/docker-compose/docFiles/tr1d1um.yaml b/deploy/docker-compose/docFiles/tr1d1um.yaml index 7081a2e..4baea72 100644 --- a/deploy/docker-compose/docFiles/tr1d1um.yaml +++ b/deploy/docker-compose/docFiles/tr1d1um.yaml @@ -26,29 +26,21 @@ subsystem: "tr1d1um" readTimeout: "15s" idleTimeout: "15s" + log: file: "stdout" level: "INFO" json: false reducedLoggingResponseCodes: [200, 504] - webhooksEnabled: true - - start: - duration: 1 - apiPath: tr1d1um:6100/hooks - - waitForDns: 0 - - aws: - accessKey: "supbro" - secretKey: "nahbro" - env: local-dev - sns: - awsEndpoint: http://goaws:4100 - region: "us-east-1" - topicArn: arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus - urlPath: "/api/v2/aws/sns" + webhook: + argus: + bucket: webhooks + address: http://argus:6600 + pullInterval: 5s + adminToken: Hzu1WpIe7S8G + auth: + Basic: Basic dXNlcjpwYXNz authHeader: ["dXNlcjpwYXNz"] targetURL: http://scytale:6300 diff --git a/deploy/docker-compose/docker-compose.yml b/deploy/docker-compose/docker-compose.yml index af0f701..d068f4e 100644 --- a/deploy/docker-compose/docker-compose.yml +++ b/deploy/docker-compose/docker-compose.yml @@ -8,7 +8,7 @@ services: - 6002:6002 - 6003:6003 depends_on: - - goaws + - argus volumes: - ./docFiles/caduceus.yaml:/etc/caduceus/caduceus.yaml networks: @@ -21,7 +21,7 @@ services: - 6101:6101 - 6102:6102 depends_on: - - goaws + - argus - scytale volumes: - ./docFiles/tr1d1um.yaml:/etc/tr1d1um/tr1d1um.yaml @@ -73,7 +73,6 @@ services: - 6302:6302 - 6303:6303 depends_on: - - goaws - petasos volumes: - ./docFiles/scytale.yaml:/etc/scytale/scytale.yaml @@ -108,16 +107,6 @@ services: networks: - xmidt - goaws: - image: goaws:local - container_name: goaws - ports: - - 4100:4100 - networks: - - xmidt - volumes: - - ./docFiles/sns.yaml:/conf/goaws.yaml - prometheus: image: prom/prometheus networks: @@ -175,6 +164,27 @@ services: volumes: - ./docFiles/consul-1.json:/consul.json command: "agent -server -bootstrap-expect 1 -ui -client 0.0.0.0 -config-file consul.json" + + argus: + image: xmidt/argus:${ARGUS_VERSION} + ports: + - 6600:6600 + - 6601:6601 + - 6602:6602 + depends_on: + - dynamodb + volumes: + - ./docFiles/argus.yaml:/etc/argus/argus.yaml + networks: + - xmidt + + dynamodb: + image: amazon/dynamodb-local + hostname: dynamodb-local + ports: + - "8000:8000" + networks: + - xmidt networks: xmidt: