hypertrace · findingrish · Aug 27, 2021 · Aug 26, 2021 · Aug 26, 2021 · Aug 26, 2021
diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml
@@ -30,7 +30,7 @@ jobs:
           docker-compose -f docker-compose.yml ps
       - name: Runs tests
         working-directory: ./.github/workflows/scripts/
-        run: ./tests.sh
+        run: ./tests.sh mongo
 
   docker-compose-verification-with-postgres:
     runs-on: ubuntu-20.04
@@ -40,13 +40,13 @@ jobs:
       - name: Verify hypertrace deployment
         working-directory: docker
         run: |
-          docker-compose -f postgres/docker-compose.yml -f docker-compose-zipkin-example.yml pull
-          docker-compose -f postgres/docker-compose.yml -f docker-compose-zipkin-example.yml up -d || { ../.github/workflows/scripts/inspect.sh ; exit 1 ; }
+          docker-compose -f docker-compose.yml -f docker-compose.postgres.yml -f docker-compose-zipkin-example.yml pull
+          docker-compose -f docker-compose.yml -f docker-compose.postgres.yml -f docker-compose-zipkin-example.yml up -d || { ../.github/workflows/scripts/inspect.sh postgres; exit 1 ; }
       - name: Waits for some stability
         working-directory: docker
         run: |
           sleep 60 # you can decrease it but never increase it
-          docker-compose -f postgres/docker-compose.yml ps
+          docker-compose -f docker-compose.yml  -f docker-compose.postgres.yml ps
       - name: Runs tests
         working-directory: ./.github/workflows/scripts/
-        run: ./tests.sh
+        run: ./tests.sh postgres
diff --git a/.github/workflows/scripts/inspect.sh b/.github/workflows/scripts/inspect.sh
@@ -5,7 +5,15 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 DOCKER_COMPOSE_FILE_DIR="$(dirname $SCRIPT_DIR)/docker"
 
-containers=$(docker-compose -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose.yml -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose-zipkin-example.yml ps -q -a)
+echo "Executing INSPECT_SCRIPT. ARGUMENT $1  SCRIPT_DIR: $SCRIPT_DIR DOCKER_COMPOSE_FILE_DIR: $DOCKER_COMPOSE_FILE_DIR"
+
+echo ""
+if [[ "$1" == "postgres" ]]; then
+   containers=$(docker-compose -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose.yml -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose.postgres.yml -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose-zipkin-example.yml ps -q -a)
+else
+   containers=$(docker-compose -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose.yml -f ${DOCKER_COMPOSE_FILE_DIR}/docker-compose-zipkin-example.yml ps -q -a)
+fi     
+
 while IFS= read -r container; do
     name=$(docker inspect $container | jq -r '.[0].Name')
     echo "=================="

diff --git a/.github/workflows/scripts/tests.sh b/.github/workflows/scripts/tests.sh
@@ -8,10 +8,10 @@
 # docker-compose -f docker/docker-compose.yml -f docker/docker-compose-zipkin-example.yml up -d
 # sh ./circleci/tests.sh
 
-TRACES_SERVER_HOST=${1:-127.0.0.1}
-FRONTEND_SERVICE_HOST=${2:-127.0.0.1}
+TRACES_SERVER_HOST=${2:-127.0.0.1}
+FRONTEND_SERVICE_HOST=${3:-127.0.0.1}
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
+echo "Executing test SCRIPT_DIR: $SCRIPT_DIR ARGUMENT: $1"
 echo ""
 echo "Making sure the traces service is up..."
 
@@ -21,7 +21,7 @@ RETRY_COUNT=0
 while : ; do
   if [[ "$RETRY_COUNT" == "$NUMBER_OF_RETRIES" ]]; then
     echo "Failed to connect to $TRACES_SERVER_HOST:2020 after $NUMBER_OF_RETRIES retries."
-    sh $SCRIPT_DIR/inspect.sh 
+    sh $SCRIPT_DIR/inspect.sh $1 
     exit 1
   fi
 
@@ -47,7 +47,7 @@ TRACES=""
 while : ; do
   if [[ "$RETRY_COUNT" == "$NUMBER_OF_RETRIES" ]]; then
     echo "Failed to retrieve traces from $TRACES_SERVER_HOST:2020 after $NUMBER_OF_RETRIES retries."
-    sh $SCRIPT_DIR/inspect.sh 
+    sh $SCRIPT_DIR/inspect.sh $1
     exit 1
   fi
 
@@ -74,7 +74,7 @@ while : ; do
   if [[ "$ERROR" != "null" ]]; then
     echo ""
     echo "Error while retrieving traces: $ERROR.";
-    sh $SCRIPT_DIR/inspect.sh 
+    sh $SCRIPT_DIR/inspect.sh $1
     exit 1;
   fi
 

diff --git a/docker/README.md b/docker/README.md
@@ -25,9 +25,23 @@ This will start all services required for Hypertrace. Once you see the service h
 
 If you are facing any issues with docker-compose setup, we have listed down common issues and resolutions [here](https://docs.hypertrace.org/troubleshooting/docker-compose/).
 
-#### To start alerting related serivces 
+### Hypertrace with postgres
+Hypertrace uses two different types of store currently
+
+- Document store (for entities) 
+- OLAP store (for timeseries data)
+By default, we run docker-compose with mongo as a document store. Recently, we have also added support for postgres as an alternative to mongo. To run hypertrace with postgres override the document-store service
+
+Run ```docker-compose -f docker-compose.yml -f docker-compose.postgres.yml up```
+
+References
+https://github.com/hypertrace/hypertrace/issues/114
+https://github.com/hypertrace/hypertrace/issues/124
+
+#### To start alerting related services
 
 Run ```docker-compose -f docker-compose.yml  -f docker-compose.alerting.yml up```
+or ```docker-compose -f docker-compose.yml -f docker-compose.postgres.yml -f docker-compose.alerting.yml up``` (for postgres as document-store)
 
 Update alerting rule definition in alert-rules.json, notification-rules.json
 

diff --git a/docker/configs/alert-rules.json b/docker/configs/alert-rules.json
@@ -29,11 +29,11 @@
             }
           }
         },
+        "ruleDuration": "PT1M",
         "violationCondition": [{
           "staticThresholdCondition": {
             "operator": "STATIC_THRESHOLD_OPERATOR_GT",
             "value": 3.0,
-            "minimumViolationDuration": "PT5M",
             "severity": "SEVERITY_CRITICAL"
           }
         }]

diff --git a/docker/docker-compose.alerting.yml b/docker/docker-compose.alerting.yml
@@ -13,8 +13,8 @@ services:
         - ATTRIBUTE_SERVICE_HOST_CONFIG=hypertrace
         - ATTRIBUTE_SERVICE_PORT_CONFIG=9001
     volumes:
-        - ../docker/configs/alert-rules.json:/app/resources/alert-rules.json:ro
-        - ../docker/configs/notification-rules.json:/app/resources/notification-rules.json:ro
+        - ./configs/alert-rules.json:/app/resources/alert-rules.json:ro
+        - ./configs/notification-rules.json:/app/resources/notification-rules.json:ro
     depends_on:
         hypertrace:
         # service_started, not service_healthy as pinot and deps can take longer than 60s to start

diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml
@@ -0,0 +1,26 @@
+version: '2.4'
+services:
+
+  hypertrace:
+    image: hypertrace/hypertrace:main
+    environment:
+      - DATA_STORE_TYPE=postgres
+      - POSTGRES_HOST=document-store
+      - ZK_CONNECT_STR=zookeeper:2181/hypertrace-views
+    ports:
+      - 2020:2020
+    healthcheck:
+      start_period: 20s
+    depends_on:
+      document-store:
+        condition: service_started
+      kafka-zookeeper:
+        condition: service_healthy
+      pinot:
+        condition: service_started
+
+  document-store:
+    image: postgres:13.1
+    environment:
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_USER=postgres
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -11,14 +11,14 @@ services:
   hypertrace:
     image: hypertrace/hypertrace:main
     environment:
-      - MONGO_HOST=mongo
+      - MONGO_HOST=document-store
       - ZK_CONNECT_STR=zookeeper:2181/hypertrace-views
     ports:
       - 2020:2020
     healthcheck:
       start_period: 20s
     depends_on:
-      mongo:
+      document-store:
         condition: service_healthy
       kafka-zookeeper:
         condition: service_healthy
@@ -88,8 +88,10 @@ services:
           - kafka
           - zookeeper
   # Stores entities like API, service and backend
-  mongo:
+  # Mongo and postgres are the supported doc store, to use postgres override this using docker-compose.postgres.yml 
+  document-store:
     image: hypertrace/mongodb:main
+
   # Stores spans and traces and provides aggregation functions
   pinot:
     image: hypertrace/pinot-servicemanager:main

diff --git a/docker/postgres/README.md b/docker/postgres/README.md
diff --git a/docker/postgres/docker-compose.alerting.yml b/docker/postgres/docker-compose.alerting.yml
diff --git a/docker/postgres/docker-compose.yml b/docker/postgres/docker-compose.yml