Sandersaarond/tracing (#83)

* Upgrade plugin sdk, overdue * Add tracing to config, some tidy up on config * Docker config tweak * Tidy-up on config * Readd license to training app * Version bump
grafana · Aug 16, 2024 · 0166476 · 0166476
1 parent 9ecb169
commit 0166476
Show file tree

Hide file tree

Showing 10 changed files with 903 additions and 250 deletions.
diff --git a/.config/grafana/config/grafana.ini b/.config/grafana/config/grafana.ini
@@ -11,3 +11,15 @@ enabled = false
 enabled = true
 [users]
 default_theme = dark
+
+[plugin.grafana-aitraining-app]
+tracing = true
+
+[log.console]
+level = debug
+
+[log]
+filters = plugin.grafana-aitraining-app:debug
+
+[tracing.opentelemetry.otlp]
+address = tempo:3200
diff --git a/.config/grafana/datasources/datasource.yml b/.config/grafana/datasources/datasource.yml
@@ -27,7 +27,10 @@ datasources:
     basicAuth: false
     isDefault: false
     version: 1
-    editable: false
+    editable: true
+    jsonData:
+      # The recommended scrape interval is 60s.
+      timeInterval: '60s'
 
   - name: Loki
     type: loki
@@ -43,4 +46,25 @@ datasources:
         - datasourceUid: tempo
           matcherRegex: tid=(\w+)
           name: TraceID
-          url: $${__value.raw}
+          url: $${__value.raw}
+
+  - name: Tempo
+    type: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo:3200
+    basicAuth: false
+    isDefault: false
+    version: 1
+    editable: true
+    apiVersion: 1
+    uid: tempo
+  - name: Pyroscope
+    type: grafana-pyroscope-datasource
+    access: proxy
+    orgId: 1
+    url: http://pyroscope:4040/
+    basicAuth: false
+    isDefault: false
+    version: 1
+    editable: true
diff --git a/.config/mimir/config/mimir.yml → .config/mimir/mimir.yml b/.config/mimir/config/mimir.yml → .config/mimir/mimir.yml
diff --git a/docker-compose-jupyter.yml b/docker-compose-jupyter.yml
@@ -9,3 +9,8 @@ services:
       - "8888:8888"
     environment:
       - GF_AI_TRAINING_CREDS=83bcaff6228b39bbe431af5e19fb4368e2a03dd3:1337@http://ai-training-api:8000
+
+networks:
+  shared-network:
+    name: my-shared-network
+    external: true
diff --git a/docker-compose-pytorch.yml b/docker-compose-pytorch.yml
@@ -9,3 +9,8 @@ services:
       - "8888:8888"
     environment:
       - GF_AI_TRAINING_CREDS=83bcaff6228b39bbe431af5e19fb4368e2a03dd3:1337@http://ai-training-api:8000
+
+networks:
+  shared-network:
+    name: my-shared-network
+    external: true
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   grafana:
-    container_name: grafana-aitraining-app
-    image: grafana/grafana-oss:10.4.1
+    container_name: grafana-with-training-app
+    image: grafana/grafana-oss:11.1.3
     environment:
       - GF_DEFAULT_APP_MODE=development # Ideally this should be moved to grafana.ini
     volumes:
@@ -16,22 +16,12 @@ services:
       sh -c "sed -i 's/<\/body><\/html>/<script src=\"http:\/\/localhost:35729\/livereload.js\"><\/script><\/body><\/html>/g' /usr/share/grafana/public/views/index.html && /run.sh"
     ports:
       - "3000:3000"
-
-  loki:
-    image: grafana/loki:3.0.0
-    command: -config.file=/etc/loki/local-config.yaml
-    ports:
-      - "3100:3100"
-
-  mimir:
-    image: grafana/mimir:2.11.0
-    volumes:
-      - ./.config/mimir/config:/etc/mimir-config
-    entrypoint:
-      - /bin/mimir
-      - -config.file=/etc/mimir-config/mimir.yml
-    ports:
-      - "9009:9009"
+    depends_on:
+      - mimir
+      - loki
+      - pyroscope
+      - tempo
+      - ai-training-api
 
   db:
     image: mysql:8.0
@@ -57,6 +47,42 @@ services:
     depends_on:
       - db
 
+  mimir:
+    image: grafana/mimir:2.12.0
+    restart: on-failure
+    command:
+      - -config.file=/etc/mimir-config/mimir.yml
+    volumes:
+      - ./.config/mimir:/etc/mimir-config
+    ports:
+      - "9009:9009"
+
+  loki:
+    image: grafana/loki:3.0.0
+    restart: on-failure
+    ports:
+      - "3100:3100"
+
+  tempo:
+    image: grafana/tempo:2.4.1
+    restart: on-failure
+    command:
+      - "-storage.trace.backend=local"                  # tell tempo where to permanently put traces
+      - "-storage.trace.local.path=/tmp/tempo/traces"
+      - "-storage.trace.wal.path=/tmp/tempo/wal"        # tell tempo where to store the wal
+      - "-auth.enabled=false"                           # disables the requirement for the X-Scope-OrgID header
+      - "-server.http-listen-port=3200"
+    ports:
+      - "3200:3200"
+      - "4317:4317"
+
+  pyroscope:
+    image: grafana/pyroscope:1.5.0
+    restart: on-failure
+    ports:
+      - "4040:4040"
+
+
 networks:
   shared-network:
     name: my-shared-network