Merge pull request 'feat: add Promtail, Loki, and Grafana log aggregation stack (#86)' (#93) from issue-86-promtail-loki-grafana into main

Reviewed-on: #93

config/grafana/datasources/loki.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://mvp-loki:3100
+    isDefault: true
+    editable: false

config/loki/config.yml

@@ -0,0 +1,32 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+
+schema_config:
+  configs:
+    - from: 2020-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  retention_period: 720h  # 30 days

config/promtail/config.yml

@@ -0,0 +1,21 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://mvp-loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: containers
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: 'service'

config/traefik/dynamic/grafana.yml

@@ -0,0 +1,8 @@
+http:
+  middlewares:
+    grafana-ipwhitelist:
+      ipWhiteList:
+        sourceRange:
+          - "10.0.0.0/8"
+          - "172.16.0.0/12"
+          - "192.168.0.0/16"

docker-compose.yml

@@ -264,6 +264,84 @@ services:
         max-size: "10m"
         max-file: "3"
 
+  # Log Aggregation - Loki
+  mvp-loki:
+    image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/grafana/loki:2.9.0
+    container_name: mvp-loki
+    restart: unless-stopped
+    volumes:
+      - ./config/loki/config.yml:/etc/loki/config.yml:ro
+      - mvp_loki_data:/loki
+    command: -config.file=/etc/loki/config.yml
+    networks:
+      - backend
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:3100/ready || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+  # Log Aggregation - Promtail
+  mvp-promtail:
+    image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/grafana/promtail:2.9.0
+    container_name: mvp-promtail
+    restart: unless-stopped
+    volumes:
+      - ./config/promtail/config.yml:/etc/promtail/config.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    command: -config.file=/etc/promtail/config.yml
+    networks:
+      - backend
+    depends_on:
+      - mvp-loki
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+  # Log Aggregation - Grafana
+  mvp-grafana:
+    image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/grafana/grafana:10.0.0
+    container_name: mvp-grafana
+    restart: unless-stopped
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - ./config/grafana/datasources:/etc/grafana/provisioning/datasources:ro
+      - mvp_grafana_data:/var/lib/grafana
+    networks:
+      - backend
+      - frontend
+    depends_on:
+      - mvp-loki
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/api/health || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    labels:
+      - "traefik.enable=true"
+      - "traefik.docker.network=motovaultpro_frontend"
+      - "traefik.http.routers.grafana.rule=Host(`logs.motovaultpro.com`)"
+      - "traefik.http.routers.grafana.entrypoints=websecure"
+      - "traefik.http.routers.grafana.tls=true"
+      - "traefik.http.routers.grafana.tls.certresolver=letsencrypt"
+      - "traefik.http.routers.grafana.middlewares=grafana-ipwhitelist@file"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
 # Network Definition
 networks:
   frontend:
@@ -302,3 +380,7 @@ volumes:
     name: mvp_postgres_data
   mvp_redis_data:
     name: mvp_redis_data
+  mvp_loki_data:
+    name: mvp_loki_data
+  mvp_grafana_data:
+    name: mvp_grafana_data

scripts/ci/mirror-base-images.sh

@@ -17,6 +17,9 @@ IMAGES=(
   "python:3.11-slim"
   "docker:24.0"
   "docker:24.0-dind"
+  "grafana/loki:2.9.0"
+  "grafana/promtail:2.9.0"
+  "grafana/grafana:10.0.0"
 )
 
 echo "========================================"