[Self-Hosted] 로그와 메트릭 모니터링 Grafana + Alloy + Prometheus + Loki를 설치해보자 (feat. Caddy)

{
  # Global options
  admin 0.0.0.0:2019 # networks: host가 아닌 도커 컨테이너를 사용 중이라면 필요함

  metrics {
    # per_host # 호스트별로 메트릭 수집
  }

  log access-json { # 이름 수정 가능함
    include http.log.access.reverse_proxy # reverse_proxy 로거를 수집함
    output file /var/log/caddy/access.log { # 경로 수정 가능함
      roll_size 1Gib
      roll_keep 5
      roll_keep_for 90d
    }
    format json
    level INFO
  }
}

# reverse_proxy options
example.com {
  log reverse_proxy # 로거 이름 수정 가능함

  reverse_proxy example:80
}

services:
  caddy:
    image: 'caddy:latest'
    container_name: caddy
    hostname: caddy
    restart: unless-stopped
    ports:
      - 2019:2019 # 포트 개방이 필요함
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile
      - ./srv:/srv
      - ./data:/data
      - ./config:/config
      - /path/to:/var/log/caddy // 원하는 경로로 수정하기 (Alloy가 사용할 log 파일)
    environment:
      - TZ=Asia/Seoul

📦 grafana-stack
├─ config
│  ├─ alloy
│  │  └─ config.alloy
│  ├─ grafana
│  │  ├─ datasources
│  │  │  └─ datasource.yml
│  │  └─ grafana.ini
│  └─ prometheus
│     └─ prometheus.yml
└─ compose.yml

loki.source.file "caddy_access_logs" {
  targets = [{
    __path__ = "/var/log/access.log",
  }]

  forward_to = [loki.process.access_logs.receiver]
}

loki.process "access_logs" {
  forward_to = [loki.write.loki.receiver]
}

loki.write "loki" {
  endpoint {
    url = "http://grafana-loki:3100/loki/api/v1/push"
  }
  external_labels = {
    app = "caddy",
  }
}

apiVersion: 1

datasources:
  - name: Loki
    type: loki
    access: proxy
    orgId: 1
    url: http://grafana-loki:3100
    basicAuth: false
    isDefault: true
    version: 1
    editable: true

  - name: Prometheus
    type: prometheus
    url: http://grafana-prometheus:9090
    isDefault: false
    access: proxy
    editable: true

[analytics]
reporting_enabled = false

[auth.anonymous]
enabled = true
org_role = Admin

[explore]
enabled = true

[users]
default_theme = dark

global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - localhost:9093
      scheme: http
      timeout: 10s
      api_version: v1

scrape_configs:
  - job_name: prometheus
    honor_timestamps: true
    scrape_interval: 15s
    scrape_timeout: 10s
    metrics_path: /metrics
    scheme: http
    static_configs:
      - targets:
          - localhost:9090

  - job_name: caddy
    static_configs:
      - targets:
          - caddy:2019

volumes:
  grafana_data:
    name: grafana_data
  grafana_prometheus_data:
    name: grafana_prometheus_data

services:
  grafana-loki:
    image: 'grafana/loki:latest'
    container_name: grafana-loki
    hostname: grafana-loki
    restart: unless-stopped
    command:
      - -config.file=/etc/loki/local-config.yaml
    ports:
      - 3100:3100
    environment:
      - TZ=Asia/Seoul

  grafana-alloy:
    image: 'grafana/alloy:latest'
    container_name: grafana-alloy
    hostname: grafana-alloy
    restart: unless-stopped
    command:
      - run
      - /etc/alloy/config.alloy
      - --storage.path=/var/lib/alloy/data
      - --server.http.listen-addr=0.0.0.0:12345
    ports:
      - 12345:12345
    volumes:
      - ./config/alloy:/etc/alloy
      - /path/to:/var/log // caddy/compose.yml에서 설정한 로그 경로
    environment:
      - TZ=Asia/Seoul
    depends_on:
      - grafana-loki

  grafana-prometheus:
    image: 'prom/prometheus:latest'
    container_name: grafana-prometheus
    hostname: grafana-prometheus
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --web.enable-lifecycle
    ports:
      - 9091:9090
    volumes:
      - ./config/prometheus:/etc/prometheus
      - grafana_prometheus_data:/prometheus
    environment:
      - TZ=Asia/Seoul

  grafana:
    image: 'grafana/grafana:latest'
    container_name: grafana
    hostname: grafana
    restart: unless-stopped
    command:
      - --config=/etc/grafana-config/grafana.ini
    ports:
      - 3000:3000
    volumes:
      - grafana_data:/var/lib/grafana
      - ./config/grafana:/etc/grafana-config
      - ./config/grafana/datasources:/etc/grafana/provisioning/datasources
    environment:
      - TZ=Asia/Seoul
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:3000/healthz']
      interval: 1s
      timeout: 10s
      retries: 5