Setting up Loki

Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream.

Run the package deployment

Run the Loki deployment Script /opt/genestack/bin/install-loki.sh

#!/bin/bash
# Description: Fetches the version for Loki from the specified YAML file and executes a helm upgrade/install command.

# Disable SC2124 (unused array), SC2145 (array expansion issue), SC2294 (eval)
# shellcheck disable=SC2124,SC2145,SC2294

# Service
SERVICE_NAME="loki"
SERVICE_NAMESPACE="grafana"

# Helm
HELM_REPO_NAME="grafana"
HELM_REPO_URL="https://grafana.github.io/helm-charts"

# Base directories provided by the environment
GENESTACK_BASE_DIR="${GENESTACK_BASE_DIR:-/opt/genestack}"
GENESTACK_OVERRIDES_DIR="${GENESTACK_OVERRIDES_DIR:-/etc/genestack}"

# Define service-specific override directories based on the framework
SERVICE_BASE_OVERRIDES="${GENESTACK_BASE_DIR}/base-helm-configs/${SERVICE_NAME}"
SERVICE_CUSTOM_OVERRIDES="${GENESTACK_OVERRIDES_DIR}/helm-configs/${SERVICE_NAME}"
GLOBAL_OVERRIDES_DIR="${GENESTACK_OVERRIDES_DIR}/helm-configs/global_overrides"

# Read the desired chart version from VERSION_FILE
VERSION_FILE="${GENESTACK_OVERRIDES_DIR}/helm-chart-versions.yaml"

if [ ! -f "$VERSION_FILE" ]; then
    echo "Error: helm-chart-versions.yaml not found at $VERSION_FILE" >&2
    exit 1
fi

# Extract version dynamically using the SERVICE_NAME variable
SERVICE_VERSION=$(grep "^[[:space:]]*${SERVICE_NAME}:" "$VERSION_FILE" | sed "s/.*${SERVICE_NAME}: *//")

if [ -z "$SERVICE_VERSION" ]; then
    echo "Error: Could not extract version for '$SERVICE_NAME' from $VERSION_FILE" >&2
    exit 1
fi

echo "Found version for $SERVICE_NAME: $SERVICE_VERSION"

# Prepare an array to collect -f arguments
overrides_args=()

# Include all YAML files from the BASE configuration directory
if [[ -d "$SERVICE_BASE_OVERRIDES" ]]; then
    echo "Including base overrides from directory: $SERVICE_BASE_OVERRIDES"
    for file in "$SERVICE_BASE_OVERRIDES"/*.yaml; do
        if [[ -e "$file" ]]; then
            echo " - $file"
            overrides_args+=("-f" "$file")
        fi
    done
else
    echo "Warning: Base override directory not found: $SERVICE_BASE_OVERRIDES"
fi

# Include all YAML files from the GLOBAL configuration directory
if [[ -d "$GLOBAL_OVERRIDES_DIR" ]]; then
    echo "Including overrides from global config directory:"
    for file in "$GLOBAL_OVERRIDES_DIR"/*.yaml; do
        if [[ -e "$file" ]]; then
            echo " - $file"
            overrides_args+=("-f" "$file")
        fi
    done
else
    echo "Warning: Global config directory not found: $GLOBAL_OVERRIDES_DIR"
fi

# Include all YAML files from the custom SERVICE configuration directory
if [[ -d "$SERVICE_CUSTOM_OVERRIDES" ]]; then
    echo "Including overrides from service config directory:"
    for file in "$SERVICE_CUSTOM_OVERRIDES"/*.yaml; do
        if [[ -e "$file" ]]; then
            echo " - $file"
            overrides_args+=("-f" "$file")
        fi
    done
else
    echo "Warning: Service config directory not found: $SERVICE_CUSTOM_OVERRIDES"
fi

echo

# --- Helm Repository and Execution ---
helm repo add "$HELM_REPO_NAME" "$HELM_REPO_URL"
helm repo update

# Collect all --set arguments.
set_args=()


helm_command=(
    helm upgrade --install "$SERVICE_NAME" "$HELM_REPO_NAME/$SERVICE_NAME"
    --version "${SERVICE_VERSION}"
    --namespace="$SERVICE_NAMESPACE"
    --timeout 120m
    --create-namespace

    "${overrides_args[@]}"
    "${set_args[@]}"

    --post-renderer "$GENESTACK_OVERRIDES_DIR/kustomize/kustomize.sh"
    --post-renderer-args "$SERVICE_NAME/overlay"

    "$@"
)

echo "Executing Helm command (arguments are quoted safely):"
printf '%q ' "${helm_command[@]}"
echo

# Execute the command directly from the array
"${helm_command[@]}"

Tip

Swift (Recommended)S3MinIO

Abstract

If you plan on using Swift as a backend for log storage see the loki-helm-swift-overrides-example.yaml file in the helm-configs/loki directory.

---
global:
  dnsService: coredns
minio:
  enabled: false
loki:
  persistence:
    enabled: true
    size: 100Gi
  auth_enabled: false
  configStorageType: Secret
  ingester:
    autoforget_unhealthy: true
  storage:
    bucketNames:
      chunks: LOKI_CHUNKS_BUCKET_NAME
      ruler: LOKI_RULER_BUCKET_NAME
      admin: LOKI_ADMIN_BUCKET_NAME
    type: swift
    swift:
      auth_url: YOUR_SWIFT_AUTH_URL
      password: YOUR_SWIFT_PASSWORD
      username: YOUR_SWIFT_USERNAME
      container_name: YOUR_SWIFT_CONTAINER_NAME
      region_name: YOUR_SWIFT_REGION
  schemaConfig:
    configs:
      - from: 2020-01-01
        store: tsdb
        object_store: swift
        schema: v13
        index:
          prefix: index_
          period: 24h
  limits_config:
    ingestion_rate_mb: 10
    ingestion_burst_size_mb: 40
    per_stream_rate_limit: 10MB
    per_stream_rate_limit_burst: 40MB

  write:
    replicas: 5
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  read:
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  canary:
    enabled: true
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 50%
    serviceMonitor:
      enabled: true

  backend:
    serviceMonitor:
      enabled: true

gateway:
  service:
    type: NodePort
    nodePort: 31776
  podDisruptionBudget:
    enabled: true
    maxUnavailable: 1
  nodeSelector:
    openstack-control-plane: enabled
  serviceMonitor:
    enabled: true

lokiCanary:
  kind: Deployment
  replicas: 1
  nodeSelector:
    openstack-control-plane: enabled

Abstract

If you plan on using S3 as a backend for log storage see the loki-helm-s3-overrides-example.yaml file in the helm-configs/loki directory.

---
global:
  dnsService: coredns
minio:
  enabled: false
loki:
  persistence:
    enabled: true
    size: 100Gi
  auth_enabled: false
  configStorageType: Secret
  ingester:
    autoforget_unhealthy: true
  storage:
    bucketNames:
      chunks: loki-chunks-bucket-name
      ruler: loki-ruler-bucket-name
      admin: loki-admin-bucket-name
    type: s3
    s3:
      endpoint: YOUR_S3_ENDPOINT_URL
      region: YOUR_S3_REGION
      access_key_id: YOUR_S3_ACCESS_KEY_ID
      secret_access_key: YOUR_S3_SECRET_ACCESS_KEY
      s3forcepathstyle: true
      insecure: false
  schemaConfig:
    configs:
      - from: 2020-01-01
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: index_
          period: 24h
  limits_config:
    ingestion_rate_mb: 10
    ingestion_burst_size_mb: 40
    per_stream_rate_limit: 10MB
    per_stream_rate_limit_burst: 40MB

  write:
    replicas: 5
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  read:
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  canary:
    enabled: true
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 50%
    serviceMonitor:
      enabled: true

  backend:
    serviceMonitor:
      enabled: true

gateway:
  service:
    type: NodePort
    nodePort: 31776
  podDisruptionBudget:
    enabled: true
    maxUnavailable: 1
  nodeSelector:
    openstack-control-plane: enabled
  serviceMonitor:
    enabled: true

lokiCanary:
  kind: Deployment
  replicas: 1
  nodeSelector:
    openstack-control-plane: enabled

Abstract

If you plan on using Minio as a backend for log storage see the loki-helm-s3-overrides-example.yaml file in the helm-configs/loki directory.

---
global:
  dnsService: coredns
minio:
  enabled: true
loki:
  persistence:
    enabled: true
    size: 100Gi
  auth_enabled: false
  configStorageType: Secret
  ingester:
    autoforget_unhealthy: true
  storage:
    bucketNames:
      chunks: LOKI_CHUNKS_BUCKET_NAME
      ruler: LOKI_RULER_BUCKET_NAME
      admin: LOKI_ADMIN_BUCKET_NAME
    # Storage type and connection details are handled implicitly by the chart
    # when MinIO is enabled, connecting via S3 protocol to the internal MinIO service.
  schemaConfig:
    configs:
      - from: 2020-01-01
        store: tsdb
        object_store: s3  # MinIO is S3-compatible
        schema: v13
        index:
          prefix: index_
          period: 24h
  limits_config:
    ingestion_rate_mb: 10
    ingestion_burst_size_mb: 40
    per_stream_rate_limit: 10MB
    per_stream_rate_limit_burst: 40MB

  write:
    replicas: 5
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  read:
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 40%
    nodeSelector:
      openstack-control-plane: enabled
    serviceMonitor:
      enabled: true

  canary:
    enabled: true
    podDisruptionBudget:
      enabled: true
      maxUnavailable: 50%
    serviceMonitor:
      enabled: true

  backend:
    serviceMonitor:
      enabled: true

gateway:
  service:
    type: NodePort
    nodePort: 31776
  podDisruptionBudget:
    enabled: true
    maxUnavailable: 1
  nodeSelector:
    openstack-control-plane: enabled
  serviceMonitor:
    enabled: true

lokiCanary:
  kind: Deployment
  replicas: 1
  nodeSelector:
    openstack-control-plane: enabled