OpenTelemetry
We are taking advantage of the Opentelemetry community opentelemetry-kube-stack as well as other various components for monitoring and observability. For more information, take a look at the Opentelemetry Kube Stack Helm Chart as well as the Opentelemetry Docs.
The Monitoring and Observability Overview documentation page for more information into it's features, components and how it's used within genestack.
Install the Opentelemetry Stack
efore installing OpenTelemetry we'll need secrets for various services we're gather metrics from.
If this is a fresh cluster deployment we'll need to create the secrets in the monitoring namespace:
Create MariaDB secret
kubectl --namespace monitoring \
create secret generic mariadb-monitoring \
--type Opaque \
--from-literal=username="monitoring" \
--from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-64};echo;)"
Create Postgres secret
kubectl --namespace monitoring \
create secret generic postgres-monitoring \
--type Opaque \
--from-literal=username="monitoring"
--from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-64}; echo;)"
Create RabbitMQ secret in openstack namespace
kubectl --namespace openstack \
create secret generic rabbitmq-monitoring-user \
--type Opaque \
--from-literal=username="monitoring" \
--from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-64}; echo;)"
Copy RabbitMQ secret to monitoring namespace
kubectl get secret rabbitmq-monitoring-user \
-n openstack -o yaml \
| sed 's/namespace: openstack/namespace: monitoring/' \
| kubectl apply -f -
Run the Opentelemetry deployment Script /opt/genestack/bin/install-opentelemetry-kube-stack.sh
#!/bin/bash
# Description: Fetches the version for SERVICE_NAME_DEFAULT from the specified
# YAML file and executes a helm upgrade/install command with dynamic values files.
# Disable SC2124 (unused array), SC2145 (array expansion issue), SC2294 (eval)
# shellcheck disable=SC2124,SC2145,SC2294
# Service
SERVICE_NAME_DEFAULT="opentelemetry-kube-stack"
SERVICE_NAMESPACE="monitoring"
# Helm
HELM_REPO_NAME_DEFAULT="open-telemetry"
HELM_REPO_URL_DEFAULT="https://open-telemetry.github.io/opentelemetry-helm-charts"
# Base directories provided by the environment
GENESTACK_BASE_DIR="${GENESTACK_BASE_DIR:-/opt/genestack}"
GENESTACK_OVERRIDES_DIR="${GENESTACK_OVERRIDES_DIR:-/etc/genestack}"
# Define service-specific override directories based on the framework
SERVICE_BASE_OVERRIDES="${GENESTACK_BASE_DIR}/base-helm-configs/monitoring/${SERVICE_NAME_DEFAULT}"
SERVICE_CUSTOM_OVERRIDES="${GENESTACK_OVERRIDES_DIR}/helm-configs/${SERVICE_NAME_DEFAULT}"
# Read the desired chart version from VERSION_FILE
VERSION_FILE="${GENESTACK_OVERRIDES_DIR}/helm-chart-versions.yaml"
if [ ! -f "$VERSION_FILE" ]; then
echo "Error: helm-chart-versions.yaml not found at $VERSION_FILE" >&2
exit 1
fi
# Extract version dynamically using the SERVICE_NAME_DEFAULT variable
SERVICE_VERSION=$(grep "^[[:space:]]*${SERVICE_NAME_DEFAULT}:" "$VERSION_FILE" | sed "s/.*${SERVICE_NAME_DEFAULT}: *//")
if [ -z "$SERVICE_VERSION" ]; then
echo "Error: Could not extract version for '$SERVICE_NAME_DEFAULT' from $VERSION_FILE" >&2
exit 1
fi
echo "Found version for $SERVICE_NAME_DEFAULT: $SERVICE_VERSION"
# Load chart metadata from custom override YAML if defined
for yaml_file in "${SERVICE_CUSTOM_OVERRIDES}"/*.yaml; do
if [ -f "$yaml_file" ]; then
HELM_REPO_URL=$(yq eval '.chart.repo_url // ""' "$yaml_file")
HELM_REPO_NAME=$(yq eval '.chart.repo_name // ""' "$yaml_file")
SERVICE_NAME=$(yq eval '.chart.service_name // ""' "$yaml_file")
break # use the first match and stop
fi
done
# Fallback to defaults if variables not set
: "${HELM_REPO_URL:=$HELM_REPO_URL_DEFAULT}"
: "${HELM_REPO_NAME:=$HELM_REPO_NAME_DEFAULT}"
: "${SERVICE_NAME:=$SERVICE_NAME_DEFAULT}"
# Determine Helm chart path
if [[ "$HELM_REPO_URL" == oci://* ]]; then
# OCI registry path
HELM_CHART_PATH="$HELM_REPO_URL/$HELM_REPO_NAME/$SERVICE_NAME"
else
# --- Helm Repository and Execution ---
helm repo add "$HELM_REPO_NAME" "$HELM_REPO_URL" # uncomment if needed
helm repo update
HELM_CHART_PATH="$HELM_REPO_NAME/$SERVICE_NAME"
fi
# Debug output
echo "[DEBUG] HELM_REPO_URL=$HELM_REPO_URL"
echo "[DEBUG] HELM_REPO_NAME=$HELM_REPO_NAME"
echo "[DEBUG] SERVICE_NAME=$SERVICE_NAME"
echo "[DEBUG] HELM_CHART_PATH=$HELM_CHART_PATH"
# Prepare an array to collect -f arguments
overrides_args=()
# Include all YAML files from the BASE configuration directory
# NOTE: Files in this directory are included first.
if [[ -d "$SERVICE_BASE_OVERRIDES" ]]; then
echo "Including base overrides from directory: $SERVICE_BASE_OVERRIDES"
for file in "$SERVICE_BASE_OVERRIDES"/*.yaml; do
# Check that there is at least one match
if [[ -e "$file" ]]; then
echo " - $file (Base Config)"
overrides_args+=("-f" "$file")
fi
done
fi
# Include all YAML files from the custom SERVICE configuration directory
# NOTE: Files here have the highest precedence.
if [[ -d "$SERVICE_CUSTOM_OVERRIDES" ]]; then
echo "Including overrides from service config directory:"
for file in "$SERVICE_CUSTOM_OVERRIDES"/*.yaml; do
if [[ -e "$file" ]]; then
echo " - $file"
overrides_args+=("-f" "$file")
fi
done
else
echo "Warning: Service config directory not found: $SERVICE_CUSTOM_OVERRIDES"
fi
echo
# Collect all --set arguments, executing commands and quoting safely
set_args=()
helm_command=(
helm upgrade --install "$SERVICE_NAME_DEFAULT" "$HELM_CHART_PATH"
--version "${SERVICE_VERSION}"
--namespace="$SERVICE_NAMESPACE"
--timeout 120m
--create-namespace
"${overrides_args[@]}"
"${set_args[@]}"
# Post-renderer configuration
--post-renderer "$GENESTACK_OVERRIDES_DIR/kustomize/kustomize.sh"
--post-renderer-args "$SERVICE_NAME_DEFAULT/overlay"
"$@"
)
echo "Executing Helm command (arguments are quoted safely):"
printf '%q ' "${helm_command[@]}"
echo
# Execute the command directly from the array
"${helm_command[@]}"