Log Cost Reduction: Loki/Grafana Playbook

# Install Loki with S3 backend helm repo add grafana https://grafana.github.io/helm-charts helm install loki grafana/loki-distributed \ --namespace logging --create-namespace \ --values loki-values.yaml

# loki-values.yaml loki: auth_enabled: false storage: type: s3 bucketNames: chunks: loki-chunks ruler: loki-ruler s3: region: us-east-1 schema_config: configs: - from: 2024-01-01 store: tsdb # Latest index format object_store: s3 schema: v13 index: prefix: index_ period: 24h limits_config: retention_period: 720h # 30 days max_query_length: 721h max_query_lookback: 720h ingestion_rate_mb: 50 ingestion_burst_size_mb: 100 compactor: working_directory: /data/compactor shared_store: s3 compaction_interval: 10m retention_enabled: true retention_delete_delay: 2h retention_delete_worker_count: 150 # Deploy with 3 components for scalability write: replicas: 3 # Ingestors read: replicas: 3 # Query frontend backend: replicas: 2 # Compactor, query scheduler

# Implement tiered retention with S3 Lifecycle # s3-lifecycle.json { "Rules": [ { "Id": "MoveToIA", "Status": "Enabled", "Transitions": [ { "Days": 7, "StorageClass": "STANDARD_IA" }, { "Days": 30, "StorageClass": "GLACIER_IR" } ], "Expiration": { "Days": 365 }, "Filter": { "Prefix": "loki/" } } ] } # Apply lifecycle policy aws s3api put-bucket-lifecycle-configuration \ --bucket loki-chunks \ --lifecycle-configuration file://s3-lifecycle.json

# Good: Search by label, filter by content {service="api", level="error"} |= "user_id=12345" # Bad: Using user_id as label (DON'T DO THIS) {service="api", level="error", user_id="12345"} # Creates millions of streams!

# Fast: Narrows to specific log streams first {service="api", level="error"} |= "payment failed" # Slow: Scans ALL logs {} |= "payment failed" # Fastest: Pre-aggregate into metrics sum by (service) ( count_over_time({level="error"}[5m]) )

# Convert logs to metrics for dashboards # Error rate by service sum by (service) ( rate({level="error"}[5m]) ) / sum by (service) ( rate({} [5m]) ) # P99 latency from structured logs quantile_over_time(0.99, {service="api"} | json | unwrap duration [5m] ) by (service) # Top 10 error messages topk(10, sum by (msg) ( count_over_time({level="error"} | json [1h]) ) )

# Python: Sample DEBUG logs at 1% import logging import random class SamplingFilter(logging.Filter): def filter(self, record): if record.levelno == logging.DEBUG: return random.random() < 0.01 # Keep 1% of DEBUG return True # Keep all ERROR/WARN/INFO logger = logging.getLogger() logger.addFilter(SamplingFilter()) # Result: 80% volume reduction, keep all errors

# Bad: Unstructured (hard to query) logger.info(f"User &#123;user_id&#125; purchased &#123;product&#125; for $&#123;amount&#125;") # Good: Structured JSON (easy to query) logger.info("Purchase completed", extra=&#123; "user_id": user_id, "product": product, "amount": amount, "payment_method": "stripe" &#125;) # Output in Loki: # {"level":"info","msg":"Purchase completed","user_id":123,"product":"widget","amount":49.99} # Query specific purchases &#123;service="api"&#125; | json | amount &gt; 100 | line_format "&#123;&#123;.user_id&#125;&#125;: $&#123;&#123;.amount&#125;&#125;"

# Track ingestion rate (bytes/sec) by service sum by (service) ( rate(loki_distributor_bytes_received_total[5m]) ) # Estimated monthly cost ($0.02/GB storage) sum(loki_ingester_chunks_stored_total) * 1024 * 0.02 / 1024 / 1024 / 1024 # Top 5 noisiest services topk(5, sum by (service) ( rate(loki_distributor_lines_received_total[1h]) ) ) # Query performance (slow queries to optimize) histogram_quantile(0.99, rate(loki_query_frontend_duration_seconds_bucket[5m]) )