OpenTelemetry in Production: Phased Rollout

# Install OTEL Collector as DaemonSet helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts helm install otel-collector open-telemetry/opentelemetry-collector \ --namespace observability --create-namespace \ --set mode=daemonset \ --values otel-values.yaml

# otel-values.yaml config: receivers: otlp: protocols: grpc: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 processors: batch: timeout: 10s send_batch_size: 1024 # Tail-based sampling (keep only interesting traces) tail_sampling: policies: - name: errors type: status_code status_code: status_codes: [ERROR] - name: slow-requests type: latency latency: threshold_ms: 1000 - name: sample-fast-requests type: probabilistic probabilistic: sampling_percentage: 10 # Keep 10% of fast requests exporters: otlp: endpoint: tempo.observability.svc:4317 tls: insecure: true prometheus: endpoint: "0.0.0.0:8889" service: pipelines: traces: receivers: [otlp] processors: [batch, tail_sampling] exporters: [otlp] metrics: receivers: [otlp] processors: [batch] exporters: [prometheus]

# requirements.txt opentelemetry-api opentelemetry-sdk opentelemetry-instrumentation-flask opentelemetry-instrumentation-requests opentelemetry-instrumentation-sqlalchemy opentelemetry-exporter-otlp # app.py from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.flask import FlaskInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor # Initialize tracing trace.set_tracer_provider(TracerProvider()) tracer = trace.get_tracer(__name__) # Export to OTEL Collector otlp_exporter = OTLPSpanExporter( endpoint="otel-collector.observability.svc:4317", insecure=True ) span_processor = BatchSpanProcessor(otlp_exporter) trace.get_tracer_provider().add_span_processor(span_processor) # Auto-instrument Flask app = Flask(__name__) FlaskInstrumentor().instrument_app(app) RequestsInstrumentor().instrument() SQLAlchemyInstrumentor().instrument(engine=db.engine) # Manual instrumentation for critical sections @app.route('/api/order') def create_order(): with tracer.start_as_current_span("create_order") as span: span.set_attribute("user.id", request.user_id) span.set_attribute("order.amount", order_data['amount']) # This will automatically be a child span result = process_payment(order_data) span.set_attribute("payment.status", result.status) return jsonify(result) def process_payment(data): with tracer.start_as_current_span("process_payment"): # Payment logic here pass

# Install Grafana Tempo (cost-effective trace storage) helm repo add grafana https://grafana.github.io/helm-charts helm install tempo grafana/tempo \ --namespace observability \ --set tempo.storage.trace.backend=s3 \ --set tempo.storage.trace.s3.bucket=tempo-traces \ --set tempo.storage.trace.s3.region=us-east-1 # Tempo stores traces in S3 ($0.023/GB vs $1/GB for APM vendors)

# Add metrics instrumentation from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.exporter.prometheus import PrometheusMetricReader # Initialize metrics prometheus_reader = PrometheusMetricReader() meter_provider = MeterProvider(metric_readers=[prometheus_reader]) metrics.set_meter_provider(meter_provider) meter = metrics.get_meter(__name__) # Create custom metrics order_counter = meter.create_counter( "orders_total", description="Total number of orders", unit="1" ) order_duration = meter.create_histogram( "order_duration_seconds", description="Order processing duration", unit="s" ) # Use metrics in code @app.route('/api/order') def create_order(): start = time.time() with tracer.start_as_current_span("create_order") as span: result = process_order(data) # Record metrics with trace context order_counter.add(1, {"status": result.status, "product": data['product']}) order_duration.record(time.time() - start, {"status": result.status}) return jsonify(result)

# Structured logging with trace context import logging from opentelemetry import trace from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter # Initialize logging logger_provider = LoggerProvider() logger_provider.add_log_record_processor( BatchLogRecordProcessor(OTLPLogExporter(endpoint="otel-collector:4317")) ) # Configure Python logging handler = LoggingHandler(logger_provider=logger_provider) logging.basicConfig(level=logging.INFO, handlers=[handler]) logger = logging.getLogger(__name__) # Logs automatically include trace context @app.route('/api/order') def create_order(): with tracer.start_as_current_span("create_order") as span: trace_id = span.get_span_context().trace_id # This log will be correlated with the trace logger.info( "Processing order", extra={ "user_id": request.user_id, "order_amount": data['amount'], "trace_id": trace_id # Key for correlation! } ) result = process_payment(data) if result.status == "failed": logger.error( "Payment failed", extra={"reason": result.error, "trace_id": trace_id} ) return jsonify(result)

# Advanced tail sampling config processors: tail_sampling: decision_wait: 10s # Wait for trace to complete num_traces: 100000 # Buffer size expected_new_traces_per_sec: 1000 policies: # ALWAYS keep errors - name: errors type: status_code status_code: status_codes: [ERROR] # ALWAYS keep slow requests - name: slow-requests type: latency latency: threshold_ms: 1000 # Keep traces for specific critical endpoints - name: critical-endpoints type: string_attribute string_attribute: key: http.route values: ["/api/payment", "/api/checkout"] # Sample 10% of everything else - name: probabilistic type: probabilistic probabilistic: sampling_percentage: 10