📡 技術調査 - OpenTelemetry CNCF Graduated

2019-05-07  OpenTracing + OpenCensus 統合 → Sandbox 入り
2021-08-26  Incubating 昇格
2026-05-11  Graduated 決定
2026-05-21  Observability Summit で公式発表

python --version
node --version
go version
java -version
docker --version
kubectl version --client

pip install opentelemetry-api \
            opentelemetry-sdk \
            opentelemetry-exporter-otlp-proto-grpc \
            opentelemetry-exporter-otlp-proto-http

# zero-code instrumentation
pip install opentelemetry-distro
opentelemetry-bootstrap -a install

npm install @opentelemetry/api \
            @opentelemetry/sdk-node \
            @opentelemetry/sdk-trace-node \
            @opentelemetry/sdk-metrics \
            @opentelemetry/auto-instrumentations-node \
            @opentelemetry/exporter-trace-otlp-grpc \
            @opentelemetry/exporter-metrics-otlp-grpc

go get go.opentelemetry.io/otel \
       go.opentelemetry.io/otel/sdk \
       go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc \
       go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc
go mod tidy

curl -L -O https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar

export JAVA_TOOL_OPTIONS="-javaagent:./opentelemetry-javaagent.jar"
export OTEL_TRACES_EXPORTER=otlp
export OTEL_METRICS_EXPORTER=otlp
export OTEL_LOGS_EXPORTER=otlp
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
java -jar myapp.jar

docker pull otel/opentelemetry-collector-contrib:0.152.0

docker run -d \
  -v $(pwd)/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml \
  -p 4317:4317 \
  -p 4318:4318 \
  -p 8888:8888 \
  -p 55679:55679 \
  otel/opentelemetry-collector-contrib:0.152.0

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm repo update

helm install otel-collector open-telemetry/opentelemetry-collector \
  --namespace observability \
  --create-namespace \
  --set mode=daemonset \
  --values otel-collector-values.yaml

OTEL_VERSION=0.152.0
OS=linux
ARCH=amd64

curl -L -o otelcol-contrib.tar.gz \
  "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_VERSION}/otelcol-contrib_${OTEL_VERSION}_${OS}_${ARCH}.tar.gz"

tar -xzf otelcol-contrib.tar.gz
./otelcol-contrib --config otel-collector-config.yaml

from opentelemetry import trace, metrics
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter

resource = Resource.create({"service.name": "my-llm-service", "service.version": "1.0.0"})

trace_exporter = OTLPSpanExporter(endpoint="http://localhost:4317")
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter))
trace.set_tracer_provider(tracer_provider)
tracer = trace.get_tracer("my-llm-service")

metric_exporter = OTLPMetricExporter(endpoint="localhost:4317")
metric_reader = PeriodicExportingMetricReader(metric_exporter, export_interval_millis=10_000)
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter("my-llm-service")

llm_call_counter = meter.create_counter(
    "gen_ai.calls",
    description="Number of LLM API calls",
)

pip install traceloop-sdk openai

import os
from traceloop.sdk import Traceloop
from openai import OpenAI

os.environ["TRACELOOP_BASE_URL"] = "http://localhost:4318"

Traceloop.init(app_name="my-llm-app", disable_batch=False)

client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "OpenTelemetry とは何ですか？"}],
)
print(response.choices[0].message.content)

# export OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental
import openai
from opentelemetry import trace

tracer = trace.get_tracer("llm-instrumentation")

def call_llm(prompt: str) -> str:
    client = openai.OpenAI()

    with tracer.start_as_current_span("openai.chat") as span:
        span.set_attribute("gen_ai.provider.name", "openai")
        span.set_attribute("gen_ai.operation.name", "chat")
        span.set_attribute("gen_ai.request.model", "gpt-4o")

        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": prompt}],
        )

        usage = response.usage
        span.set_attribute("gen_ai.usage.input_tokens", usage.prompt_tokens)
        span.set_attribute("gen_ai.usage.output_tokens", usage.completion_tokens)
        span.set_attribute("gen_ai.response.finish_reason", response.choices[0].finish_reason)

        return response.choices[0].message.content

# gRPC (4317) — デフォルト
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
export OTEL_EXPORTER_OTLP_PROTOCOL=grpc

# HTTP/Protobuf (4318)
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf

export OTEL_EXPORTER_OTLP_COMPRESSION=gzip
export OTEL_SERVICE_NAME=my-llm-service

export OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  memory_limiter:
    check_interval: 5s
    limit_mib: 4000
    spike_limit_mib: 500

  batch:
    send_batch_size: 1024
    timeout: 10s
    send_batch_max_size: 2048

exporters:
  otlp/backend:
    endpoint: https://your-backend:4317
    compression: gzip
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 1000

  debug:
    verbosity: detailed

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp/backend]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp/backend]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp/backend]

pip install traceloop-sdk opentelemetry-instrumentation-langchain langchain langchain-openai

import os
from traceloop.sdk import Traceloop
from traceloop.sdk.decorators import workflow
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate

os.environ["TRACELOOP_BASE_URL"] = "http://localhost:4318"
Traceloop.init(app_name="langchain-app")

llm = ChatOpenAI(model="gpt-4o")
prompt = ChatPromptTemplate.from_messages([
    ("system", "You are a helpful assistant."),
    ("human", "{input}"),
])
chain = prompt | llm

@workflow(name="qa-workflow")
def run_qa(question: str) -> str:
    result = chain.invoke({"input": question})
    return result.content

export LANGSMITH_TRACING=true
export LANGSMITH_ENDPOINT=https://api.smith.langchain.com
export LANGSMITH_API_KEY=<your-key>
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318

from langsmith import traceable
from langchain_openai import ChatOpenAI

llm = ChatOpenAI(model="gpt-4o")

@traceable
def answer_question(question: str) -> str:
    return llm.invoke(question).content

services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel/config.yaml"]
    volumes:
      - ./otel-config.yaml:/etc/otel/config.yaml
    ports:
      - "4317:4317"
      - "4318:4318"
      - "8888:8888"
      - "13133:13133"
    restart: unless-stopped

extensions:
  health_check:
    endpoint: "0.0.0.0:13133"
  pprof:
    endpoint: "0.0.0.0:1777"
  zpages:
    endpoint: "0.0.0.0:55679"

service:
  extensions: [health_check, pprof, zpages]

docker logs -f otel-collector

curl -s http://localhost:8888/metrics | grep otelcol_receiver_accepted_spans
curl -s http://localhost:8888/metrics | grep otelcol_exporter_sent_spans
curl -s http://localhost:8888/metrics | grep otelcol_exporter_send_failed_spans
curl -s http://localhost:8888/metrics | grep otelcol_processor_tail_sampling

processors:
  tail_sampling:
    decision_wait: 120s
    num_traces: 200000
    expected_new_traces_per_sec: 100

    policies:
      - name: high_token_usage
        type: numeric_attribute
        numeric_attribute:
          key: gen_ai.usage.input_tokens
          min_value: 8000

      - name: content_filter
        type: string_attribute
        string_attribute:
          key: gen_ai.response.finish_reason
          values: ["content_filter", "error"]

      - name: error_status
        type: status_code
        status_code:
          status_codes: ["ERROR"]

      - name: probabilistic_fallback
        type: probabilistic
        probabilistic:
          sampling_percentage: 5

各ポッド / ノード (Agent モード)
  otel-collector-agent (DaemonSet)
    receivers: [otlp]
    processors: [memory_limiter, batch]
    exporters: [otlp → Gateway]
                ↓ OTLP gRPC
Gateway モード (Deployment + HPA)
  otel-collector-gateway
    receivers: [otlp]
    processors: [memory_limiter, tail_sampling, batch]
    exporters: [datadog, langfuse, prometheus_remote]

exporters:
  loadbalancing:
    protocol:
      otlp:
        tls:
          insecure: true
    resolver:
      dns:
        hostname: otel-collector-gateway-headless
        port: 4317

processors:
  redaction:
    allow_all_keys: false
    allowed_keys:
      - gen_ai.operation.name
      - gen_ai.provider.name
      - gen_ai.request.model
      - gen_ai.usage.input_tokens
      - gen_ai.usage.output_tokens
      - gen_ai.usage.cache_read.input_tokens
      - gen_ai.usage.reasoning.output_tokens
      - gen_ai.response.finish_reason
      - http.status_code
      - error.type
    blocked_values:
      - "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b"
      - "\\b\\d{4}[- ]?\\d{4}[- ]?\\d{4}[- ]?\\d{4}\\b"
    summary: "redacted"

from opentelemetry.instrumentation.openai import OpenAIInstrumentor

OpenAIInstrumentor().instrument(
    capture_message_content=False,
)

resource = Resource({"semconv.version": "1.38.0", "service.name": "my-llm-app"})

resource = Resource({
    "service.name": "llm-api",
    "service.version": "2.1.0",
    "deployment.environment": "production",
    "ai.framework": "strands",
    "semconv.version": "1.38.0",
    "team": "platform-ai",
})

processors:
  filter/prod_only:
    spans:
      include:
        match_type: strict
        attributes:
          - key: deployment.environment
            value: production

exporters:
  datadog/prod:
    api:
      key: ${DD_API_KEY_PROD}
  datadog/dev:
    api:
      key: ${DD_API_KEY_DEV}

service:
  pipelines:
    traces/prod:
      processors: [memory_limiter, filter/prod_only, tail_sampling, batch]
      exporters: [datadog/prod]
    traces/dev:
      processors: [memory_limiter, batch]
      exporters: [datadog/dev]

# 誤った実装例 (retry してしまう)
response = requests.post(url, data=payload)
if response.status_code == 200:
    body = response.json()
    if body.get("partialSuccess", {}).get("rejectedSpans", 0) > 0:
        retry_export(payload)  # NG

# 正しい実装例
response = requests.post(url, data=payload)
if response.status_code == 200:
    body = response.json()
    rejected = body.get("partialSuccess", {}).get("rejectedSpans", 0)
    if rejected > 0:
        logger.warning(f"OTLP partial_success: {rejected} spans rejected. Do NOT retry.")
        metrics.increment("otlp.partial_success.rejected_spans", rejected)

num_traces の目安:
  = expected_new_traces_per_sec × decision_wait_seconds × safety_factor

例:
  TPS = 200 trace/s
  decision_wait = 120s
  safety_factor = 2.0
  num_traces = 200 × 120 × 2.0 = 48,000 → 60,000 に設定

メモリ消費の目安:
  1 trace ≒ 平均 span 数 × 平均 span size
  LLM trace: 5〜20 spans × 10KB (content なし) = 50〜200KB
  60,000 traces × 100KB = 6GB → limit_mib も連動して増やす

from opentelemetry import trace
from opentelemetry.trace import Link, SpanKind

tracer = trace.get_tracer("my-agent")

with tracer.start_as_current_span(
    "invoke_workflow start",
    kind=SpanKind.INTERNAL,
    attributes={
        "session.id": session_id,
        "gen_ai.operation.name": "invoke_workflow",
    }
) as start_span:
    start_ctx = start_span.get_span_context()

# 数時間後

with tracer.start_as_current_span(
    "invoke_workflow complete",
    kind=SpanKind.INTERNAL,
    links=[Link(start_ctx)],
    attributes={
        "session.id": session_id,
        "gen_ai.operation.name": "invoke_workflow",
        "workflow.status": "completed",
    }
) as complete_span:
    pass