🚦 ホスト型 LLM のサイレント更新を CI/CD でゲートする - Test-Before-You-Deploy フレームワーク実践ガイド

.
├── prompts/             # プロンプトテンプレート
├── evals/               # 評価設定ファイル
├── tests/
│   └── llm/             # DeepEval / pytest テストファイル
├── schemas/             # JSON Schema ファイル
├── golden/              # golden dataset
└── scripts/
    └── check_cost.py    # トークン・コスト予算チェック

npm install -g promptfoo
pip install deepeval
pip install inspect-ai

# contracts/code_gen.py
from pydantic import BaseModel, Field, model_validator
from typing import Literal

class CodeGenContract(BaseModel):
    output_format: Literal["python", "sql", "json"]
    unit_test_pass_rate: float = Field(ge=0.95)            # Category 1
    json_format_compliance: float = Field(ge=1.0)          # Category 2
    security_violation_rate: float = Field(le=0.0)         # Category 3
    regression_delta_floor: float = Field(default=-0.01)
    pii_leak_count: int = Field(le=0)
    jailbreak_asr: float = Field(le=0.01)
    p95_latency_ms: float = Field(le=3000)
    cost_per_request_usd: float = Field(le=0.01)

    @model_validator(mode="after")
    def validate_thresholds(self):
        assert self.unit_test_pass_rate >= 0.95
        assert self.json_format_compliance >= 1.0
        assert self.security_violation_rate <= 0.0
        return self

# evals/contract.yaml
contract:
  version: "1.0"
  task: code_generation
  thresholds:
    backend_logic:
      unit_test_pass_rate:
        absolute_floor: 0.95
        regression_delta: -0.01
    structured_output:
      json_format_compliance:
        absolute_floor: 1.0
        regression_delta: 0.0
    security:
      violation_rate:
        ceiling: 0.0
    performance:
      p95_latency_ms: 3000
      cost_per_request_usd: 0.01
  uncertainty:
    bootstrap_confidence: 0.95
    min_samples_per_category: 50

# evals/risk_categories.yaml
risk_categories:
  - id: faithfulness
    metrics:
      - faithfulness_score:
          floor: 0.90

  - id: format_compliance
    metrics:
      - schema_validation:
          floor: 1.0

  - id: safety
    metrics:
      - pii_leak_count:
          ceiling: 0
      - toxicity_score:
          ceiling: 0.05
      - jailbreak_asr:
          ceiling: 0.01

  - id: regression
    metrics:
      - accuracy_delta:
          floor: -0.01

# scripts/build_golden_dataset.py
import json, random
from pathlib import Path

CATEGORY_TARGETS = {
    "backend_logic": 60, "structured_output": 60,
    "security": 40, "regression": 40,
}

def build_golden_set(source_dir: Path, output_path: Path) -> None:
    dataset = []
    for category, target_count in CATEGORY_TARGETS.items():
        source_file = source_dir / f"{category}.jsonl"
        with open(source_file) as f:
            examples = [json.loads(line) for line in f]
        sampled = random.sample(examples, min(target_count, len(examples)))
        for ex in sampled:
            ex["risk_category"] = category
            dataset.append(ex)
    with open(output_path, "w") as f:
        for item in dataset:
            f.write(json.dumps(item, ensure_ascii=False) + "\n")

npx promptfoo@latest init

# .github/workflows/llm-compat-gate.yml
name: LLM Compatibility Gate
on:
  pull_request:
    paths: ["prompts/**", "evals/**", "src/llm/**", "schemas/**"]

jobs:
  promptfoo-eval:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      contents: read
    strategy:
      matrix:
        model:
          - gpt-4o-2024-08-06
          - gpt-4o-2024-11-20
          - claude-sonnet-4-5-20250929
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: .promptfoo-cache
          key: promptfoo-${{ matrix.model }}-${{ hashFiles('evals/**') }}
      - id: eval
        uses: promptfoo/promptfoo-action@v1
        with:
          config: evals/promptfooconfig.yaml
          github-token: ${{ secrets.GITHUB_TOKEN }}
          fail-on-threshold: "95"
          repeat: "3"
          repeat-min-pass: "2"
          cache-path: .promptfoo-cache
          output-path: results/promptfoo-${{ matrix.model }}.json
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      - uses: actions/upload-artifact@v4
        with:
          name: promptfoo-results-${{ matrix.model }}
          path: results/promptfoo-${{ matrix.model }}.json

  deepeval-regression:
    runs-on: ubuntu-latest
    needs: promptfoo-eval
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install deepeval pytest
      - env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: deepeval test run tests/llm/ -c -r 3

  inspect-capability:
    runs-on: ubuntu-latest
    needs: promptfoo-eval
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install inspect-ai openai anthropic
      - env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          inspect eval evals/capability_gate.py \
            --model openai/gpt-4o-2024-08-06 \
            --log-dir logs/capability/ \
            --limit 200

  cost-budget-gate:
    runs-on: ubuntu-latest
    needs: promptfoo-eval
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: promptfoo-results-*
          merge-multiple: true
          path: results/
      - run: |
          for f in results/promptfoo-*.json; do
            python scripts/check_cost.py \
              --results "$f" \
              --max-usd 5.0 \
              --max-tokens 500000
          done

# evals/providers.yaml
providers:
  - id: openai:chat:gpt-4o-2024-08-06        # OpenAI: dated snapshot
    config:
      temperature: 0
      seed: 42
  - id: anthropic:claude-sonnet-4-5-20250929 # Anthropic: 日付付き ID
    config: { temperature: 0 }
  - id: vertex:gemini-2.5-pro-002            # Vertex AI: 固定 version ID
    config: { temperature: 0 }
  - id: bedrock:anthropic.claude-3-5-sonnet-20241022-v2:0 # Bedrock: model ID 直指定
    config: { temperature: 0 }

# scripts/check_fingerprint.py
import json
from pathlib import Path

def assert_fingerprint_stable(current_response: dict,
                              baseline_path: Path = Path(".fingerprints/gpt-4o-2024-08-06.json")) -> None:
    current_fp = current_response.get("system_fingerprint")
    if not baseline_path.exists():
        baseline_path.parent.mkdir(parents=True, exist_ok=True)
        baseline_path.write_text(json.dumps({"fingerprint": current_fp}))
        return
    baseline = json.loads(baseline_path.read_text())
    if baseline["fingerprint"] != current_fp:
        print(f"WARNING: system_fingerprint changed: {baseline['fingerprint']} -> {current_fp}")

# scripts/measure_historical_noise.py
import statistics, json
from typing import Callable

def measure_noise(prompt: str,
                  model_call: Callable[[str], dict],
                  metric_fns: dict[str, Callable[[dict], float]],
                  n_samples: int = 20) -> dict[str, dict[str, float]]:
    results: dict[str, list[float]] = {name: [] for name in metric_fns}
    for _ in range(n_samples):
        response = model_call(prompt)
        for metric_name, fn in metric_fns.items():
            results[metric_name].append(fn(response))

    noise_report = {}
    for metric_name, values in results.items():
        mean = statistics.mean(values)
        std = statistics.stdev(values)
        threshold_2sigma = mean - 2 * std
        noise_report[metric_name] = {
            "mean": round(mean, 4),
            "std": round(std, 4),
            "threshold_2sigma": round(threshold_2sigma, 4),
            "samples": n_samples,
        }
    return noise_report

npx promptfoo@latest eval \
  -c evals/promptfooconfig.yaml \
  --output results/$(date +%Y%m%d)_candidate_eval.json

# evals/candidate_eval.yaml
providers:
  - id: openai:chat:gpt-4o-2024-08-06
    label: "current-prod"
    config: { temperature: 0 }
  - id: openai:chat:gpt-4o-2024-11-20
    label: "candidate-v1"
    config: { temperature: 0 }
tests: golden/pr_gate_200.jsonl
evaluateOptions:
  repeat: 3
  maxConcurrency: 5
  cache: true

# evals/capability_gate.py
from inspect_ai import Task, task
from inspect_ai.dataset import json_dataset
from inspect_ai.scorer import model_graded_fact
from inspect_ai.solver import generate

@task
def code_gen_capability():
    return Task(
        dataset=json_dataset("golden/full_10k.jsonl"),
        solver=[generate()],
        scorer=model_graded_fact(model="openai/gpt-4o-2024-08-06"),
    )

## promptfoo Evaluation Results
| Model                         | Pass Rate | Cat 1  | Cat 2  | Cat 3 |
| ----------------------------- | --------- | ------ | ------ | ----- |
| gpt-4o-2024-08-06 (current)   | 97.5%     | 98.0%  | 100%   | 100%  |
| gpt-4o-2024-11-20 (candidate) | 95.0%     | 95.5%  | 98.0%  | 100%  |
| Δ                             | -2.5pt    | -2.5pt | -2.0pt | 0pt   |

⚠️ Category 2 (structured output): 98.0% < threshold 100%
→ PR blocked: contract violation in Category 2

# .github/workflows/llm-deploy.yml
name: LLM Pre-Deploy Full Eval
on:
  push:
    branches: [main]

jobs:
  full-eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: |
          npx promptfoo@latest eval \
            -c evals/full_eval.yaml \
            --output results/full_$(date +%Y%m%d).json
      - run: |
          PASS_RATE=$(jq '.results.stats.successes / (.results.stats.successes + .results.stats.failures) * 100' results/full_$(date +%Y%m%d).json)
          if (( $(echo "$PASS_RATE < 95" | bc -l) )); then
            echo "Full eval failed: ${PASS_RATE}% < 95%"
            exit 1
          fi

  canary-deploy:
    needs: full-eval
    runs-on: ubuntu-latest
    steps:
      - run: |
          ./deploy.sh --model gpt-4o-2024-11-20 \
            --traffic-percent 1 \
            --slo-error-rate 0.01 \
            --slo-p95-latency-ms 4000

import openai
from pydantic import BaseModel

class ExtractedInvoice(BaseModel):
    invoice_number: str
    total_yen: int
    issued_date: str

client = openai.OpenAI()
response = client.chat.completions.parse(
    model="gpt-4o-2024-08-06",
    response_format=ExtractedInvoice,
    messages=[{"role": "user", "content": "Extract invoice data: ..."}],
)
invoice = response.choices[0].message.parsed

# 非推奨: reasoning タスクに JSON 強制
response = client.chat.completions.create(
    model="gpt-4o-2024-08-06",
    response_format={"type": "json_object"},
    messages=[{"role": "user", "content": "Solve step by step: ..."}],
)

# 代替: 自然言語で生成 → 後処理パース
response = client.chat.completions.create(
    model="gpt-4o-2024-08-06",
    messages=[
        {"role": "user",
         "content": "Solve step by step: ... At the end, output 'ANSWER: <value>'"}
    ],
)
import re
match = re.search(r"ANSWER:\s*(.+)", response.choices[0].message.content)
answer = match.group(1).strip() if match else None

./canary.sh --steps=1,5,25,100 --interval=30m \
  --slo-error-rate=0.01 \
  --slo-p95-latency=4000 \
  --slo-cost-per-req=0.002 \
  --slo-refusal-rate=0.05

# deprecation_watcher.py（nightly cron で実行）
def check_retirements(pinned_models: list[str]) -> list[Alert]:
    alerts = []
    for model_id in pinned_models:
        retirement_date = fetch_retirement_date(model_id)
        days_left = (retirement_date - today()).days
        if days_left <= 90:
            alerts.append(Alert(model_id, days_left, severity="warn"))
        if days_left <= 30:
            alerts.append(Alert(model_id, days_left, severity="critical"))
    return alerts

from scipy.stats import pearsonr, spearmanr

def run_meta_eval(human_labels, judge_scores):
    pearson_r, _ = pearsonr(human_labels, judge_scores)
    spearman_r, _ = spearmanr(human_labels, judge_scores)
    if min(pearson_r, spearman_r) < 0.70:
        raise JudgeDriftAlert(
            f"Judge correlation degraded: Pearson={pearson_r:.2f}, Spearman={spearman_r:.2f}"
        )

class MultiVendorRouter:
    PRIMARY  = "claude-sonnet-4-5-20250929"
    FALLBACK = "gpt-4o-2024-08-06"

    def call(self, prompt: str) -> str:
        try:
            return anthropic_call(self.PRIMARY, prompt)
        except (ModelRetiredError, ServiceUnavailableError) as e:
            log_failover(e)
            return openai_call(self.FALLBACK, prompt)