📊 技術調査 - OfficeQA Pro / 企業文書エージェント評価ハーネス

# Python 3.10 以上を確認
python --version

# 必須パッケージ
pip install datasets huggingface_hub openai

# PDF 前処理ツール (brute-force OCR パイプライン用)
# macOS
brew install tesseract poppler

# Ubuntu/Debian
apt-get install -y tesseract-ocr poppler-utils

# Python バインディング
pip install Pillow pdf2image

git clone https://github.com/databricks/officeqa.git
cd officeqa

# ディレクトリ構成を確認
ls -la
# reward.py        — スコアリングの中核
# corpus_scripts/  — Treasury Bulletin 前処理ユーティリティ
# *.ipynb          — 可視化ノートブック (Primary Lang: Jupyter Notebook)

# HF アカウントでログイン (ブラウザが開く)
huggingface-cli login
# または環境変数で渡す
export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxx

from datasets import load_dataset
from huggingface_hub import snapshot_download

# ベンチマーク CSV の読み込み
# officeqa_pro.csv  — 133 問 (frontier model 評価用デフォルト)
# officeqa_full.csv — 246 問 (Easy 113 + Pro 133)
ds = load_dataset(
    "databricks/officeqa",
    data_files="officeqa_pro.csv",
    split="train",
)
print(ds.column_names)
# ['uid', 'question', 'answer', 'source_docs', 'source_files', 'difficulty']

# コーパスのダウンロード (transformed TXT 形式を推奨、約 460 MB)
# PDF は約 4 GB、Parsed JSON は約 730 MB のため TXT が最小構成
snapshot_download(
    repo_id="databricks/officeqa",
    repo_type="dataset",
    allow_patterns="treasury_bulletins_parsed/transformed/*.txt",
    local_dir="./data",
)

# schema.py  (実装案)
from __future__ import annotations
from enum import Enum
from typing import Optional, List
from pydantic import BaseModel, Field
from sqlalchemy import Column, String, Float, JSON, Enum as SAEnum
from sqlalchemy.orm import DeclarativeBase

class Difficulty(str, Enum):
    easy   = "easy"    # フロンティア 2 モデルが両方正解した問題
    pro    = "pro"     # どちらかが不正解だった難問サブセット

class QuestionRow(BaseModel):
    uid:          str
    question:     str
    answer:       str                        # 数値文字列 or テキスト
    source_docs:  List[str]                  # Federal Reserve Archive URL 相当
    source_files: List[str]                  # 対応ローカルパス
    difficulty:   Difficulty = Difficulty.pro
    domain:       str        = "finance"     # 自社拡張: ドメイン分類

class Base(DeclarativeBase):
    pass

class Question(Base):
    __tablename__ = "questions"
    uid          = Column(String, primary_key=True)
    question     = Column(String, nullable=False)
    answer       = Column(String, nullable=False)
    source_docs  = Column(JSON)
    source_files = Column(JSON)
    difficulty   = Column(SAEnum(Difficulty), default=Difficulty.pro)
    domain       = Column(String, default="finance")

# runner.py  (実装案)
import os
import time
from enum import Enum
from openai import OpenAI
from schema import QuestionRow

class HarnessMode(str, Enum):
    prompt_only  = "prompt_only"   # LLM Prompt Only
    oracle_pages = "oracle_pages"  # Oracle Pages PDF
    agent        = "agent"         # Agent Harness (full corpus)

def run_question(
    client: OpenAI,
    q: QuestionRow,
    mode: HarnessMode,
    oracle_context: str = "",
    model: str = "gpt-5.5",
    reasoning_effort: str = "high",
    max_output_tokens: int = 50_000,
) -> dict:
    """1 問を指定モードで実行し、予測回答・latency・コストを返す (実装案)"""

    system = "Answer the question using only information from the provided documents."

    if mode == HarnessMode.prompt_only:
        user_msg = q.question
    elif mode == HarnessMode.oracle_pages:
        user_msg = f"Documents:\n{oracle_context}\n\nQuestion: {q.question}"
    else:  # agent mode — tool 呼び出しで自律検索 (別途 tool 定義が必要)
        user_msg = q.question

    t0 = time.monotonic()
    resp = client.responses.create(
        model=model,
        input=user_msg,
        reasoning={"effort": reasoning_effort},
        max_output_tokens=max_output_tokens,
    )
    latency = time.monotonic() - t0

    return {
        "uid":       q.uid,
        "mode":      mode,
        "predicted": resp.output_text.strip(),
        "latency_s": latency,
        "in_tokens": resp.usage.input_tokens,
        "out_tokens": resp.usage.output_tokens,
    }

# scorer.py  (実装案)
import re
import math

_PUNCT = re.compile(r"[,\s%$€£¥]")

def normalize(s: str) -> str:
    """句読点・通貨記号・空白を除去して比較可能な文字列に正規化する"""
    return _PUNCT.sub("", s).lower().strip()

def score_answer(
    ground_truth: str,
    predicted: str,
    tolerance: float = 0.0,
) -> bool:
    """
    決定論的 exact match。数値の場合は相対誤差で比較する。
    tolerance: 0.0 / 0.001 / 0.01 / 0.05 を想定
    式: |ŷ − y| / |y| <= tolerance  (OfficeQA Pro arXiv §3 の定義に準拠)
    """
    gt = normalize(ground_truth)
    pr = normalize(predicted)

    # 文字列一致を先行チェック
    if gt == pr:
        return True

    # 数値として比較
    try:
        y   = float(gt)
        y_hat = float(pr)
        if y == 0:
            return y_hat == 0
        return abs(y_hat - y) / abs(y) <= tolerance
    except ValueError:
        return False

TOLERANCES = [0.0, 0.001, 0.01, 0.05]

def score_batch(results: list[dict], questions: dict[str, str]) -> dict:
    """
    results: [{"uid": ..., "predicted": ...}, ...]
    questions: {uid: ground_truth}
    """
    totals = {t: 0 for t in TOLERANCES}
    n = len(results)
    for r in results:
        gt = questions[r["uid"]]
        for t in TOLERANCES:
            if score_answer(gt, r["predicted"], tolerance=t):
                totals[t] += 1
    return {f"acc@{t*100:.1f}%": v / n for t, v in totals.items()}

# .github/workflows/eval-gate.yml  (実装案)
name: Agent Eval Gate

on:
  pull_request:
    branches: [main]

jobs:
  eval:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      HF_TOKEN:       ${{ secrets.HF_TOKEN }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install datasets huggingface_hub openai

      - name: Run eval (smoke — 20 questions)
        run: |
          python eval/run_smoke.py \
            --mode prompt_only oracle_pages agent \
            --n 20 \
            --out results/pr-${{ github.run_number }}.json

      - name: Check accuracy gate
        run: |
          python eval/check_gate.py \
            --file results/pr-${{ github.run_number }}.json \
            --min-acc-oracle 0.50 \
            --min-acc-agent  0.30

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results-${{ github.run_number }}
          path: results/

# リポ直下でスコアラーが動作するか確認
python - <<'EOF'
from reward import score_answer
assert score_answer("123.45", "123.45", tolerance=0.0)
assert score_answer("100",    "100.5",  tolerance=0.01)
assert not score_answer("100", "106",   tolerance=0.05)
print("reward.py: OK")
EOF

# HF 認証
export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxx

# Pro 133 問をロードして内容確認
python - <<'EOF'
from datasets import load_dataset
ds = load_dataset("databricks/officeqa",
                  data_files="officeqa_pro.csv",
                  split="train",
                  token=True)
print(f"Total questions: {len(ds)}")           # 133
print(ds)
EOF

# 実装案 — 論文実験条件の再現イメージ
python -m officeqa.run \
  --model        gpt-5.5 \
  --mode         agent \
  --reasoning    high \
  --max-tokens   50000 \
  --dataset      officeqa_pro \
  --corpus-dir   ./data/treasury_bulletins_parsed/transformed/ \
  --output       ./results/gpt55_agent_pro.json

# ※ officeqa.run は 2026-05 時点で公開未確認。
#    上記は論文の実験条件 (arXiv:2603.08655 §3) を元にした実装案。

python - <<'EOF'
import json
from openai import OpenAI
from datasets import load_dataset
from runner import run_question, HarnessMode
from scorer import score_batch
from schema import QuestionRow

client = OpenAI()

ds = load_dataset("databricks/officeqa",
                  data_files="officeqa_pro.csv",
                  split="train",
                  token=True)

results = []
for row in ds.select(range(5)):   # まず 5 問でテスト
    q = QuestionRow(**row)
    r = run_question(client, q, HarnessMode.prompt_only)
    results.append(r)

questions = {row["uid"]: row["answer"] for row in ds}
print(json.dumps(score_batch(results, questions), indent=2))
EOF

# databricks_client.py  (実装案)
import os
from openai import OpenAI

def get_databricks_client(endpoint_name: str = "gpt-5-5-endpoint") -> OpenAI:
    """
    Unity AI Gateway 経由で GPT-5.5 を呼ぶクライアントを返す。
    認証: Service Principal の OAuth (M2M) トークンを DATABRICKS_TOKEN に設定する。
    Unity Catalog の Serving Endpoint に CAN_QUERY 権限が必要。
    """
    return OpenAI(
        api_key=os.environ["DATABRICKS_TOKEN"],   # dapi-... or SP OAuth token
        base_url=(
            f"https://{os.environ['DATABRICKS_WORKSPACE']}"
            ".cloud.databricks.com/serving-endpoints"
        ),
    )

# 呼び出し例 (Chat Completions 互換)
client = get_databricks_client()
resp = client.chat.completions.create(
    model="gpt-5-5-endpoint",
    messages=[{"role": "user", "content": "Summarize Q3 1945 cash receipts."}],
)
print(resp.choices.message.content)

# gateway_config_example.py  (実装案 / Terraform 等で実際には設定)
gateway_config = {
    "endpoint_name": "gpt-5-5-endpoint",
    "model": {
        "provider":    "openai",
        "name":        "gpt-5.5",
        "openai_config": {
            "openai_api_key": "{{secrets/openai-scope/api-key}}",  # Secret Scope
        },
    },
    "rate_limits": [
        {
            "calls":    100,
            "key":      "user",
            "renewal_period": "minute",
        }
    ],
    "auto_capture_config": {
        "catalog_name":    "main",
        "schema_name":     "eval_logs",
        "table_name_prefix": "officeqa_run",
        "enabled": True,          # Delta tables に identity/tokens/latency/cost を記録
    },
    # failover: rate limit 発火時に自動でバックアップモデルへルーティング
    "fallback_config": {
        "fallback_endpoint": "gpt-5-4-endpoint",
    },
}

# retry_example.py  (実装案)
import time
import random
from openai import OpenAI, RateLimitError, APIStatusError

def call_with_backoff(client: OpenAI, **kwargs) -> object:
    """Exponential backoff: initial 1s, base 2, max 60s, jitter ±20%"""
    max_retries = 5
    delay = 1.0
    for attempt in range(max_retries):
        try:
            return client.responses.create(**kwargs)
        except (RateLimitError, APIStatusError) as e:
            if attempt == max_retries - 1:
                raise
            jitter = delay * random.uniform(-0.2, 0.2)
            time.sleep(min(delay + jitter, 60.0))
            delay = min(delay * 2, 60.0)

# weekly_batch.py  (実装案)
"""
週次バッチ: 全 133 問を Batch API (50% 割引) で実行し、
結果を Delta tables / JSON に保存して Slack 通知する。
Batch API: 24h 以内処理、最大 50,000 req/batch。
GPT-5.5 Batch 単価: $2.50 / $15.00 per 1M tokens (input/output)。
"""
import json
import datetime
import os
from openai import OpenAI
from datasets import load_dataset

client = OpenAI()

def prepare_batch(ds, mode: str = "prompt_only") -> list[dict]:
    requests = []
    for row in ds:
        requests.append({
            "custom_id": f"{row['uid']}_{mode}",
            "method":    "POST",
            "url":       "/v1/responses",
            "body": {
                "model": "gpt-5.5",
                "input": row["question"],
                "reasoning": {"effort": "high"},
                "max_output_tokens": 50_000,
            },
        })
    return requests

def run_weekly_batch():
    ds = load_dataset("databricks/officeqa",
                      data_files="officeqa_pro.csv",
                      split="train",
                      token=True)

    for mode in ["prompt_only", "oracle_pages", "agent"]:
        requests = prepare_batch(ds, mode)

        # JSONL ファイルに書き出して Batch API に投入
        fname = f"/tmp/batch_{mode}_{datetime.date.today()}.jsonl"
        with open(fname, "w") as f:
            for req in requests:
                f.write(json.dumps(req) + "\n")

        batch_file = client.files.create(
            file=open(fname, "rb"),
            purpose="batch",
        )
        batch = client.batches.create(
            input_file_id=batch_file.id,
            endpoint="/v1/responses",
            completion_window="24h",
        )
        print(f"Batch submitted: {batch.id} (mode={mode})")

if __name__ == "__main__":
    run_weekly_batch()

# notify_slack.py  (実装案)
import json
import urllib.request
import urllib.parse

def notify_slack(webhook_url: str, scores: dict, run_date: str) -> None:
    """
    週次評価結果を Slack に投稿する。
    scores: {"prompt_only": {"acc@0.0%": 0.008, ...}, "agent": {...}, ...}
    """
    lines = [f"*OfficeQA Pro 週次評価 {run_date}*\n"]
    for mode, acc in scores.items():
        a0 = acc.get("acc@0.0%", 0)
        a5 = acc.get("acc@5.0%", 0)
        lines.append(f"• `{mode}`: acc@0% = {a0:.1%}  /  acc@5% = {a5:.1%}")

    payload = {"text": "\n".join(lines)}
    data = json.dumps(payload).encode()
    req = urllib.request.Request(
        webhook_url,
        data=data,
        headers={"Content-Type": "application/json"},
    )
    urllib.request.urlopen(req)

# .github/workflows/weekly-eval.yml  (実装案)
name: Weekly OfficeQA Eval

on:
  schedule:
    - cron: "0 2 * * 1"   # 毎週月曜 02:00 UTC

jobs:
  weekly-eval:
    runs-on: ubuntu-latest
    timeout-minutes: 120   # Batch API は 24h 以内だが投入+ポーリングで余裕を持つ
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      HF_TOKEN:       ${{ secrets.HF_TOKEN }}
      SLACK_WEBHOOK:  ${{ secrets.SLACK_EVAL_WEBHOOK }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install datasets huggingface_hub openai

      - name: Submit weekly batch
        run: python eval/weekly_batch.py

      - name: Wait for batch completion
        run: python eval/poll_batch.py --timeout 7200   # 2h ポーリング上限

      - name: Score and notify Slack
        run: python eval/score_and_notify.py

      - name: Upload results to Delta table
        run: python eval/upload_delta.py \
          --catalog main --schema eval_logs --table officeqa_weekly

# DeepEval による PR ゲートの最小構成例
from deepeval import evaluate
from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric
from deepeval.test_case import LLMTestCase

test_cases = [
    LLMTestCase(
        input="2024年度の総資産額は？",
        actual_output=agent.run(q),
        retrieval_context=agent.get_retrieved_chunks(q),
        expected_output=golden["answer"],
    )
    for q, golden in load_golden_set()
]

evaluate(
    test_cases=test_cases,
    metrics=[
        AnswerRelevancyMetric(threshold=0.8),
        FaithfulnessMetric(threshold=0.85),
    ],
)

# .github/workflows/eval.yml の骨格
name: LLM Eval Gate
on: [pull_request]

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run golden set (L1)
        run: |
          python -m pytest tests/eval/ \
            --deepeval \
            --threshold-faithfulness 0.85 \
            --threshold-answer-relevancy 0.80
      - name: Post results to PR
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              body: `## Eval Results\n${process.env.EVAL_SUMMARY}`
            })

# Langfuse を使ったカナリア比較の骨格
from langfuse import Langfuse

lf = Langfuse()

def route_query(query: str, canary_ratio: float = 0.05) -> dict:
    import random
    model = "gpt-5-5" if random.random() < canary_ratio else "gpt-5-4"
    with lf.trace(name="canary_eval", metadata={"model": model}) as trace:
        result = call_agent(query, model=model)
        trace.score(name="faithfulness", value=score_faithfulness(result))
    return result

# cron で日次実行する batch eval の例 (OpenAI Batch API 活用)
#!/bin/bash
DATE=$(date +%Y%m%d)
python scripts/create_batch.py \
  --golden-set data/golden/current.jsonl \
  --model gpt-5-5 \
  --output-file "results/batch_${DATE}.jsonl"

# Batch API はリクエスト単価が通常比 50% 割引
openai api batches.create \
  --input-file "results/batch_${DATE}.jsonl" \
  --endpoint "/v1/chat/completions" \
  --completion-window "24h"

# doc_version を golden set に埋め込む例
golden_entry = {
    "question": "2024年度の売上総利益は？",
    "expected_answer": "1,234億円",
    "source_chunk_ids": ["doc_v20240401_page12_table3"],
    "valid_until": "2025-03-31",  # 次年度更新予定日
}

# Promptfoo での multi-vendor 並走設定 (promptfooconfig.yaml)
providers:
  - id: openai:gpt-5-5
    config:
      temperature: 0
  - id: anthropic:claude-opus-4-7
    config:
      temperature: 0
  - id: vertex:gemini-2-5-pro
    config:
      temperature: 0

tests:
  - description: "財務数値抽出"
    vars:
      query: "2024年度の純利益は？"
    assert:
      - type: contains
        value: "{{expected_answer}}"
      - type: latency
        threshold: 10000

# cached input 最大化のプロンプト構成
messages = [
    {
        "role": "system",
        "content": SYSTEM_PROMPT,  # 固定 → キャッシュ対象
    },
    {
        "role": "user",
        "content": [
            {"type": "text", "text": CORPUS_SUMMARY},   # 固定 → キャッシュ対象
            {"type": "text", "text": f"質問: {question}"},  # 変動部分
        ],
    },
]

# Claude Anthropic SDK でのキャッシュ制御
import anthropic
client = anthropic.Anthropic()
response = client.messages.create(
    model="claude-opus-4-7",
    max_tokens=2048,
    system=[
        {"type": "text", "text": SYSTEM_PROMPT,
         "cache_control": {"type": "ephemeral"}},   # キャッシュ指定
    ],
    messages=[{"role": "user", "content": question}],
)

# タスク単位のコスト記録
cost_per_task = {
    "input_tokens": response.usage.prompt_tokens,
    "output_tokens": response.usage.completion_tokens,
    "cached_tokens": response.usage.prompt_tokens_details.cached_tokens,
    "cost_usd": (
        response.usage.prompt_tokens * 5e-6
        + response.usage.completion_tokens * 30e-6
        - response.usage.prompt_tokens_details.cached_tokens * 4.5e-6  # キャッシュ割引
    ),
}

# 誤差許容を階段で記録する exact match 評価
def evaluate_numeric(predicted: float, expected: float) -> dict:
    abs_rel_err = abs(predicted - expected) / max(abs(expected), 1e-9)
    return {
        "exact_0pct":  abs_rel_err == 0.0,
        "exact_01pct": abs_rel_err <= 0.001,
        "exact_1pct":  abs_rel_err <= 0.01,
        "exact_5pct":  abs_rel_err <= 0.05,
        "abs_rel_err": abs_rel_err,
    }

# PoC stage gate の判定条件例
POC_GATE = {
    "min_golden_accuracy": 0.70,      # 自社 golden set で 70% 以上
    "max_cost_per_task_usd": 0.50,    # タスク単価 $0.50 以下
    "max_p95_latency_ms": 15_000,     # p95 レイテンシ 15 秒以下
    "min_faithfulness": 0.80,         # 根拠付け 80% 以上
    "required_vendors": 2,            # 最低 2 ベンダーで比較
}

# YomiToku を前処理として挟む評価パイプラインの骨格
from yomitoku import DocumentAnalyzer

def extract_text_jp(pdf_path: str) -> str:
    analyzer = DocumentAnalyzer(
        lang="ja",
        figure=False,  # 図表をスキップして速度優先
    )
    results, _ = analyzer(pdf_path)
    return "\n".join(
        element.content
        for page in results
        for element in page.paragraphs + page.tables
    )

requests.exceptions.HTTPError: 403 Client Error: Forbidden for url:
https://huggingface.co/datasets/databricks/officeqa/resolve/main/data.tar.gz

# 1. CLI でログイン (ブラウザが開く)
huggingface-cli login

# 2. HuggingFace のデータセットページで access request を承認する
# https://huggingface.co/datasets/<org>/<dataset> → "Access repository" ボタン

# 3. トークンを環境変数に設定してから再取得
export HF_TOKEN=$(huggingface-cli env | grep token | awk '{print $2}')
python -c "
from datasets import load_dataset
ds = load_dataset('databricks/officeqa', token='${HF_TOKEN}')
print(ds)
"

# 4. 企業環境でプロキシを経由する場合
export HF_DATASETS_CACHE=/path/to/shared/cache
export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt

openai.RateLimitError: Rate limit reached for gpt-5-5 in organization org-xxxx
on tokens per min. Limit: 800000, Used: 800000, Requested: 42000.

# 短期対処: 指数バックオフでリトライ
python scripts/eval_with_backoff.py \
  --max-retries 5 \
  --base-delay 10 \
  --golden-set data/golden/current.jsonl

# 中期対処: Batch API に切替 (非同期・50% 割引)
openai api batches.create \
  --input-file batch_input.jsonl \
  --endpoint "/v1/chat/completions" \
  --completion-window "24h"

# 根本対処: Tier アップグレード申請
# https://platform.openai.com/account/limits → "Request increase" フォーム
# Databricks 経由の場合は Unity AI Gateway の quota 設定も合わせて変更する

# 指数バックオフの実装例
import time
import openai

def call_with_backoff(client, **kwargs):
    for attempt in range(5):
        try:
            return client.chat.completions.create(**kwargs)
        except openai.RateLimitError:
            wait = 10 * (2 ** attempt)
            print(f"Rate limit hit. Waiting {wait}s...")
            time.sleep(wait)
    raise RuntimeError("Max retries exceeded")

# 1. YomiToku (日本語特化) に切替
pip install yomitoku

python -c "
from yomitoku import DocumentAnalyzer
analyzer = DocumentAnalyzer(lang='ja')
results, _ = analyzer('problem_doc.pdf')
for page in results:
    for para in page.paragraphs:
        print(para.content)
"

# 2. Databricks ai_parse_document との比較検証
# ※ OfficeQA Pro では ai_parse_document が +16.1pt の改善を出したが
#   Databricks 製品依存になるためコスト・ロックインを評価すること
curl -X POST "https://<databricks-host>/api/2.0/serving-endpoints/databricks-gte-large-en/invocations" \
  -H "Authorization: Bearer $DATABRICKS_TOKEN" \
  -d '{"inputs": {"pdf_url": "https://example.com/doc.pdf"}}'

# 3. 解像度アップサンプリング (前処理)
python -c "
from pdf2image import convert_from_path
import pytesseract

pages = convert_from_path('low_res.pdf', dpi=300)
for i, page in enumerate(pages):
    text = pytesseract.image_to_string(page, lang='jpn')
    print(f'--- Page {i+1} ---')
    print(text)
"

# GPT-5.5 での reasoning_effort 設定
response = client.chat.completions.create(
    model="gpt-5-5",
    messages=messages,
    max_tokens=8192,          # 出力上限を引き上げる (デフォルト 4096 から)
    reasoning_effort="high",  # medium / high / max から選択
)

# Codex CLI での設定
# codex --model gpt-5-5 --reasoning-effort high --max-tokens 8192 "質問文"

# Claude Opus 4.7 での extended thinking 有効化
response = anthropic_client.messages.create(
    model="claude-opus-4-7",
    max_tokens=16000,
    thinking={
        "type": "enabled",
        "budget_tokens": 10000,  # reasoning に使える token 予算
    },
    messages=[{"role": "user", "content": question}],
)

# 1. OpenAI Models API でデプリケーション日を確認する (定期実行推奨)
curl https://api.openai.com/v1/models \
  -H "Authorization: Bearer $OPENAI_API_KEY" | \
  jq '.data[] | select(.id | startswith("gpt-5")) | {id, created, deprecated}'

# 2. Deprecation 通知をモニタリングする
# https://platform.openai.com/docs/deprecations (公式ページをブックマーク)
# または RSS フィードで変更をウォッチする

# 3. Databricks Unity AI Gateway での model alias 設定
# alias を使うと model 変更時にコード修正が不要になる
cat << 'EOF' > gateway_config.yaml
routes:
  - name: doc-agent-model
    route_type: llm/v1/chat
    model:
      provider: openai
      name: gpt-5-5          # ← 移行時はここだけ変える
      openai_api_key: "{{secrets/openai/api_key}}"
EOF

# PoC stage gate: ベンチスコアだけでなく ROI 指標を必須にする
POC_SCORECARD = {
    # 技術指標 (L1 eval)
    "golden_accuracy_pct": 75.0,       # 自社 golden で 75% 以上
    "faithfulness_score": 0.82,
    "cost_per_task_usd": 0.30,

    # ビジネス価値指標 (PoC 必須)
    "hours_saved_per_week": 20,        # 週 20 時間削減 (LayerX 事例: 年 570h)
    "error_rate_vs_manual_pct": -30,   # 手作業比 30% 以上の誤り削減
    "user_adoption_rate_pct": 60,      # パイロットユーザーの 60% が継続利用

    # リスク指標
    "pii_leak_incidents": 0,           # PII 漏洩ゼロ
    "prompt_injection_bypass_pct": 0,
}

def evaluate_poc(results: dict, scorecard: dict) -> bool:
    failures = [k for k, threshold in scorecard.items()
                if results.get(k, 0) < threshold]
    if failures:
        print(f"PoC GATE FAILED: {failures}")
        return False
    print("PoC GATE PASSED")
    return True