📑 技術調査 - OpenAI×Databricks 企業文書エージェント評価ハーネス設計

import json
import pathlib
from openai import OpenAI

client = OpenAI()

SYSTEM_PROMPT = """企業文書から評価用 QA ペアを生成する専門家。
制約:
- 文書に明示的な根拠がある質問のみ
- unanswerable を 15% 混入
- 数値・日付を含む質問を 40% 以上
- 回答には引用 (page_ref, span) を付ける
出力は JSON: {"qa_pairs": [{"question": ..., "answer": ..., "page_ref": N, "span": ..., "answerable": true/false}]}
"""

def generate_silver_qa(doc_text: str, doc_id: str, n: int = 10) -> list[dict]:
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": f"文書ID: {doc_id}\n\n{doc_text[:8000]}"},
        ],
        response_format={"type": "json_object"},
    )
    items = json.loads(response.choices[0].message.content).get("qa_pairs", [])[:n]
    for item in items:
        item["doc_id"] = doc_id
        item["status"] = "silver"
    return items

import mlflow
import pandas as pd

golden_df = pd.read_json("./golden_set/gold.jsonl", lines=True)
eval_df = golden_df[["question", "answer", "doc_id"]].rename(
    columns={"question": "inputs", "answer": "ground_truth"}
)

def run_agent(question: str) -> dict:
    response = client.chat.completions.create(
        model="gpt-5.5",  # Unity AI Gateway endpoint 名
        messages=[{"role": "user", "content": question}],
    )
    return {"answer": response.choices[0].message.content, "retrieved_context": ""}

with mlflow.start_run(run_name="golden_set_eval_v1"):
    results = mlflow.evaluate(
        model=run_agent,
        data=eval_df,
        targets="ground_truth",
        model_type="question-answering",
        evaluators="databricks",
        extra_metrics=[
            mlflow.metrics.genai.faithfulness(),
            mlflow.metrics.genai.relevance_to_query(),
            mlflow.metrics.genai.answer_similarity(),
        ],
    )

# RAGAS
from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy, context_precision, context_recall
from datasets import Dataset

ragas_data = Dataset.from_list([
    {
        "question": q["question"],
        "answer": q["agent_answer"],
        "contexts": q["retrieved_chunks"],
        "ground_truth": q["gold_answer"],
    }
    for q in eval_records
])
ragas_result = evaluate(
    dataset=ragas_data,
    metrics=[faithfulness, answer_relevancy, context_precision, context_recall],
)

# TruLens RAG Triad
from trulens.apps.langchain import TruChain
from trulens.providers.openai import OpenAI as TruOpenAI
provider = TruOpenAI(model_engine="gpt-4o")
tru_rag = TruChain(
    my_rag_chain,
    app_name="DocumentAgent",
    feedbacks=[
        provider.groundedness_measure_with_cot_reasons,
        provider.context_relevance,
        provider.relevance,
    ],
)

# Phoenix OTel
import phoenix as px
from openinference.instrumentation.openai import OpenAIInstrumentor
px.launch_app()
OpenAIInstrumentor().instrument()

import pytest
import json
from deepeval import assert_test
from deepeval.metrics import (
    FaithfulnessMetric,
    AnswerRelevancyMetric,
    HallucinationMetric,
)
from deepeval.test_case import LLMTestCase

def load_golden_set():
    with open("golden_set/gold.jsonl") as f:
        return [json.loads(line) for line in f if line.strip()]

@pytest.mark.parametrize("gold", load_golden_set())
def test_document_agent_golden(gold, document_agent):
    actual_output, retrieved_context = document_agent(gold["question"])
    test_case = LLMTestCase(
        input=gold["question"],
        actual_output=actual_output,
        expected_output=gold["answer"],
        retrieval_context=retrieved_context,
    )
    assert_test(test_case, [
        FaithfulnessMetric(threshold=0.75, model="gpt-4o", include_reason=True),
        AnswerRelevancyMetric(threshold=0.70, model="gpt-4o"),
        HallucinationMetric(threshold=0.3),
    ])

name: Document Agent — Golden Set Evaluation
on:
  pull_request:
    paths: ["src/agent/**", "prompts/**", "golden_set/**"]
  workflow_dispatch:

jobs:
  golden-eval:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install -r requirements-eval.txt
      - run: python scripts/run_ragas_eval.py --golden golden_set/gold.jsonl --output reports/ragas_result.json
      - run: pytest tests/test_golden.py -v --json-report --json-report-file=reports/pytest_result.json
      - uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const r = JSON.parse(fs.readFileSync('reports/ragas_result.json', 'utf8'));
            const passed = (r.faithfulness ?? 0) >= 0.75;
            const body = passed
              ? `✅ Golden eval passed — Faithfulness: ${r.faithfulness.toFixed(3)}`
              : `❌ Golden eval FAILED — Faithfulness: ${r.faithfulness.toFixed(3)} (threshold: 0.75)`;
            await github.rest.issues.createComment({
              owner: context.repo.owner, repo: context.repo.repo,
              issue_number: context.issue.number, body,
            });
            if (!passed) core.setFailed(body);
      - uses: actions/upload-artifact@v4
        if: always()
        with: { name: eval-reports, path: reports/, retention-days: 30 }

SELECT
  eval_date,
  model_version,
  golden_version,
  ROUND(faithfulness, 3)      AS faithfulness,
  ROUND(answer_relevancy, 3)  AS answer_relevancy,
  ROUND(context_precision, 3) AS context_precision,
  ROUND(context_recall, 3)    AS context_recall,
  ROUND(pass_rate, 3)         AS pass_rate
FROM main.llmops.agent_eval_results
WHERE eval_date >= DATEADD(DAY, -30, CURRENT_DATE())
ORDER BY eval_date DESC, model_version;

# unity_ai_gateway_config.yaml (概念例 / 実構文は Databricks docs を参照)
routes:
  - name: document-agent-gpt55
    model:
      provider: openai
      name: gpt-5.5
    rate_limits:
      - { calls: 100,  renewal_period: minute, key: user }
      - { calls: 2000, renewal_period: day,    key: workspace }
    guardrails:
      input:
        pii:
          behavior: BLOCK
          pii_entities: [CREDIT_CARD, SSN, EMAIL_ADDRESS, PHONE_NUMBER]
        toxic_content: { behavior: BLOCK }
      output:
        pii: { behavior: MASK }
    logging:
      enabled: true
      log_input: true
      log_output: true
      log_metadata:
        - user_id
        - session_id
        - document_source_path
        - token_count_input
        - token_count_output
        - long_context_surcharge
    context_window_limit:
      max_input_tokens: 270000     # 272K long-context 課金回避
      on_exceed: TRUNCATE_AND_ALERT

SELECT
  DATE(request_timestamp) AS log_date,
  user_id,
  COUNT(*) AS call_count,
  SUM(token_count_input)  AS total_input_tokens,
  SUM(token_count_output) AS total_output_tokens,
  COUNT(CASE WHEN long_context_surcharge = TRUE THEN 1 END) AS long_ctx_calls,
  ROUND(
    SUM(token_count_input)  * 10.0 / 1e6
    + SUM(token_count_output) * 45.0 / 1e6,
    2
  ) AS estimated_cost_usd
FROM unity_ai_gateway_logs
WHERE model_name = 'gpt-5.5'
  AND request_timestamp >= CURRENT_DATE - INTERVAL 7 DAYS
GROUP BY log_date, user_id
HAVING long_ctx_calls > 0
ORDER BY estimated_cost_usd DESC;