Prompt Engineering 实验：同一抽取任务下 4 版 Prompt 的 Token 消耗与输出质量对比

你是一个评论分析助手，请从下面的评论中提取情感、类别、关键词，以及是否有退款意图。
评论：{text}

import json, time, statistics
from openai import OpenAI

client = OpenAI(api_key="sk-xxx")
MODEL = "gpt-4o-mini-2024-07-18"

PROMPTS = {
    "v1": "你是一个评论分析助手，请从下面的评论中提取情感、类别、关键词，以及是否有退款意图。\n评论：{text}",
    "v3": (
        "从评论中抽取字段，只输出 JSON，不要代码块、不要解释。\n"
        "字段：sentiment∈{positive,negative,neutral}；"
        "category∈{logistics,quality,price,service,other}；"
        "keywords: 最多3个名词短语的数组；has_refund_intent: boolean。\n"
        "评论：{text}"
    ),
}

def run(prompt_tpl, texts):
    rows = []
    for t in texts:
        t0 = time.time()
        resp = client.chat.completions.create(
            model=MODEL,
            temperature=0,
            max_tokens=256,
            messages=[{"role": "user", "content": prompt_tpl.format(text=t)}],
        )
        dt = (time.time() - t0) * 1000
        raw = resp.choices[0].message.content
        ok = True
        try:
            obj = json.loads(raw)
        except Exception:
            ok, obj = False, None
        rows.append({
            "raw": raw, "ok": ok, "obj": obj, "latency_ms": dt,
            "in_tok": resp.usage.prompt_tokens,
            "out_tok": resp.usage.completion_tokens,
        })
    return rows

def summarize(rows):
    ok_rate = sum(r["ok"] for r in rows) / len(rows)
    lat = sorted(r["latency_ms"] for r in rows)
    return {
        "parse_ok": round(ok_rate, 4),
        "avg_in_tok": round(statistics.mean(r["in_tok"] for r in rows), 1),
        "avg_out_tok": round(statistics.mean(r["out_tok"] for r in rows), 1),
        "p95_latency_ms": round(lat[int(len(lat) * 0.95) - 1], 1),
    }

def field_f1(pred, gold):
    if not pred:
        return 0.0
    scores = []
    for f in ["sentiment", "category", "has_refund_intent"]:
        scores.append(1.0 if pred.get(f) == gold[f] else 0.0)
    p, g = set(pred.get("keywords", [])), set(gold["keywords"])
    tp = len(p & g)
    prec = tp / len(p) if p else 0
    rec = tp / len(g) if g else 0
    scores.append(2 * prec * rec / (prec + rec) if prec + rec else 0.0)
    return sum(scores) / len(scores)