我用3种Prompt把GPT-4o准确率从61%拉到94%：一次结构化信息抽取的Token与质量对比实验

从下面的合同文本中抽取信息，输出JSON：
{contract_text}

import json
import time
from openai import OpenAI
from typing import Callable

client = OpenAI(api_key="sk-xxx")

FIELDS = ["contract_no", "party_a", "party_b",
          "sign_date", "amount", "penalty_rate"]

def call_llm(prompt: str, use_json_mode: bool = False) -> dict:
    kwargs = dict(
        model="gpt-4o-2024-08-06",
        messages=[{"role": "user", "content": prompt}],
        temperature=0,
        top_p=1,
        max_tokens=1024,
    )
    if use_json_mode:
        kwargs["response_format"] = {"type": "json_object"}

    t0 = time.time()
    resp = client.chat.completions.create(**kwargs)
    latency = time.time() - t0

    return {
        "content": resp.choices[0].message.content,
        "prompt_tokens": resp.usage.prompt_tokens,
        "completion_tokens": resp.usage.completion_tokens,
        "latency": latency,
    }

def evaluate(predict_fn: Callable[[str], dict], testset: list) -> dict:
    total, correct = 0, 0
    total_prompt_tok, total_completion_tok = 0, 0
    field_hit = {f: 0 for f in FIELDS}

    for item in testset:
        text, gold = item["text"], item["label"]
        result = predict_fn(text)
        total += 1
        total_prompt_tok += result["prompt_tokens"]
        total_completion_tok += result["completion_tokens"]

        try:
            pred = json.loads(result["content"])
        except json.JSONDecodeError:
            continue

        all_ok = True
        for f in FIELDS:
            if str(pred.get(f)).strip() == str(gold.get(f)).strip():
                field_hit[f] += 1
            else:
                all_ok = False
        if all_ok:
            correct += 1

    n = len(testset)
    return {
        "accuracy": round(correct / n, 4),
        "field_acc": {f: round(hit / n, 4) for f, hit in field_hit.items()},
        "avg_prompt_tokens": round(total_prompt_tok / n, 1),
        "avg_completion_tokens": round(total_completion_tok / n, 1),
    }

FIELD_SPEC = """
字段规范：
- contract_no: 字符串，合同编号，原样保留不含"合同编号："前缀
- party_a: 字符串，甲方全称
- party_b: 字符串，乙方全称
- sign_date: 字符串，格式必须为 YYYY-MM-DD，无法确定填 null
- amount: 数字，单位元，去掉货币符号和千分位逗号
- penalty_rate: 字符串，违约金比例，统一转为"日万分之X"或"合同总额的X%"
"""

def prompt_a(text: str) -> str:
    return f"从下面的合同文本中抽取信息，输出JSON：\n{text}"

def prompt_b(text: str, examples: list) -> str:
    shots = "\n\n".join(
        f"输入：\n{ex['text']}\n输出：\n{json.dumps(ex['label'], ensure_ascii=False)}"
        for ex in examples
    )
    return (
        f"你是一个合同信息抽取助手。请严格按字段规范抽取信息，"
        f"未出现的字段填 null，不要编造。\n{FIELD_SPEC}\n\n"
        f"示例：\n{shots}\n\n"
        f"现在抽取：\n{text}\n只输出JSON。"
    )

def prompt_c(text: str, examples: list) -> str:
    base = prompt_b(text, examples)
    return base + (
        "\n\n请先用一句话说明每个字段的抽取依据（找不到的写'未提及'），"
        "然后再输出最终JSON。输出格式：\n"
        '{"reasoning": "...", "result": {...}}'
    )