Prompt Engineering实测：商品评论情感分类任务中6种提示词设计的Token消耗与准确率对比

import openai, json, time
from collections import defaultdict

openai.api_key = "sk-xxxx"  # 换成你自己的

def call_model(prompt: str, review: str) -> dict:
    resp = openai.ChatCompletion.create(
        model="gpt-3.5-turbo-0613",
        messages=[{"role": "user", "content": prompt.format(review=review)}],
        temperature=0,
        top_p=1,
        max_tokens=16,
        request_timeout=30,
    )
    usage = resp["usage"]
    content = resp["choices"][0]["message"]["content"].strip()
    return {"output": content, "prompt_tokens": usage["prompt_tokens"],
            "completion_tokens": usage["completion_tokens"]}

def normalize(label: str) -> str:
    # 模型偶尔输出"正面。"或"正面情感"，统一映射
    for k in ["正面", "负面", "中性"]:
        if k in label:
            return k
    return "未知"

def evaluate(prompt: str, dataset: list, name: str) -> dict:
    correct, total_pt, total_ct, fails = 0, 0, 0, 0
    pred_dist = defaultdict(int)
    for item in dataset:
        try:
            r = call_model(prompt, item["review"])
        except Exception as e:
            fails += 1
            continue
        pred = normalize(r["output"])
        pred_dist[pred] += 1
        if pred == item["label"]:
            correct += 1
        total_pt += r["prompt_tokens"]
        total_ct += r["completion_tokens"]
        time.sleep(0.05)  # 简单限速，避免触发429
    n = len(dataset)
    return {
        "name": name,
        "acc": round(correct / n * 100, 2),
        "avg_prompt_tokens": round(total_pt / n, 1),
        "avg_completion_tokens": round(total_ct / n, 1),
        "avg_total_tokens": round((total_pt + total_ct) / n, 1),
        "api_fail": fails,
        "pred_dist": dict(pred_dist),
    }

if __name__ == "__main__":
    with open("reviews_500.json", encoding="utf-8") as f:
        data = json.load(f)  # [{"review": "...", "label": "正面"}, ...]
    results = []
    for name, p in PROMPTS.items():  # PROMPTS 定义见下
        results.append(evaluate(p, data, name))
        print(results[-1])
    with open("result.json", "w", encoding="utf-8") as f:
        json.dump(results, f, ensure_ascii=False, indent=2)

# P1: 极简零样本
P1 = "判断这条商品评论的情感倾向：{review}"

# P2: 加标签定义
P2 = """判断以下商品评论的情感倾向，只输出"正面"、"负面"或"中性"之一。
定义：正面=明确满意；负面=明确不满；中性=褒贬混合或态度不明确。
评论：{review}
输出："""

# P3: Few-shot 3例（每类1个，都是典型样本）
P3 = """判断商品评论情感，只输出"正面"、"负面"或"中性"。

评论：质量很好，物流也快，会回购
输出：正面

评论：收到就是坏的，客服还不理人
输出：负面

评论：东西还行，就是有点贵
输出：中性

评论：{review}
输出："""

# P5: CoT
P5 = """请分析以下商品评论的情感倾向。先简要分析评论中的褒贬信息，再给出结论。
结论只能是"正面"、"负面"或"中性"之一。

评论：{review}

分析："""