Prompt Engineering实测：商品评论情感分类任务中6版Prompt的Token消耗与准确率对比

import json
import time
import tiktoken
from openai import OpenAI

client = OpenAI(api_key="sk-xxx")
ENC = tiktoken.encoding_for_model("gpt-4o-mini")

def count_tokens(text: str) -> int:
    return len(ENC.encode(text))

def classify(prompt_template: str, review: str, retries: int = 3) -> dict:
    prompt = prompt_template.format(review=review)
    for attempt in range(retries):
        try:
            t0 = time.time()
            resp = client.chat.completions.create(
                model="gpt-4o-mini-2024-07-18",
                messages=[{"role": "user", "content": prompt}],
                temperature=0,
                max_tokens=64,
                top_p=1,
            )
            latency = time.time() - t0
            content = resp.choices[0].message.content.strip()
            return {
                "raw": content,
                "input_tokens": resp.usage.prompt_tokens,
                "output_tokens": resp.usage.completion_tokens,
                "latency": latency,
            }
        except Exception as e:
            if attempt == retries - 1:
                return {"raw": "ERROR", "input_tokens": 0,
                        "output_tokens": 0, "latency": 0, "error": str(e)}
            time.sleep(1.5 * (attempt + 1))

def parse_label(raw: str) -> str:
    # 统一从输出里抽标签，兼容"正面"、"positive"、"结论：负面"等格式
    raw_l = raw.lower()
    mapping = {
        "负面": ["负面", "negative", "差评"],
        "正面": ["正面", "positive", "好评"],
        "中立": ["中立", "neutral", "中性"],
    }
    for label, keys in mapping.items():
        if any(k in raw_l for k in keys):
            return label
    return "解析失败"

PROMPT_V6 = """判断商品评论的情感倾向，输出格式严格为：情感：正面/负面/中立

判定规则：
1. 含"但是/不过/可惜"等转折词时，以转折后的内容为准
2. 仅描述事实、无明显褒贬（如"第二天就到了"）判为中立
3. 反讽、阴阳怪气按实际贬义判为负面
4. 物流、客服等非商品本身的抱怨，若不影响商品评价，判为中立

示例1：
评论：包装挺精致的，送人也好看，就是用了三天屏幕就出现竖线了，客服还不给换。
情感：负面

示例2：
评论：价格比平时便宜二十块，发货速度正常，东西和描述一致。
情感：中立

评论：{review}
情感："""

def evaluate(prompt_template: str, dataset: list, name: str) -> dict:
    correct, total_in, total_out, latencies = 0, 0, 0, []
    errors = []
    for item in dataset:
        r = classify(prompt_template, item["review"])
        label = parse_label(r["raw"])
        if label == item["label"]:
            correct += 1
        else:
            errors.append({"review": item["review"],
                           "gold": item["label"],
                           "pred": label, "raw": r["raw"]})
        total_in += r["input_tokens"]
        total_out += r["output_tokens"]
        latencies.append(r["latency"])
    n = len(dataset)
    latencies.sort()
    return {
        "name": name,
        "accuracy": round(correct / n, 4),
        "avg_input_tokens": round(total_in / n, 1),
        "avg_output_tokens": round(total_out / n, 1),
        "p50_latency": round(latencies[n // 2], 3),
        "errors_sample": errors[:5],
    }