LoRA与QLoRA微调7B大模型：从数据准备到推理效果对比的完整实践

Python: 3.10.13
PyTorch: 2.1.2 + cu121
transformers: 4.41.2
peft: 0.11.1
bitsandbytes: 0.43.1
datasets: 2.19.1
accelerate: 0.30.1
trl: 0.8.6
GPU: RTX 4090 24GB
CUDA: 12.1

import json
from datasets import Dataset

def build_sample(user, assistant):
    return {
        "messages": [
            {"role": "system", "content": "你是一个专业的电商客服助手。"},
            {"role": "user", "content": user},
            {"role": "assistant", "content": assistant},
        ]
    }

raw = [
    ("我的订单为什么还没发货？", "您好，订单一般在付款后48小时内发货，如遇大促可能延迟至72小时。您可以在“我的订单”中查看物流状态。"),
    ("优惠券可以叠加使用吗？", "平台优惠券与店铺优惠券通常不可叠加，具体以券面说明为准。如券面标注“可叠加”，则可同时使用。"),
    # ... 实际使用8000条
]

data = [build_sample(u, a) for u, a in raw]
dataset = Dataset.from_list(data)
dataset = dataset.train_test_split(test_size=0.05, seed=42)
print(dataset)

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training

model_name = "Qwen/Qwen2-7B-Instruct"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
model = prepare_model_for_kbit_training(model)

lora_config = LoraConfig(
    r=8,
    lora_alpha=16,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# 输出：trainable params: 4,194,304 || all params: 7,619,340,288 || trainable%: 0.055

from transformers import Trainer, TrainingArguments, DataCollatorForSeq2Seq

def preprocess(example):
    messages = example["messages"]
    prompt = tokenizer.apply_chat_template(
        messages[:-1], tokenize=False, add_generation_prompt=True
    )
    answer = messages[-1]["content"] + tokenizer.eos_token
    prompt_ids = tokenizer(prompt, add_special_tokens=False).input_ids
    answer_ids = tokenizer(answer, add_special_tokens=False).input_ids
    input_ids = prompt_ids + answer_ids
    labels = [-100] * len(prompt_ids) + answer_ids
    return {"input_ids": input_ids[:1024], "labels": labels[:1024]}

train_ds = dataset["train"].map(preprocess, remove_columns=dataset["train"].column_names)
eval_ds = dataset["test"].map(preprocess, remove_columns=dataset["test"].column_names)

collator = DataCollatorForSeq2Seq(tokenizer, padding=True, return_tensors="pt")

args = TrainingArguments(
    output_dir="./qwen2-7b-qlora",
    per_device_train_batch_size=4,
    per_device_eval_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=3,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    logging_steps=10,
    eval_strategy="steps",
    eval_steps=50,
    save_strategy="steps",
    save_steps=100,
    save_total_limit=2,
    bf16=True,
    gradient_checkpointing=True,
    optim="paged_adamw_8bit",
    report_to="none",
)

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=train_ds,
    eval_dataset=eval_ds,
    data_collator=collator,
)
trainer.train()
trainer.save_model("./qwen2-7b-qlora-final")