Milvus 2.4 与 Qdrant 1.9 在千万级图文检索场景下的部署、性能与资源占用实测对比

# docker-compose-milvus.yml
version: '3.5'
services:
  etcd:
    image: quay.io/coreos/etcd:v3.5.5
    environment:
      - ETCD_AUTO_COMPACTION_MODE=revision
      - ETCD_AUTO_COMPACTION_RETENTION=1000
      - ETCD_QUOTA_BACKEND_BYTES=4294967296
      - ETCD_SNAPSHOT_COUNT=50000
    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
    volumes:
      - ./volumes/etcd:/etcd

  minio:
    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
    environment:
      MINIO_ACCESS_KEY: minioadmin
      MINIO_SECRET_KEY: minioadmin
    command: minio server /minio_data --console-address ":9001"
    volumes:
      - ./volumes/minio:/minio_data

  standalone:
    image: milvusdb/milvus:v2.4.6
    command: ["milvus", "run", "standalone"]
    environment:
      ETCD_ENDPOINTS: etcd:2379
      MINIO_ADDRESS: minio:9000
    volumes:
      - ./volumes/milvus:/var/lib/milvus
    ports:
      - "19530:19530"
      - "9091:9091"
    depends_on:
      - etcd
      - minio

docker run -d --name qdrant \
  -p 6333:6333 -p 6334:6334 \
  -v $(pwd)/volumes/qdrant:/qdrant/storage \
  -e QDRANT__SERVICE__GRPC_PORT=6334 \
  -e QDRANT__STORAGE__OPTIMIZERS__DEFAULT_SEGMENT_NUMBER=4 \
  qdrant/qdrant:v1.9.2

# bench.py
import time, random, numpy as np, threading
from concurrent.futures import ThreadPoolExecutor
from pymilvus import MilvusClient
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct, HnswConfigDiff, SearchParams

DIM, N = 768, 10_000_000
BATCH = 5000
rng = np.random.default_rng(42)

def gen_batch(n):
    v = rng.standard_normal((n, DIM)).astype(np.float32)
    v /= np.linalg.norm(v, axis=1, keepdims=True)
    return v

def bench_milvus():
    c = MilvusClient(uri="http://localhost:19530")
    if c.has_collection("bench"): c.drop_collection("bench")
    c.create_collection("bench", dimension=DIM, metric_type="COSINE",
        index_params={"index_type": "HNSW", "metric_type": "COSINE",
                      "params": {"M": 16, "efConstruction": 200}})
    t0 = time.time()
    for i in range(0, N, BATCH):
        data = gen_batch(BATCH)
        c.insert("bench", [{"id": i + j, "vector": data[j].tolist()} for j in range(BATCH)])
    c.flush("bench")
    print(f"[Milvus] insert+flush: {time.time()-t0:.1f}s")
    return c

def bench_qdrant():
    c = QdrantClient(host="localhost", port=6333, timeout=120)
    c.recreate_collection("bench",
        vectors_config=VectorParams(size=DIM, distance=Distance.COSINE),
        hnsw_config=HnswConfigDiff(m=16, ef_construct=200))
    t0 = time.time()
    for i in range(0, N, BATCH):
        data = gen_batch(BATCH)
        c.upsert("bench", points=[PointStruct(id=i+j, vector=data[j].tolist())
                                  for j in range(BATCH)], wait=False)
    c.upsert("bench", points=[], wait=True)  # 等优化器落盘
    print(f"[Qdrant] upsert+wait: {time.time()-t0:.1f}s")
    return c

def query_bench(search_fn, queries, threads=16):
    lat = []
    lock = threading.Lock()
    def worker(qs):
        for q in qs:
            t = time.perf_counter()
            search_fn(q)
            with lock: lat.append((time.perf_counter()-t)*1000)
    chunks = np.array_split(queries, threads)
    t0 = time.time()
    with ThreadPoolExecutor(threads) as ex:
        list(ex.map(worker, chunks))
    total = time.time()-t0
    lat.sort()
    print(f"QPS={len(queries)/total:.0f} P50={lat[len(lat)//2]:.2f}ms "
          f"P99={lat[int(len(lat)*0.99)]:.2f}ms")