Milvus与Qdrant在亿级向量场景下的选型实测：从部署到查询延迟

# docker-compose-milvus.yml
version: '3.5'
services:
  etcd:
    image: quay.io/coreos/etcd:v3.5.5
    environment:
      - ETCD_AUTO_COMPACTION_MODE=revision
      - ETCD_AUTO_COMPACTION_RETENTION=1000
      - ETCD_QUOTA_BACKEND_BYTES=4294967296  # 4G quota，防止数据量大时写入阻塞
    volumes:
      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd

  minio:
    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
    environment:
      MINIO_ACCESS_KEY: minioadmin
      MINIO_SECRET_KEY: minioadmin
    command: minio server /minio_data
    volumes:
      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data

  standalone:
    image: milvusdb/milvus:v2.4.1
    command: ["milvus", "run", "standalone"]
    environment:
      ETCD_ENDPOINTS: etcd:2379
      MINIO_ADDRESS: minio:9000
    ports:
      - "19530:19530"
      - "9091:9091"
    volumes:
      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus

# import_milvus.py
from pymilvus import connections, CollectionSchema, FieldSchema, DataType, Collection, utility
import numpy as np
import time

connections.connect(host='localhost', port='19530')

# 创建collection
fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
    FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=768)
]
schema = CollectionSchema(fields, description="poi_vectors")
col = Collection("poi_embedding", schema)

# 构建CAGRA索引（注意：CAGRA需要GPU，如果没有GPU会报错）
index_params = {
    "index_type": "CAGRA",
    "metric_type": "IP",
    "params": {
        "M": 32,
        "build_algo": "IVF_PQ",  # CAGRA内部会做PQ压缩，这个参数官方文档没细说，但实测影响构建速度
    }
}
col.create_index("embedding", index_params)

# 分批插入，每批10000条，避免内存爆掉
BATCH = 10000
total = 0
for i in range(0, 2500):
    vectors = np.random.rand(BATCH, 768).astype(np.float32)
    ids = np.arange(i*BATCH, (i+1)*BATCH)
    col.insert([ids, vectors])
    total += BATCH
    if i % 100 == 0:
        print(f"inserted {total} rows, time elapsed: {time.time() - start:.2f}s")

col.flush()
print(f"total rows: {col.num_entities}")

# 启动Qdrant
docker run -d \
  --name qdrant \
  -p 6333:6333 -p 6334:6334 \
  -v $(pwd)/qdrant_storage:/qdrant/storage \
  qdrant/qdrant:v1.9.0

# 导入数据（Python client）

# import_qdrant.py
from qdrant_client import QdrantClient
from qdrant_client.models import VectorParams, Distance, PointStruct
import numpy as np
import time

client = QdrantClient(host="localhost", port=6333)

# 创建collection，注意Qdrant的HNSW参数命名和Milvus不同
client.create_collection(
    collection_name="poi_vectors",
    vectors_config=VectorParams(size=768, distance=Distance.DOT, 
                                hnsw_config={"m": 32, "ef_construct": 200})
)

# 批量导入
BATCH = 10000
start = time.time()
points = []
for i in range(2500):
    vectors = np.random.rand(BATCH, 768).astype(np.float32)
    for j in range(BATCH):
        points.append(PointStruct(id=i*BATCH+j, vector=vectors[j].tolist()))
    if len(points) >= 10000:
        client.upsert(collection_name="poi_vectors", points=points)
        points = []
    if i % 100 == 0:
        print(f"inserted {i*BATCH} rows, time: {time.time()-start:.2f}s")