向量数据库选型：Milvus与Qdrant在百万级影视素材检索中的实测对比

# 下载standalone的docker-compose.yml
wget https://github.com/milvus-io/milvus/releases/download/v2.4.1/milvus-standalone-docker-compose.yml -O docker-compose.yml

# 修改配置：限制内存使用，防止OOM
# 在docker-compose.yml中为standalone服务添加：
# environment:
#   - GOGC=200
#   - CACHE_SIZE=1024  # MB

docker compose up -d

# 直接运行容器，挂载数据卷
docker run -d --name qdrant \
  -p 6333:6333 -p 6334:6334 \
  -v $(pwd)/qdrant_storage:/qdrant/storage \
  qdrant/qdrant:v1.9.2

# 初始化collection时需要指定量化配置
curl -X PUT 'http://localhost:6333/collections/movies' \
  -H 'Content-Type: application/json' \
  -d '{
    "vectors": {
      "size": 768,
      "distance": "Cosine",
      "quantization_config": {
        "scalar": { "type": "int8", "always_ram": true }
      }
    }
  }'

# milvus_test.py
from pymilvus import connections, Collection, CollectionSchema, FieldSchema, DataType, utility
import numpy as np
import time

connections.connect(alias="default", host="localhost", port="19530")

# 定义schema
fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
    FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=768)
]
schema = CollectionSchema(fields)
collection = Collection(name="movies", schema=schema)

# 批量写入（模拟100万条）
vectors = np.random.rand(1000000, 768).astype(np.float32)
t0 = time.time()
for i in range(0, 1000000, 256):
    batch_ids = list(range(i, i+256))
    collection.insert([batch_ids, vectors[i:i+256]])
print(f"Milvus插入耗时: {time.time()-t0:.2f}s")

# 创建HNSW索引
index_params = {"index_type": "HNSW", "metric_type": "COSINE", "params": {"M": 16, "efConstruction": 200}}
collection.create_index(field_name="embedding", index_params=index_params)
collection.load()

# 查询
query_vec = vectors[0].reshape(1, -1)
t0 = time.time()
results = collection.search(data=query_vec, anns_field="embedding", param={"metric_type": "COSINE", "params": {"ef": 64}}, limit=10)
print(f"Milvus单次查询延迟: {(time.time()-t0)*1000:.2f}ms")

# qdrant_test.py
from qdrant_client import QdrantClient
from qdrant_client.models import VectorParams, Distance, PointStruct, HnswConfigDiff
import numpy as np
import time

client = QdrantClient(host="localhost", port=6333)

# 调整HNSW参数并开启量化
client.update_collection(
    collection_name="movies",
    hnsw_config=HnswConfigDiff(m=16, ef_construct=200)
)

# 批量upsert（每批512条，Qdrant对批量大小更宽容）
vectors = np.random.rand(1000000, 768).astype(np.float32)
points = [
    PointStruct(id=i, vector=vectors[i].tolist()) for i in range(1000)  # 示例前1000条
]
t0 = time.time()
for i in range(0, 1000000, 512):
    batch_points = [
        PointStruct(id=j, vector=vectors[j].tolist()) for j in range(i, min(i+512, 1000000))
    ]
    client.upsert(collection_name="movies", points=batch_points)
print(f"Qdrant插入耗时: {time.time()-t0:.2f}s")

# 查询
t0 = time.time()
hits = client.search(
    collection_name="movies",
    query_vector=vectors[0].tolist(),
    limit=10,
    search_params={"ef": 64}
)
print(f"Qdrant单次查询延迟: {(time.time()-t0)*1000:.2f}ms")