faiss 向量库使用；指定自定义id；对外serving http接口

下载：mac或liunx使用

参考：/weixin_42357472/article/details/109243957

使用参考

/p/269907779/zlb872551601/article/details/103704874

注意点

faiss是用的二维ndarray，需要不对的需要reshape

1、再add数据时候

ValueError: too many values to unpack (expected 2)

解决方法就是reshape

np.reshape(np.array(imgs2),(-1,512)) ##512是你向量维度大小

2、search的时候

TypeError: in method ‘IndexFlat_search’, argument 3 of type ‘float const *’

解决方法是把输入转化成float32

a1.reshape(-1,512).astype(‘float32’)

代码

import faissimport numpy as npimport torch## 加载torch tensor数据imgs1 = np.load("/mn***es_embs.npy",allow_pickle=True)## 转化成array，然后再转成二维imgs2=[(item / item.norm(dim=-1, keepdim=True)).cpu().numpy() for item in imgs1]imgs3 = np.reshape(np.array(imgs2),(-1,512))### 创建faiss索引dim = 512# 向量维度 k = 10 # 定义召回向量个数 index = faiss.IndexFlatL2(dim) # L2距离，即欧式距离（越小越好） # index=faiss.IndexFlatIP(dim) # 点乘，归一化的向量点乘即cosine相似度（越大越好）### 索引里添加数据index.add(imgs3) # 添加训练时的样本faiss.write_index(index, '/***/tests1.faiss')## 查询query，也转成二维a1 = np.array([-1.6135e-03, 6.1526e-03, 2.5185e-02, 1.4978e-02, 6.2750e-03,-4.5656e-03, -7.8408e-03, -1.1010e-01, -1.7680e-02, 1.1410e-02,1.9577e-02, -1.4683e-02, 1.2894e-02, -9.9095e-03, 1.5662e-02,-9.2513e-03, 3.5585e-02, 8.1605e-03, 1.4446e-02, -1.9918e-02,1.5668e-02, -1.3067e-02, -6.1856e-03, -6.8425e-03, -1.6277e-02,-1.5883e-03, 6.0463e-03, -8.4852e-03, 4.3100e-03, 1.3474e-02,-1.8023e-03, -8.6439e-03, -1.6139e-02, 7.0169e-03, -8.7095e-03,1.2031e-02, 1.6851e-02, -1.2659e-02, -1.1850e-03, -2.1572e-02,3.4236e-03, -1.2205e-02, 4.9338e-04, -1.1429e-02, 1.3062e-03,4.7159e-02, -9.6514e-03, -2.1358e-03, 1.7201e-02, -5.9182e-03,-1.9796e-03, -2.8365e-03, -1.3234e-02, -2.9237e-02, 1.8063e-02,7.7699e-03, -6.7131e-04, -5.1597e-04, -3.6758e-02, -3.2200e-03,4.e-03, -1.1084e-02, 6.1378e-03, 5.5161e-03, -1.5331e-02,-2.4345e-02, 6.9667e-03, 5.2047e-03, 4.0733e-03, -3.5713e-03,3.1985e-03, -2.2585e-02, 1.4869e-03, 5.5577e-03, -1.1904e-02,-5.9605e-03, 8.5666e-06, 9.3678e-04, -1.7986e-02, 3.5209e-03,2.6181e-03, -4.0341e-03, -3.3970e-03, 1.9925e-02, 2.1615e-03,2.5095e-02, -1.7045e-02, -1.5802e-02, 1.9989e-04, 2.9729e-04,1.2976e-02, -2.6175e-03, -1.4921e-01, 8.0902e-03, 1.9631e-03,1.8781e-02, 1.8078e-02, 1.2302e-02, -5.2962e-04, 1.7031e-02,-5.8983e-03, 4.9108e-03, 7.3916e-03, 1.5125e-03, -2.7008e-02,8.3710e-03, 1.0673e-03, 3.0579e-03, -1.2527e-02, -6.4894e-03,6.5960e-03, 5.5670e-02, -3.5310e-03, 2.0328e-04, 1.2250e-02,-1.0867e-02, 8.9006e-04, 1.3119e-02, -1.9625e-03, 9.7306e-03,6.5921e-03, -2.3217e-02, -2.8782e-02, 1.3316e-03, 1.8779e-02,4.0654e-03, 1.4688e-02, 4.4120e-03, 1.7675e-02, 7.8542e-03,-4.9073e-03, 1.0992e-02, -9.8296e-03, 6.3323e-01, -2.0654e-02,3.5992e-02, -1.8231e-03, -2.3037e-02, 2.7939e-03, 1.0834e-03,-2.9149e-02, -1.9654e-02, -2.1427e-03, 1.7366e-02, 5.6344e-03,-7.1132e-03, -5.0703e-03, 1.2484e-02, -1.6282e-02, 1.3723e-02,1.9881e-02, -1.1774e-02, 2.7111e-02, -1.3049e-02, -1.6459e-04,7.6603e-03, 4.0284e-03, 8.9308e-03, -2.3976e-02, 9.5381e-03,-2.5485e-02, -6.8038e-03, 9.3994e-03, -2.9602e-03, -9.1713e-03,4.1727e-03, -1.1986e-02, -2.4143e-03, 2.5649e-02, -1.1914e-02,1.2253e-02, 5.4045e-03, -4.6767e-04, -8.0852e-03, 1.7791e-02,9.1432e-03, 6.2547e-03, 1.3582e-02, 7.6576e-05, -1.3660e-02,1.4324e-03, 1.1834e-02, -6.3317e-03, 1.8725e-02, -5.7790e-03,7.2861e-03, -5.6955e-03, -1.9022e-02, -1.4089e-02, 1.5589e-02,-2.4922e-03, -1.2894e-02, -3.0975e-04, 1.4137e-02, 1.2805e-02,-1.0152e-03, 8.2071e-03, 7.5843e-03, -7.0602e-03, -4.5651e-03,2.3972e-03, 1.0509e-02, 4.9439e-03, -1.1613e-02, -1.0624e-02,4.7919e-03, -1.4091e-02, -2.3123e-03, -8.9199e-03, -9.8764e-03,6.8790e-03, 2.5924e-02, -2.0357e-02, -1.2985e-02, -1.6051e-02,7.6852e-03, 1.5877e-02, -1.1363e-02, 3.9967e-03, 5.9453e-04,9.1136e-03, 8.8958e-03, 1.2909e-02, -1.3445e-02, -7.0824e-03,-1.5924e-02, 2.1556e-02, 9.8549e-03, 2.4709e-03, -4.1877e-04,1.3899e-02, 1.4676e-02, -1.4603e-03, 8.9065e-03, 1.2485e-02,-1.2827e-04, -4.3024e-03, -1.8105e-02, -6.6219e-03, 9.7098e-03,1.2541e-02, 1.0112e-02, -1.2859e-02, 8.5914e-04, 8.3273e-03,1.0884e-02, -1.2448e-02, 2.7684e-02, 7.7927e-03, -4.6589e-04,-2.1644e-02, 8.4616e-03, -5.2601e-03, -5.0115e-04, -5.1695e-03,-1.2965e-03, 8.2417e-03, -1.5047e-02, -2.4089e-03, -8.4488e-03,1.8661e-02, -8.7175e-03, 5.4111e-03, 1.0800e-02, 1.1520e-02,1.0093e-02, 1.3177e-03, -1.2543e-03, -6.5828e-03, -1.2601e-03,-1.2253e-02, -4.8685e-03, 1.9724e-02, -1.6586e-02, 9.9319e-03,-4.9152e-03, -2.7814e-03, 4.8142e-04, -8.2368e-03, -6.8973e-03,2.3372e-02, 1.5286e-02, -5.5229e-04, -1.2409e-03, 1.4874e-02,-2.3169e-03, 1.4013e-02, -2.8084e-02, -3.2836e-03, 1.3919e-03,-5.0962e-03, -1.1291e-02, 3.4799e-03, 2.6452e-02, 5.2030e-03,-1.8576e-02, -1.6978e-02, -1.3849e-02, -2.7388e-03, 1.2103e-02,1.3315e-02, 7.6825e-03, -8.8126e-03, 6.0293e-03, -1.5018e-02,1.0982e-02, 1.4156e-03, -1.1665e-02, -1.9467e-02, 1.7890e-03,2.4781e-02, 8.0220e-03, 6.3223e-01, -1.5195e-02, -8.7269e-03,2.0717e-02, 1.2941e-02, -1.1308e-02, 1.7480e-03, 3.8277e-02,1.5216e-02, 7.4206e-03, 6.3749e-03, 2.6863e-03, -9.5829e-03,4.0324e-04, -1.5226e-02, -6.5413e-03, -4.6104e-03, -2.6882e-01,-8.8758e-03, 1.1077e-02, -1.2184e-02, -6.9019e-03, -5.4372e-03,-1.4190e-03, -1.0375e-02, 1.5531e-02, 1.9881e-03, 8.4441e-03,1.0767e-02, -1.6926e-02, -5.2742e-03, -7.1148e-04, -9.5662e-03,2.6844e-03, 1.0767e-02, 2.0270e-02, 1.1346e-02, 6.9011e-03,2.2358e-02, 9.1316e-03, -2.0739e-03, -7.3354e-03, 2.4359e-02,2.7133e-02, 4.5612e-03, 8.0810e-03, -2.2221e-03, 3.1086e-02,-8.4999e-03, -1.0824e-03, 6.3306e-03, 5.5410e-03, -1.1954e-02,-1.1339e-02, 2.2676e-02, 2.2205e-02, -9.8124e-03, 2.0691e-02,1.5414e-02, -1.6097e-02, 1.2325e-02, 1.9335e-03, -2.8254e-02,2.9626e-02, -2.3636e-02, -9.5312e-03, -2.1991e-02, 2.6658e-03,-2.0500e-02, -4.0802e-03, -5.6075e-03, 1.5054e-02, -2.4843e-03,1.8199e-02, -1.9825e-02, -8.2771e-03, 2.2264e-02, 1.5191e-02,-4.6921e-03, -1.5965e-02, 8.4242e-03, 1.7309e-02, 2.7541e-02,-5.1904e-03, 1.6142e-02, -1.5974e-02, -1.8812e-03, -6.3679e-04,-8.4971e-03, -3.8564e-03, -2.0782e-02, -5.1108e-04, -4.5805e-03,-3.2035e-02, 1.9102e-02, 6.1913e-03, 3.0655e-03, 2.9454e-03,4.2798e-03, -1.6380e-02, -1.2336e-02, -1.2465e-02, -1.0507e-02,-4.0164e-03, -1.8573e-02, -8.5779e-03, -1.5583e-02, 1.5603e-02,-1.2849e-02, -1.7189e-02, -9.5142e-03, 6.3270e-03, -1.6096e-02,8.0913e-03, 6.3659e-03, -1.1888e-02, 9.0989e-03, 9.7781e-03,-7.4295e-03, -1.2599e-02, -4.6298e-03, -9.7783e-03, -1.7343e-02,3.0361e-03, -4.4915e-02, 4.1937e-03, 5.8446e-03, 5.7702e-03,1.2147e-02, 2.5410e-02, 2.7014e-02, 8.7954e-03, 1.4540e-02,-3.7720e-03, -5.4728e-03, -1.5467e-02, 5.8019e-04, 4.7111e-04,1.0072e-02, -1.2770e-02, 2.1306e-02, 3.4574e-03, 1.1501e-02,9.2801e-03, 1.e-02, -2.9342e-03, -3.9477e-04, 3.7866e-03,-5.4005e-03, 1.5335e-02, 1.1379e-02, -4.0515e-03, 3.5547e-04,2.1120e-03, -6.9303e-03, -7.9791e-03, -2.0967e-02, 7.7576e-03,1.2412e-02, -2.5669e-03, -5.3622e-03, -7.9149e-03, 9.7494e-03,4.7953e-03, 1.7233e-03, 9.6419e-03, 4.4269e-03, 1.0419e-02,-2.5379e-03, -7.3149e-02, 3.8080e-03, -7.7298e-03, -1.4290e-02,9.8190e-03, 2.7394e-03, 8.3593e-03, 2.2444e-04, -9.2171e-03,-5.2412e-03, 1.5812e-02, 3.6178e-03, 8.2209e-04, 3.9101e-03,-1.3362e-02, 2.2686e-02, -6.1439e-03, 5.5911e-03, 3.1641e-03,-2.4614e-02, -7.6460e-03, 1.8715e-02, -1.5930e-02, -7.7982e-04,-9.6896e-03, 9.1894e-03, -5.3417e-03, -1.7233e-02, 6.3065e-03,1.4615e-02, 1.5300e-02])a2 = a1.reshape(-1,512).astype('float32')## 搜索最近cosineD, I = index.search(a2, k) # 寻找相似向量， I表示相似用户ID矩阵， D表示距离矩阵

与直接tensor求结果一致

指定自定义id

参考：/houkai/p/9316155.html

/yhzhou/p/10568728.html

自定义id：ids为整数字符串不可以

最后结果索引I ：为什么很多0和很大内存值？，间隔，其他加100000能对应上

### 测试自定义idids = np.arange(100000, 137843) index1 = faiss.IndexFlatL2(dim)index3 = faiss.IndexIDMap(index1)index3.add_with_ids(imgs3, ids)index1.add(imgs3)D, I = index3.search(a1.reshape(-1,512).astype('float32'), 20) # 寻找相似向量， I表示相似用户ID矩阵， D表示距离矩阵

对外serving http接口

参考：/scatterlab/faiss-serving

生成faiss 文件

faiss.write_index(index, '/***/tests1.faiss')

docker 运行

docker run -p 8080:8080 -v D:/t**ex/tests1.faiss:/index/tests1.faiss scatterlab/faiss-serving --index-file /index/tests1.faiss

curl 验证

curl localhost:8080/v1/search \-d '{"queries":[[ 7.3085e-03, -4.6353e-03, -4.4408e-04, -3.3634e-04, -1.4829e-02,-2.7456e-02, -1.0324e-02, -1.4206e-01, 8.3942e-03, 4.9810e-03,1.2072e-02, -3.7897e-03, -2.8862e-03, 3.2950e-03, -2.6619e-03,-4.8818e-03, 2.2135e-02, -3.2677e-04, -1.5755e-02, -2.0045e-02,9.9448e-03, -3.3146e-03, 1.1625e-02, 1.8703e-02, -6.4526e-03,1.0457e-03, -5.9497e-03, -9.7896e-03, 1.9700e-03, -6.4454e-03,-2.5071e-02, -1.6847e-02, 2.8318e-03, 9.6513e-03, -1.1618e-02,1.0626e-02, 1.2928e-02, 2.1819e-03, -1.3969e-02, 1.3454e-02,-4.5545e-03, 5.0375e-03, 3.2696e-03, -1.0707e-02, -4.0825e-03,3.2296e-02, 1.2704e-02, 2.6181e-02, 2.1077e-02, 1.7445e-02,1.4241e-02, -5.4372e-03, 1.3411e-02, -3.0425e-02, -4.2241e-03,7.4228e-03, -1.8870e-03, 4.8402e-03, -1.9162e-02, -4.9534e-03,1.5652e-02, 6.7666e-03, 9.0648e-03, -4.9025e-03, -4.5077e-03,-1.5952e-02, 5.8742e-03, -2.5665e-02, 1.6273e-02, -3.7294e-04,5.2161e-03, 6.6520e-03, -2.6140e-02, 8.4355e-03, 3.0880e-05,-1.7359e-02, 2.8472e-03, -4.2389e-03, -1.4569e-02, -1.7418e-02,-1.7773e-02, -3.1625e-03, -2.5610e-02, 2.9700e-03, 3.1136e-03,7.7301e-03, 1.7498e-02, -1.8270e-02, 2.8468e-02, -5.7182e-03,-1.7443e-03, 8.3888e-03, -1.5365e-01, 9.2144e-03, 7.2510e-03,1.1288e-02, -5.9218e-03, 2.1798e-02, -1.2795e-02, -1.7796e-02,1.0269e-02, 9.1577e-03, 9.0181e-03, 1.0456e-02, 2.4344e-03,4.6265e-03, 5.6966e-02, -1.0212e-03, -1.1346e-02, -1.2175e-02,-1.1278e-02, 3.0670e-02, -8.5659e-03, 6.4355e-03, -7.5599e-04,-4.8674e-03, 1.6917e-02, 2.4903e-02, 3.4145e-02, -6.7437e-03,1.7535e-02, 5.8753e-03, -9.9181e-03, -6.7377e-04, -8.9304e-03,-5.2707e-03, 7.7407e-03, -5.6767e-03, 1.2428e-03, 2.2054e-02,3.1943e-02, 1.6942e-02, -8.7350e-03, 6.3862e-01, 1.2686e-02,3.0839e-02, -1.7029e-03, -8.3889e-03, -1.8593e-03, 6.1738e-03,1.7674e-02, -5.5610e-03, 5.0874e-03, 7.0389e-04, -1.9809e-02,1.1335e-02, 6.7522e-03, -1.6445e-03, -1.2855e-02, -1.2721e-02,9.9355e-03, -3.1388e-02, -1.6257e-02, -1.9840e-02, -1.8985e-02,1.0873e-02, -1.e-02, 9.1951e-03, -6.0484e-03, 1.4649e-02,5.5614e-03, 1.0983e-02, 8.5162e-03, -7.3627e-03, -5.1069e-03,-7.0275e-03, 9.7484e-03, -4.7364e-03, 1.5896e-02, 5.2478e-03,-1.7393e-02, -1.5866e-02, 4.9922e-03, -2.7392e-02, -1.5921e-02,1.4461e-02, -2.4244e-02, 1.9156e-02, -1.0526e-02, 4.7988e-03,7.0213e-04, 7.6714e-03, 1.1333e-02, -1.9360e-03, -5.7769e-03,-1.0216e-02, 1.7380e-02, -1.6150e-03, 1.2054e-02, -4.2982e-03,-1.3849e-02, -2.9835e-02, -3.5999e-03, 3.4162e-03, -1.6784e-02,1.2114e-02, 7.4834e-03, 1.4978e-02, 1.2631e-02, -2.5660e-04,6.4626e-03, -4.5177e-03, -1.0836e-02, -1.0800e-03, -1.3658e-03,-1.0902e-03, -2.7910e-02, -4.0745e-03, -5.6879e-03, 5.9956e-03,3.4220e-02, 2.3980e-03, -1.5363e-03, 2.2087e-03, 5.3201e-04,6.7227e-03, 1.1851e-02, -1.1044e-03, -5.7496e-03, 5.6624e-03,-3.9450e-02, -2.1106e-02, 1.1475e-02, -1.1815e-02, 8.4447e-03,1.9530e-02, 1.5334e-02, 1.9508e-02, -7.6016e-03, -3.5144e-03,1.5208e-02, 2.2831e-03, -4.2248e-03, 1.2131e-02, -3.0158e-03,2.3440e-02, -8.5046e-03, 1.5710e-03, -8.0973e-03, -9.9754e-03,1.3329e-02, 2.4785e-03, 4.1647e-03, -1.6831e-03, 6.0369e-03,1.4145e-03, 8.8724e-03, 2.6906e-02, -1.0865e-02, 4.5856e-03,-8.5550e-03, 6.8099e-03, -1.4174e-02, 8.7767e-03, 1.5745e-02,8.6514e-04, 6.5093e-03, -6.9840e-03, 1.8041e-02, -1.0809e-02,1.2335e-03, 1.5234e-02, 4.0324e-03, 3.0829e-03, -1.7889e-03,-1.7815e-05, 1.6386e-02, -2.6454e-03, -1.1920e-02, -3.4991e-03,1.3945e-03, -2.0398e-02, -1.9783e-03, -3.2363e-02, 1.9394e-02,-5.3849e-04, -1.8205e-02, -1.5242e-03, 1.5408e-02, 5.0961e-03,2.3142e-02, -1.4586e-02, 1.6434e-02, 9.6912e-03, -1.6438e-03,3.4175e-02, 1.1746e-02, 1.2542e-02, -1.1904e-02, -8.7490e-03,6.6509e-03, -5.0126e-03, 2.4992e-03, 1.5660e-02, 6.5289e-03,9.2534e-04, -1.1778e-02, -1.0911e-02, -2.2644e-04, -1.0628e-03,-1.1750e-02, -2.0066e-02, 3.4047e-03, 1.4525e-02, -1.0974e-02,2.9513e-02, -5.8881e-04, -3.5587e-03, -2.1363e-02, 7.4650e-03,9.2625e-03, -3.1942e-03, 6.3776e-01, 6.6852e-03, 1.6750e-02,1.1994e-02, -1.7569e-02, 2.2896e-02, -2.6475e-03, 2.2442e-02,2.5537e-02, 3.8603e-02, 5.0819e-03, 6.1190e-03, 1.7971e-03,4.5396e-03, 1.0780e-02, 4.7449e-03, 4.7517e-03, -2.1612e-01,4.8340e-03, -1.6210e-03, -2.4634e-03, -2.8124e-02, -1.3283e-02,-4.2895e-03, -2.0994e-03, 8.8114e-04, -2.1283e-02, 1.8092e-03,5.9424e-04, -1.7993e-02, -1.4776e-02, 6.4218e-03, 6.2324e-03,-1.2590e-02, 5.8211e-03, 1.8629e-02, 6.5436e-03, -1.4351e-03,-1.7231e-02, -1.1930e-02, 1.0128e-02, 7.2143e-03, -1.2411e-02,4.2067e-02, -2.3527e-03, 5.4889e-03, 4.9218e-03, -3.7314e-03,8.7904e-04, -6.6602e-04, -1.0349e-04, -5.3710e-03, -1.5883e-02,-1.3397e-02, 8.5489e-03, 2.9856e-02, 7.2624e-03, -1.0937e-03,-1.7870e-03, 5.7600e-03, -2.0354e-03, -7.2364e-04, -3.0990e-02,1.4195e-02, 8.1073e-03, 2.1951e-02, -9.9118e-03, -6.7451e-04,-1.1753e-02, 4.3679e-03, 2.1099e-02, -1.4304e-02, -7.8660e-03,-8.8881e-03, 7.8771e-03, 3.8313e-03, 3.7284e-03, 2.4129e-02,2.2434e-02, -3.0874e-02, 1.4902e-02, -6.5396e-03, 9.2859e-03,2.4560e-02, 1.8782e-02, -2.9251e-03, -9.9732e-03, 3.8261e-02,-9.4069e-03, 1.8983e-03, 1.1038e-02, -2.2499e-03, 1.1631e-02,-1.4429e-04, 2.1892e-02, -1.3974e-03, 1.7860e-02, -2.5137e-02,1.3441e-02, -2.9762e-03, 1.5916e-02, 1.3762e-02, -9.7347e-03,-1.5565e-03, 3.1618e-03, -1.5910e-03, 3.6944e-03, 1.1395e-02,8.1018e-03, -5.0805e-04, -1.0689e-02, 4.2169e-03, -4.3550e-03,8.7160e-03, -4.6717e-03, -1.2337e-02, 5.3276e-03, -5.5379e-04,4.7590e-03, -5.1222e-03, 1.0061e-02, -9.4191e-03, -7.2576e-03,1.2232e-03, -2.8143e-02, 2.1499e-02, 1.6424e-02, -1.6685e-03,-1.7169e-02, 1.6592e-02, 3.8382e-04, 1.2361e-02, 1.7880e-02,3.0675e-03, -1.2032e-02, -1.3708e-02, 9.0455e-03, -6.8327e-03,4.0794e-03, -1.1344e-02, 1.6198e-03, -1.5118e-03, -4.6432e-04,-3.7913e-03, -4.3649e-03, 9.3436e-03, 2.4821e-04, -1.6864e-03,3.9419e-03, -1.1874e-02, 1.5172e-03, 1.0948e-02, 1.2832e-03,4.1755e-04, -4.4705e-03, 2.3464e-02, -1.1158e-02, 6.4721e-03,-2.0909e-03, -1.0397e-02, 1.1618e-02, -4.5632e-03, -4.7449e-03,-3.8893e-03, 1.0814e-03, 2.4207e-02, 7.0664e-03, -1.0485e-02,-3.5137e-03, -3.1732e-02, 4.1115e-03, 3.6619e-03, -1.8879e-02,-2.0408e-03, 1.3219e-02, 1.2778e-02, 1.2741e-02, 1.0342e-02,-5.5712e-03, 1.0855e-02, -1.6590e-02, 6.3717e-02, -2.4155e-02,-2.1811e-03, 1.3202e-02, -2.7181e-02, -8.4490e-03, -7.1806e-03,-8.8565e-03, -1.5422e-02, 2.0632e-03, -3.9416e-03, -2.0525e-02,3.6169e-03, 1.2179e-02, 9.3075e-03, -4.3982e-03, -3.4500e-02,-1.2487e-02, 2.8299e-03]], "top_k": 1}'