dnrops
/
qdrant_demo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update client

This commit is contained in:
Andrey Vasnetsov 2021-02-09 23:49:47 +01:00
parent 236a8d2f39
commit 400c49c2ce
10 changed files with 193 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker-compose.yaml
View File

@ -17,8 +17,8 @@ services:
qdrant:
image: generall/qdrant:latest
# mem_limit: 450m
ports:
- 6333:6333
# ports:
# - 6333:6333
volumes:
- ./data/qdarnt_storage:/qdrant/storage

10
frontend/src/views/Home.vue
View File

@ -1,6 +1,6 @@
<template>
<q-page>
<div class="q-pa-md q-col-gutter-sm">
<div class="q-pa-md q-col-gutter-sm items-stretch">
<div class="row justify-evenly">
<div class="col-10">
<q-input
@ -68,8 +68,8 @@
<p :style="{ fontSize: '16pt' }">
This demo uses short descriptions of startups to perform a
<b>semantic search</b>. Each startup description converted into
a vector using a <b>pre-trained</b> SentenceTransformer model and
uploaded to the Qdrant vector search engine. Demo service
a vector using a <b>pre-trained</b> SentenceTransformer model
and uploaded to the Qdrant vector search engine. Demo service
processes text input with the same model and uses its output to
query Qdrant for similar vectors.
</p>
@ -92,7 +92,7 @@
</div>
</div>
</div>
<q-page-sticky position="bottom-left" :offset="[18, 18]" class="text-grey">
<q-footer class="bg-white text-grey q-ml-md">
<ul>
<li>
Data source:
@ -106,7 +106,7 @@
></a>
</li>
</ul>
</q-page-sticky>
</q-footer>
</q-page>
</template>

243
poetry.lock generated
View File

@ -80,6 +80,39 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "httpcore"
version = "0.12.3"
description = "A minimal low-level HTTP client."
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
h11 = "<1.0.0"
sniffio = ">=1.0.0,<2.0.0"
[package.extras]
http2 = ["h2 (>=3,<5)"]
[[package]]
name = "httpx"
version = "0.16.1"
description = "The next generation HTTP client."
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
certifi = "*"
httpcore = ">=0.12.0,<0.13.0"
rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
sniffio = "*"
[package.extras]
brotli = ["brotlipy (>=0.7.0,<0.8.0)"]
http2 = ["h2 (>=3.0.0,<4.0.0)"]
[[package]]
name = "idna"
version = "2.10"
@ -133,21 +166,13 @@ plot = ["matplotlib"]
tgrep = ["pyparsing"]
twitter = ["twython"]
[[package]]
name = "nulltype"
version = "2.3.1"
description = "Null values and sentinels like (but not) None, False & True"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "numpy"
version = "1.19.4"
version = "1.20.1"
description = "NumPy is the fundamental package for array computing with Python."
category = "main"
optional = false
python-versions = ">=3.6"
python-versions = ">=3.7"
[[package]]
name = "packaging"
@ -200,7 +225,7 @@ test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"]
[[package]]
name = "pydantic"
version = "1.7.2"
version = "1.7.3"
description = "Data validation and settings management using python 3.6 type hinting"
category = "main"
optional = false
@ -239,25 +264,18 @@ optional = false
python-versions = "*"
[[package]]
name = "qdrant-openapi-client"
version = "1.0.0"
description = ""
name = "qdrant-client"
version = "0.1.4"
description = "Client library for the Qdrant vector search engine"
category = "main"
optional = false
python-versions = ">=3.5"
develop = false
python-versions = ">=3.8"
[package.dependencies]
certifi = "*"
nulltype = "*"
python-dateutil = "*"
urllib3 = ">=1.15"
[package.source]
type = "git"
url = "https://github.com/qdrant/qdrant_python_client.git"
reference = "master"
resolved_reference = "5e3c620a43af1f5051fe7f119d416df16cb7b0cc"
httpx = ">=0.16.1,<0.17.0"
numpy = ">=1.20.1,<2.0.0"
pydantic = ">=1.7.3,<2.0.0"
tqdm = ">=4.56.0,<5.0.0"
[[package]]
name = "regex"
@ -269,7 +287,7 @@ python-versions = "*"
[[package]]
name = "requests"
version = "2.25.0"
version = "2.25.1"
description = "Python HTTP for Humans."
category = "main"
optional = false
@ -277,7 +295,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<4"
chardet = ">=3.0.2,<5"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.27"
@ -285,6 +303,20 @@ urllib3 = ">=1.21.1,<1.27"
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
[[package]]
name = "rfc3986"
version = "1.4.0"
description = "Validating URI References per RFC 3986"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
[package.extras]
idna2008 = ["idna"]
[[package]]
name = "sacremoses"
version = "0.0.43"
@ -361,6 +393,14 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "sniffio"
version = "1.2.0"
description = "Sniff out which async library your code is running under"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "starlette"
version = "0.13.6"
@ -407,14 +447,15 @@ typing-extensions = "*"
[[package]]
name = "tqdm"
version = "4.54.0"
version = "4.56.1"
description = "Fast, Extensible Progress Meter"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
[package.extras]
dev = ["py-make (>=0.1.0)", "twine", "argopt", "pydoc-markdown", "wheel"]
dev = ["py-make (>=0.1.0)", "twine", "wheel"]
telegram = ["requests"]
[[package]]
name = "transformers"
@ -503,8 +544,8 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
[metadata]
lock-version = "1.1"
python-versions = "~3.8"
content-hash = "5b54f736a3e5496563451c802a67a4787ac67db2aa9f10c66696f19145e759aa"
python-versions = ">=3.8"
content-hash = "58d44cab38ec66125ad97b7992f758264de9591ce765ed2ee91b68186103126c"
[metadata.files]
certifi = [
@ -542,6 +583,14 @@ h11 = [
{file = "h11-0.11.0-py2.py3-none-any.whl", hash = "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87"},
{file = "h11-0.11.0.tar.gz", hash = "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab"},
]
httpcore = [
{file = "httpcore-0.12.3-py3-none-any.whl", hash = "sha256:93e822cd16c32016b414b789aeff4e855d0ccbfc51df563ee34d4dbadbb3bcdc"},
{file = "httpcore-0.12.3.tar.gz", hash = "sha256:37ae835fb370049b2030c3290e12ed298bf1473c41bb72ca4aa78681eba9b7c9"},
]
httpx = [
{file = "httpx-0.16.1-py3-none-any.whl", hash = "sha256:9cffb8ba31fac6536f2c8cde30df859013f59e4bcc5b8d43901cb3654a8e0a5b"},
{file = "httpx-0.16.1.tar.gz", hash = "sha256:126424c279c842738805974687e0518a94c7ae8d140cd65b9c4f77ac46ffa537"},
]
idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
@ -557,45 +606,31 @@ loguru = [
nltk = [
{file = "nltk-3.5.zip", hash = "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35"},
]
nulltype = [
{file = "nulltype-2.3.1-py2.py3-none-any.whl", hash = "sha256:16ae565745118e37e0558441f5821c76351d8c3a789640b5bca277cf65b2271b"},
{file = "nulltype-2.3.1.zip", hash = "sha256:64aa3cb2ab5e904d1b37175b9b922bea268c13f9ce32e3d373313150ab5ef272"},
]
numpy = [
{file = "numpy-1.19.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949"},
{file = "numpy-1.19.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4"},
{file = "numpy-1.19.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad"},
{file = "numpy-1.19.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83"},
{file = "numpy-1.19.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764"},
{file = "numpy-1.19.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6"},
{file = "numpy-1.19.4-cp36-cp36m-win32.whl", hash = "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1"},
{file = "numpy-1.19.4-cp36-cp36m-win_amd64.whl", hash = "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb"},
{file = "numpy-1.19.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2"},
{file = "numpy-1.19.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2"},
{file = "numpy-1.19.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9"},
{file = "numpy-1.19.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757"},
{file = "numpy-1.19.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15"},
{file = "numpy-1.19.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387"},
{file = "numpy-1.19.4-cp37-cp37m-win32.whl", hash = "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36"},
{file = "numpy-1.19.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c"},
{file = "numpy-1.19.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909"},
{file = "numpy-1.19.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c"},
{file = "numpy-1.19.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893"},
{file = "numpy-1.19.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab"},
{file = "numpy-1.19.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9"},
{file = "numpy-1.19.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db"},
{file = "numpy-1.19.4-cp38-cp38-win32.whl", hash = "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac"},
{file = "numpy-1.19.4-cp38-cp38-win_amd64.whl", hash = "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce"},
{file = "numpy-1.19.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63"},
{file = "numpy-1.19.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37"},
{file = "numpy-1.19.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414"},
{file = "numpy-1.19.4-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc"},
{file = "numpy-1.19.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3"},
{file = "numpy-1.19.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753"},
{file = "numpy-1.19.4-cp39-cp39-win32.whl", hash = "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f"},
{file = "numpy-1.19.4-cp39-cp39-win_amd64.whl", hash = "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b"},
{file = "numpy-1.19.4-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08"},
{file = "numpy-1.19.4.zip", hash = "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512"},
{file = "numpy-1.20.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ae61f02b84a0211abb56462a3b6cd1e7ec39d466d3160eb4e1da8bf6717cdbeb"},
{file = "numpy-1.20.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:65410c7f4398a0047eea5cca9b74009ea61178efd78d1be9847fac1d6716ec1e"},
{file = "numpy-1.20.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2d7e27442599104ee08f4faed56bb87c55f8b10a5494ac2ead5c98a4b289e61f"},
{file = "numpy-1.20.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:4ed8e96dc146e12c1c5cdd6fb9fd0757f2ba66048bf94c5126b7efebd12d0090"},
{file = "numpy-1.20.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ecb5b74c702358cdc21268ff4c37f7466357871f53a30e6f84c686952bef16a9"},
{file = "numpy-1.20.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b9410c0b6fed4a22554f072a86c361e417f0258838957b78bd063bde2c7f841f"},
{file = "numpy-1.20.1-cp37-cp37m-win32.whl", hash = "sha256:3d3087e24e354c18fb35c454026af3ed8997cfd4997765266897c68d724e4845"},
{file = "numpy-1.20.1-cp37-cp37m-win_amd64.whl", hash = "sha256:89f937b13b8dd17b0099c7c2e22066883c86ca1575a975f754babc8fbf8d69a9"},
{file = "numpy-1.20.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a1d7995d1023335e67fb070b2fae6f5968f5be3802b15ad6d79d81ecaa014fe0"},
{file = "numpy-1.20.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:60759ab15c94dd0e1ed88241fd4fa3312db4e91d2c8f5a2d4cf3863fad83d65b"},
{file = "numpy-1.20.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:125a0e10ddd99a874fd357bfa1b636cd58deb78ba4a30b5ddb09f645c3512e04"},
{file = "numpy-1.20.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c26287dfc888cf1e65181f39ea75e11f42ffc4f4529e5bd19add57ad458996e2"},
{file = "numpy-1.20.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7199109fa46277be503393be9250b983f325880766f847885607d9b13848f257"},
{file = "numpy-1.20.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:72251e43ac426ff98ea802a931922c79b8d7596480300eb9f1b1e45e0543571e"},
{file = "numpy-1.20.1-cp38-cp38-win32.whl", hash = "sha256:c91ec9569facd4757ade0888371eced2ecf49e7982ce5634cc2cf4e7331a4b14"},
{file = "numpy-1.20.1-cp38-cp38-win_amd64.whl", hash = "sha256:13adf545732bb23a796914fe5f891a12bd74cf3d2986eed7b7eba2941eea1590"},
{file = "numpy-1.20.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:104f5e90b143dbf298361a99ac1af4cf59131218a045ebf4ee5990b83cff5fab"},
{file = "numpy-1.20.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:89e5336f2bec0c726ac7e7cdae181b325a9c0ee24e604704ed830d241c5e47ff"},
{file = "numpy-1.20.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:032be656d89bbf786d743fee11d01ef318b0781281241997558fa7950028dd29"},
{file = "numpy-1.20.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:66b467adfcf628f66ea4ac6430ded0614f5cc06ba530d09571ea404789064adc"},
{file = "numpy-1.20.1-cp39-cp39-win32.whl", hash = "sha256:12e4ba5c6420917571f1a5becc9338abbde71dd811ce40b37ba62dec7b39af6d"},
{file = "numpy-1.20.1-cp39-cp39-win_amd64.whl", hash = "sha256:9c94cab5054bad82a70b2e77741271790304651d584e2cdfe2041488e753863b"},
{file = "numpy-1.20.1-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:9eb551d122fadca7774b97db8a112b77231dcccda8e91a5bc99e79890797175e"},
{file = "numpy-1.20.1.zip", hash = "sha256:3bc63486a870294683980d76ec1e3efc786295ae00128f9ea38e2c6e74d5a60a"},
]
packaging = [
{file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"},
@ -661,28 +696,28 @@ psutil = [
{file = "psutil-5.7.3.tar.gz", hash = "sha256:af73f7bcebdc538eda9cc81d19db1db7bf26f103f91081d780bbacfcb620dee2"},
]
pydantic = [
{file = "pydantic-1.7.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dfaa6ed1d509b5aef4142084206584280bb6e9014f01df931ec6febdad5b200a"},
{file = "pydantic-1.7.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:2182ba2a9290964b278bcc07a8d24207de709125d520efec9ad6fa6f92ee058d"},
{file = "pydantic-1.7.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:0fe8b45d31ae53d74a6aa0bf801587bd49970070eac6a6326f9fa2a302703b8a"},
{file = "pydantic-1.7.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:01f0291f4951580f320f7ae3f2ecaf0044cdebcc9b45c5f882a7e84453362420"},
{file = "pydantic-1.7.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4ba6b903e1b7bd3eb5df0e78d7364b7e831ed8b4cd781ebc3c4f1077fbcb72a4"},
{file = "pydantic-1.7.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b11fc9530bf0698c8014b2bdb3bbc50243e82a7fa2577c8cfba660bcc819e768"},
{file = "pydantic-1.7.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a3c274c49930dc047a75ecc865e435f3df89715c775db75ddb0186804d9b04d0"},
{file = "pydantic-1.7.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:c68b5edf4da53c98bb1ccb556ae8f655575cb2e676aef066c12b08c724a3f1a1"},
{file = "pydantic-1.7.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:95d4410c4e429480c736bba0db6cce5aaa311304aea685ebcf9ee47571bfd7c8"},
{file = "pydantic-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a2fc7bf77ed4a7a961d7684afe177ff59971828141e608f142e4af858e07dddc"},
{file = "pydantic-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9572c0db13c8658b4a4cb705dcaae6983aeb9842248b36761b3fbc9010b740f"},
{file = "pydantic-1.7.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f83f679e727742b0c465e7ef992d6da4a7e5268b8edd8fdaf5303276374bef52"},
{file = "pydantic-1.7.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:e5fece30e80087d9b7986104e2ac150647ec1658c4789c89893b03b100ca3164"},
{file = "pydantic-1.7.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ce2d452961352ba229fe1e0b925b41c0c37128f08dddb788d0fd73fd87ea0f66"},
{file = "pydantic-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:fc21a37ff3f545de80b166e1735c4172b41b017948a3fb2d5e2f03c219eac50a"},
{file = "pydantic-1.7.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c9760d1556ec59ff745f88269a8f357e2b7afc75c556b3a87b8dda5bc62da8ba"},
{file = "pydantic-1.7.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c1673633ad1eea78b1c5c420a47cd48717d2ef214c8230d96ca2591e9e00958"},
{file = "pydantic-1.7.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:388c0c26c574ff49bad7d0fd6ed82fbccd86a0473fa3900397d3354c533d6ebb"},
{file = "pydantic-1.7.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:ab1d5e4d8de00575957e1c982b951bffaedd3204ddd24694e3baca3332e53a23"},
{file = "pydantic-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:f045cf7afb3352a03bc6cb993578a34560ac24c5d004fa33c76efec6ada1361a"},
{file = "pydantic-1.7.2-py3-none-any.whl", hash = "sha256:6665f7ab7fbbf4d3c1040925ff4d42d7549a8c15fe041164adfe4fc2134d4cce"},
{file = "pydantic-1.7.2.tar.gz", hash = "sha256:c8200aecbd1fb914e1bd061d71a4d1d79ecb553165296af0c14989b89e90d09b"},
{file = "pydantic-1.7.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c59ea046aea25be14dc22d69c97bee629e6d48d2b2ecb724d7fe8806bf5f61cd"},
{file = "pydantic-1.7.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a4143c8d0c456a093387b96e0f5ee941a950992904d88bc816b4f0e72c9a0009"},
{file = "pydantic-1.7.3-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:d8df4b9090b595511906fa48deda47af04e7d092318bfb291f4d45dfb6bb2127"},
{file = "pydantic-1.7.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:514b473d264671a5c672dfb28bdfe1bf1afd390f6b206aa2ec9fed7fc592c48e"},
{file = "pydantic-1.7.3-cp36-cp36m-win_amd64.whl", hash = "sha256:dba5c1f0a3aeea5083e75db9660935da90216f8a81b6d68e67f54e135ed5eb23"},
{file = "pydantic-1.7.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:59e45f3b694b05a69032a0d603c32d453a23f0de80844fb14d55ab0c6c78ff2f"},
{file = "pydantic-1.7.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:5b24e8a572e4b4c18f614004dda8c9f2c07328cb5b6e314d6e1bbd536cb1a6c1"},
{file = "pydantic-1.7.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:b2b054d095b6431cdda2f852a6d2f0fdec77686b305c57961b4c5dd6d863bf3c"},
{file = "pydantic-1.7.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:025bf13ce27990acc059d0c5be46f416fc9b293f45363b3d19855165fee1874f"},
{file = "pydantic-1.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6e3874aa7e8babd37b40c4504e3a94cc2023696ced5a0500949f3347664ff8e2"},
{file = "pydantic-1.7.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e682f6442ebe4e50cb5e1cfde7dda6766fb586631c3e5569f6aa1951fd1a76ef"},
{file = "pydantic-1.7.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:185e18134bec5ef43351149fe34fda4758e53d05bb8ea4d5928f0720997b79ef"},
{file = "pydantic-1.7.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:f5b06f5099e163295b8ff5b1b71132ecf5866cc6e7f586d78d7d3fd6e8084608"},
{file = "pydantic-1.7.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:24ca47365be2a5a3cc3f4a26dcc755bcdc9f0036f55dcedbd55663662ba145ec"},
{file = "pydantic-1.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:d1fe3f0df8ac0f3a9792666c69a7cd70530f329036426d06b4f899c025aca74e"},
{file = "pydantic-1.7.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f6864844b039805add62ebe8a8c676286340ba0c6d043ae5dea24114b82a319e"},
{file = "pydantic-1.7.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:ecb54491f98544c12c66ff3d15e701612fc388161fd455242447083350904730"},
{file = "pydantic-1.7.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:ffd180ebd5dd2a9ac0da4e8b995c9c99e7c74c31f985ba090ee01d681b1c4b95"},
{file = "pydantic-1.7.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8d72e814c7821125b16f1553124d12faba88e85405b0864328899aceaad7282b"},
{file = "pydantic-1.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:475f2fa134cf272d6631072554f845d0630907fce053926ff634cc6bc45bf1af"},
{file = "pydantic-1.7.3-py3-none-any.whl", hash = "sha256:38be427ea01a78206bcaf9a56f835784afcba9e5b88fbdce33bbbfbcd7841229"},
{file = "pydantic-1.7.3.tar.gz", hash = "sha256:213125b7e9e64713d16d988d10997dabc6a1f73f3991e1ff8e35ebb1409c7dc9"},
]
pyparsing = [
{file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
@ -696,7 +731,10 @@ pytz = [
{file = "pytz-2020.4-py2.py3-none-any.whl", hash = "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"},
{file = "pytz-2020.4.tar.gz", hash = "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268"},
]
qdrant-openapi-client = []
qdrant-client = [
{file = "qdrant_client-0.1.4-py3-none-any.whl", hash = "sha256:a2ba8681b9a39dd82b2e79c5556363d6728afa1bcab8d8cdbc250f1faa7f9bc3"},
{file = "qdrant_client-0.1.4.tar.gz", hash = "sha256:1a685cc689c7a3f2176649a7dbbef0bded7193665f4d1d01fc380ac82247daf3"},
]
regex = [
{file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"},
@ -741,8 +779,12 @@ regex = [
{file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"},
]
requests = [
{file = "requests-2.25.0-py2.py3-none-any.whl", hash = "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"},
{file = "requests-2.25.0.tar.gz", hash = "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8"},
{file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
{file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
]
rfc3986 = [
{file = "rfc3986-1.4.0-py2.py3-none-any.whl", hash = "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50"},
{file = "rfc3986-1.4.0.tar.gz", hash = "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d"},
]
sacremoses = [
{file = "sacremoses-0.0.43.tar.gz", hash = "sha256:123c1bf2664351fb05e16f87d3786dbe44a050cfd7b85161c09ad9a63a8e2948"},
@ -825,6 +867,10 @@ six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
]
sniffio = [
{file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"},
{file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"},
]
starlette = [
{file = "starlette-0.13.6-py3-none-any.whl", hash = "sha256:bd2ffe5e37fb75d014728511f8e68ebf2c80b0fa3d04ca1479f4dc752ae31ac9"},
{file = "starlette-0.13.6.tar.gz", hash = "sha256:ebe8ee08d9be96a3c9f31b2cb2a24dbdf845247b745664bd8a3f9bd0c977fdbc"},
@ -854,15 +900,18 @@ tokenizers = [
]
torch = [
{file = "torch-1.7.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6b0c9b56cb56afe3ecbac79351d21c6f7172dffc7b7daa8c365f660541baf1a5"},
{file = "torch-1.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:b609b30b09627409612461b247531cf50894256e8b026105f1bc3858d67c2fa7"},
{file = "torch-1.7.0-cp36-none-macosx_10_9_x86_64.whl", hash = "sha256:e8cc3b2c3937b7ae036a3b447a189af049bfc006bca054fc1d8ae78766ca3105"},
{file = "torch-1.7.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1520c48430dea38e5845b7b3defc9054edad45f1f245808aa268ade840bb2c2a"},
{file = "torch-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a29a7aab6fa161259219c8aaa88b25abbf214ec098a002c4499660c23e0f6c33"},
{file = "torch-1.7.0-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:89cb8774243750bd3fd2b3b3d09bab6e3be68b1785ad48b8411f1eb4fc7acdba"},
{file = "torch-1.7.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:11054f26eee5c3114d217201dba5b3a35f1745d11133c123c077c5981bc95997"},
{file = "torch-1.7.0-cp38-cp38m-win_amd64.whl", hash = "sha256:83538fd7920bd02c8b6403ebe4ecee3e0641fe7bb6a18efc4585d2e1375ad1de"},
{file = "torch-1.7.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:b8000e39600e101b2f19dbbab75de663a3b78e3979c3e1720b7136aae1c35ce2"},
]
tqdm = [
{file = "tqdm-4.54.0-py2.py3-none-any.whl", hash = "sha256:9e7b8ab0ecbdbf0595adadd5f0ebbb9e69010e0bd48bbb0c15e550bf2a5292df"},
{file = "tqdm-4.54.0.tar.gz", hash = "sha256:5c0d04e06ccc0da1bd3fa5ae4550effcce42fcad947b4a6cafa77bdc9b09ff22"},
{file = "tqdm-4.56.1-py2.py3-none-any.whl", hash = "sha256:ab9b659241d82b8b51b2269ee243ec95286046bf06015c4e15a947cc15914211"},
{file = "tqdm-4.56.1.tar.gz", hash = "sha256:2874fa525c051177583ec59c0fb4583e91f28ccd3f217ffad2acdb32d2c789ac"},
]
transformers = [
{file = "transformers-3.5.1-py3-none-any.whl", hash = "sha256:cefb06c8892e0ce2de1bfc782d2c49af020960ebada8340317306ff606190341"},

5
pyproject.toml
View File

@ -5,15 +5,16 @@ description = "Qdrant vector similarity engine demo"
authors = ["Andrey Vasnetsov <andrey@vasnetsov.com>"]
[tool.poetry.dependencies]
python = "~3.8"
python = ">=3.8"
fastapi = "^0.61.2"
uvicorn = "^0.12.3"
sentence-transformers = "^0.3.9"
psutil = "^5.7.3"
nltk = "^3.5"
qdrant-openapi-client = {git = "https://github.com/qdrant/qdrant_python_client.git", rev = "master"}
pandas = "^1.1.5"
loguru = "^0.5.3"
requests = "^2.25.1"
qdrant-client = "0.1.5"
[tool.poetry.dev-dependencies]

4
qdrant_demo/config.py
View File

@ -5,3 +5,7 @@ ROOT_DIR = os.path.dirname(CODE_DIR)
DATA_DIR = os.path.join(ROOT_DIR, 'data')
COLLECTION_NAME = "startups"
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost")
QDRANT_PORT = os.environ.get("QDRANT_PORT", 6333)

4
qdrant_demo/init_text_search_index.py
View File

@ -1,7 +1,7 @@
import pandas as pd
import os
from typing import Iterable, List
from typing import List
import pandas as pd
from qdrant_demo.config import DATA_DIR
from qdrant_demo.sqlite_searcher import SqliteSearch

72
qdrant_demo/init_vector_search_index.py
View File

@ -1,61 +1,31 @@
import json
import logging
import os
from itertools import islice
from typing import Iterable
import numpy as np
from qdrant_openapi_client.model.point_struct import PointStruct
from tqdm import tqdm
from qdrant_demo.config import COLLECTION_NAME, DATA_DIR
from qdrant_demo.qdrant_client import QdrantClient
from qdrant_client import QdrantClient
from qdrant_demo.config import DATA_DIR, COLLECTION_NAME, QDRANT_HOST, QDRANT_PORT
BATCH_SIZE = 256
def iter_batch(iterable, size):
"""
>>> list(iter_batch([1,2,3,4,5], 3))
[[1, 2, 3], [4, 5]]
"""
source_iter = iter(iterable)
while source_iter:
b = list(islice(source_iter, size))
if len(b) == 0:
break
yield b
class VectorUploader:
def __init__(self):
self.qdrant_client = QdrantClient()
def read_points(self, data: Iterable[dict], vectors: np.ndarray) -> Iterable[PointStruct]:
for idx, (record, vector) in tqdm(enumerate(zip(data, vectors))):
yield self.qdrant_client.make_point(idx=idx, vector=vector.tolist(), payload=record)
def upload_to_qdrant(self, data: Iterable[dict], vectors: np.ndarray):
vector_size = vectors.shape[1]
self.qdrant_client.recreate_collection(COLLECTION_NAME, vector_size=vector_size)
for point_batch in iter_batch(self.read_points(data, vectors), BATCH_SIZE):
self.qdrant_client.upload_point(COLLECTION_NAME, points=point_batch)
def read_and_upload(self, payload_path: str, vectors_path: str):
vectors = np.load(vectors_path)
with open(payload_path) as fd:
data = map(json.loads, fd)
self.upload_to_qdrant(data, vectors)
if __name__ == '__main__':
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
logging.basicConfig(level=LOGLEVEL)
uploader = VectorUploader()
uploader.read_and_upload(
payload_path=os.path.join(DATA_DIR, 'startups.json'),
vectors_path=os.path.join(DATA_DIR, 'startup_vectors.npy')
)
qdrant_client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
vectors_path = os.path.join(DATA_DIR, 'startup_vectors.npy')
vectors = np.load(vectors_path)
vector_size = vectors.shape[1]
payload_path = os.path.join(DATA_DIR, 'startups.json')
with open(payload_path) as fd:
payload = list(map(json.loads, fd))
qdrant_client.recreate_collection(collection_name=COLLECTION_NAME, vector_size=vector_size)
qdrant_client.upload_collection(
collection_name=COLLECTION_NAME,
vectors=vectors,
payload=payload,
ids=None,
batch_size=BATCH_SIZE
)

29
qdrant_demo/neural_searcher.py
View File

@ -1,14 +1,12 @@
import logging
import os
from typing import List, Optional
from typing import List
import psutil
from qdrant_openapi_client.model.filter import Filter
from qdrant_openapi_client.model.scored_point import ScoredPoint
from qdrant_client import QdrantClient
from qdrant_openapi_client.models.models import Filter
from sentence_transformers import SentenceTransformer
from qdrant_demo.qdrant_client import QdrantClient
class NeuralSearcher:
@ -25,22 +23,13 @@ class NeuralSearcher:
self.model = self.load_model(device='cpu')
self.qdrant_client = QdrantClient()
@classmethod
def _convert_filters(cls, filter_: dict) -> Optional[Filter]:
if not filter_:
return None
return Filter(**filter_)
def search(self, text: str, filter_: dict = None) -> List[dict]:
vector = self.model.encode(text).tolist()
res: List[ScoredPoint] = self.qdrant_client.search(
search_result = self.qdrant_client.search(
collection_name=self.collection_name,
vector=vector,
filter_=filter_
).result
payloads = self.qdrant_client.lookup(self.collection_name, [point.id for point in res])
query_vector=vector,
query_filter=Filter(**filter_) if filter_ else None,
top=5
)
payloads = [payload for point, payload in search_result]
return payloads

126
qdrant_demo/qdrant_client.py
View File

@ -1,126 +0,0 @@
import logging
import os
from typing import Dict, List, Optional
import qdrant_openapi_client
from qdrant_openapi_client.api.collections_api import CollectionsApi
from qdrant_openapi_client.api.points_api import PointsApi
from qdrant_openapi_client.model.collection_update_operations import CollectionUpdateOperations
from qdrant_openapi_client.model.distance import Distance
from qdrant_openapi_client.model.filter import Filter
from qdrant_openapi_client.model.payload_interface import PayloadInterface
from qdrant_openapi_client.model.point_insert_ops import PointInsertOps
from qdrant_openapi_client.model.point_request import PointRequest
from qdrant_openapi_client.model.point_struct import PointStruct
from qdrant_openapi_client.model.record import Record
from qdrant_openapi_client.model.search_request import SearchRequest
from qdrant_openapi_client.model.storage_ops import StorageOps
from qdrant_openapi_client.model.storage_ops_any_of_create_collection import StorageOpsAnyOfCreateCollection
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost")
QDRANT_PORT = os.environ.get("QDRANT_PORT", 6333)
class QdrantClient:
@classmethod
def _get_qdrant_client(cls):
configuration = qdrant_openapi_client.Configuration(
host=f"http://{QDRANT_HOST}:{QDRANT_PORT}"
)
api_client = qdrant_openapi_client.ApiClient(configuration)
return api_client
@classmethod
def _get_collection_api(cls):
api_client = cls._get_qdrant_client()
collections_api = CollectionsApi(api_client)
return collections_api
@classmethod
def _get_points_api(cls):
api_client = cls._get_qdrant_client()
points_api = PointsApi(api_client)
return points_api
def __init__(self):
self.collection_api = self._get_collection_api()
self.point_api = self._get_points_api()
def recreate_collection(self, collection_name, vector_size):
delete_ops = StorageOps(delete_collection=collection_name)
try:
self.collection_api.update_collections(storage_ops=delete_ops)
except Exception as e:
print("Exception when calling CollectionsApi->update_collections: %s\n" % e)
create_ops = StorageOps(create_collection=StorageOpsAnyOfCreateCollection(
distance=Distance(value="Cosine"),
name=collection_name,
vector_size=vector_size,
))
try:
self.collection_api.update_collections(storage_ops=create_ops)
except qdrant_openapi_client.ApiException as e:
print("Exception when calling CollectionsApi->update_collections: %s\n" % e)
def upload_point(self, collection_name: str, points: List[PointStruct]):
self.point_api.update_points(
collection_name,
collection_update_operations=CollectionUpdateOperations(
upsert_points=PointInsertOps(points=points)
), async_req=True)
def search(self, collection_name: str, vector, filter_: Optional[Filter] = None, top=5):
search_result = self.point_api.search_points(
collection_name,
search_request=SearchRequest(
top=top,
vector=vector,
filter=filter_
)
)
logging.info(f"search time: {search_result.time}")
return search_result
@classmethod
def _simplify_payload(cls, payload: dict) -> dict:
res = {}
for key, val in payload.items():
if len(val["value"]) == 1:
res[key] = val['value'][0]
else:
res[key] = val['value']
return res
def lookup(self, collection_name: str, ids: List[int]) -> List[dict]:
records: List[Record] = self.point_api.get_points(collection_name, point_request=PointRequest(ids=ids)).result
payloads = dict(
(record.id, record.to_dict()['payload'])
for record in records
)
return [self._simplify_payload(payloads[idx]) for idx in ids if idx in payloads]
@classmethod
def data_to_payload_request(cls, obj: dict) -> Dict[str, PayloadInterface]:
"""
>>> QdrantClient.data_to_payload_request({"idx": 123})['idx'].to_dict()
{'type': 'integer', 'value': 123}
"""
res = {}
for key, val in obj.items():
if isinstance(val, int):
res[key] = PayloadInterface(value=val, type="integer", _check_type=False)
if isinstance(val, float):
res[key] = PayloadInterface(value=val, type="float", _check_type=False)
if isinstance(val, str):
res[key] = PayloadInterface(value=val, type="keyword", _check_type=False)
return res
@classmethod
def make_point(cls, idx, vector, payload):
return PointStruct(
id=idx,
vector=vector,
payload=cls.data_to_payload_request(payload)
)

2
qdrant_demo/service.py
View File

@ -17,7 +17,7 @@ text_searcher = TextSearcher()
@app.get("/api/search")
def read_item(q: str, neural: bool = True):
async def read_item(q: str, neural: bool = True):
return {
"result": neural_searcher.search(text=q)
if neural else text_searcher.search(query=q)