From 6078063cd0a118a95bba92f99c08295dc58a8f48 Mon Sep 17 00:00:00 2001 From: clelia Date: Thu, 20 Aug 2026 21:01:33 +0200 Subject: [PATCH 1/2] feat: revamp multimodal search notebook --- .../Multimodal_Search_with_LlamaIndex.ipynb | 979 ------------------ .../Multimodal_Search_with_Qwen.ipynb | 244 +++++ 2 files changed, 244 insertions(+), 979 deletions(-) delete mode 100644 multimodal-search/Multimodal_Search_with_LlamaIndex.ipynb create mode 100644 multimodal-search/Multimodal_Search_with_Qwen.ipynb diff --git a/multimodal-search/Multimodal_Search_with_LlamaIndex.ipynb b/multimodal-search/Multimodal_Search_with_LlamaIndex.ipynb deleted file mode 100644 index fc30ecf..0000000 --- a/multimodal-search/Multimodal_Search_with_LlamaIndex.ipynb +++ /dev/null @@ -1,979 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "YIVBTkiDAlfy" - }, - "source": [ - "### Tutorial\n", - "\n", - "We will use [**LlamaIndex**](https://huggingface.co/llamaindex/vdr-2b-multi-v1/tree/main) for generating multimodal embeddings and [**Qdrant**](http://qdrant.tech) for storing and retrieving them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TrkMVQD0H6Qu", - "outputId": "f264747c-51bd-4d78-d34b-0a7596c79189" - }, - "outputs": [], - "source": [ - "%pip install llama-index-embeddings-huggingface qdrant-client " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "Jn6xx48IJf7D" - }, - "outputs": [], - "source": [ - "from qdrant_client import QdrantClient, models\n", - "\n", - "# docker run -p 6333:6333 qdrant/qdrant\n", - "client = QdrantClient(url=\"http://localhost:6333/\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vfkXbWRhBbPT" - }, - "source": [ - "Let's embed a very short selection of images and their captions in the **shared embedding space**." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 205, - "referenced_widgets": [ - "a614636be04d4063a7b1a402b56c83e3", - "5d04e84cb0a346dd98e8028ebcf95c6e", - "161c7935d4854077ad9292e9e2ec1279", - "4b9696ef699c46a4aefce756ba64d61b", - "c9dd792f85204a2d9397c9fa6895c12e", - "411a90b1abbe41248218fa4e24058f69", - "0c058c78e1e44c60be3ef50f1ad70712", - "1e862c55ee124adcb5c5268a56c5e37e", - "accc4b7b05d64a66b7b8fe7652b79f21", - "1f4694a6cbbb4f2ca1e3ffa3afdcbc17", - "54ace692885a46b0a4e0c8e081417229", - "5995173ff9a247b9a8d68f92d5205f19", - "da0dfb1fb80b4c90872ebfa452729cb5", - "87850e2da76c4af58949c96c034dde2d", - "f83c630b5ddf4ab7ad0d5c0f63706c9f", - "0c1793a3fd924225a503ec1ed8e36d36", - "7efc69c1a4214c0bbe399a6fd79cac61", - "9c957c2bf0724d6fab30d733c2cf17e0", - "56813987a9ff4f5397d75b027a6693ed", - "7e3dd20df5cf47b2b578aac295790ce0", - "4421a17644bc4e6baa6e0a1c246aa2cf", - "4c925e41e49b4bd6b6762dc81059f7a9" - ] - }, - "id": "7ZZYV2aqKD5b", - "outputId": "69c4530a-cbef-456a-92e3-2feda3a61f72" - }, - "outputs": [], - "source": [ - "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n", - "\n", - "model = HuggingFaceEmbedding(\n", - " model_name=\"llamaindex/vdr-2b-multi-v1\",\n", - " device=\"cpu\", # \"mps\" for mac, \"cuda\" for nvidia GPUs\n", - " trust_remote_code=True,\n", - ")\n", - "\n", - "documents = [\n", - " {\"caption\": \"An image about plane emergency safety.\", \"image\": \"images/image-1.png\"},\n", - " {\"caption\": \"An image about airplane components.\", \"image\": \"images/image-2.png\"},\n", - " {\"caption\": \"An image about COVID safety restrictions.\", \"image\": \"images/image-3.png\"},\n", - " {\"caption\": \"An confidential image about UFO sightings.\", \"image\": \"images/image-4.png\"},\n", - " {\"caption\": \"An image about unusual footprints on Aralar 2011.\", \"image\": \"images/image-5.png\"},\n", - "]\n", - "\n", - "text_embeddings = model.get_text_embedding_batch([doc[\"caption\"] for doc in documents])\n", - "image_embeddings = model.get_image_embedding_batch([doc[\"image\"] for doc in documents])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yEn3HWTiYl_u" - }, - "source": [ - "Create a **Collection**" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "98oNUlicaiXK" - }, - "outputs": [], - "source": [ - "COLLECTION_NAME = \"llama-multi\"\n", - "\n", - "if not client.collection_exists(COLLECTION_NAME):\n", - " client.create_collection(\n", - " collection_name=COLLECTION_NAME,\n", - " vectors_config={\n", - " \"image\": models.VectorParams(size=len(image_embeddings[0]), distance=models.Distance.COSINE),\n", - " \"text\": models.VectorParams(size=len(text_embeddings[0]), distance=models.Distance.COSINE),\n", - " }\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4vwr7UbRFV5M" - }, - "source": [ - "Now let's upload our images with captions to the **Collection**. Each image with its caption will create a [Point](https://qdrant.tech/documentation/concepts/points/) in Qdrant." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "mpqu-qzbP8Eh" - }, - "outputs": [], - "source": [ - "client.upload_points(\n", - " collection_name=COLLECTION_NAME,\n", - " points=[\n", - " models.PointStruct(\n", - " id=idx,\n", - " vector={\n", - " \"text\": text_embeddings[idx],\n", - " \"image\": image_embeddings[idx],\n", - " },\n", - " payload=doc\n", - " )\n", - " for idx, doc in enumerate(documents)\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pAkj-XFVGm9N" - }, - "source": [ - "Let'see what image we will get to the query \"*Adventures on snow hills*\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "DY2PvcZVnZwN", - "outputId": "ad87d1c1-918e-4f8e-eac2-f7c0be08fb53" - }, - "outputs": [], - "source": [ - "from PIL import Image\n", - "\n", - "find_image = model.get_query_embedding(\"Adventures on snow hills\")\n", - "\n", - "Image.open(client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=find_image,\n", - " using=\"image\",\n", - " with_payload=[\"image\"],\n", - " limit=1\n", - ").points[0].payload['image'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's also run the same query in Italian and compare the results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "Image.open(client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=model.get_query_embedding(\"Avventure sulle colline innevate\"),\n", - " using=\"image\",\n", - " with_payload=[\"image\"],\n", - " limit=1\n", - ").points[0].payload['image'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IihSWFsnHfwx" - }, - "source": [ - "Now let's do a reverse search for the follwing image:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 529 - }, - "id": "2ScFbT38I2rO", - "outputId": "bdc415ff-9ae5-4f96-e4d0-aba541e87a42" - }, - "outputs": [], - "source": [ - "Image.open('images/image-2.png')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 35 - }, - "id": "HLmPdjktJic7", - "outputId": "7b743094-9ee0-478e-f3bc-0f59bf0606e6" - }, - "outputs": [], - "source": [ - "client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=model.get_image_embedding(\"images/image-2.png\"), \n", - " # Now we are searching only among text vectors with our image query\n", - " using=\"text\",\n", - " with_payload=[\"caption\"],\n", - " limit=1\n", - ").points[0].payload['caption']" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "0c058c78e1e44c60be3ef50f1ad70712": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0c1793a3fd924225a503ec1ed8e36d36": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "161c7935d4854077ad9292e9e2ec1279": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1e862c55ee124adcb5c5268a56c5e37e", - "max": 5, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_accc4b7b05d64a66b7b8fe7652b79f21", - "value": 5 - } - }, - "1e862c55ee124adcb5c5268a56c5e37e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1f4694a6cbbb4f2ca1e3ffa3afdcbc17": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "411a90b1abbe41248218fa4e24058f69": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4421a17644bc4e6baa6e0a1c246aa2cf": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4b9696ef699c46a4aefce756ba64d61b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1f4694a6cbbb4f2ca1e3ffa3afdcbc17", - "placeholder": "​", - "style": "IPY_MODEL_54ace692885a46b0a4e0c8e081417229", - "value": " 5/5 [00:00<00:00, 93.94it/s]" - } - }, - "4c925e41e49b4bd6b6762dc81059f7a9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "54ace692885a46b0a4e0c8e081417229": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "56813987a9ff4f5397d75b027a6693ed": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5995173ff9a247b9a8d68f92d5205f19": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_da0dfb1fb80b4c90872ebfa452729cb5", - "IPY_MODEL_87850e2da76c4af58949c96c034dde2d", - "IPY_MODEL_f83c630b5ddf4ab7ad0d5c0f63706c9f" - ], - "layout": "IPY_MODEL_0c1793a3fd924225a503ec1ed8e36d36" - } - }, - "5d04e84cb0a346dd98e8028ebcf95c6e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_411a90b1abbe41248218fa4e24058f69", - "placeholder": "​", - "style": "IPY_MODEL_0c058c78e1e44c60be3ef50f1ad70712", - "value": "Fetching 5 files: 100%" - } - }, - "7e3dd20df5cf47b2b578aac295790ce0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7efc69c1a4214c0bbe399a6fd79cac61": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "87850e2da76c4af58949c96c034dde2d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_56813987a9ff4f5397d75b027a6693ed", - "max": 3, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_7e3dd20df5cf47b2b578aac295790ce0", - "value": 3 - } - }, - "9c957c2bf0724d6fab30d733c2cf17e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a614636be04d4063a7b1a402b56c83e3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5d04e84cb0a346dd98e8028ebcf95c6e", - "IPY_MODEL_161c7935d4854077ad9292e9e2ec1279", - "IPY_MODEL_4b9696ef699c46a4aefce756ba64d61b" - ], - "layout": "IPY_MODEL_c9dd792f85204a2d9397c9fa6895c12e" - } - }, - "accc4b7b05d64a66b7b8fe7652b79f21": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "c9dd792f85204a2d9397c9fa6895c12e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "da0dfb1fb80b4c90872ebfa452729cb5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7efc69c1a4214c0bbe399a6fd79cac61", - "placeholder": "​", - "style": "IPY_MODEL_9c957c2bf0724d6fab30d733c2cf17e0", - "value": "Fetching 3 files: 100%" - } - }, - "f83c630b5ddf4ab7ad0d5c0f63706c9f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4421a17644bc4e6baa6e0a1c246aa2cf", - "placeholder": "​", - "style": "IPY_MODEL_4c925e41e49b4bd6b6762dc81059f7a9", - "value": " 3/3 [00:00<00:00, 46.61it/s]" - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/multimodal-search/Multimodal_Search_with_Qwen.ipynb b/multimodal-search/Multimodal_Search_with_Qwen.ipynb new file mode 100644 index 0000000..50b7150 --- /dev/null +++ b/multimodal-search/Multimodal_Search_with_Qwen.ipynb @@ -0,0 +1,244 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "YIVBTkiDAlfy" + }, + "source": [ + "### Tutorial\n", + "\n", + "We will use [**Qwen3-VL-Embeddings-2B**](https://huggingface.co/Qwen/Qwen3-VL-Embedding-2B) through [`sentence-transformers`](https://sbert.net) for generating multimodal embeddings and [**Qdrant**](http://qdrant.tech) for storing and retrieving them." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "TrkMVQD0H6Qu" + }, + "outputs": [], + "source": [ + "! pip install -q sentence-transformers qdrant-client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Jn6xx48IJf7D" + }, + "outputs": [], + "source": [ + "from qdrant_client import QdrantClient, models\n", + "from getpass import getpass\n", + "\n", + "# docker run -p 6333:6333 qdrant/qdrant\n", + "client = QdrantClient(url=\"http://localhost:6333\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vfkXbWRhBbPT" + }, + "source": [ + "Let's embed a very short selection of images and their captions in the **shared embedding space**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7ZZYV2aqKD5b" + }, + "outputs": [], + "source": [ + "from sentence_transformers import SentenceTransformer\n", + "\n", + "model = SentenceTransformer(\"Qwen/Qwen3-VL-Embedding-2B\")\n", + "\n", + "documents = [\n", + " {\"caption\": \"An image about plane emergency safety.\", \"image\": \"images/image-1.png\"},\n", + " {\"caption\": \"An image about airplane components.\", \"image\": \"images/image-2.png\"},\n", + " {\"caption\": \"An image about COVID safety restrictions.\", \"image\": \"images/image-3.png\"},\n", + " {\"caption\": \"A confidential image about UFO sightings.\", \"image\": \"images/image-4.png\"},\n", + " {\"caption\": \"An image about unusual footprints on Aralar 2011.\", \"image\": \"images/image-5.png\"},\n", + "]\n", + "\n", + "text_embeddings = model.encode_document([doc[\"caption\"] for doc in documents])\n", + "image_embeddings = model.encode_document([doc[\"image\"] for doc in documents])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yEn3HWTiYl_u" + }, + "source": [ + "Create a **Collection**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "98oNUlicaiXK" + }, + "outputs": [], + "source": [ + "COLLECTION_NAME = \"multimodal-embeddings\"\n", + "\n", + "if not client.collection_exists(COLLECTION_NAME):\n", + " client.create_collection(\n", + " collection_name=COLLECTION_NAME,\n", + " vectors_config={\n", + " \"image\": models.VectorParams(size=len(image_embeddings[0]), distance=models.Distance.COSINE),\n", + " \"text\": models.VectorParams(size=len(text_embeddings[0]), distance=models.Distance.COSINE),\n", + " }\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4vwr7UbRFV5M" + }, + "source": [ + "Now let's upload our images with captions to the **Collection**. Each image with its caption will create a [Point](https://qdrant.tech/documentation/concepts/points/) in Qdrant." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "mpqu-qzbP8Eh" + }, + "outputs": [], + "source": [ + "client.upload_points(\n", + " collection_name=COLLECTION_NAME,\n", + " points=[\n", + " models.PointStruct(\n", + " id=idx,\n", + " vector={\n", + " \"text\": text_embeddings[idx],\n", + " \"image\": image_embeddings[idx],\n", + " },\n", + " payload=doc\n", + " )\n", + " for idx, doc in enumerate(documents)\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pAkj-XFVGm9N" + }, + "source": [ + "Let'see what image we will get to the query \"*Adventures on snow hills*\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DY2PvcZVnZwN" + }, + "outputs": [], + "source": [ + "from PIL import Image\n", + "\n", + "find_image = model.encode_query(\"Plane components\")\n", + "\n", + "Image.open(client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=find_image,\n", + " using=\"image\",\n", + " with_payload=[\"image\"],\n", + " limit=1\n", + ").points[0].payload['image'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mg77HNQv7T3s" + }, + "source": [ + "Let's also run the same query in Italian (one of the 30+ languages supported by the model) and compare the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C7m5bOOM7T3s" + }, + "outputs": [], + "source": [ + "Image.open(client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=model.encode_query(\"Componenti di un aereo\"),\n", + " using=\"image\",\n", + " with_payload=[\"image\"],\n", + " limit=1\n", + ").points[0].payload['image'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IihSWFsnHfwx" + }, + "source": [ + "Now let's do a reverse search for the follwing image:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2ScFbT38I2rO" + }, + "outputs": [], + "source": [ + "Image.open('images/image-2.png')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HLmPdjktJic7" + }, + "outputs": [], + "source": [ + "client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=model.encode_query(\"images/image-2.png\"),\n", + " # Now we are searching only among text vectors with our image query\n", + " using=\"text\",\n", + " with_payload=[\"caption\"],\n", + " limit=1\n", + ").points[0].payload['caption']" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file From 722d1a164ad252497f60c082b6cb27d51c54abbe Mon Sep 17 00:00:00 2001 From: clelia Date: Fri, 21 Aug 2026 12:31:57 +0200 Subject: [PATCH 2/2] chore: switch to cloud inference --- ...arch_with_Cohere_and_Cloud_Inference.ipynb | 289 ++++++++++++++++++ .../Multimodal_Search_with_Qwen.ipynb | 244 --------------- 2 files changed, 289 insertions(+), 244 deletions(-) create mode 100644 multimodal-search/Multimodal_Search_with_Cohere_and_Cloud_Inference.ipynb delete mode 100644 multimodal-search/Multimodal_Search_with_Qwen.ipynb diff --git a/multimodal-search/Multimodal_Search_with_Cohere_and_Cloud_Inference.ipynb b/multimodal-search/Multimodal_Search_with_Cohere_and_Cloud_Inference.ipynb new file mode 100644 index 0000000..680c1b2 --- /dev/null +++ b/multimodal-search/Multimodal_Search_with_Cohere_and_Cloud_Inference.ipynb @@ -0,0 +1,289 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "YIVBTkiDAlfy" + }, + "source": [ + "### Tutorial\n", + "\n", + "We will use [**Cohere Embed 4.0**](https://cohere.com/blog/embed-4) through [Qdrant Cloud Inference](https://qdrant.tech/documentation/inference/inference-api/) for generating multimodal embeddings and a [**Qdrant Collection**](qdrant.tech/documentation/manage-data/collections/) for storing and retrieving them.\n", + "\n", + "> _To follow along with this example, you need a Cohere API key. Create a free one [here](https://dashboard.cohere.com/api-keys)_" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "TrkMVQD0H6Qu" + }, + "outputs": [], + "source": [ + "! pip install -q qdrant-client" + ] + }, + { + "cell_type": "markdown", + "source": "We will be using a [Qdrant Cloud Free Tier Cluster](/documentation/cloud/create-cluster/#free-clusters).\n\n[Create a free cluster](https://cloud.qdrant.io/), save the associated API key and endpoint URL, and instantiate the Qdrant Client (make sure to set `cloud_inference=True` to enable Cloud Inference):", + "metadata": { + "id": "-wTBQHkedYIp" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Jn6xx48IJf7D" + }, + "outputs": [], + "source": [ + "from qdrant_client import QdrantClient, models\n", + "from getpass import getpass\n", + "\n", + "client = QdrantClient(url=getpass(\"Qdrant URL: \"), api_key=getpass(\"Qdrant API key: \"), cloud_inference=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vfkXbWRhBbPT" + }, + "source": [ + "\n", + "Let's embed a very short selection of images and their captions in the **shared embedding space**." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "7ZZYV2aqKD5b" + }, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "def image_to_base64_url(image_path: str) -> str:\n", + " prefix = \"data:image/png;base64\"\n", + " with open(image_path, \"rb\") as image_file:\n", + " return prefix + \",\" + base64.b64encode(image_file.read()).decode(\"utf-8\")\n", + "\n", + "documents = [\n", + " {\"caption\": \"An image about plane emergency safety.\", \"image\": \"images/image-1.png\"},\n", + " {\"caption\": \"An image about airplane components.\", \"image\": \"images/image-2.png\"},\n", + " {\"caption\": \"An image about COVID safety restrictions.\", \"image\": \"images/image-3.png\"},\n", + " {\"caption\": \"A confidential image about UFO sightings.\", \"image\": \"images/image-4.png\"},\n", + " {\"caption\": \"An image about unusual footprints on Aralar 2011.\", \"image\": \"images/image-5.png\"},\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yEn3HWTiYl_u" + }, + "source": [ + "Create a **Collection**" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "98oNUlicaiXK" + }, + "outputs": [], + "source": [ + "COLLECTION_NAME = \"multimodal-embeddings\"\n", + "\n", + "if not client.collection_exists(COLLECTION_NAME):\n", + " client.create_collection(\n", + " collection_name=COLLECTION_NAME,\n", + " vectors_config={\n", + " \"image\": models.VectorParams(size=512, distance=models.Distance.COSINE),\n", + " \"text\": models.VectorParams(size=512, distance=models.Distance.COSINE),\n", + " }\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4vwr7UbRFV5M" + }, + "source": [ + "Now let's upload our images with captions to the **Collection**. Each image with its caption will be embedded by the Cohere model, [through Cloud Inference](https://qdrant.tech/documentation/inference/external-inference-providers/#cohere), and uploaded, as a [Point](https://qdrant.tech/documentation/concepts/points/), to the collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mpqu-qzbP8Eh" + }, + "outputs": [], + "source": [ + "from qdrant_client.context_headers import headers\n", + "\n", + "cohere_api_key = getpass(\"Cohere API key: \")\n", + "\n", + "with headers({\"cohere-api-key\": cohere_api_key}):\n", + " client.upsert(\n", + " collection_name=COLLECTION_NAME,\n", + " points=[\n", + " models.PointStruct(\n", + " id=idx,\n", + " vector={\n", + " \"text\": models.Document(\n", + " text=doc[\"caption\"],\n", + " model=\"cohere/embed-v4.0\",\n", + " options={\n", + " \"output_dimension\": 512\n", + " }\n", + " ),\n", + " \"image\": models.Image(\n", + " image=image_to_base64_url(doc[\"image\"]),\n", + " model=\"cohere/embed-v4.0\",\n", + " options={\n", + " \"output_dimension\": 512\n", + " }\n", + " ),\n", + " },\n", + " payload=doc\n", + " )\n", + " for idx, doc in enumerate(documents)\n", + " ]\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pAkj-XFVGm9N" + }, + "source": "Let's see what image we get for the query \"*Plane components*\"" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DY2PvcZVnZwN" + }, + "outputs": [], + "source": [ + "from PIL import Image\n", + "\n", + "with headers({\"cohere-api-key\": cohere_api_key}):\n", + " image_path = client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=models.Document(\n", + " text=\"Plane components\",\n", + " model=\"cohere/embed-v4.0\",\n", + " options={\n", + " \"output_dimension\": 512\n", + " }\n", + " ),\n", + " using=\"image\",\n", + " with_payload=[\"image\"],\n", + " limit=1\n", + " ).points[0].payload['image']\n", + "\n", + "Image.open(image_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mg77HNQv7T3s" + }, + "source": [ + "Let's also run the same query in Italian (one of the 30+ languages supported by the model) and compare the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C7m5bOOM7T3s" + }, + "outputs": [], + "source": [ + "with headers({\"cohere-api-key\": cohere_api_key}):\n", + " image_path = client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=models.Document(\n", + " text=\"Componenti di un aereo\",\n", + " model=\"cohere/embed-v4.0\",\n", + " options={\n", + " \"output_dimension\": 512\n", + " }\n", + " ),\n", + " using=\"image\",\n", + " with_payload=[\"image\"],\n", + " limit=1\n", + " ).points[0].payload['image']\n", + "\n", + "Image.open(image_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IihSWFsnHfwx" + }, + "source": "Now let's do a reverse search for the following image:" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2ScFbT38I2rO" + }, + "outputs": [], + "source": [ + "Image.open(\"images/image-2.png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HLmPdjktJic7" + }, + "outputs": [], + "source": [ + "with headers({\"cohere-api-key\": cohere_api_key}):\n", + " client.query_points(\n", + " collection_name=COLLECTION_NAME,\n", + " query=models.Image(\n", + " image=image_to_base64_url(\"images/image-2.png\"),\n", + " model=\"cohere/embed-v4.0\",\n", + " options={\n", + " \"output_dimension\": 512\n", + " }\n", + " ),\n", + " # Now we are searching only among text vectors with our image query\n", + " using=\"text\",\n", + " with_payload=[\"caption\"],\n", + " limit=1\n", + " ).points[0].payload['caption']" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/multimodal-search/Multimodal_Search_with_Qwen.ipynb b/multimodal-search/Multimodal_Search_with_Qwen.ipynb deleted file mode 100644 index 50b7150..0000000 --- a/multimodal-search/Multimodal_Search_with_Qwen.ipynb +++ /dev/null @@ -1,244 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "YIVBTkiDAlfy" - }, - "source": [ - "### Tutorial\n", - "\n", - "We will use [**Qwen3-VL-Embeddings-2B**](https://huggingface.co/Qwen/Qwen3-VL-Embedding-2B) through [`sentence-transformers`](https://sbert.net) for generating multimodal embeddings and [**Qdrant**](http://qdrant.tech) for storing and retrieving them." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "TrkMVQD0H6Qu" - }, - "outputs": [], - "source": [ - "! pip install -q sentence-transformers qdrant-client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Jn6xx48IJf7D" - }, - "outputs": [], - "source": [ - "from qdrant_client import QdrantClient, models\n", - "from getpass import getpass\n", - "\n", - "# docker run -p 6333:6333 qdrant/qdrant\n", - "client = QdrantClient(url=\"http://localhost:6333\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vfkXbWRhBbPT" - }, - "source": [ - "Let's embed a very short selection of images and their captions in the **shared embedding space**." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "7ZZYV2aqKD5b" - }, - "outputs": [], - "source": [ - "from sentence_transformers import SentenceTransformer\n", - "\n", - "model = SentenceTransformer(\"Qwen/Qwen3-VL-Embedding-2B\")\n", - "\n", - "documents = [\n", - " {\"caption\": \"An image about plane emergency safety.\", \"image\": \"images/image-1.png\"},\n", - " {\"caption\": \"An image about airplane components.\", \"image\": \"images/image-2.png\"},\n", - " {\"caption\": \"An image about COVID safety restrictions.\", \"image\": \"images/image-3.png\"},\n", - " {\"caption\": \"A confidential image about UFO sightings.\", \"image\": \"images/image-4.png\"},\n", - " {\"caption\": \"An image about unusual footprints on Aralar 2011.\", \"image\": \"images/image-5.png\"},\n", - "]\n", - "\n", - "text_embeddings = model.encode_document([doc[\"caption\"] for doc in documents])\n", - "image_embeddings = model.encode_document([doc[\"image\"] for doc in documents])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yEn3HWTiYl_u" - }, - "source": [ - "Create a **Collection**" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "98oNUlicaiXK" - }, - "outputs": [], - "source": [ - "COLLECTION_NAME = \"multimodal-embeddings\"\n", - "\n", - "if not client.collection_exists(COLLECTION_NAME):\n", - " client.create_collection(\n", - " collection_name=COLLECTION_NAME,\n", - " vectors_config={\n", - " \"image\": models.VectorParams(size=len(image_embeddings[0]), distance=models.Distance.COSINE),\n", - " \"text\": models.VectorParams(size=len(text_embeddings[0]), distance=models.Distance.COSINE),\n", - " }\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4vwr7UbRFV5M" - }, - "source": [ - "Now let's upload our images with captions to the **Collection**. Each image with its caption will create a [Point](https://qdrant.tech/documentation/concepts/points/) in Qdrant." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "mpqu-qzbP8Eh" - }, - "outputs": [], - "source": [ - "client.upload_points(\n", - " collection_name=COLLECTION_NAME,\n", - " points=[\n", - " models.PointStruct(\n", - " id=idx,\n", - " vector={\n", - " \"text\": text_embeddings[idx],\n", - " \"image\": image_embeddings[idx],\n", - " },\n", - " payload=doc\n", - " )\n", - " for idx, doc in enumerate(documents)\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pAkj-XFVGm9N" - }, - "source": [ - "Let'see what image we will get to the query \"*Adventures on snow hills*\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DY2PvcZVnZwN" - }, - "outputs": [], - "source": [ - "from PIL import Image\n", - "\n", - "find_image = model.encode_query(\"Plane components\")\n", - "\n", - "Image.open(client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=find_image,\n", - " using=\"image\",\n", - " with_payload=[\"image\"],\n", - " limit=1\n", - ").points[0].payload['image'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mg77HNQv7T3s" - }, - "source": [ - "Let's also run the same query in Italian (one of the 30+ languages supported by the model) and compare the results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "C7m5bOOM7T3s" - }, - "outputs": [], - "source": [ - "Image.open(client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=model.encode_query(\"Componenti di un aereo\"),\n", - " using=\"image\",\n", - " with_payload=[\"image\"],\n", - " limit=1\n", - ").points[0].payload['image'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IihSWFsnHfwx" - }, - "source": [ - "Now let's do a reverse search for the follwing image:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2ScFbT38I2rO" - }, - "outputs": [], - "source": [ - "Image.open('images/image-2.png')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HLmPdjktJic7" - }, - "outputs": [], - "source": [ - "client.query_points(\n", - " collection_name=COLLECTION_NAME,\n", - " query=model.encode_query(\"images/image-2.png\"),\n", - " # Now we are searching only among text vectors with our image query\n", - " using=\"text\",\n", - " with_payload=[\"caption\"],\n", - " limit=1\n", - ").points[0].payload['caption']" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file