Merge pull request #41 from epsilla-cloud/auto-embedding

Support auto embedding

Author: richard-epsilla

examples/hello_epsilla_cloud_embedding.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+# Try this simple example for epsilla cloud
+# 1. create project and db on epsilla cloud
+# 2. create a table with schema in db
+# 3. get the api key with project id, run this program
+
+import sys
+
+from pyepsilla import cloud
+
+# Connect to Epsilla Cloud
+client = cloud.Client(
+    project_id="7a68814c-f839-4a67-9ec6-93c027c865e6",
+    api_key="epsilla-cloud-api-key",
+)
+
+# Connect to Vectordb
+db = client.vectordb(db_id="6accafb1-476d-43b0-aa64-12ebfbf7442d")
+
+
+# Create a table with schema
+status_code, response = db.create_table(
+    table_name="MyTable",
+    table_fields=[
+        {"name": "ID", "dataType": "INT", "primaryKey": True},
+        {"name": "Doc", "dataType": "STRING"},
+    ],
+    indices=[
+      {"name": "Index", "field": "Doc"},
+    ]
+)
+print(status_code, response)
+
+
+# Insert new vector records into table
+status_code, response = db.insert(
+    table_name="MyTable",
+    records=[
+        {"ID": 1, "Doc": "The garden was blooming with vibrant flowers, attracting butterflies and bees with their sweet nectar."},
+        {"ID": 2, "Doc": "In the busy city streets, people rushed to and fro, hardly noticing the beauty of the day."},
+        {"ID": 3, "Doc": "The library was a quiet haven, filled with the scent of old books and the soft rustling of pages."},
+        {"ID": 4, "Doc": "High in the mountains, the air was crisp and clear, revealing breathtaking views of the valley below."},
+        {"ID": 5, "Doc": "At the beach, children played joyfully in the sand, building castles and chasing the waves."},
+        {"ID": 6, "Doc": "Deep in the forest, a deer cautiously stepped out, its ears alert for any signs of danger."},
+        {"ID": 7, "Doc": "The old town's historical architecture spoke volumes about its rich cultural past."},
+        {"ID": 8, "Doc": "Night fell, and the sky was a canvas of stars, shining brightly in the moon's soft glow."},
+        {"ID": 9, "Doc": "A cozy cafe on the corner served the best hot chocolate, warming the hands and hearts of its visitors."},
+        {"ID": 10, "Doc": "The artist's studio was cluttered but inspiring, filled with unfinished canvases and vibrant paints."},
+    ],
+)
+print(status_code, response)
+
+# Query Vectors with specific response field, otherwise it will return all fields
+status_code, response = db.query(
+    table_name="MyTable",
+    query_text="Where can I find a serene environment, ideal for relaxation and introspection?",
+)
+print(status_code, response)
+
+
+# Delete specific records from table
+status_code, response = db.delete(table_name="MyTable", primary_keys=[4, 5])
+status_code, response = db.delete(table_name="MyTable", filter="Doc <> 'A cozy cafe on the corner served the best hot chocolate, warming the hands and hearts of its visitors.'")
+print(status_code, response)
+
+# Drop table
+status_code, response = db.drop_table(table_name="MyTable")
+print(status_code, response)

examples/hello_epsilla_embedding.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+# Try this simple example
+# 1. docker run --pull=always -d -p 8888:8888 epsilla/vectordb
+# 2. pip3 install --upgrade pyepsilla
+# 3. python3 simple_example.py
+#
+
+from pyepsilla import vectordb
+
+# Connect to Epsilla VectorDB
+client = vectordb.Client(protocol="http", host="127.0.0.1", port="8888")
+
+# You can also use Epsilla Cloud
+# client = vectordb.Client(protocol='https', host='demo.epsilla.com', port='443')
+
+# Load DB with path
+## pay attention to change db_path to persistent volume for production environment
+status_code, response = client.load_db(db_name="MyDB", db_path="/data/epsilla_demo")
+print(response)
+
+# Set DB to current DB
+client.use_db(db_name="MyDB")
+
+# Create a table with schema in current DB
+status_code, response = client.create_table(
+    table_name="MyTable",
+    table_fields=[
+        {"name": "ID", "dataType": "INT", "primaryKey": True},
+        {"name": "Doc", "dataType": "STRING"},
+    ],
+    indices=[
+      {"name": "Index", "field": "Doc"},
+    ]
+)
+print(response)
+
+# Get a list of table names in current DB
+status_code, response = client.list_tables()
+print(response)
+
+# Insert new vector records into table
+status_code, response = client.insert(
+    table_name="MyTable",
+    records=[
+        {"ID": 1, "Doc": "The garden was blooming with vibrant flowers, attracting butterflies and bees with their sweet nectar."},
+        {"ID": 2, "Doc": "In the busy city streets, people rushed to and fro, hardly noticing the beauty of the day."},
+        {"ID": 3, "Doc": "The library was a quiet haven, filled with the scent of old books and the soft rustling of pages."},
+        {"ID": 4, "Doc": "High in the mountains, the air was crisp and clear, revealing breathtaking views of the valley below."},
+        {"ID": 5, "Doc": "At the beach, children played joyfully in the sand, building castles and chasing the waves."},
+        {"ID": 6, "Doc": "Deep in the forest, a deer cautiously stepped out, its ears alert for any signs of danger."},
+        {"ID": 7, "Doc": "The old town's historical architecture spoke volumes about its rich cultural past."},
+        {"ID": 8, "Doc": "Night fell, and the sky was a canvas of stars, shining brightly in the moon's soft glow."},
+        {"ID": 9, "Doc": "A cozy cafe on the corner served the best hot chocolate, warming the hands and hearts of its visitors."},
+        {"ID": 10, "Doc": "The artist's studio was cluttered but inspiring, filled with unfinished canvases and vibrant paints."},
+    ],
+)
+print(response)
+
+# Query Vectors with specific response field
+status_code, response = client.query(
+    table_name="MyTable",
+    query_text="Where can I find a serene environment, ideal for relaxation and introspection?",
+    response_fields=["Doc"],
+    with_distance=True,
+    limit=3
+)
+print(response)
+
+# Query Vectors without specific response field, then it will return all fields
+status_code, response = client.query(
+    table_name="MyTable",
+    query_text="Where can I find a serene environment, ideal for relaxation and introspection?",
+)
+print(response)
+
+# Get Vectors
+status_code, response = client.get(table_name="MyTable", limit=2)
+print(response)
+
+# status_code, response =  client.delete(table_name="MyTable", ids=[3])
+status_code, response = client.delete(table_name="MyTable", primary_keys=[1, 3, 4])
+# status_code, response =  client.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
+print(response)
+
+
+# Drop table
+status_code, response = client.drop_table("MyTable")
+print(response)
+
+# Unload db
+status_code, response = client.unload_db("MyDB")
+print(response)

pyepsilla/cloud/client.py

@@ -16,7 +16,7 @@
 
 
 class Client(object):
-    def __init__(self, project_id: str, api_key: str):
+    def __init__(self, project_id: str, api_key: str, headers: dict = None):
         self._project_id = project_id
         self._apikey = api_key
         self._baseurl = "https://dispatch.epsilla.com/api/v3/project/{}".format(
@@ -28,6 +28,8 @@ def __init__(self, project_id: str, api_key: str):
             "Connection": "close",
             "X-API-Key": api_key,
         }
+        if headers is not None:
+            self._header.update(headers)
 
     def validate(self):
         res = requests.get(
@@ -85,7 +87,7 @@ def vectordb(self, db_id: str):
 
 
 class Vectordb(Client):
-    def __init__(self, project_id: str, db_id: str, api_key: str, public_endpoint: str):
+    def __init__(self, project_id: str, db_id: str, api_key: str, public_endpoint: str, headers: dict = None):
         self._project_id = project_id
         self._db_id = db_id
         self._api_key = api_key
@@ -94,6 +96,8 @@ def __init__(self, project_id: str, db_id: str, api_key: str, public_endpoint: s
             self._public_endpoint, self._project_id, self._db_id
         )
         self._header = {"Content-type": "application/json", "X-API-Key": self._api_key}
+        if headers is not None:
+            self._header.update(headers)
 
     # List table
     def list_tables(self):
@@ -107,13 +111,15 @@ def list_tables(self):
         return status_code, body
 
     # Create table
-    def create_table(self, table_name: str, table_fields: list[str] = None):
+    def create_table(self, table_name: str, table_fields: list[dict] = None, indices: list[dict] = None):
         if self._db_id is None:
             raise Exception("[ERROR] db_id is None!")
         if table_fields is None:
             table_fields = []
         req_url = "{}/table/create".format(self._baseurl)
         req_data = {"name": table_name, "fields": table_fields}
+        if indices is not None:
+            req_data["indices"] = indices
         res = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )
@@ -152,6 +158,8 @@ def insert(self, table_name: str, records: list[dict]):
     def query(
         self,
         table_name: str,
+        query_text: str = None,
+        query_index: str = None,
         query_field: str = None,
         query_vector: Union[list, dict] = None,
         response_fields: Optional[list] = None,
@@ -161,6 +169,10 @@ def query(
     ):
         req_url = "{}/data/query".format(self._baseurl)
         req_data = {"table": table_name}
+        if query_text is not None:
+            req_data["query"] = query_text
+        if query_index is not None:
+            req_data["queryIndex"] = query_index
         if query_field != None:
             req_data["queryField"] = query_field
         if query_vector != None:

pyepsilla/enterprise/client.py

@@ -25,7 +25,7 @@ class DbModel(BaseModel):
 
 
 class Client(cloud.Client):
-    def __init__(self, base_url: str, project_id: Optional[str] = "default"):
+    def __init__(self, base_url: str, project_id: Optional[str] = "default", headers: dict = None):
         self._project_id = project_id
         self._baseurl = f"{base_url}/api/v3/project/{project_id}"
         self._timeout = 10
@@ -34,6 +34,8 @@ def __init__(self, base_url: str, project_id: Optional[str] = "default"):
             "Connection": "close",
             "accept": "application/json",
         }
+        if headers is not None:
+            self._header.update(headers)
         self._db = None
 
     def hello(self):
@@ -173,13 +175,15 @@ def list_tables(self):
         return status_code, body
 
     # Create table
-    def create_table(self, table_name: str, table_fields: list[str] = None):
+    def create_table(self, table_name: str, table_fields: list[dict] = None, indices: list[dict] = None):
         if self._db_id is None:
             raise Exception("[ERROR] db_id is None!")
         if table_fields is None:
             table_fields = []
         req_url = "{}/table/create".format(self._baseurl)
-        req_data = {"table_name": table_name, "fields": table_fields}
+        req_data = {"name": table_name, "fields": table_fields}
+        if indices is not None:
+            req_data["indices"] = indices
         res = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )
@@ -222,6 +226,8 @@ def insert(self, table_name: str, records: list[dict]):
     def query(
         self,
         table_name: str,
+        query_text: str = None,
+        query_index: str = None,
         query_field: str = None,
         query_vector: Union[list, dict] = None,
         response_fields: Optional[list] = None,
@@ -231,6 +237,10 @@ def query(
     ):
         req_url = "{}/data/query".format(self._baseurl)
         req_data = {"table": table_name}
+        if query_text is not None:
+            req_data["query"] = query_text
+        if query_index is not None:
+            req_data["queryIndex"] = query_index
         if query_field != None:
             req_data["queryField"] = query_field
         if query_vector != None:

pyepsilla/vectordb/client.py

@@ -17,7 +17,7 @@
 
 class Client:
     def __init__(
-        self, protocol: str = "http", host: str = "localhost", port: str = "8888"
+        self, protocol: str = "http", host: str = "localhost", port: str = "8888", headers: dict = None
     ):
         self._protocol = protocol
         self._host = host
@@ -26,6 +26,8 @@ def __init__(
         self._db = None
         self._timeout = 10
         self._header = {"Content-type": "application/json", "Connection": "close"}
+        if headers is not None:
+            self._header.update(headers)
         self.check_networking()
 
     def check_networking(self):
@@ -100,13 +102,15 @@ def unload_db(self, db_name: str):
         res.close()
         return status_code, body
 
-    def create_table(self, table_name: str, table_fields: list[str] = None):
+    def create_table(self, table_name: str, table_fields: list[dict] = None, indices: list[dict] = None):
         if self._db is None:
             raise Exception("[ERROR] Please use_db() first!")
         if table_fields is None:
             table_fields = []
         req_url = "{}/api/{}/schema/tables".format(self._baseurl, self._db)
         req_data = {"name": table_name, "fields": table_fields}
+        if indices is not None:
+            req_data["indices"] = indices
         res = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )
@@ -203,29 +207,35 @@ def rebuild(self, timeout: int = 7200):
     def query(
         self,
         table_name: str,
-        query_field: str = "",
+        query_text: str = None,
+        query_index: str = None,
+        query_field: str = None,
         query_vector: Union[list, dict] = None,
         response_fields: list = None,
-        limit: int = 1,
+        limit: int = 2,
         filter: str = "",
         with_distance: bool = False,
     ):
         if self._db is None:
             raise Exception("[ERROR] Please use_db() first!")
-        if query_vector is None:
-            query_vector = []
         if response_fields is None:
             response_fields = []
         req_url = "{}/api/{}/data/query".format(self._baseurl, self._db)
         req_data = {
             "table": table_name,
-            "queryField": query_field,
-            "queryVector": query_vector,
             "response": response_fields,
             "limit": limit,
             "filter": filter,
             "withDistance": with_distance,
         }
+        if query_text is not None:
+            req_data["query"] = query_text
+        if query_index is not None:
+            req_data["queryIndex"] = query_index
+        if query_field is not None:
+            req_data["queryField"] = query_field
+        if query_vector is not None:
+            req_data["queryVector"] = query_vector
         res = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )

pyepsilla/vectordb/version.py

@@ -1 +1 @@
-__version__ = "0.2.7"
+__version__ = "0.3.0"