Merge pull request #55 from epsilla-cloud/dev

support epsilla cloud into langchain

Author: eric-epsilla

examples/Question_Answering_Pipeline_with_LangChain_and_Epsilla.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+
+# Question Answering Pipeline with LangChain and Epsilla
+# Step1. Install the required packages
+"""
+pip install langchain
+pip install openai
+pip install tiktoken
+pip install pyepsilla
+pip install -U langchain-community
+pip install -U langchain-openai
+"""
+
+
+# Step2. Configure the OpenAI API Key
+import os
+os.environ["OPENAI_API_KEY"] = "Your-OpenAI-API-Key"
+
+
+# Step3. Load the documents
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+
+loader = WebBaseLoader("https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt")
+documents = loader.load()
+documents = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0).split_documents(documents)
+embeddings = OpenAIEmbeddings()
+
+
+# Step4. Load the vector store
+from langchain_community.vectorstores import Epsilla
+from pyepsilla import vectordb
+
+db_client = vectordb.Client(protocol="https", host="demo.epsilla.com", port="443")
+
+status_code, response = db_client.load_db("MyDB", "/data/MyDB")
+print(status_code, response)
+
+vector_store = Epsilla.from_documents(
+    documents,
+    embeddings,
+    db_client,
+    db_path="/data/MyDB",
+    db_name="MyDB",
+    collection_name="MyCollection",
+)
+
+
+
+
+# Step4. Create the QA for Retrieval
+from langchain.chains import RetrievalQA
+from langchain_openai import OpenAI
+
+qa = RetrievalQA.from_chain_type(
+    llm=OpenAI(), chain_type="stuff", retriever=vector_store.as_retriever()
+)
+query = "What did the president say about Ketanji Brown Jackson"
+resp = qa.invoke(query)
+print("resp:", resp)

examples/Question_Answering_Pipeline_with_LangChain_and_EpsillaCloud.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+
+# Question Answering Pipeline with LangChain and Epsilla
+# Step1. Install the required packages
+"""
+pip install langchain
+pip install openai
+pip install tiktoken
+pip install pyepsilla
+pip install -U langchain-openai
+pip install -U langchain-community
+"""
+
+
+# Step2. Configure the OpenAI API Key
+import os
+
+os.environ["OPENAI_API_KEY"] = "Your-OpenAI-API-Key"
+epsilla_api_key = os.getenv("EPSILLA_API_KEY", "Your-Epsilla-API-Key")
+project_id = os.getenv("EPSILLA_PROJECT_ID", "Your-Project-ID")
+db_id = os.getenv("EPSILLA_DB_ID", "Your-DB-ID")
+db_sharding_id = os.getenv("EPSILLA_DB_SHARDING_ID", 0)
+
+
+# Step3. Load the documents
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+
+loader = WebBaseLoader("https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt")
+documents = loader.load()
+documents = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0).split_documents(documents)
+embeddings = OpenAIEmbeddings()
+
+
+# Step4. Load the vector store
+from langchain_community.vectorstores import Epsilla
+from pyepsilla import cloud, vectordb
+
+db_name = f"db_{db_id.replace('-', '_')}"
+db_path = f"/data/{project_id}/{db_name}/s{db_sharding_id}"
+table_name = "MyCollection"
+
+# Connect to Epsilla Cloud
+cloud_client = cloud.Client(
+    project_id=project_id,
+    api_key=epsilla_api_key,
+)
+
+# Connect to Vectordb
+db_client = cloud_client.vectordb(db_id)
+
+vector_store = Epsilla.from_documents(
+    documents,
+    embeddings,
+    db_client,
+    db_path=db_path,
+    db_name=db_name,
+    collection_name=table_name,
+)
+
+# Step4. Create the QA for Retrieval
+from langchain.chains import RetrievalQA
+from langchain_openai import OpenAI
+
+qa = RetrievalQA.from_chain_type(
+    llm=OpenAI(), chain_type="stuff", retriever=vector_store.as_retriever()
+)
+query = "What did the president say about Ketanji Brown Jackson"
+resp = qa.invoke(query)
+print("resp:", resp)

pyepsilla/cloud/client.py

@@ -27,10 +27,11 @@ def __init__(self, project_id: str, api_key: str, headers: dict = None):
         }
         if headers is not None:
             self._header.update(headers)
+        self._db_id = None
 
     def validate(self):
         resp = requests.get(
-            url=self._baseurl + "/vectordb/list",
+            url=f"{self._baseurl}/vectordb/list",
             data=None,
             headers=self._header,
             verify=False,
@@ -42,7 +43,7 @@ def validate(self):
 
     def get_db_list(self):
         db_list = []
-        req_url = "{}/vectordb/list".format(self._baseurl)
+        req_url = f"{self._baseurl}/vectordb/list"
         resp = requests.get(url=req_url, data=None, headers=self._header, verify=False)
         status_code = resp.status_code
         body = resp.json()
@@ -52,8 +53,22 @@ def get_db_list(self):
         del resp
         return db_list
 
+    def load_db(self, db_name: str, db_path: str):
+        db_id = db_name.lstrip("db_").replace("_", "-")
+        req_url = f"{self._baseurl}/vectordb/{db_id}/load"
+        resp = requests.post(url=req_url, data=None, headers=self._header, verify=False)
+        status_code = resp.status_code
+        body = resp.json()
+        resp.close()
+        del resp
+        return status_code, body
+
+    def use_db(self, db_name: str):
+        self._db_id = db_name.lstrip("db_").replace("_", "-")
+        return 200, {"statusCode": 200, "message": "", "result": {}}
+
     def get_db_info(self, db_id: str):
-        req_url = "{}/vectordb/{}".format(self._baseurl, db_id)
+        req_url = f"{self._baseurl}/vectordb/{db_id}"
         resp = requests.get(url=req_url, data=None, headers=self._header, verify=False)
         status_code = resp.status_code
         body = resp.json()
@@ -62,7 +77,7 @@ def get_db_info(self, db_id: str):
         return status_code, body
 
     def get_db_statistics(self, db_id: str):
-        req_url = "{}/vectordb/{}/statistics".format(self._baseurl, db_id)
+        req_url = f"{self._baseurl}/vectordb/{db_id}/statistics"
         req_data = None
         resp = requests.get(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
@@ -121,7 +136,7 @@ def __init__(
     def list_tables(self):
         if self._db_id is None:
             raise Exception("[ERROR] db_id is None!")
-        req_url = "{}/table/list".format(self._baseurl)
+        req_url = f"{self._baseurl}/table/list"
         resp = requests.get(url=req_url, headers=self._header, verify=False)
         status_code = resp.status_code
         body = resp.json()
@@ -140,7 +155,7 @@ def create_table(
             raise Exception("[ERROR] db_id is None!")
         if table_fields is None:
             table_fields = []
-        req_url = "{}/table/create".format(self._baseurl)
+        req_url = f"{self._baseurl}/table/create"
         req_data = {"name": table_name, "fields": table_fields}
         if indices is not None:
             req_data["indices"] = indices
@@ -157,7 +172,7 @@ def create_table(
     def drop_table(self, table_name: str):
         if self._db_id is None:
             raise Exception("[ERROR] db_id is None!")
-        req_url = "{}/table/delete?table_name={}".format(self._baseurl, table_name)
+        req_url = f"{self._baseurl}/table/delete?table_name={table_name}"
         req_data = {}
         resp = requests.delete(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
@@ -170,7 +185,7 @@ def drop_table(self, table_name: str):
 
     # Insert data into table
     def insert(self, table_name: str, records: list[dict]):
-        req_url = "{}/data/insert".format(self._baseurl)
+        req_url = f"{self._baseurl}/data/insert"
         req_data = {"table": table_name, "data": records}
         resp = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
@@ -182,7 +197,7 @@ def insert(self, table_name: str, records: list[dict]):
         return status_code, body
 
     def upsert(self, table_name: str, records: list[dict]):
-        req_url = "{}/data/insert".format(self._baseurl)
+        req_url = f"{self._baseurl}/data/insert"
         req_data = {"table": table_name, "data": records, "upsert": True}
         resp = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
@@ -207,7 +222,7 @@ def query(
         with_distance: Optional[bool] = False,
         facets: Optional[list[dict]] = None,
     ):
-        req_url = "{}/data/query".format(self._baseurl)
+        req_url = f"{self._baseurl}/data/query"
         req_data = {"table": table_name, "limit": limit}
 
         if response_fields is None:
@@ -272,7 +287,7 @@ def delete(
                 "[WARN] Both primary_keys and ids are prvoided, will use primary keys by default!"
             )
 
-        req_url = "{}/data/delete".format(self._baseurl)
+        req_url = f"{self._baseurl}/data/delete"
         req_data = {"table": table_name}
         if primary_keys is not None:
             req_data["primaryKeys"] = primary_keys
@@ -335,7 +350,7 @@ def get(
             else:
                 req_data["facets"] = facets
 
-        req_url = "{}/data/get".format(self._baseurl)
+        req_url = f"{self._baseurl}/data/get"
         resp = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )

pyepsilla/vectordb/version.py

@@ -1 +1 @@
-__version__ = "0.3.8"
+__version__ = "0.3.9"