From 3f6db58772c12e1071c08fa1ae2a9ab248fdcf43 Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Sat, 20 Jun 2026 07:23:29 +0300 Subject: [PATCH 1/6] docs: update todo list --- todo.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/todo.md b/todo.md index 2075c72..b897220 100644 --- a/todo.md +++ b/todo.md @@ -1,17 +1,14 @@ -# todo +# TODO List + +Status legend: + - [ ] todo + - [>] in progress + - [x] done + +## todo - [ ] address todo comments -- [ ] **US-001: User can submit a natural language query and receive a "no index" error** - *As a user, I want to submit a natural language question and get a clear error message when the documentation index has not been built, so that I know the system is not ready and I can take action (e.g., trigger a sync).* - - [ ] **Scenario 1.1:** Query submitted with no index built - Given the documentation index has not been built - When the user submits a natural language question - Then the system returns an error message: "Documentation index is not available. Please sync a repository first." - - [ ] **Scenario 1.2:** Query submitted after index is built - Given the documentation index has been built - When the user submits a natural language question - Then the system does not return the "no index" error - [ ] **US-002: User can submit a natural language query and receive a validation error for empty or malformed input** *As a user, I want to receive a validation error when I submit an empty or malformed query, so that I know my input was not accepted and I can correct it.* @@ -118,12 +115,24 @@ Then the web page displays the error message: "Documentation index is not available. Please sync a repository first." -# in progress - +## in progress -# done +## done - [x] brainstorm agent implementation - [x] refactor to use commands/events/message-bus - [x] Implement lexical search over document index + +## Discarded + +- [x] **US-001: User can submit a natural language query and receive a "no index" error** (this should be caught by bootstrap) + *As a user, I want to submit a natural language question and get a clear error message when the documentation index has not been built, so that I know the system is not ready and I can take action (e.g., trigger a sync).* + - [ ] **Scenario 1.1:** Query submitted with no index built + Given the documentation index has not been built + When the user submits a natural language question + Then the system returns an error message: "Documentation index is not available. Please sync a repository first." + - [ ] **Scenario 1.2:** Query submitted after index is built + Given the documentation index has been built + When the user submits a natural language question + Then the system does not return the "no index" error From 2343048a0dee80e44dc6f30cad73752ada6857b2 Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Sat, 20 Jun 2026 12:41:55 +0300 Subject: [PATCH 2/6] feat: validate user query --- docs/design.md | 7 ++++- src/docs_buddy/entrypoints/cli/__main__.py | 32 ++++++++-------------- src/docs_buddy/services/use_cases.py | 3 ++ todo.md | 31 ++++++++++----------- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/docs/design.md b/docs/design.md index 4de90b4..4de6431 100644 --- a/docs/design.md +++ b/docs/design.md @@ -2,6 +2,8 @@ ## Repository Structure +It matches the domain driven design architecture of the application + ### Main package The package is rooted at `src/docs_buddy`. @@ -18,12 +20,15 @@ Use case handlers, adapter interfaces, events and commands #### adapters -Infrastructure-level implementations +Infrastructure-level abstractions #### entrypoints This is the presentation layer exposed to the external world. +Bootstrapping happens here where the appropriate dependencies are +injected, event and command handlers registered by the message bus. + ### Tests Tests are structured as follows: diff --git a/src/docs_buddy/entrypoints/cli/__main__.py b/src/docs_buddy/entrypoints/cli/__main__.py index 33e2044..317e25f 100644 --- a/src/docs_buddy/entrypoints/cli/__main__.py +++ b/src/docs_buddy/entrypoints/cli/__main__.py @@ -16,6 +16,8 @@ datefmt="%Y-%m-%d %H:%M:%S", ) +log = logging.getLogger(__name__) + def _get_data_dir() -> Path: """Return the XDG data directory""" @@ -75,33 +77,23 @@ def main() -> None: message_bus.send(sync_repo_command) elif args.query: - # Use the first repo ID for search (multi-repo search not yet implemented) + # todo: Use the first repo ID for search (multi-repo search not yet implemented) repo_id = args.repo_ids[0] - if len(args.repo_ids) > 1: - print( - "Warning: multiple --repo-id given, searching only in repository:", - repo_id, - ) index_path = data_dir / "whoosh" / repo_id if not index_path.exists(): - print(f"No index found for {repo_id}. Run --update-sources first.") + log.error("No index found for %s. Run --update-sources first", repo_id) sys.exit(1) document_index = adapters.WhooshDocumentIndex(index_location=index_path) - query = domain.Query(args.query) - - results = services.search_index(query, document_index, max_results=5) - - if not results: - print("No results found.") - else: - for i, result in enumerate(results, start=1): - print("\nSearch Result:", i) - print("*" * 7) - print("Content:", result.content) - print("Path:", result.path) - print("Metadata:", result.metadata) + + try: + query = domain.Query(args.query) + except domain.InvalidQueryError as exc: + log.exception("Invalid query detected") + sys.exit(1) + + services.find_answer(query) else: parser.print_help() diff --git a/src/docs_buddy/services/use_cases.py b/src/docs_buddy/services/use_cases.py index dc5b5eb..d4a354f 100644 --- a/src/docs_buddy/services/use_cases.py +++ b/src/docs_buddy/services/use_cases.py @@ -186,3 +186,6 @@ def search_index( raise SearchIndexError("max results must be at least 1") return index.search(query, max_results) + + +def find_answer(query: domain.Query) -> None: ... diff --git a/todo.md b/todo.md index b897220..e5a3669 100644 --- a/todo.md +++ b/todo.md @@ -10,21 +10,6 @@ Status legend: - [ ] address todo comments -- [ ] **US-002: User can submit a natural language query and receive a validation error for empty or malformed input** - *As a user, I want to receive a validation error when I submit an empty or malformed query, so that I know my input was not accepted and I can correct it.* - - [ ] **Scenario 2.1:** Empty query submitted - Given the documentation index has been built - When the user submits an empty string as the query - Then the system returns a validation error: "Query cannot be empty." - - [ ] **Scenario 2.2:** Malformed query submitted (e.g., only whitespace) - Given the documentation index has been built - When the user submits a query consisting only of whitespace - Then the system returns a validation error: "Query cannot be empty." - - [ ] **Scenario 2.3:** Valid query submitted - Given the documentation index has been built - When the user submits a non-empty, non-whitespace query - Then the system does not return a validation error - - [ ] **US-003: User can submit a query and receive a hardcoded answer with a hardcoded citation** *As a user, I want to submit a query and receive a hardcoded answer with a hardcoded citation, so that I can see the end-to-end flow working (even if the answer is not real) and validate the response structure.* - [ ] **Scenario 3.1:** Query submitted returns hardcoded answer @@ -120,11 +105,25 @@ Status legend: ## done +- [x] **US-002: User can submit a natural language query and receive a validation error for empty or malformed input** + *As a user, I want to receive a validation error when I submit an empty or malformed query, so that I know my input was not accepted and I can correct it.* + - [x] **Scenario 2.1:** Empty query submitted + Given the documentation index has been built + When the user submits an empty string as the query + Then the system returns a validation error: "Query cannot be empty." + - [x] **Scenario 2.2:** Malformed query submitted (e.g., only whitespace) + Given the documentation index has been built + When the user submits a query consisting only of whitespace + Then the system returns a validation error: "Query cannot be empty." + - [x] **Scenario 2.3:** Valid query submitted + Given the documentation index has been built + When the user submits a non-empty, non-whitespace query + Then the system does not return a validation error - [x] brainstorm agent implementation - [x] refactor to use commands/events/message-bus - [x] Implement lexical search over document index -## Discarded +## Parked - [x] **US-001: User can submit a natural language query and receive a "no index" error** (this should be caught by bootstrap) *As a user, I want to submit a natural language question and get a clear error message when the documentation index has not been built, so that I know the system is not ready and I can take action (e.g., trigger a sync).* From 473a7b1eba7a7278346cb1074c853cb745faa4bc Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Sat, 20 Jun 2026 13:20:27 +0300 Subject: [PATCH 3/6] feat: return hard coded answer to query --- src/docs_buddy/domain/__init__.py | 11 +++++++++++ src/docs_buddy/entrypoints/cli/__main__.py | 8 ++++++-- src/docs_buddy/services/use_cases.py | 7 ++++++- tests/unit/test_services.py | 12 +++++++++++ todo.md | 23 +++++++++------------- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/docs_buddy/domain/__init__.py b/src/docs_buddy/domain/__init__.py index 571a5b0..730a5d0 100644 --- a/src/docs_buddy/domain/__init__.py +++ b/src/docs_buddy/domain/__init__.py @@ -92,6 +92,17 @@ def __str__(self): return json.dumps(asdict(self)) +@dataclass(frozen=True) +class QueryResponse: + """Structured response for a query""" + + answer: str + citations: list[str] + + def __str__(self): + return json.dumps(asdict(self)) + + def sliding_window(seq: Sequence, size: int, step: int) -> Iterator[dict]: """Returns chunks from the sequence""" return ({"chunk": seq[i : i + size], "index": i} for i in range(0, len(seq), step)) diff --git a/src/docs_buddy/entrypoints/cli/__main__.py b/src/docs_buddy/entrypoints/cli/__main__.py index 317e25f..785068e 100644 --- a/src/docs_buddy/entrypoints/cli/__main__.py +++ b/src/docs_buddy/entrypoints/cli/__main__.py @@ -90,10 +90,14 @@ def main() -> None: try: query = domain.Query(args.query) except domain.InvalidQueryError as exc: - log.exception("Invalid query detected") + log.error("Invalid query detected: %s", exc) sys.exit(1) - services.find_answer(query) + response = services.find_answer(query) + print(f"Answer: {response.answer}") + print("Citations:") + for citation in response.citations: + print(f" - {citation}") else: parser.print_help() diff --git a/src/docs_buddy/services/use_cases.py b/src/docs_buddy/services/use_cases.py index d4a354f..0f0a5ec 100644 --- a/src/docs_buddy/services/use_cases.py +++ b/src/docs_buddy/services/use_cases.py @@ -188,4 +188,9 @@ def search_index( return index.search(query, max_results) -def find_answer(query: domain.Query) -> None: ... +def find_answer(query: domain.Query) -> domain.QueryResponse: + """Return a hardcoded answer with a hardcoded citation.""" + return domain.QueryResponse( + answer="This is a placeholder answer.", + citations=["https://example.com/doc"], + ) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 403844c..5aa23a5 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -263,3 +263,15 @@ def test_can_search_index() -> None: for bad_value in bad_values: with pytest.raises(services.SearchIndexError): _ = services.search_index(query, index, max_results=bad_value) + + +def test_find_answer_returns_structured_response() -> None: + query = domain.Query("any query") + response = services.find_answer(query) + + assert isinstance(response, domain.QueryResponse) + assert isinstance(response.answer, str) + assert len(response.answer) > 0 + assert isinstance(response.citations, list) + assert len(response.citations) > 0 + assert all(isinstance(c, str) for c in response.citations) diff --git a/todo.md b/todo.md index e5a3669..b787871 100644 --- a/todo.md +++ b/todo.md @@ -10,20 +10,6 @@ Status legend: - [ ] address todo comments -- [ ] **US-003: User can submit a query and receive a hardcoded answer with a hardcoded citation** - *As a user, I want to submit a query and receive a hardcoded answer with a hardcoded citation, so that I can see the end-to-end flow working (even if the answer is not real) and validate the response structure.* - - [ ] **Scenario 3.1:** Query submitted returns hardcoded answer - Given the documentation index has been built - And the query is valid - When the user submits any query - Then the system returns a structured response containing: - - A hardcoded answer text (e.g., "This is a placeholder answer.") - - A list with one hardcoded citation (e.g., `["https://example.com/doc"]`) - - [ ] **Scenario 3.2:** Response structure is correct - Given the documentation index has been built - When the user submits a valid query - Then the response contains exactly two fields: `answer` (string) and `citations` (list of strings) - - [ ] **US-004: User can submit a query and receive a real answer from the document index (no AI agent yet)** *As a user, I want to submit a query and receive an answer synthesized from the document index using a simple deterministic strategy (e.g., return the top chunk text as the answer), so that I can validate the search and retrieval pipeline end-to-end.* - [ ] **Scenario 4.1:** Query returns top chunk as answer @@ -105,6 +91,15 @@ Status legend: ## done +- [x] **US-003: User can submit a query and receive a hardcoded answer with a hardcoded citation** + *As a user, I want to submit a query and receive a hardcoded answer with a hardcoded citation, so that I can see the end-to-end flow working (even if the answer is not real) and validate the response structure.* + - [x] **Scenario 3.1:** Query submitted returns hardcoded answer + Given the documentation index has been built + And the query is valid + When the user submits any query + Then the system returns a structured response containing: + - A hardcoded answer text (e.g., "This is a placeholder answer.") + - One hardcoded citation in the output text (e.g., `"https://example.com/doc"`) - [x] **US-002: User can submit a natural language query and receive a validation error for empty or malformed input** *As a user, I want to receive a validation error when I submit an empty or malformed query, so that I know my input was not accepted and I can correct it.* - [x] **Scenario 2.1:** Empty query submitted From 99e02e4860c9ba4cbf00423661021ac0b77e2ad4 Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Sat, 20 Jun 2026 14:13:21 +0300 Subject: [PATCH 4/6] feat: query returns answer from document index --- Makefile | 7 ++++-- src/docs_buddy/domain/__init__.py | 8 ++++-- src/docs_buddy/entrypoints/cli/__main__.py | 14 +++++++---- src/docs_buddy/services/use_cases.py | 21 +++++++++++----- tests/unit/test_services.py | 19 ++++++++------ todo.md | 29 +++++++++++----------- 6 files changed, 61 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 9cb9169..2d26456 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: all test mypy test_verbose +.PHONY: all test mypy test_verbose black_check -all: test mypy +all: test mypy black_check test: pytest tests/ @@ -10,3 +10,6 @@ mypy: test_verbose: pytest -s tests/ + +black_check: + black --check . diff --git a/src/docs_buddy/domain/__init__.py b/src/docs_buddy/domain/__init__.py index 730a5d0..7322712 100644 --- a/src/docs_buddy/domain/__init__.py +++ b/src/docs_buddy/domain/__init__.py @@ -82,7 +82,7 @@ def __str__(self): @dataclass(frozen=True) class QueryResult: - """Result of a user query""" + """Result of an index query""" content: str path: str @@ -94,7 +94,7 @@ def __str__(self): @dataclass(frozen=True) class QueryResponse: - """Structured response for a query""" + """Structured response for a user query""" answer: str citations: list[str] @@ -102,6 +102,10 @@ class QueryResponse: def __str__(self): return json.dumps(asdict(self)) + def __repr__(self): + cls_name = type(self).__name__ + return f"{cls_name}({self.answer!r}, {self.citations!r})" + def sliding_window(seq: Sequence, size: int, step: int) -> Iterator[dict]: """Returns chunks from the sequence""" diff --git a/src/docs_buddy/entrypoints/cli/__main__.py b/src/docs_buddy/entrypoints/cli/__main__.py index 785068e..42fa22a 100644 --- a/src/docs_buddy/entrypoints/cli/__main__.py +++ b/src/docs_buddy/entrypoints/cli/__main__.py @@ -93,11 +93,15 @@ def main() -> None: log.error("Invalid query detected: %s", exc) sys.exit(1) - response = services.find_answer(query) - print(f"Answer: {response.answer}") - print("Citations:") - for citation in response.citations: - print(f" - {citation}") + base_url = f"https://github.com/{repo_id}/blob/main/" + response = services.find_answer(query, document_index, base_url) + if not response.answer: + print("No results found.") + else: + print(f"Answer: {response.answer}") + print("Citations:") + for citation in response.citations: + print(f" - {citation}") else: parser.print_help() diff --git a/src/docs_buddy/services/use_cases.py b/src/docs_buddy/services/use_cases.py index 0f0a5ec..202587d 100644 --- a/src/docs_buddy/services/use_cases.py +++ b/src/docs_buddy/services/use_cases.py @@ -188,9 +188,18 @@ def search_index( return index.search(query, max_results) -def find_answer(query: domain.Query) -> domain.QueryResponse: - """Return a hardcoded answer with a hardcoded citation.""" - return domain.QueryResponse( - answer="This is a placeholder answer.", - citations=["https://example.com/doc"], - ) +def find_answer( + query: domain.Query, + index: DocumentIndex, + base_url: str | None = None, +) -> domain.QueryResponse: + """Return the top search result as answer + citation. + + If no results, returns an empty answer and citations list. + """ + results = search_index(query, index, max_results=1) + if not results: + return domain.QueryResponse(answer="", citations=[]) + top = results[0] + citation = f"{base_url}{top.path}" if base_url else top.path + return domain.QueryResponse(answer=top.content, citations=[citation]) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 5aa23a5..8fd3dbb 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -265,13 +265,18 @@ def test_can_search_index() -> None: _ = services.search_index(query, index, max_results=bad_value) -def test_find_answer_returns_structured_response() -> None: - query = domain.Query("any query") - response = services.find_answer(query) +def test_find_answer_returns_top_chunk() -> None: + source = ".chunks/test-repo" + dest = ".index/test-repo" + pipeline = adapters.FakeDocumentChunksPipeline(source, dest) + index = adapters.FakeIndex(pipeline) + services.index_document_chunks(pipeline, index) + + query = domain.Query("provider") + base_url = "https://github.com/test/docs/blob/main/" + response = services.find_answer(query, index, base_url) assert isinstance(response, domain.QueryResponse) - assert isinstance(response.answer, str) assert len(response.answer) > 0 - assert isinstance(response.citations, list) - assert len(response.citations) > 0 - assert all(isinstance(c, str) for c in response.citations) + assert len(response.citations) == 1 + assert response.citations[0].startswith(base_url) diff --git a/todo.md b/todo.md index b787871..cdc2fc9 100644 --- a/todo.md +++ b/todo.md @@ -10,21 +10,6 @@ Status legend: - [ ] address todo comments -- [ ] **US-004: User can submit a query and receive a real answer from the document index (no AI agent yet)** - *As a user, I want to submit a query and receive an answer synthesized from the document index using a simple deterministic strategy (e.g., return the top chunk text as the answer), so that I can validate the search and retrieval pipeline end-to-end.* - - [ ] **Scenario 4.1:** Query returns top chunk as answer - Given the documentation index has been built and contains indexed chunks - When the user submits a valid query - Then the system returns a structured response where: - - The `answer` field contains the text of the top-ranked chunk - - The `citations` field contains the document URL of that chunk - - [ ] **Scenario 4.2:** Query with no matching chunks - Given the documentation index has been built but contains no chunks matching the query - When the user submits a valid query - Then the system returns a structured response where: - - The `answer` field is an empty string - - The `citations` field is an empty list - - [ ] **US-005: User can submit a query and receive an answer synthesized by an AI agent using the search tool** *As a user, I want to submit a query and receive an answer synthesized by an AI agent that uses the search tool to retrieve relevant chunks, so that I get a natural language answer grounded in the documentation.* - [ ] **Scenario 5.1:** AI agent returns answer with citations @@ -91,6 +76,20 @@ Status legend: ## done +- [x] **US-004: User can submit a query and receive a real answer from the document index (no AI agent yet)** + *As a user, I want to submit a query and receive an answer synthesized from the document index using a simple deterministic strategy (e.g., return the top chunk text as the answer), so that I can validate the search and retrieval pipeline end-to-end.* + - [x] **Scenario 4.1:** Query returns top chunk as answer + Given the documentation index has been built and contains indexed chunks + When the user submits a valid query + Then the system returns a structured response where: + - The `answer` field contains the text of the top-ranked chunk + - The `citations` field contains the document URL of that chunk + - [x] **Scenario 4.2:** Query with no matching chunks + Given the documentation index has been built but contains no chunks matching the query + When the user submits a valid query + Then the system returns a structured response where: + - The `answer` field is an empty string + - The `citations` field is an empty list - [x] **US-003: User can submit a query and receive a hardcoded answer with a hardcoded citation** *As a user, I want to submit a query and receive a hardcoded answer with a hardcoded citation, so that I can see the end-to-end flow working (even if the answer is not real) and validate the response structure.* - [x] **Scenario 3.1:** Query submitted returns hardcoded answer From 3e47bf86b10cca3a382645e4889a0fcbdfa5e706 Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Sat, 20 Jun 2026 14:40:55 +0300 Subject: [PATCH 5/6] style: adopt gherkin style commentary in service tests --- tests/unit/test_services.py | 97 +++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 26 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 8fd3dbb..ef53791 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -9,23 +9,33 @@ def test_syncing_existing_repository() -> None: + # Given an already-cloned repository location = ".repo/programmer-ke/akash-docs-buddy" storage = adapters.FakeRepoStorage(location) storage.fake_is_cloned = True github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + + # When we synchronise services.sync_repository(github_url, storage) + + # Then a pull is performed assert len(storage.actions) == 1 [(action,)] = storage.actions assert action == "PULL" def test_syncing_non_existent_repo_and_can_clone() -> None: + # Given a non‑existent repository that can be cloned location = ".repo/programmer-ke/akash-docs-buddy" storage = adapters.FakeRepoStorage(location) storage.fake_is_cloned = False storage.fake_can_clone = True github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + + # When we synchronise services.sync_repository(github_url, storage) + + # Then a clone is performed with the correct URL and target assert len(storage.actions) == 1 [(action, url, target)] = storage.actions assert action == "CLONE" @@ -34,19 +44,24 @@ def test_syncing_non_existent_repo_and_can_clone() -> None: def test_syncing_non_existent_repo_and_cannot_clone() -> None: + # Given a non‑existent repository that cannot be cloned location = ".repo/programmer-ke/akash-docs-buddy" storage = adapters.FakeRepoStorage(location) storage.fake_is_cloned = False storage.fake_can_clone = False github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + # When we try to synchronise + # Then a RepositorySyncError is raised with pytest.raises(services.RepositorySyncError): services.sync_repository(github_url, storage) + # And no storage action was recorded assert len(storage.actions) == 0 def test_document_artifact_update_existing_content_replaced() -> None: + # Given existing processed artifacts in the destination destination = ".docs/programmer-ke/akash-docs-buddy" source = ".repo/programmer-ke/akash-docs-buddy" storage = adapters.FakeDocsStorage(source, destination) @@ -59,11 +74,12 @@ def test_document_artifact_update_existing_content_replaced() -> None: {"content": "old_foo", "path": "old_path_2.json"} ), } - storage.sink[destination] = existing_content + # When we update the document artifacts services.update_document_artifacts(storage, services.process_raw_document) + # Then a fresh temporary location is created assert len(storage.actions) == 3 [(action0, target0), (action1, target1), (action2, src, target2)] = storage.actions @@ -72,17 +88,21 @@ def test_document_artifact_update_existing_content_replaced() -> None: assert action0 == "MKDIR" assert target0 == expected_tmp_dir + # Then the old destination is removed assert action1 == "RMRF" assert target1 == destination + # Then the temporary location replaces the destination assert action2 == "MV" assert src == expected_tmp_dir assert target2 == destination + # And the new content differs from the old content assert storage.sink[destination] != existing_content def test_artifact_updates_existing_content_preserved_on_error() -> None: + # Given existing processed artifacts and a pipeline that will fail destination = ".docs/programmer-ke/akash-docs-buddy" source = ".repo/programmer-ke/akash-docs-buddy" storage = adapters.FakeDocsStorage(source, destination) @@ -95,38 +115,42 @@ def test_artifact_updates_existing_content_preserved_on_error() -> None: {"content": "old_foo", "path": "old_path_2.json"} ), } - storage.sink[destination] = existing_content - # create non json-serializable sources to trigger exception + # Corrupt the sources so processing raises an exception for k in storage.sources: storage.sources[k] = object() # type: ignore + # When updating document artifacts with pytest.raises(TypeError): services.update_document_artifacts(storage, services.process_raw_document) - # existing content preserved + # Then the existing content remains untouched assert storage.sink[destination] == existing_content - # temporary destination should have been cleared + # And the temporary directory was cleaned up expected_tmp_dir = destination + ".tmp" assert expected_tmp_dir not in storage.sink def test_raw_document_processing() -> None: - + # Given a source document source_key = "path/to/file.mdx" content = "some file content" + # When we process it as a raw document [(raw_doc, dest_key)] = list(services.process_raw_document(content, source_key)) + # Then the destination path is transformed correctly assert str(dest_key) == source_key.replace("/", "_").replace("mdx", "json") + + # And the document content and original path are preserved assert raw_doc.content == content assert raw_doc.path == source_key def test_metadata_extraction() -> None: - + # Given a document with metadata embedded as a prefix source_key = "path/to/file.md" source_path = "path_to_file.json" content = "some content" @@ -138,58 +162,62 @@ def fake_extractor(content): metadata, text = content.split("|") return ast.literal_eval(metadata), text + # When we annotate the document [(annotated_doc, dest_key)] = list( services.annotate_document( str(raw_document), source_path, metadata_extractor=fake_extractor ) ) + # Then the destination path is unchanged assert str(dest_key) == str(source_path) + + # Then the metadata is extracted correctly + assert annotated_doc.metadata == metadata + + # Then the content and original path are preserved assert annotated_doc.content == content assert annotated_doc.path == source_key - assert annotated_doc.metadata == metadata def test_document_chunking() -> None: """Test that documents are properly chunked with metadata preserved.""" - # Create an annotated document as a string + # Given an annotated document source_path = "docs/intro.json" metadata = {"title": "Introduction", "author": "Alice"} - content = "This is a sample document. " * 100 # Make it long enough to chunk + content = "This is a sample document. " * 100 - # Create an AnnotatedDocument and convert to string annotated_doc = domain.AnnotatedDocument( content=content, path=source_path, metadata=metadata ) raw_content = str(annotated_doc) - # Test chunking + # When we chunk the document results = list(services.chunk_document(raw_content, source_path)) - # Verify we got chunks + # Then multiple chunks are produced assert len(results) > 1 for chunk, dest_path in results: + # And each chunk is a DocumentChunk with preserved metadata and path assert isinstance(chunk, domain.DocumentChunk) assert chunk.metadata == metadata assert chunk.path == source_path - # check that chuck index is appended to path + # And the destination path includes the chunk index prefix, extension = source_path.rsplit(".", 1) assert str(dest_path).startswith(prefix) assert re.match(f"{prefix}_{chunk.index}\\.json", str(dest_path)) def test_composed_pipeline() -> None: - - # raw document + # Given a document with metadata and content source_key = "path/to/file.mdx" content = "some file content" * 1000 metadata = {"title": "foo", "author": "bar"} source_text = f"{metadata}|{content}" - # processors def fake_extractor(content): metadata, text = content.split("|") return ast.literal_eval(metadata), text @@ -197,44 +225,48 @@ def fake_extractor(content): annotate_document = functools.partial( services.annotate_document, metadata_extractor=fake_extractor ) - process_document = services.composed_processor( services.process_raw_document, annotate_document, services.chunk_document ) + # When we run the composed pipeline chunk_data = list(process_document(source_text, source_key)) + + # Then we get at least one chunk assert len(chunk_data) > 0 for chunk, path in chunk_data: + # And each chunk is a DocumentChunk with required fields assert isinstance(chunk, domain.DocumentChunk) assert isinstance(chunk.index, int) assert chunk.metadata == metadata assert str(path).endswith(".json") - # confirm that the paths are unique + # And all output paths are unique paths = {path for _, path in chunk_data} assert len(paths) == len(chunk_data) def test_can_index_documents() -> None: + # Given a fake chunks pipeline and an in‑memory index source = ".chunks/programmmer-ke/akash-docs-buddy" dest = ".index/programmer-ke/akash-docs-buddy" - pipeline = adapters.FakeDocumentChunksPipeline(source, dest) index = adapters.FakeIndex(pipeline) assert dest not in pipeline.sink + # When we index the document chunks services.index_document_chunks(pipeline, index) + # Then the index destination contains DocumentChunk items assert len(pipeline.sink[dest]) > 0 - for item in pipeline.sink[dest]: assert isinstance(item, domain.DocumentChunk) + # And the intermediate storage performed the expected operations [(action1, arg1), (action2, arg2), (action3, arg3_1, arg3_2)] = pipeline.actions - # assert correct order of operations tmp_location = f"{dest}.tmp" assert (action1, arg1) == ("MKDIR", tmp_location) assert (action2, arg2) == ("RMRF", dest) @@ -242,23 +274,26 @@ def test_can_index_documents() -> None: def test_can_search_index() -> None: + # Given an index built from a fake pipeline source = ".chunks/programmmer-ke/akash-docs-buddy" dest = ".index/programmer-ke/akash-docs-buddy" - pipeline = adapters.FakeDocumentChunksPipeline(source, dest) index = adapters.FakeIndex(pipeline) - services.index_document_chunks(pipeline, index) query = domain.Query(text="provider") + + # When we search without a limit results = services.search_index(query, index) + # Then we get at least one result assert len(results) > 0 - # can specify max results + # When we request exactly one result results = services.search_index(query, index, max_results=1) + # Then exactly one result is returned assert len(results) == 1 - # max results must be > 0 + # When max_results is <= 0, an error is raised bad_values = [0, -1, -30] for bad_value in bad_values: with pytest.raises(services.SearchIndexError): @@ -266,6 +301,7 @@ def test_can_search_index() -> None: def test_find_answer_returns_top_chunk() -> None: + # Given an indexed document set with a known query matching source = ".chunks/test-repo" dest = ".index/test-repo" pipeline = adapters.FakeDocumentChunksPipeline(source, dest) @@ -274,9 +310,18 @@ def test_find_answer_returns_top_chunk() -> None: query = domain.Query("provider") base_url = "https://github.com/test/docs/blob/main/" + + # When we retrieve an answer response = services.find_answer(query, index, base_url) + # Then the response is a QueryResponse assert isinstance(response, domain.QueryResponse) + + # Then the answer is non‑empty assert len(response.answer) > 0 + + # Then there is exactly one citation assert len(response.citations) == 1 + + # Then the citation starts with the provided base URL assert response.citations[0].startswith(base_url) From cecc3e19aab58ae8b235e9f76355cc50cd118fb6 Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Thu, 2 Jul 2026 12:44:16 +0300 Subject: [PATCH 6/6] agent returns structed response to user query --- notebooks/docs_searching_agent.org | 306 +++++++++++++++++++++ notebooks/openai_agent_sdk_intro.org | 101 +++++++ pyproject.toml | 1 + src/docs_buddy/adapters/__init__.py | 1 + src/docs_buddy/adapters/agent.py | 165 +++++++++++ src/docs_buddy/domain/__init__.py | 6 + src/docs_buddy/entrypoints/cli/__main__.py | 26 +- src/docs_buddy/services/use_cases.py | 39 ++- tests/integration/test_adapters.py | 32 ++- tests/unit/test_services.py | 60 ++-- tests/unit/test_tools.py | 36 +++ todo.md | 52 ++-- 12 files changed, 739 insertions(+), 86 deletions(-) create mode 100644 notebooks/docs_searching_agent.org create mode 100644 notebooks/openai_agent_sdk_intro.org create mode 100644 src/docs_buddy/adapters/agent.py create mode 100644 tests/unit/test_tools.py diff --git a/notebooks/docs_searching_agent.org b/notebooks/docs_searching_agent.org new file mode 100644 index 0000000..03568d3 --- /dev/null +++ b/notebooks/docs_searching_agent.org @@ -0,0 +1,306 @@ +#+PROPERTY: header-args :results output :session *docs-agent* + +Initialize agent with custom provider: + +#+begin_src python + +import asyncio +from dataclasses import dataclass, asdict +import os + +from openai import AsyncOpenAI + +from agents import ( + Agent, + Model, + ModelProvider, + OpenAIChatCompletionsModel, + RunConfig, + Runner, + RunHooks, + function_tool, + set_tracing_disabled, +) +from agents.agent import StopAtTools + +# set environment variables if necessary +#os.environ["OPENAI_API_KEY"] = "sk-..." +#os.environ["OPENAI_BASE_URL"] = "https://openrouter.ai/api/v1" + +BASE_URL = os.getenv("OPENAI_BASE_URL") or "" +API_KEY = os.getenv("OPENAI_API_KEY") or "" +MODEL_NAME = "deepseek/deepseek-v4-flash" + +if not BASE_URL or not API_KEY or not MODEL_NAME: + raise ValueError( + "Please set BASE_URL, API_KEY, MODEL_NAME" + ) + + +client = AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY) +set_tracing_disabled(disabled=True) + + +class CustomModelProvider(ModelProvider): + def get_model(self, model_name: str | None) -> Model: + return OpenAIChatCompletionsModel(model=model_name or MODEL_NAME, openai_client=client) + + +CUSTOM_MODEL_PROVIDER = CustomModelProvider() + +#+end_src + +#+RESULTS: + +Create searching tools: + +#+begin_src python +from pathlib import Path +import json + +from docs_buddy import adapters + +index_path = Path.home() / ".local" / "share" / "docs-buddy" / "whoosh" / "akash-network/website" + +index = adapters.WhooshDocumentIndex(index_path) + +search_tool = function_tool(adapters.make_search_tool(index)) + +@dataclass +class QueryResponse: + """A response to the user's query""" + final_answer: str + citations: list[str] + + def __str__(self): + return json.dumps(asdict(self)) + + + +@function_tool +def generate_final_answer(final_answer: str, citations: list[str]) -> QueryResponse: + """Generates the final answer to the user's query + + Args: + final_answer (str): The final answer to the user's query + citations (list[str]): A list of paths from the search tool results that were instrumental + in generating the answer + + Returns: + QueryResponse: A structured response to the user's query + """ + + return QueryResponse(final_answer, citations) + +#+end_src + +#+RESULTS: + +Trigger agentic search: + +#+RESULTS: + +#+begin_src python + +agent_instructions = """ +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +""" + + +class LoggingHooks(RunHooks): + async def on_agent_start(self, context, agent): + print(f"Starting agent: {agent.name}, usage: {context.usage.output_tokens}") + + async def on_llm_end(self, context, agent, response): + print(f"llm ended: agent {agent.name} produced output length{len(response.output)}, usage: {context.usage.output_tokens}") + + async def on_llm_start(self, context, agent, system_prompt, input_items): + print(f"llm started with: {agent.name}, usage: {context.usage.output_tokens}, prompt: {system_prompt}, inputs length {len(input_items)}") + + async def on_agent_end(self, context, agent, output): + print(f"agent ended: {agent.name} finished with usage: {context.usage.output_tokens}") + + async def on_tool_start(self, context, agent, tool): + print(f"tool {tool.name} started with: agent {agent.name}, usage: {context.usage.output_tokens}") + + async def on_tool_end(self, context, agent, tool, result): + print(f"tool {tool.name} ended with: agent {agent.name}, usage: {context.usage.output_tokens}") + + +async def main(user_query): + agent = Agent( + name="Docs Buddy Agent", + instructions=agent_instructions, + tools=[search_tool, generate_final_answer], + tool_use_behavior=StopAtTools(stop_at_tool_names=["generate_final_answer"]), + ) + + # This will use the custom model provider + result = await Runner.run( + agent, + user_query, + hooks=LoggingHooks(), + run_config=RunConfig(model_provider=CUSTOM_MODEL_PROVIDER), + ) + response = result.final_output + d = json.loads(response) + print(d['final_answer']) + for path in d['citations']: + print(f"https://github.com/akash-network/website/blob/main/{path}\n") + + +user_query="Hello, I have some spare dedicated servers I'd hopefully like to cover my costs on until I have a better use. I'm trying to figure out how much I can make from them but it's not very clear. I found a calculator but I have to enter the price per resource, which doesn't really help when I don't know market prices?" + +asyncio.run(main(user_query)) + +#+end_src + +#+RESULTS: +#+begin_example +Starting agent: Docs Buddy Agent, usage: 0 +llm started with: Docs Buddy Agent, usage: 0, prompt: +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +, inputs length 1 +llm ended: agent Docs Buddy Agent produced output length3, usage: 148 +tool search_document_index started with: agent Docs Buddy Agent, usage: 148 +tool search_document_index started with: agent Docs Buddy Agent, usage: 148 +search query: dedicated server pricing calculator earnings, results: 5 +search query: market prices resources compute cost, results: 5 +tool search_document_index ended with: agent Docs Buddy Agent, usage: 148 +tool search_document_index ended with: agent Docs Buddy Agent, usage: 148 +llm started with: Docs Buddy Agent, usage: 148, prompt: +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +, inputs length 6 +llm ended: agent Docs Buddy Agent produced output length3, usage: 292 +tool search_document_index started with: agent Docs Buddy Agent, usage: 292 +tool search_document_index started with: agent Docs Buddy Agent, usage: 292 +search query: Provider Earn Calculator how to use price per resource, results: 5search query: CPU memory GPU pricing market rates Akash provider, results: 5 + +tool search_document_index ended with: agent Docs Buddy Agent, usage: 292 +tool search_document_index ended with: agent Docs Buddy Agent, usage: 292 +llm started with: Docs Buddy Agent, usage: 292, prompt: +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +, inputs length 11 +llm ended: agent Docs Buddy Agent produced output length3, usage: 435 +tool search_document_index started with: agent Docs Buddy Agent, usage: 435 +tool search_document_index started with: agent Docs Buddy Agent, usage: 435 +search query: bid price scale CPU RAM disk pricing strategy provider earnings, results: 5search query: provider explorer active providers lease prices market, results: 5 + +tool search_document_index ended with: agent Docs Buddy Agent, usage: 435 +tool search_document_index ended with: agent Docs Buddy Agent, usage: 435 +llm started with: Docs Buddy Agent, usage: 435, prompt: +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +, inputs length 16 +llm ended: agent Docs Buddy Agent produced output length2, usage: 521 +tool search_document_index started with: agent Docs Buddy Agent, usage: 521 +search query: GPU pricing page akash.network/gpus market rates, results: 5 +tool search_document_index ended with: agent Docs Buddy Agent, usage: 521 +llm started with: Docs Buddy Agent, usage: 521, prompt: +You are a documentation assistant equipped to answer user queries +by searching the docs. + +IMPORTANT: Always call generate_final_answer as your final output. +, inputs length 19 +llm ended: agent Docs Buddy Agent produced output length2, usage: 1644 +tool generate_final_answer started with: agent Docs Buddy Agent, usage: 1644 +tool generate_final_answer ended with: agent Docs Buddy Agent, usage: 1644 +agent ended: Docs Buddy Agent finished with usage: 1644 +## Understanding Your Potential Earnings as an Akash Provider + +Great question! It's a common concern that the calculator asks you to input prices without giving you market context. Let me break down what the docs tell us. + +### 📊 The Provider Earn Calculator + +You can find it here: **[Provider Earn Calculator](https://akash.network/pricing/provider-calculator/)** + +It was specifically built by the Praetor team because many users struggled with the exact same issue — not knowing what prices to set for CPU, RAM, and Disk. The calculator uses the most recent Osmosis price for AKT-to-USD conversion to help you estimate your earnings. + +### 💰 General Revenue Estimates (from the docs) + +Based on the documentation's **"Should I Run a Provider?"** guide, here are the ballpark ranges: + +| Resource | Estimated Monthly Revenue | +|---|---| +| **CPU/Memory** | **$10–$100+/month** (highly variable) | +| **GPUs** | **$100–$1,000+/month** (high demand) | +| **Persistent Storage** | **$10–$50+/month** | + +> **Note:** Revenue depends heavily on your capacity, uptime, pricing strategy, and current market demand. + +### 🔧 How Pricing Actually Works (Pricing Strategy) + +You have **two options** for setting your pricing in the provider configuration: + +,**1. Simple Scale-Based Pricing** (easiest to start with) +You set multipliers in your `provider.yaml` like: +```yaml +bidpricecpuscale: 1.0 +bidpricememoryscale: 1.0 +bidpricestoragescale: 1.0 +bidpriceendpointscale: 1.0 +``` +The price is calculated as: `(cpu_units × cpu_scale) + (memory_units × memory_scale) + (storage_units × storage_scale) + (endpoints × endpoint_scale)` + +,**2. Custom Shell Script Pricing** (advanced) +Write a custom `price_script.sh` that takes order details as input and outputs your price in uact/block. This gives you maximum flexibility — you can dynamically adjust based on demand, time of day, etc. + +### 👀 How to See What Other Providers Are Charging + +This is the best way to figure out competitive pricing without guessing: + +- **Browser Active Providers** at the [Provider Explorer](https://console.akash.network/providers) — you can see active providers, their resources, and lease counts +- **Check the GPU Pricing Page** at [akash.network/gpus](https://akash.network/pricing/gpus/) — shows real-time pricing from providers for different GPU models +- For reference, GPU prices on Akash are roughly **3-5x cheaper** than AWS/GCP/Azure, e.g.: + - RTX 4090: ~$0.50–$1.50/hr + - A100 40GB: ~$1.50–$2.50/hr (vs $4.10/hr on AWS) + - H100: ~$2.50–$4.00/hr (vs $8.03/hr on AWS) + +### 🎯 Recommendations for Starting + +1. **Start conservative** — Use the **Provider Earn Calculator** with modest multipliers (try 0.5–1.0 for the scale values) to see baseline estimates +2. **Check existing providers** — Browse the Provider Explorer to see what other providers with similar hardware are offering +3. **Join the community** — The docs strongly recommend asking in the **#providers channel on Discord** ([discord.akash.network](https://discord.akash.network)) where experienced providers can give you specific advice for your hardware setup +4. **Consider GPUs if you have them** — GPU compute has the highest demand and best returns + +### ⚠️ Also Consider Your Costs + +Don't forget to subtract your ongoing expenses: +- **Electricity:** ~$20–$200+/month depending on hardware +- **Internet:** ~$50–$100/month (business connection recommended) +- **Maintenance:** Your time (2–4 hours/week average) +- **Bid fees:** ~0.005 AKT per bid submission +- **AKT for gas:** Have some AKT available for transaction fees + +Hope this helps you get a clearer picture! The key is to start with reasonable pricing, then adjust based on how many leases you're winning. +https://github.com/akash-network/website/blob/main/src/content/Docs/providers/getting-started/should-i-run-a-provider/index.md + +https://github.com/akash-network/website/blob/main/src/content/Docs/providers/getting-started/hardware-requirements/index.md + +https://github.com/akash-network/website/blob/main/src/content/Docs/providers/architecture/bid-engine/index.md + +https://github.com/akash-network/website/blob/main/src/content/Docs/learn/core-concepts/gpu-deployments/index.md + +https://github.com/akash-network/website/blob/main/src/content/Blog/how-to-become-an-akash-provider-in-20-minutes-or-less/index.md +#+end_example + + +#+begin_src python +#+end_src + +#+RESULTS: + diff --git a/notebooks/openai_agent_sdk_intro.org b/notebooks/openai_agent_sdk_intro.org new file mode 100644 index 0000000..551c351 --- /dev/null +++ b/notebooks/openai_agent_sdk_intro.org @@ -0,0 +1,101 @@ +#+PROPERTY: header-args :results output :session *shared* + +#+begin_src python + +import asyncio +import os + +from openai import AsyncOpenAI + +from agents import ( + Agent, + Model, + ModelProvider, + OpenAIChatCompletionsModel, + RunConfig, + Runner, + function_tool, + set_tracing_disabled, +) + +# set environment variables if necessary + +#os.environ["OPENAI_API_KEY"] = "sk-" +#os.environ["OPENAI_BASE_URL"] = "https://openrouter.ai/api/v1" + +BASE_URL = os.getenv("OPENAI_BASE_URL") or "" +API_KEY = os.getenv("OPENAI_API_KEY") or "" +MODEL_NAME = "deepseek/deepseek-v4-flash" + +if not BASE_URL or not API_KEY or not MODEL_NAME: + raise ValueError( + "Please set BASE_URL, API_KEY, MODEL_NAME" + ) + + +"""This example uses a custom provider for some calls to Runner.run(), and direct calls to OpenAI for +others. Steps: +1. Create a custom OpenAI client. +2. Create a ModelProvider that uses the custom client. +3. Use the ModelProvider in calls to Runner.run(), only when we want to use the custom LLM provider. + +Note that in this example, we disable tracing under the assumption that you don't have an API key +from platform.openai.com. If you do have one, you can either set the `OPENAI_API_KEY` env var +or call set_tracing_export_api_key() to set a tracing specific key. +""" +client = AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY) +set_tracing_disabled(disabled=True) + + +class CustomModelProvider(ModelProvider): + def get_model(self, model_name: str | None) -> Model: + return OpenAIChatCompletionsModel(model=model_name or MODEL_NAME, openai_client=client) + + +CUSTOM_MODEL_PROVIDER = CustomModelProvider() + +#+end_src + +#+begin_src python +@function_tool +def get_weather(city: str): + print(f"[debug] getting weather for {city}") + return f"The weather in {city} is sunny." + + +async def main(): + agent = Agent(name="Assistant", instructions="You only respond in haikus.", tools=[get_weather]) + + # This will use the custom model provider + result = await Runner.run( + agent, + "What's the weather in Nairobi?", + run_config=RunConfig(model_provider=CUSTOM_MODEL_PROVIDER), + ) + print(result.final_output) + + # If you uncomment this, it will use OpenAI directly, not the custom provider + # result = await Runner.run( + # agent, + # "What's the weather in Tokyo?", + # ) + # print(result.final_output) + + +if __name__ == "__main__": + asyncio.run(main()) + +#+end_src + +#+RESULTS: +: [debug] getting weather for Nairobi +: Sunny skies above +: Nairobi warms in the light +: Hope you enjoy it + + +#+begin_src python +#+end_src + +#+RESULTS: + diff --git a/pyproject.toml b/pyproject.toml index 436e86e..5c78c86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ requires-python = ">=3.11" dependencies = [ "python-frontmatter>=1.1.0", "Whoosh-Reloaded>=2.7.5", + "openai-agents>=0.17.7", ] [project.optional-dependencies] diff --git a/src/docs_buddy/adapters/__init__.py b/src/docs_buddy/adapters/__init__.py index dc70992..d5a66f0 100644 --- a/src/docs_buddy/adapters/__init__.py +++ b/src/docs_buddy/adapters/__init__.py @@ -16,6 +16,7 @@ from docs_buddy import domain, services from docs_buddy.services import events, commands from .whoosh_index import WhooshDocumentIndex, WhooshIndexError +from .agent import * log = logging.getLogger(__name__) diff --git a/src/docs_buddy/adapters/agent.py b/src/docs_buddy/adapters/agent.py new file mode 100644 index 0000000..7bc600f --- /dev/null +++ b/src/docs_buddy/adapters/agent.py @@ -0,0 +1,165 @@ +"""Functionality required by agents""" + +import asyncio +import json +from typing import Callable +import textwrap +import os + +import openai +import agents + +from docs_buddy import domain, services +from docs_buddy.common import DocsBuddyError + + +class AgentError(DocsBuddyError): + pass + + +class ToolError(DocsBuddyError): + pass + + +def make_search_tool(index: services.DocumentIndex) -> Callable: + """Creates a search tool over the index""" + + def search_document_index(phrase: str, max_results: int = 5) -> list[str]: + """Search the document index for the given phrase + + Args: + phrase (string): The phrase to search + max_results (int): The maximum number of results to return + + Returns: + list[str]: A list of JSON formatted strings representing the + the results. Each result has associated content, + path and metadata fields + """ + + if not max_results > 0: + raise ToolError( + f"maximum results should be greater than 0, got {max_results}" + ) + + query = domain.Query(phrase) + results = index.search(query, max_results) + return [str(r) for r in results] + + return search_document_index + + +def make_fake_research_agent(prompt: str) -> services.Agent: + """Creates a fake agent""" + + def fake_agent(query: domain.Query, tools: list[Callable]) -> domain.QueryResponse: + """Uses the search tool provided to get results matching the query""" + + search_tool, *_ = tools + results = search_tool(str(query)) + + try: + first_result, *_ = results + json_result = json.loads(first_result) + answer, citation = json_result["content"], json_result["path"] + except (TypeError, ValueError) as exc: + raise AgentError("Something went wrong") from exc + return domain.QueryResponse(answer, [citation]) + + return fake_agent + + +@agents.function_tool +def generate_final_response( + final_answer: str, citations: list[str] +) -> domain.QueryResponse: + """Generates the final response to the user's query in a structed format + + Args: + final_answer (str): The final answer to the user's query + citations (list[str]): A list of paths from the search tool results that were instrumental + in generating the answer + + Returns: + QueryResponse: A structured response to the user's query + """ + + try: + final_response = domain.QueryResponse(final_answer, citations) + except (TypeError, ValueError) as exc: + raise AgentError("Could not create response") from exc + + return final_response + + +def make_openai_research_agent(prompt: str) -> services.Agent: + """Creates an openai agent""" + + BASE_URL = os.getenv("OPENAI_BASE_URL") or "" + API_KEY = os.getenv("OPENAI_API_KEY") or "" + MODEL_NAME = os.getenv("DOCS_BUDDY_MODEL_NAME") or "" + + if not BASE_URL or not API_KEY or not MODEL_NAME: + raise AgentError( + "Please set OPENAI_BASE_URL, OPENAI_API_KEY, DOCS_BUDDY_MODEL_NAME" + ) + + client = openai.AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY) + agents.set_tracing_disabled(disabled=True) + + class CustomModelProvider(agents.ModelProvider): + def get_model(self, model_name: str | None) -> agents.Model: + return agents.OpenAIChatCompletionsModel( + model=model_name or MODEL_NAME, openai_client=client + ) + + CUSTOM_MODEL_PROVIDER = CustomModelProvider() + + def openai_agent( + query: domain.Query, tools: list[Callable] + ) -> domain.QueryResponse: + """OpenaAI compatible agent that answers user's query""" + + final_response_callout = ( + "IMPORTANT: Always call generate_final_response as your final output." + ) + + system_instructions = f"""\ + {prompt} + + {final_response_callout} + """ + + agent = agents.Agent( + name="Docs Buddy Agent", + instructions=textwrap.dedent(system_instructions), + tools=[agents.function_tool(func) for func in tools] # type: ignore[arg-type] + + [generate_final_response], + tool_use_behavior=agents.agent.StopAtTools( + stop_at_tool_names=["generate_final_response"] + ), + ) + + result = asyncio.run( + run_agent( + agent, + str(query), + agents.RunConfig(model_provider=CUSTOM_MODEL_PROVIDER), + ) + ) + + output: str = result.final_output + try: + response = domain.QueryResponse.fromstring(output) + except json.JSONDecodeError as exc: + raise AgentError("Could not parse query response from llm") from exc + return response + + return openai_agent + + +async def run_agent( + agent: agents.Agent, query: str, config: agents.RunConfig +) -> agents.RunResult: + """Run agent in event loop""" + return await agents.Runner.run(agent, query, run_config=config) diff --git a/src/docs_buddy/domain/__init__.py b/src/docs_buddy/domain/__init__.py index 7322712..5a4720c 100644 --- a/src/docs_buddy/domain/__init__.py +++ b/src/docs_buddy/domain/__init__.py @@ -106,6 +106,12 @@ def __repr__(self): cls_name = type(self).__name__ return f"{cls_name}({self.answer!r}, {self.citations!r})" + @classmethod + def fromstring(cls, text: str) -> "QueryResponse": + # todo: factor out json serialization + dict_ = json.loads(text) + return cls(**dict_) + def sliding_window(seq: Sequence, size: int, step: int) -> Iterator[dict]: """Returns chunks from the sequence""" diff --git a/src/docs_buddy/entrypoints/cli/__main__.py b/src/docs_buddy/entrypoints/cli/__main__.py index 42fa22a..219c6a0 100644 --- a/src/docs_buddy/entrypoints/cli/__main__.py +++ b/src/docs_buddy/entrypoints/cli/__main__.py @@ -5,6 +5,7 @@ import argparse import logging import sys +import textwrap from docs_buddy import adapters, services, domain from docs_buddy.services import commands @@ -94,14 +95,23 @@ def main() -> None: sys.exit(1) base_url = f"https://github.com/{repo_id}/blob/main/" - response = services.find_answer(query, document_index, base_url) - if not response.answer: - print("No results found.") - else: - print(f"Answer: {response.answer}") - print("Citations:") - for citation in response.citations: - print(f" - {citation}") + + tools = [adapters.make_search_tool(document_index)] + + system_prompt = textwrap.dedent("""\ + You are a documentation assistant equipped to answer user queries + by searching the docs. + """) + + research_user_query = adapters.make_openai_research_agent(system_prompt) + + response = services.find_answer(query, research_user_query, tools) + + print("Docs Buddy Response:\n") + print(response.answer) + print("\nReferences:\n") + for path in response.citations: + print(f"{base_url}{path}") else: parser.print_help() diff --git a/src/docs_buddy/services/use_cases.py b/src/docs_buddy/services/use_cases.py index 202587d..3300be6 100644 --- a/src/docs_buddy/services/use_cases.py +++ b/src/docs_buddy/services/use_cases.py @@ -1,8 +1,9 @@ """Use case handlers, adapter interfaces""" import functools +import logging from dataclasses import dataclass -from typing import Protocol, Iterator, ContextManager, Callable +from typing import Protocol, Iterator, ContextManager, Callable, TypeAlias from pathlib import Path from docs_buddy.common import PathLike, DocsBuddyError @@ -10,6 +11,10 @@ DEFAULT_MAX_RESULTS = 10 +log = logging.getLogger(__name__) + +Agent: TypeAlias = Callable[[domain.Query, list[Callable]], domain.QueryResponse] + class RepositorySyncError(DocsBuddyError): pass @@ -178,28 +183,18 @@ def index_document_chunks( pipeline.replace_destination(tmp_location) -def search_index( - query: domain.Query, index: DocumentIndex, max_results: int = DEFAULT_MAX_RESULTS -) -> list[domain.QueryResult]: - """Returns search results from the index""" - if max_results < 1: - raise SearchIndexError("max results must be at least 1") - - return index.search(query, max_results) - - def find_answer( query: domain.Query, - index: DocumentIndex, - base_url: str | None = None, + research_query: Agent, + tools: list[Callable], ) -> domain.QueryResponse: - """Return the top search result as answer + citation. + """Uses the agent and tools to find an answer to the user query""" - If no results, returns an empty answer and citations list. - """ - results = search_index(query, index, max_results=1) - if not results: - return domain.QueryResponse(answer="", citations=[]) - top = results[0] - citation = f"{base_url}{top.path}" if base_url else top.path - return domain.QueryResponse(answer=top.content, citations=[citation]) + try: + response = research_query(query, tools) + except DocsBuddyError as exc: + log.exception("Something went wrong, could not generate response") + msg = "Something went wrong, please try again." + response = domain.QueryResponse(msg, []) + + return response diff --git a/tests/integration/test_adapters.py b/tests/integration/test_adapters.py index 4750acd..7bf5009 100644 --- a/tests/integration/test_adapters.py +++ b/tests/integration/test_adapters.py @@ -8,7 +8,7 @@ import pytest import whoosh.index -from docs_buddy import adapters, domain, common +from docs_buddy import adapters, domain, common, services def test_get_temp_location_creates_and_cleans_up() -> None: @@ -146,6 +146,36 @@ def test_can_search_whoosh_index() -> None: assert len(document_index.search(query, max_results=1)) == 1 +def test_can_create_whoosh_search_tool() -> None: + + with tempfile.TemporaryDirectory() as temp_dir: + + indexer = adapters.WhooshDocumentIndex() + + chunks = [ + domain.DocumentChunk.fromstring(json.dumps(_SAMPLE_CHUNK_1)), + domain.DocumentChunk.fromstring(json.dumps(_SAMPLE_CHUNK_2)), + ] + + indexer.fit(iter(chunks), temp_dir) + + document_index = adapters.WhooshDocumentIndex(temp_dir) + + search = adapters.make_search_tool(document_index) + + search_phrase = "providers" + results = search(search_phrase) + assert len(results) > 1 + + for result in results: + # assert required keys in json string + d = json.loads(result) + assert all(k in d for k in ["content", "path", "metadata"]) + + # it should be possible to specify max length of results + assert len(search(search_phrase, max_results=1)) == 1 + + def test_whoosh_document_index_fitting_for_empty_documents() -> None: """Test that WhooshDocumentIndex creates an index from DocumentChunks.""" diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index ef53791..230e385 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -273,55 +273,53 @@ def test_can_index_documents() -> None: assert (action3, arg3_1, arg3_2) == ("MV", tmp_location, dest) -def test_can_search_index() -> None: - # Given an index built from a fake pipeline - source = ".chunks/programmmer-ke/akash-docs-buddy" - dest = ".index/programmer-ke/akash-docs-buddy" +def test_find_answer_with_configured_agent() -> None: + # Given the documentation index has been built + source = ".chunks/test-repo" + dest = ".index/test-repo" pipeline = adapters.FakeDocumentChunksPipeline(source, dest) index = adapters.FakeIndex(pipeline) services.index_document_chunks(pipeline, index) - query = domain.Query(text="provider") + # And an agent has been configured with a search tool + tools = [adapters.make_search_tool(index)] + research_user_query = adapters.make_fake_research_agent(prompt="some prompt") + + # When user submits valid query + query = domain.Query("provider") + response = services.find_answer(query, research_user_query, tools) - # When we search without a limit - results = services.search_index(query, index) - # Then we get at least one result - assert len(results) > 0 + # Then the system returns a structured response + assert isinstance(response, domain.QueryResponse) - # When we request exactly one result - results = services.search_index(query, index, max_results=1) - # Then exactly one result is returned - assert len(results) == 1 + # Then the answer is non‑empty + assert response.answer - # When max_results is <= 0, an error is raised - bad_values = [0, -1, -30] - for bad_value in bad_values: - with pytest.raises(services.SearchIndexError): - _ = services.search_index(query, index, max_results=bad_value) + # Then there is at least one citation + assert len(response.citations) > 0 -def test_find_answer_returns_top_chunk() -> None: - # Given an indexed document set with a known query matching +def test_agent_error_gracefully_handled() -> None: + # Given the documentation index has been built source = ".chunks/test-repo" dest = ".index/test-repo" pipeline = adapters.FakeDocumentChunksPipeline(source, dest) index = adapters.FakeIndex(pipeline) services.index_document_chunks(pipeline, index) - query = domain.Query("provider") - base_url = "https://github.com/test/docs/blob/main/" + # And an agent has been configured with a search tool + tools = [adapters.make_search_tool(index)] + research_user_query = adapters.make_fake_research_agent(prompt="some prompt") - # When we retrieve an answer - response = services.find_answer(query, index, base_url) + # When user submits query likely to fail + query = domain.Query("nonexistent") + response = services.find_answer(query, research_user_query, tools) - # Then the response is a QueryResponse + # Then the system returns a structured response assert isinstance(response, domain.QueryResponse) # Then the answer is non‑empty - assert len(response.answer) > 0 - - # Then there is exactly one citation - assert len(response.citations) == 1 + assert "went wrong" in response.answer - # Then the citation starts with the provided base URL - assert response.citations[0].startswith(base_url) + # Then citation list is empty + assert len(response.citations) == 0 diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py new file mode 100644 index 0000000..70db695 --- /dev/null +++ b/tests/unit/test_tools.py @@ -0,0 +1,36 @@ +import pytest +from docs_buddy import adapters, services, domain + + +def test_can_create_search_tool_over_index() -> None: + # Given an index built from a fake pipeline + source = ".chunks/programmmer-ke/akash-docs-buddy" + dest = ".index/programmer-ke/akash-docs-buddy" + pipeline = adapters.FakeDocumentChunksPipeline(source, dest) + index = adapters.FakeIndex(pipeline) + services.index_document_chunks(pipeline, index) + + search_index_tool = adapters.make_search_tool(index) + phrase = "provider" + + # When we search without a limit + results = search_index_tool(phrase) + # Then we get at least one result + assert len(results) > 0 + + # When we request exactly one result + results = search_index_tool(phrase, max_results=1) + # Then exactly one result is returned + assert len(results) == 1 + + # When max_results is <= 0, then an error is raised + bad_values = [0, -1, -30] + for bad_value in bad_values: + with pytest.raises(adapters.ToolError): + _ = search_index_tool(phrase, max_results=bad_value) + + # When search phrase is invalid, then an error is raised + bad_phrases = ["", " ", "\t"] + for phrase in bad_phrases: + with pytest.raises(domain.InvalidQueryError): + _ = search_index_tool(phrase) diff --git a/todo.md b/todo.md index cdc2fc9..1b48784 100644 --- a/todo.md +++ b/todo.md @@ -10,30 +10,6 @@ Status legend: - [ ] address todo comments -- [ ] **US-005: User can submit a query and receive an answer synthesized by an AI agent using the search tool** - *As a user, I want to submit a query and receive an answer synthesized by an AI agent that uses the search tool to retrieve relevant chunks, so that I get a natural language answer grounded in the documentation.* - - [ ] **Scenario 5.1:** AI agent returns answer with citations - Given the documentation index has been built and contains indexed chunks - And an AI agent is configured with the search tool - When the user submits a valid query - Then the system returns a structured response where: - - The `answer` field contains a natural language answer synthesized by the agent - - The `citations` field contains document URLs of chunks the agent selected - - [ ] **Scenario 5.2:** AI agent decides no chunk is relevant - Given the documentation index has been built and contains indexed chunks - And an AI agent is configured with the search tool - When the user submits a valid query - And the agent decides none of the returned chunks are relevant - Then the system returns a structured response where: - - The `answer` field contains a message like "I could not find relevant information in the documentation." - - The `citations` field is an empty list - - [ ] **Scenario 5.3:** Search returns zero chunks - Given the documentation index has been built but contains no chunks matching the query - And an AI agent is configured with the search tool - When the user submits a valid query - Then the system returns a structured response where: - - The `answer` field contains a message like "I could not find relevant information in the documentation." - - The `citations` field is an empty list - [ ] **US-006: Operator can observe query and result logs** *As an operator, I want to see logs of each query submitted and the corresponding result (answer + citations), so that I can monitor usage, debug issues, and audit the system.* @@ -73,6 +49,34 @@ Status legend: ## in progress +- [>] **US-005: User can submit a query and receive an answer + synthesized by an AI agent using the search tool* + *As a user, I want to submit a query and receive an answer + synthesized by an AI agent that uses the search tool to retrieve + relevant chunks, so that I get a natural language answer grounded in + the documentation.* + - [x] **Scenario 5.1:** AI agent returns answer with citations + Given the documentation index has been built and contains indexed chunks + And an AI agent is configured with the search tool + When the user submits a valid query + Then the system returns a structured response where: + - The `answer` field contains a natural language answer synthesized by the agent + - The `citations` field contains document URLs of chunks the agent selected + - [x] **Scenario 5.2:** AI agent decides no chunk is relevant + Given the documentation index has been built and contains indexed chunks + And an AI agent is configured with the search tool + When the user submits a valid query + And the agent decides none of the returned chunks are relevant + Then the system returns a structured response where: + - The `answer` field contains a message like "I could not find relevant information in the documentation." + - The `citations` field is an empty list + - [x] **Scenario 5.3:** Search returns zero chunks + Given the documentation index has been built but contains no chunks matching the query + And an AI agent is configured with the search tool + When the user submits a valid query + Then the system returns a structured response where: + - The `answer` field contains a message like "I could not find relevant information in the documentation." + - The `citations` field is an empty list ## done