diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 55d20255..ff261bad 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,7 +3,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} USER vscode -RUN curl -sSf https://rye.astral.sh/get | RYE_VERSION="0.35.0" RYE_INSTALL_OPTION="--yes" bash +RUN curl -sSf https://rye.astral.sh/get | RYE_VERSION="0.44.0" RYE_INSTALL_OPTION="--yes" bash ENV PATH=/home/vscode/.rye/shims:$PATH RUN echo "[[ -d .venv ]] && source .venv/bin/activate || export PATH=\$PATH" >> /home/vscode/.bashrc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8a8a4f7..0ea316c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,18 @@ name: CI on: push: - branches: - - main - pull_request: - branches: - - main - - next + branches-ignore: + - 'generated' + - 'codegen/**' + - 'integrated/**' + - 'stl-preview-head/**' + - 'stl-preview-base/**' jobs: lint: + timeout-minutes: 10 name: lint - runs-on: ubuntu-latest - + runs-on: ${{ github.repository == 'stainless-sdks/sunrise-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -21,7 +21,7 @@ jobs: curl -sSf https://rye.astral.sh/get | bash echo "$HOME/.rye/shims" >> $GITHUB_PATH env: - RYE_VERSION: '0.35.0' + RYE_VERSION: '0.44.0' RYE_INSTALL_OPTION: '--yes' - name: Install dependencies @@ -31,9 +31,9 @@ jobs: run: ./scripts/lint test: + timeout-minutes: 10 name: test - runs-on: ubuntu-latest - + runs-on: ${{ github.repository == 'stainless-sdks/sunrise-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: curl -sSf https://rye.astral.sh/get | bash echo "$HOME/.rye/shims" >> $GITHUB_PATH env: - RYE_VERSION: '0.35.0' + RYE_VERSION: '0.44.0' RYE_INSTALL_OPTION: '--yes' - name: Bootstrap diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index a959f412..91803357 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -21,7 +21,7 @@ jobs: curl -sSf https://rye.astral.sh/get | bash echo "$HOME/.rye/shims" >> $GITHUB_PATH env: - RYE_VERSION: '0.35.0' + RYE_VERSION: '0.44.0' RYE_INSTALL_OPTION: '--yes' - name: Publish to PyPI diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d04f223f..4208b5cb 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.5.1" + ".": "0.6.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 50cd37d6..525d18cd 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,4 @@ -configured_endpoints: 46 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/contextual-ai%2Fsunrise-194878b194cd507d7c5418ff38cc0fc53441ef618f991990d334b4b75775cd8f.yml +configured_endpoints: 52 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/contextual-ai%2Fsunrise-17bdb8a33fb4fcade827bba868bd65cd30c64b1d09b4a6d83c3e37a8439ed37f.yml +openapi_spec_hash: bc325b52f3b20d8c56e0be5de88f2dc3 +config_hash: 1ecef0ff4fd125bbc00eec65e3dd4798 diff --git a/CHANGELOG.md b/CHANGELOG.md index fcecd19e..09892b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,56 @@ # Changelog +## 0.6.0 (2025-05-08) + +Full Changelog: [v0.5.1...v0.6.0](https://github.com/ContextualAI/contextual-client-python/compare/v0.5.1...v0.6.0) + +### Features + +* **api:** update via SDK Studio ([2024a46](https://github.com/ContextualAI/contextual-client-python/commit/2024a46629ceca81d6c146b2e4d92ed4afb72e4b)) +* **api:** update via SDK Studio ([dba986f](https://github.com/ContextualAI/contextual-client-python/commit/dba986f3194a37160064270836d15a88ed0f8ee4)) +* **api:** update via SDK Studio ([a707edc](https://github.com/ContextualAI/contextual-client-python/commit/a707edc06c74353788bfa182d07682f7352a7a02)) +* **api:** update via SDK Studio ([#68](https://github.com/ContextualAI/contextual-client-python/issues/68)) ([7b8f948](https://github.com/ContextualAI/contextual-client-python/commit/7b8f9488ff9ba324210a23694065830a25985edc)) +* **api:** update via SDK Studio ([#70](https://github.com/ContextualAI/contextual-client-python/issues/70)) ([25477c7](https://github.com/ContextualAI/contextual-client-python/commit/25477c7d93934f2b5b72f5b24857f023c17349cf)) +* **api:** update via SDK Studio ([#73](https://github.com/ContextualAI/contextual-client-python/issues/73)) ([e07435e](https://github.com/ContextualAI/contextual-client-python/commit/e07435e7ab08a53cd13f4f8d91f2baca1ec2c28d)) + + +### Bug Fixes + +* **ci:** ensure pip is always available ([#79](https://github.com/ContextualAI/contextual-client-python/issues/79)) ([ec1e2ce](https://github.com/ContextualAI/contextual-client-python/commit/ec1e2ce6de25021983b0a48b90069f24e3ee8def)) +* **ci:** remove publishing patch ([#80](https://github.com/ContextualAI/contextual-client-python/issues/80)) ([9e32578](https://github.com/ContextualAI/contextual-client-python/commit/9e32578922eb4dbad057231999add02c8aca3eb1)) +* **perf:** optimize some hot paths ([b88026d](https://github.com/ContextualAI/contextual-client-python/commit/b88026d6bfee7100a5663a95dd9800ed0059b353)) +* **perf:** skip traversing types for NotGiven values ([5bd2eab](https://github.com/ContextualAI/contextual-client-python/commit/5bd2eabd88852941a46d70823cb6163db08558eb)) +* **pydantic v1:** more robust ModelField.annotation check ([ce1ecab](https://github.com/ContextualAI/contextual-client-python/commit/ce1ecab62f47913665a51e3116232f65de95a3f3)) +* testing value for tune endpoints. ([7be555f](https://github.com/ContextualAI/contextual-client-python/commit/7be555fe0f39430923a9473420a88fc8c065a299)) +* **types:** handle more discriminated union shapes ([#78](https://github.com/ContextualAI/contextual-client-python/issues/78)) ([473adf4](https://github.com/ContextualAI/contextual-client-python/commit/473adf4d731241a2c34271d39a37d0ac2bc99d4e)) + + +### Chores + +* broadly detect json family of content-type headers ([f4f3951](https://github.com/ContextualAI/contextual-client-python/commit/f4f39513b3f9a1c1cb5f323d2c0da2b0d04eff06)) +* **ci:** add timeout thresholds for CI jobs ([542b4ad](https://github.com/ContextualAI/contextual-client-python/commit/542b4adaefef61d93d6d7ec971c50d3d87490c17)) +* **ci:** only use depot for staging repos ([973153b](https://github.com/ContextualAI/contextual-client-python/commit/973153b08c9780b0d27ee107f71045c5921ee4f5)) +* **client:** minor internal fixes ([379b18e](https://github.com/ContextualAI/contextual-client-python/commit/379b18e3382f4cb3cbf2f4fb768a0d23885ec562)) +* fix typos ([#81](https://github.com/ContextualAI/contextual-client-python/issues/81)) ([9ba43be](https://github.com/ContextualAI/contextual-client-python/commit/9ba43bed2b39d60d599b90e624a2a40e57584749)) +* **internal:** base client updates ([1c44fea](https://github.com/ContextualAI/contextual-client-python/commit/1c44fea55a67de0d11f00fd3b63f64302f5eee51)) +* **internal:** bump pyright version ([6878eae](https://github.com/ContextualAI/contextual-client-python/commit/6878eae3717d1076836f59dafcab08a44ec573c8)) +* **internal:** bump rye to 0.44.0 ([#77](https://github.com/ContextualAI/contextual-client-python/issues/77)) ([520ba3a](https://github.com/ContextualAI/contextual-client-python/commit/520ba3a8e069a19543238009a241579ede90c2fe)) +* **internal:** codegen related update ([ddb9f6c](https://github.com/ContextualAI/contextual-client-python/commit/ddb9f6c3be981908de24cb485b4787a2fa969b80)) +* **internal:** codegen related update ([#74](https://github.com/ContextualAI/contextual-client-python/issues/74)) ([6e8bc46](https://github.com/ContextualAI/contextual-client-python/commit/6e8bc46fab20d9babe7b047298b55b0565ba4a8b)) +* **internal:** expand CI branch coverage ([fce3ddf](https://github.com/ContextualAI/contextual-client-python/commit/fce3ddf98a13402dc63da54b1042e625247e1e72)) +* **internal:** fix list file params ([561214d](https://github.com/ContextualAI/contextual-client-python/commit/561214d491c29833c6babc1ad1f5d6cc4367f794)) +* **internal:** import reformatting ([a9e8ae2](https://github.com/ContextualAI/contextual-client-python/commit/a9e8ae26c8f15d4bc385890eb0954d41475f5fba)) +* **internal:** minor formatting changes ([d036bee](https://github.com/ContextualAI/contextual-client-python/commit/d036bee6b1e9e9bf16d775f1f48732d5cf0bd206)) +* **internal:** reduce CI branch coverage ([b10d32e](https://github.com/ContextualAI/contextual-client-python/commit/b10d32ecd725652eba23d9e14714f82af0ead691)) +* **internal:** refactor retries to not use recursion ([7689427](https://github.com/ContextualAI/contextual-client-python/commit/7689427fe4667c1efcf66ba7ee6ace7e2dbd05f3)) +* **internal:** remove extra empty newlines ([#75](https://github.com/ContextualAI/contextual-client-python/issues/75)) ([8117197](https://github.com/ContextualAI/contextual-client-python/commit/81171975661f4a03f22820a36773bdff14b79e20)) +* **internal:** remove trailing character ([#82](https://github.com/ContextualAI/contextual-client-python/issues/82)) ([72018c8](https://github.com/ContextualAI/contextual-client-python/commit/72018c8784cf5d9974fca682b2b9998e2c4d341c)) +* **internal:** slight transform perf improvement ([#83](https://github.com/ContextualAI/contextual-client-python/issues/83)) ([29e9d80](https://github.com/ContextualAI/contextual-client-python/commit/29e9d80b25a84deb927098fd9c7bc0341a8e165f)) +* **internal:** update models test ([c3fcd9c](https://github.com/ContextualAI/contextual-client-python/commit/c3fcd9c8c1ef3de1b9681a8298ac508b758bf98c)) +* **internal:** update pyright settings ([9a560f4](https://github.com/ContextualAI/contextual-client-python/commit/9a560f4f6234c724a0426356e1c154ddfbccfa71)) +* slight wording improvement in README ([#84](https://github.com/ContextualAI/contextual-client-python/issues/84)) ([d5d7f2a](https://github.com/ContextualAI/contextual-client-python/commit/d5d7f2a5ece5735e83e431c9f231b94eb7d41773)) +* use lazy imports for resources ([e41fd1c](https://github.com/ContextualAI/contextual-client-python/commit/e41fd1c3869c16d18ba7f9151a1b3f1a463f0fd6)) + ## 0.5.1 (2025-03-11) Full Changelog: [v0.5.0...v0.5.1](https://github.com/ContextualAI/contextual-client-python/compare/v0.5.0...v0.5.1) diff --git a/README.md b/README.md index a1c16de3..4be5749c 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,51 @@ for agent in first_page.agents: # Remove `await` for non-async usage. ``` +## Nested params + +Nested parameters are dictionaries, typed using `TypedDict`, for example: + +```python +from contextual import ContextualAI + +client = ContextualAI() + +create_agent_output = client.agents.create( + name="xxx", + agent_configs={ + "filter_and_rerank_config": { + "rerank_instructions": "rerank_instructions", + "reranker_score_filter_threshold": 0, + "top_k_reranked_chunks": 0, + }, + "generate_response_config": { + "avoid_commentary": True, + "calculate_groundedness": True, + "frequency_penalty": 0, + "max_new_tokens": 0, + "seed": 0, + "temperature": 0, + "top_p": 0, + }, + "global_config": { + "enable_filter": True, + "enable_multi_turn": True, + "enable_rerank": True, + "should_check_retrieval_need": True, + }, + "retrieval_config": { + "lexical_alpha": 0, + "semantic_alpha": 0, + "top_k_retrieved_chunks": 0, + }, + }, +) +print(create_agent_output.agent_configs) +``` + ## File uploads -Request parameters that correspond to file uploads can be passed as `bytes`, a [`PathLike`](https://docs.python.org/3/library/os.html#os.PathLike) instance or a tuple of `(filename, contents, media type)`. +Request parameters that correspond to file uploads can be passed as `bytes`, or a [`PathLike`](https://docs.python.org/3/library/os.html#os.PathLike) instance or a tuple of `(filename, contents, media type)`. ```python from pathlib import Path diff --git a/api.md b/api.md index 295a1b11..34b9ceda 100644 --- a/api.md +++ b/api.md @@ -9,6 +9,7 @@ from contextual.types import ( DatastoreMetadata, ListDatastoresResponse, DatastoreDeleteResponse, + DatastoreResetResponse, ) ``` @@ -18,6 +19,7 @@ Methods: - client.datastores.list(\*\*params) -> SyncDatastoresPage[Datastore] - client.datastores.delete(datastore_id) -> object - client.datastores.metadata(datastore_id) -> DatastoreMetadata +- client.datastores.reset(datastore_id) -> object ## Documents @@ -48,11 +50,18 @@ Types: ```python from contextual.types import ( Agent, + AgentConfigs, AgentMetadata, CreateAgentOutput, + FilterAndRerankConfig, + GenerateResponseConfig, + GlobalConfig, ListAgentsResponse, + RetrievalConfig, AgentUpdateResponse, AgentDeleteResponse, + AgentMetadataResponse, + AgentResetResponse, ) ``` @@ -62,7 +71,8 @@ Methods: - client.agents.update(agent_id, \*\*params) -> object - client.agents.list(\*\*params) -> SyncPage[Agent] - client.agents.delete(agent_id) -> object -- client.agents.metadata(agent_id) -> AgentMetadata +- client.agents.metadata(agent_id) -> AgentMetadataResponse +- client.agents.reset(agent_id) -> object ## Query @@ -250,3 +260,23 @@ from contextual.types import GenerateCreateResponse Methods: - client.generate.create(\*\*params) -> GenerateCreateResponse + +# Parse + +Types: + +```python +from contextual.types import ( + ParseCreateResponse, + ParseJobResultsResponse, + ParseJobStatusResponse, + ParseJobsResponse, +) +``` + +Methods: + +- client.parse.create(\*\*params) -> ParseCreateResponse +- client.parse.job_results(job_id, \*\*params) -> ParseJobResultsResponse +- client.parse.job_status(job_id) -> ParseJobStatusResponse +- client.parse.jobs(\*\*params) -> ParseJobsResponse diff --git a/bin/publish-pypi b/bin/publish-pypi index 05bfccbb..826054e9 100644 --- a/bin/publish-pypi +++ b/bin/publish-pypi @@ -3,7 +3,4 @@ set -eux mkdir -p dist rye build --clean -# Patching importlib-metadata version until upstream library version is updated -# https://github.com/pypa/twine/issues/977#issuecomment-2189800841 -"$HOME/.rye/self/bin/python3" -m pip install 'importlib-metadata==7.2.1' rye publish --yes --token=$PYPI_TOKEN diff --git a/pyproject.toml b/pyproject.toml index 7858c91b..dd86d924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "contextual-client" -version = "0.5.1" +version = "0.6.0" description = "The official Python library for the Contextual AI API" dynamic = ["readme"] license = "Apache-2.0" @@ -38,12 +38,11 @@ Homepage = "https://github.com/ContextualAI/contextual-client-python" Repository = "https://github.com/ContextualAI/contextual-client-python" - [tool.rye] managed = true # version pins are in requirements-dev.lock dev-dependencies = [ - "pyright>=1.1.359", + "pyright==1.1.399", "mypy", "respx", "pytest", @@ -87,7 +86,7 @@ typecheck = { chain = [ "typecheck:mypy" = "mypy ." [build-system] -requires = ["hatchling", "hatch-fancy-pypi-readme"] +requires = ["hatchling==1.26.3", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [tool.hatch.build] @@ -148,11 +147,11 @@ exclude = [ ] reportImplicitOverride = true +reportOverlappingOverload = false reportImportCycles = false reportPrivateUsage = false - [tool.ruff] line-length = 120 output-format = "grouped" diff --git a/requirements-dev.lock b/requirements-dev.lock index 83d02e00..42c2907d 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -69,7 +69,7 @@ pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via rich -pyright==1.1.392.post0 +pyright==1.1.399 pytest==8.3.3 # via pytest-asyncio pytest-asyncio==0.24.0 diff --git a/scripts/test b/scripts/test index ae2957b4..9ed47da4 100755 --- a/scripts/test +++ b/scripts/test @@ -52,6 +52,8 @@ else echo fi +export DEFER_PYDANTIC_BUILD=false + echo "==> Running tests" rye run pytest "$@" diff --git a/src/contextual/_base_client.py b/src/contextual/_base_client.py index ee4d0c95..7369dcab 100644 --- a/src/contextual/_base_client.py +++ b/src/contextual/_base_client.py @@ -98,7 +98,11 @@ _AsyncStreamT = TypeVar("_AsyncStreamT", bound=AsyncStream[Any]) if TYPE_CHECKING: - from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT + from httpx._config import ( + DEFAULT_TIMEOUT_CONFIG, # pyright: ignore[reportPrivateImportUsage] + ) + + HTTPX_DEFAULT_TIMEOUT = DEFAULT_TIMEOUT_CONFIG else: try: from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT @@ -115,6 +119,7 @@ class PageInfo: url: URL | NotGiven params: Query | NotGiven + json: Body | NotGiven @overload def __init__( @@ -130,19 +135,30 @@ def __init__( params: Query, ) -> None: ... + @overload + def __init__( + self, + *, + json: Body, + ) -> None: ... + def __init__( self, *, url: URL | NotGiven = NOT_GIVEN, + json: Body | NotGiven = NOT_GIVEN, params: Query | NotGiven = NOT_GIVEN, ) -> None: self.url = url + self.json = json self.params = params @override def __repr__(self) -> str: if self.url: return f"{self.__class__.__name__}(url={self.url})" + if self.json: + return f"{self.__class__.__name__}(json={self.json})" return f"{self.__class__.__name__}(params={self.params})" @@ -191,6 +207,19 @@ def _info_to_options(self, info: PageInfo) -> FinalRequestOptions: options.url = str(url) return options + if not isinstance(info.json, NotGiven): + if not is_mapping(info.json): + raise TypeError("Pagination is only supported with mappings") + + if not options.json_data: + options.json_data = {**info.json} + else: + if not is_mapping(options.json_data): + raise TypeError("Pagination is only supported with mappings") + + options.json_data = {**options.json_data, **info.json} + return options + raise ValueError("Unexpected PageInfo state") @@ -408,8 +437,8 @@ def _build_headers(self, options: FinalRequestOptions, *, retries_taken: int = 0 headers = httpx.Headers(headers_dict) idempotency_header = self._idempotency_header - if idempotency_header and options.method.lower() != "get" and idempotency_header not in headers: - headers[idempotency_header] = options.idempotency_key or self._idempotency_key() + if idempotency_header and options.idempotency_key and idempotency_header not in headers: + headers[idempotency_header] = options.idempotency_key # Don't set these headers if they were already set or removed by the caller. We check # `custom_headers`, which can contain `Omit()`, instead of `headers` to account for the removal case. @@ -873,7 +902,6 @@ def request( self, cast_to: Type[ResponseT], options: FinalRequestOptions, - remaining_retries: Optional[int] = None, *, stream: Literal[True], stream_cls: Type[_StreamT], @@ -884,7 +912,6 @@ def request( self, cast_to: Type[ResponseT], options: FinalRequestOptions, - remaining_retries: Optional[int] = None, *, stream: Literal[False] = False, ) -> ResponseT: ... @@ -894,7 +921,6 @@ def request( self, cast_to: Type[ResponseT], options: FinalRequestOptions, - remaining_retries: Optional[int] = None, *, stream: bool = False, stream_cls: Type[_StreamT] | None = None, @@ -904,121 +930,109 @@ def request( self, cast_to: Type[ResponseT], options: FinalRequestOptions, - remaining_retries: Optional[int] = None, *, stream: bool = False, stream_cls: type[_StreamT] | None = None, ) -> ResponseT | _StreamT: - if remaining_retries is not None: - retries_taken = options.get_max_retries(self.max_retries) - remaining_retries - else: - retries_taken = 0 - - return self._request( - cast_to=cast_to, - options=options, - stream=stream, - stream_cls=stream_cls, - retries_taken=retries_taken, - ) + cast_to = self._maybe_override_cast_to(cast_to, options) - def _request( - self, - *, - cast_to: Type[ResponseT], - options: FinalRequestOptions, - retries_taken: int, - stream: bool, - stream_cls: type[_StreamT] | None, - ) -> ResponseT | _StreamT: # create a copy of the options we were given so that if the # options are mutated later & we then retry, the retries are # given the original options input_options = model_copy(options) + if input_options.idempotency_key is None and input_options.method.lower() != "get": + # ensure the idempotency key is reused between requests + input_options.idempotency_key = self._idempotency_key() - cast_to = self._maybe_override_cast_to(cast_to, options) - options = self._prepare_options(options) - - remaining_retries = options.get_max_retries(self.max_retries) - retries_taken - request = self._build_request(options, retries_taken=retries_taken) - self._prepare_request(request) - - kwargs: HttpxSendArgs = {} - if self.custom_auth is not None: - kwargs["auth"] = self.custom_auth + response: httpx.Response | None = None + max_retries = input_options.get_max_retries(self.max_retries) - log.debug("Sending HTTP Request: %s %s", request.method, request.url) + retries_taken = 0 + for retries_taken in range(max_retries + 1): + options = model_copy(input_options) + options = self._prepare_options(options) - try: - response = self._client.send( - request, - stream=stream or self._should_stream_response_body(request=request), - **kwargs, - ) - except httpx.TimeoutException as err: - log.debug("Encountered httpx.TimeoutException", exc_info=True) + remaining_retries = max_retries - retries_taken + request = self._build_request(options, retries_taken=retries_taken) + self._prepare_request(request) - if remaining_retries > 0: - return self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - stream=stream, - stream_cls=stream_cls, - response_headers=None, - ) + kwargs: HttpxSendArgs = {} + if self.custom_auth is not None: + kwargs["auth"] = self.custom_auth - log.debug("Raising timeout error") - raise APITimeoutError(request=request) from err - except Exception as err: - log.debug("Encountered Exception", exc_info=True) + log.debug("Sending HTTP Request: %s %s", request.method, request.url) - if remaining_retries > 0: - return self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - stream=stream, - stream_cls=stream_cls, - response_headers=None, + response = None + try: + response = self._client.send( + request, + stream=stream or self._should_stream_response_body(request=request), + **kwargs, ) + except httpx.TimeoutException as err: + log.debug("Encountered httpx.TimeoutException", exc_info=True) + + if remaining_retries > 0: + self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=None, + ) + continue + + log.debug("Raising timeout error") + raise APITimeoutError(request=request) from err + except Exception as err: + log.debug("Encountered Exception", exc_info=True) + + if remaining_retries > 0: + self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=None, + ) + continue + + log.debug("Raising connection error") + raise APIConnectionError(request=request) from err + + log.debug( + 'HTTP Response: %s %s "%i %s" %s', + request.method, + request.url, + response.status_code, + response.reason_phrase, + response.headers, + ) - log.debug("Raising connection error") - raise APIConnectionError(request=request) from err - - log.debug( - 'HTTP Response: %s %s "%i %s" %s', - request.method, - request.url, - response.status_code, - response.reason_phrase, - response.headers, - ) + try: + response.raise_for_status() + except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code + log.debug("Encountered httpx.HTTPStatusError", exc_info=True) + + if remaining_retries > 0 and self._should_retry(err.response): + err.response.close() + self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=response, + ) + continue - try: - response.raise_for_status() - except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code - log.debug("Encountered httpx.HTTPStatusError", exc_info=True) - - if remaining_retries > 0 and self._should_retry(err.response): - err.response.close() - return self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - response_headers=err.response.headers, - stream=stream, - stream_cls=stream_cls, - ) + # If the response is streamed then we need to explicitly read the response + # to completion before attempting to access the response text. + if not err.response.is_closed: + err.response.read() - # If the response is streamed then we need to explicitly read the response - # to completion before attempting to access the response text. - if not err.response.is_closed: - err.response.read() + log.debug("Re-raising status error") + raise self._make_status_error_from_response(err.response) from None - log.debug("Re-raising status error") - raise self._make_status_error_from_response(err.response) from None + break + assert response is not None, "could not resolve response (should never happen)" return self._process_response( cast_to=cast_to, options=options, @@ -1028,37 +1042,20 @@ def _request( retries_taken=retries_taken, ) - def _retry_request( - self, - options: FinalRequestOptions, - cast_to: Type[ResponseT], - *, - retries_taken: int, - response_headers: httpx.Headers | None, - stream: bool, - stream_cls: type[_StreamT] | None, - ) -> ResponseT | _StreamT: - remaining_retries = options.get_max_retries(self.max_retries) - retries_taken + def _sleep_for_retry( + self, *, retries_taken: int, max_retries: int, options: FinalRequestOptions, response: httpx.Response | None + ) -> None: + remaining_retries = max_retries - retries_taken if remaining_retries == 1: log.debug("1 retry left") else: log.debug("%i retries left", remaining_retries) - timeout = self._calculate_retry_timeout(remaining_retries, options, response_headers) + timeout = self._calculate_retry_timeout(remaining_retries, options, response.headers if response else None) log.info("Retrying request to %s in %f seconds", options.url, timeout) - # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a - # different thread if necessary. time.sleep(timeout) - return self._request( - options=options, - cast_to=cast_to, - retries_taken=retries_taken + 1, - stream=stream, - stream_cls=stream_cls, - ) - def _process_response( self, *, @@ -1402,7 +1399,6 @@ async def request( options: FinalRequestOptions, *, stream: Literal[False] = False, - remaining_retries: Optional[int] = None, ) -> ResponseT: ... @overload @@ -1413,7 +1409,6 @@ async def request( *, stream: Literal[True], stream_cls: type[_AsyncStreamT], - remaining_retries: Optional[int] = None, ) -> _AsyncStreamT: ... @overload @@ -1424,7 +1419,6 @@ async def request( *, stream: bool, stream_cls: type[_AsyncStreamT] | None = None, - remaining_retries: Optional[int] = None, ) -> ResponseT | _AsyncStreamT: ... async def request( @@ -1434,116 +1428,111 @@ async def request( *, stream: bool = False, stream_cls: type[_AsyncStreamT] | None = None, - remaining_retries: Optional[int] = None, - ) -> ResponseT | _AsyncStreamT: - if remaining_retries is not None: - retries_taken = options.get_max_retries(self.max_retries) - remaining_retries - else: - retries_taken = 0 - - return await self._request( - cast_to=cast_to, - options=options, - stream=stream, - stream_cls=stream_cls, - retries_taken=retries_taken, - ) - - async def _request( - self, - cast_to: Type[ResponseT], - options: FinalRequestOptions, - *, - stream: bool, - stream_cls: type[_AsyncStreamT] | None, - retries_taken: int, ) -> ResponseT | _AsyncStreamT: if self._platform is None: # `get_platform` can make blocking IO calls so we # execute it earlier while we are in an async context self._platform = await asyncify(get_platform)() + cast_to = self._maybe_override_cast_to(cast_to, options) + # create a copy of the options we were given so that if the # options are mutated later & we then retry, the retries are # given the original options input_options = model_copy(options) + if input_options.idempotency_key is None and input_options.method.lower() != "get": + # ensure the idempotency key is reused between requests + input_options.idempotency_key = self._idempotency_key() - cast_to = self._maybe_override_cast_to(cast_to, options) - options = await self._prepare_options(options) + response: httpx.Response | None = None + max_retries = input_options.get_max_retries(self.max_retries) - remaining_retries = options.get_max_retries(self.max_retries) - retries_taken - request = self._build_request(options, retries_taken=retries_taken) - await self._prepare_request(request) + retries_taken = 0 + for retries_taken in range(max_retries + 1): + options = model_copy(input_options) + options = await self._prepare_options(options) - kwargs: HttpxSendArgs = {} - if self.custom_auth is not None: - kwargs["auth"] = self.custom_auth + remaining_retries = max_retries - retries_taken + request = self._build_request(options, retries_taken=retries_taken) + await self._prepare_request(request) - try: - response = await self._client.send( - request, - stream=stream or self._should_stream_response_body(request=request), - **kwargs, - ) - except httpx.TimeoutException as err: - log.debug("Encountered httpx.TimeoutException", exc_info=True) - - if remaining_retries > 0: - return await self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - stream=stream, - stream_cls=stream_cls, - response_headers=None, - ) + kwargs: HttpxSendArgs = {} + if self.custom_auth is not None: + kwargs["auth"] = self.custom_auth - log.debug("Raising timeout error") - raise APITimeoutError(request=request) from err - except Exception as err: - log.debug("Encountered Exception", exc_info=True) + log.debug("Sending HTTP Request: %s %s", request.method, request.url) - if remaining_retries > 0: - return await self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - stream=stream, - stream_cls=stream_cls, - response_headers=None, + response = None + try: + response = await self._client.send( + request, + stream=stream or self._should_stream_response_body(request=request), + **kwargs, ) + except httpx.TimeoutException as err: + log.debug("Encountered httpx.TimeoutException", exc_info=True) + + if remaining_retries > 0: + await self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=None, + ) + continue + + log.debug("Raising timeout error") + raise APITimeoutError(request=request) from err + except Exception as err: + log.debug("Encountered Exception", exc_info=True) + + if remaining_retries > 0: + await self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=None, + ) + continue + + log.debug("Raising connection error") + raise APIConnectionError(request=request) from err + + log.debug( + 'HTTP Response: %s %s "%i %s" %s', + request.method, + request.url, + response.status_code, + response.reason_phrase, + response.headers, + ) - log.debug("Raising connection error") - raise APIConnectionError(request=request) from err + try: + response.raise_for_status() + except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code + log.debug("Encountered httpx.HTTPStatusError", exc_info=True) + + if remaining_retries > 0 and self._should_retry(err.response): + await err.response.aclose() + await self._sleep_for_retry( + retries_taken=retries_taken, + max_retries=max_retries, + options=input_options, + response=response, + ) + continue - log.debug( - 'HTTP Request: %s %s "%i %s"', request.method, request.url, response.status_code, response.reason_phrase - ) + # If the response is streamed then we need to explicitly read the response + # to completion before attempting to access the response text. + if not err.response.is_closed: + await err.response.aread() - try: - response.raise_for_status() - except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code - log.debug("Encountered httpx.HTTPStatusError", exc_info=True) - - if remaining_retries > 0 and self._should_retry(err.response): - await err.response.aclose() - return await self._retry_request( - input_options, - cast_to, - retries_taken=retries_taken, - response_headers=err.response.headers, - stream=stream, - stream_cls=stream_cls, - ) + log.debug("Re-raising status error") + raise self._make_status_error_from_response(err.response) from None - # If the response is streamed then we need to explicitly read the response - # to completion before attempting to access the response text. - if not err.response.is_closed: - await err.response.aread() - - log.debug("Re-raising status error") - raise self._make_status_error_from_response(err.response) from None + break + assert response is not None, "could not resolve response (should never happen)" return await self._process_response( cast_to=cast_to, options=options, @@ -1553,35 +1542,20 @@ async def _request( retries_taken=retries_taken, ) - async def _retry_request( - self, - options: FinalRequestOptions, - cast_to: Type[ResponseT], - *, - retries_taken: int, - response_headers: httpx.Headers | None, - stream: bool, - stream_cls: type[_AsyncStreamT] | None, - ) -> ResponseT | _AsyncStreamT: - remaining_retries = options.get_max_retries(self.max_retries) - retries_taken + async def _sleep_for_retry( + self, *, retries_taken: int, max_retries: int, options: FinalRequestOptions, response: httpx.Response | None + ) -> None: + remaining_retries = max_retries - retries_taken if remaining_retries == 1: log.debug("1 retry left") else: log.debug("%i retries left", remaining_retries) - timeout = self._calculate_retry_timeout(remaining_retries, options, response_headers) + timeout = self._calculate_retry_timeout(remaining_retries, options, response.headers if response else None) log.info("Retrying request to %s in %f seconds", options.url, timeout) await anyio.sleep(timeout) - return await self._request( - options=options, - cast_to=cast_to, - retries_taken=retries_taken + 1, - stream=stream, - stream_cls=stream_cls, - ) - async def _process_response( self, *, diff --git a/src/contextual/_client.py b/src/contextual/_client.py index 820e0f04..9665f1b9 100644 --- a/src/contextual/_client.py +++ b/src/contextual/_client.py @@ -3,7 +3,7 @@ from __future__ import annotations import os -from typing import Any, Union, Mapping +from typing import TYPE_CHECKING, Any, Union, Mapping from typing_extensions import Self, override import httpx @@ -19,12 +19,9 @@ ProxiesTypes, RequestOptions, ) -from ._utils import ( - is_given, - get_async_library, -) +from ._utils import is_given, get_async_library +from ._compat import cached_property from ._version import __version__ -from .resources import users, lmunit, rerank, generate from ._streaming import Stream as Stream, AsyncStream as AsyncStream from ._exceptions import APIStatusError, ContextualAIError from ._base_client import ( @@ -32,8 +29,16 @@ SyncAPIClient, AsyncAPIClient, ) -from .resources.agents import agents -from .resources.datastores import datastores + +if TYPE_CHECKING: + from .resources import parse, users, agents, lmunit, rerank, generate, datastores + from .resources.parse import ParseResource, AsyncParseResource + from .resources.users import UsersResource, AsyncUsersResource + from .resources.lmunit import LMUnitResource, AsyncLMUnitResource + from .resources.rerank import RerankResource, AsyncRerankResource + from .resources.generate import GenerateResource, AsyncGenerateResource + from .resources.agents.agents import AgentsResource, AsyncAgentsResource + from .resources.datastores.datastores import DatastoresResource, AsyncDatastoresResource __all__ = [ "Timeout", @@ -48,15 +53,6 @@ class ContextualAI(SyncAPIClient): - datastores: datastores.DatastoresResource - agents: agents.AgentsResource - users: users.UsersResource - lmunit: lmunit.LMUnitResource - rerank: rerank.RerankResource - generate: generate.GenerateResource - with_raw_response: ContextualAIWithRawResponse - with_streaming_response: ContextualAIWithStreamedResponse - # client options api_key: str | None = None is_snowflake: bool = False @@ -119,14 +115,55 @@ def __init__( _strict_response_validation=_strict_response_validation, ) - self.datastores = datastores.DatastoresResource(self) - self.agents = agents.AgentsResource(self) - self.users = users.UsersResource(self) - self.lmunit = lmunit.LMUnitResource(self) - self.rerank = rerank.RerankResource(self) - self.generate = generate.GenerateResource(self) - self.with_raw_response = ContextualAIWithRawResponse(self) - self.with_streaming_response = ContextualAIWithStreamedResponse(self) + @cached_property + def datastores(self) -> DatastoresResource: + from .resources.datastores import DatastoresResource + + return DatastoresResource(self) + + @cached_property + def agents(self) -> AgentsResource: + from .resources.agents import AgentsResource + + return AgentsResource(self) + + @cached_property + def users(self) -> UsersResource: + from .resources.users import UsersResource + + return UsersResource(self) + + @cached_property + def lmunit(self) -> LMUnitResource: + from .resources.lmunit import LMUnitResource + + return LMUnitResource(self) + + @cached_property + def rerank(self) -> RerankResource: + from .resources.rerank import RerankResource + + return RerankResource(self) + + @cached_property + def generate(self) -> GenerateResource: + from .resources.generate import GenerateResource + + return GenerateResource(self) + + @cached_property + def parse(self) -> ParseResource: + from .resources.parse import ParseResource + + return ParseResource(self) + + @cached_property + def with_raw_response(self) -> ContextualAIWithRawResponse: + return ContextualAIWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ContextualAIWithStreamedResponse: + return ContextualAIWithStreamedResponse(self) @property @override @@ -239,15 +276,6 @@ def _make_status_error( class AsyncContextualAI(AsyncAPIClient): - datastores: datastores.AsyncDatastoresResource - agents: agents.AsyncAgentsResource - users: users.AsyncUsersResource - lmunit: lmunit.AsyncLMUnitResource - rerank: rerank.AsyncRerankResource - generate: generate.AsyncGenerateResource - with_raw_response: AsyncContextualAIWithRawResponse - with_streaming_response: AsyncContextualAIWithStreamedResponse - # client options api_key: str | None = None is_snowflake: bool = False @@ -310,14 +338,55 @@ def __init__( _strict_response_validation=_strict_response_validation, ) - self.datastores = datastores.AsyncDatastoresResource(self) - self.agents = agents.AsyncAgentsResource(self) - self.users = users.AsyncUsersResource(self) - self.lmunit = lmunit.AsyncLMUnitResource(self) - self.rerank = rerank.AsyncRerankResource(self) - self.generate = generate.AsyncGenerateResource(self) - self.with_raw_response = AsyncContextualAIWithRawResponse(self) - self.with_streaming_response = AsyncContextualAIWithStreamedResponse(self) + @cached_property + def datastores(self) -> AsyncDatastoresResource: + from .resources.datastores import AsyncDatastoresResource + + return AsyncDatastoresResource(self) + + @cached_property + def agents(self) -> AsyncAgentsResource: + from .resources.agents import AsyncAgentsResource + + return AsyncAgentsResource(self) + + @cached_property + def users(self) -> AsyncUsersResource: + from .resources.users import AsyncUsersResource + + return AsyncUsersResource(self) + + @cached_property + def lmunit(self) -> AsyncLMUnitResource: + from .resources.lmunit import AsyncLMUnitResource + + return AsyncLMUnitResource(self) + + @cached_property + def rerank(self) -> AsyncRerankResource: + from .resources.rerank import AsyncRerankResource + + return AsyncRerankResource(self) + + @cached_property + def generate(self) -> AsyncGenerateResource: + from .resources.generate import AsyncGenerateResource + + return AsyncGenerateResource(self) + + @cached_property + def parse(self) -> AsyncParseResource: + from .resources.parse import AsyncParseResource + + return AsyncParseResource(self) + + @cached_property + def with_raw_response(self) -> AsyncContextualAIWithRawResponse: + return AsyncContextualAIWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncContextualAIWithStreamedResponse: + return AsyncContextualAIWithStreamedResponse(self) @property @override @@ -430,43 +499,199 @@ def _make_status_error( class ContextualAIWithRawResponse: + _client: ContextualAI + def __init__(self, client: ContextualAI) -> None: - self.datastores = datastores.DatastoresResourceWithRawResponse(client.datastores) - self.agents = agents.AgentsResourceWithRawResponse(client.agents) - self.users = users.UsersResourceWithRawResponse(client.users) - self.lmunit = lmunit.LMUnitResourceWithRawResponse(client.lmunit) - self.rerank = rerank.RerankResourceWithRawResponse(client.rerank) - self.generate = generate.GenerateResourceWithRawResponse(client.generate) + self._client = client + + @cached_property + def datastores(self) -> datastores.DatastoresResourceWithRawResponse: + from .resources.datastores import DatastoresResourceWithRawResponse + + return DatastoresResourceWithRawResponse(self._client.datastores) + + @cached_property + def agents(self) -> agents.AgentsResourceWithRawResponse: + from .resources.agents import AgentsResourceWithRawResponse + + return AgentsResourceWithRawResponse(self._client.agents) + + @cached_property + def users(self) -> users.UsersResourceWithRawResponse: + from .resources.users import UsersResourceWithRawResponse + + return UsersResourceWithRawResponse(self._client.users) + + @cached_property + def lmunit(self) -> lmunit.LMUnitResourceWithRawResponse: + from .resources.lmunit import LMUnitResourceWithRawResponse + + return LMUnitResourceWithRawResponse(self._client.lmunit) + + @cached_property + def rerank(self) -> rerank.RerankResourceWithRawResponse: + from .resources.rerank import RerankResourceWithRawResponse + + return RerankResourceWithRawResponse(self._client.rerank) + + @cached_property + def generate(self) -> generate.GenerateResourceWithRawResponse: + from .resources.generate import GenerateResourceWithRawResponse + + return GenerateResourceWithRawResponse(self._client.generate) + + @cached_property + def parse(self) -> parse.ParseResourceWithRawResponse: + from .resources.parse import ParseResourceWithRawResponse + + return ParseResourceWithRawResponse(self._client.parse) class AsyncContextualAIWithRawResponse: + _client: AsyncContextualAI + def __init__(self, client: AsyncContextualAI) -> None: - self.datastores = datastores.AsyncDatastoresResourceWithRawResponse(client.datastores) - self.agents = agents.AsyncAgentsResourceWithRawResponse(client.agents) - self.users = users.AsyncUsersResourceWithRawResponse(client.users) - self.lmunit = lmunit.AsyncLMUnitResourceWithRawResponse(client.lmunit) - self.rerank = rerank.AsyncRerankResourceWithRawResponse(client.rerank) - self.generate = generate.AsyncGenerateResourceWithRawResponse(client.generate) + self._client = client + + @cached_property + def datastores(self) -> datastores.AsyncDatastoresResourceWithRawResponse: + from .resources.datastores import AsyncDatastoresResourceWithRawResponse + + return AsyncDatastoresResourceWithRawResponse(self._client.datastores) + + @cached_property + def agents(self) -> agents.AsyncAgentsResourceWithRawResponse: + from .resources.agents import AsyncAgentsResourceWithRawResponse + + return AsyncAgentsResourceWithRawResponse(self._client.agents) + + @cached_property + def users(self) -> users.AsyncUsersResourceWithRawResponse: + from .resources.users import AsyncUsersResourceWithRawResponse + + return AsyncUsersResourceWithRawResponse(self._client.users) + + @cached_property + def lmunit(self) -> lmunit.AsyncLMUnitResourceWithRawResponse: + from .resources.lmunit import AsyncLMUnitResourceWithRawResponse + + return AsyncLMUnitResourceWithRawResponse(self._client.lmunit) + + @cached_property + def rerank(self) -> rerank.AsyncRerankResourceWithRawResponse: + from .resources.rerank import AsyncRerankResourceWithRawResponse + + return AsyncRerankResourceWithRawResponse(self._client.rerank) + + @cached_property + def generate(self) -> generate.AsyncGenerateResourceWithRawResponse: + from .resources.generate import AsyncGenerateResourceWithRawResponse + + return AsyncGenerateResourceWithRawResponse(self._client.generate) + + @cached_property + def parse(self) -> parse.AsyncParseResourceWithRawResponse: + from .resources.parse import AsyncParseResourceWithRawResponse + + return AsyncParseResourceWithRawResponse(self._client.parse) class ContextualAIWithStreamedResponse: + _client: ContextualAI + def __init__(self, client: ContextualAI) -> None: - self.datastores = datastores.DatastoresResourceWithStreamingResponse(client.datastores) - self.agents = agents.AgentsResourceWithStreamingResponse(client.agents) - self.users = users.UsersResourceWithStreamingResponse(client.users) - self.lmunit = lmunit.LMUnitResourceWithStreamingResponse(client.lmunit) - self.rerank = rerank.RerankResourceWithStreamingResponse(client.rerank) - self.generate = generate.GenerateResourceWithStreamingResponse(client.generate) + self._client = client + + @cached_property + def datastores(self) -> datastores.DatastoresResourceWithStreamingResponse: + from .resources.datastores import DatastoresResourceWithStreamingResponse + + return DatastoresResourceWithStreamingResponse(self._client.datastores) + + @cached_property + def agents(self) -> agents.AgentsResourceWithStreamingResponse: + from .resources.agents import AgentsResourceWithStreamingResponse + + return AgentsResourceWithStreamingResponse(self._client.agents) + + @cached_property + def users(self) -> users.UsersResourceWithStreamingResponse: + from .resources.users import UsersResourceWithStreamingResponse + + return UsersResourceWithStreamingResponse(self._client.users) + + @cached_property + def lmunit(self) -> lmunit.LMUnitResourceWithStreamingResponse: + from .resources.lmunit import LMUnitResourceWithStreamingResponse + + return LMUnitResourceWithStreamingResponse(self._client.lmunit) + + @cached_property + def rerank(self) -> rerank.RerankResourceWithStreamingResponse: + from .resources.rerank import RerankResourceWithStreamingResponse + + return RerankResourceWithStreamingResponse(self._client.rerank) + + @cached_property + def generate(self) -> generate.GenerateResourceWithStreamingResponse: + from .resources.generate import GenerateResourceWithStreamingResponse + + return GenerateResourceWithStreamingResponse(self._client.generate) + + @cached_property + def parse(self) -> parse.ParseResourceWithStreamingResponse: + from .resources.parse import ParseResourceWithStreamingResponse + + return ParseResourceWithStreamingResponse(self._client.parse) class AsyncContextualAIWithStreamedResponse: + _client: AsyncContextualAI + def __init__(self, client: AsyncContextualAI) -> None: - self.datastores = datastores.AsyncDatastoresResourceWithStreamingResponse(client.datastores) - self.agents = agents.AsyncAgentsResourceWithStreamingResponse(client.agents) - self.users = users.AsyncUsersResourceWithStreamingResponse(client.users) - self.lmunit = lmunit.AsyncLMUnitResourceWithStreamingResponse(client.lmunit) - self.rerank = rerank.AsyncRerankResourceWithStreamingResponse(client.rerank) - self.generate = generate.AsyncGenerateResourceWithStreamingResponse(client.generate) + self._client = client + + @cached_property + def datastores(self) -> datastores.AsyncDatastoresResourceWithStreamingResponse: + from .resources.datastores import AsyncDatastoresResourceWithStreamingResponse + + return AsyncDatastoresResourceWithStreamingResponse(self._client.datastores) + + @cached_property + def agents(self) -> agents.AsyncAgentsResourceWithStreamingResponse: + from .resources.agents import AsyncAgentsResourceWithStreamingResponse + + return AsyncAgentsResourceWithStreamingResponse(self._client.agents) + + @cached_property + def users(self) -> users.AsyncUsersResourceWithStreamingResponse: + from .resources.users import AsyncUsersResourceWithStreamingResponse + + return AsyncUsersResourceWithStreamingResponse(self._client.users) + + @cached_property + def lmunit(self) -> lmunit.AsyncLMUnitResourceWithStreamingResponse: + from .resources.lmunit import AsyncLMUnitResourceWithStreamingResponse + + return AsyncLMUnitResourceWithStreamingResponse(self._client.lmunit) + + @cached_property + def rerank(self) -> rerank.AsyncRerankResourceWithStreamingResponse: + from .resources.rerank import AsyncRerankResourceWithStreamingResponse + + return AsyncRerankResourceWithStreamingResponse(self._client.rerank) + + @cached_property + def generate(self) -> generate.AsyncGenerateResourceWithStreamingResponse: + from .resources.generate import AsyncGenerateResourceWithStreamingResponse + + return AsyncGenerateResourceWithStreamingResponse(self._client.generate) + + @cached_property + def parse(self) -> parse.AsyncParseResourceWithStreamingResponse: + from .resources.parse import AsyncParseResourceWithStreamingResponse + + return AsyncParseResourceWithStreamingResponse(self._client.parse) Client = ContextualAI diff --git a/src/contextual/_models.py b/src/contextual/_models.py index c4401ff8..798956f1 100644 --- a/src/contextual/_models.py +++ b/src/contextual/_models.py @@ -19,7 +19,6 @@ ) import pydantic -import pydantic.generics from pydantic.fields import FieldInfo from ._types import ( @@ -65,7 +64,7 @@ from ._constants import RAW_RESPONSE_HEADER if TYPE_CHECKING: - from pydantic_core.core_schema import ModelField, LiteralSchema, ModelFieldsSchema + from pydantic_core.core_schema import ModelField, ModelSchema, LiteralSchema, ModelFieldsSchema __all__ = ["BaseModel", "GenericModel"] @@ -627,8 +626,8 @@ def _build_discriminated_union_meta(*, union: type, meta_annotations: tuple[Any, # Note: if one variant defines an alias then they all should discriminator_alias = field_info.alias - if field_info.annotation and is_literal_type(field_info.annotation): - for entry in get_args(field_info.annotation): + if (annotation := getattr(field_info, "annotation", None)) and is_literal_type(annotation): + for entry in get_args(annotation): if isinstance(entry, str): mapping[entry] = variant @@ -646,15 +645,18 @@ def _build_discriminated_union_meta(*, union: type, meta_annotations: tuple[Any, def _extract_field_schema_pv2(model: type[BaseModel], field_name: str) -> ModelField | None: schema = model.__pydantic_core_schema__ + if schema["type"] == "definitions": + schema = schema["schema"] + if schema["type"] != "model": return None + schema = cast("ModelSchema", schema) fields_schema = schema["schema"] if fields_schema["type"] != "model-fields": return None fields_schema = cast("ModelFieldsSchema", fields_schema) - field = fields_schema["fields"].get(field_name) if not field: return None @@ -678,7 +680,7 @@ def set_pydantic_config(typ: Any, config: pydantic.ConfigDict) -> None: setattr(typ, "__pydantic_config__", config) # noqa: B010 -# our use of subclasssing here causes weirdness for type checkers, +# our use of subclassing here causes weirdness for type checkers, # so we just pretend that we don't subclass if TYPE_CHECKING: GenericModel = BaseModel diff --git a/src/contextual/_response.py b/src/contextual/_response.py index 51fc249d..819adec3 100644 --- a/src/contextual/_response.py +++ b/src/contextual/_response.py @@ -235,7 +235,7 @@ def _parse(self, *, to: type[_T] | None = None) -> R | _T: # split is required to handle cases where additional information is included # in the response, e.g. application/json; charset=utf-8 content_type, *_ = response.headers.get("content-type", "*").split(";") - if content_type != "application/json": + if not content_type.endswith("json"): if is_basemodel(cast_to): try: data = response.json() diff --git a/src/contextual/_utils/_transform.py b/src/contextual/_utils/_transform.py index 18afd9d8..b0cc20a7 100644 --- a/src/contextual/_utils/_transform.py +++ b/src/contextual/_utils/_transform.py @@ -5,13 +5,15 @@ import pathlib from typing import Any, Mapping, TypeVar, cast from datetime import date, datetime -from typing_extensions import Literal, get_args, override, get_type_hints +from typing_extensions import Literal, get_args, override, get_type_hints as _get_type_hints import anyio import pydantic from ._utils import ( is_list, + is_given, + lru_cache, is_mapping, is_iterable, ) @@ -108,6 +110,7 @@ class Params(TypedDict, total=False): return cast(_T, transformed) +@lru_cache(maxsize=8096) def _get_annotated_type(type_: type) -> type | None: """If the given type is an `Annotated` type then it is returned, if not `None` is returned. @@ -126,7 +129,7 @@ def _get_annotated_type(type_: type) -> type | None: def _maybe_transform_key(key: str, type_: type) -> str: """Transform the given `data` based on the annotations provided in `type_`. - Note: this function only looks at `Annotated` types that contain `PropertInfo` metadata. + Note: this function only looks at `Annotated` types that contain `PropertyInfo` metadata. """ annotated_type = _get_annotated_type(type_) if annotated_type is None: @@ -142,6 +145,10 @@ def _maybe_transform_key(key: str, type_: type) -> str: return key +def _no_transform_needed(annotation: type) -> bool: + return annotation == float or annotation == int + + def _transform_recursive( data: object, *, @@ -184,6 +191,15 @@ def _transform_recursive( return cast(object, data) inner_type = extract_type_arg(stripped_type, 0) + if _no_transform_needed(inner_type): + # for some types there is no need to transform anything, so we can get a small + # perf boost from skipping that work. + # + # but we still need to convert to a list to ensure the data is json-serializable + if is_list(data): + return data + return list(data) + return [_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] if is_union_type(stripped_type): @@ -245,6 +261,11 @@ def _transform_typeddict( result: dict[str, object] = {} annotations = get_type_hints(expected_type, include_extras=True) for key, value in data.items(): + if not is_given(value): + # we don't need to include `NotGiven` values here as they'll + # be stripped out before the request is sent anyway + continue + type_ = annotations.get(key) if type_ is None: # we do not have a type annotation for this field, leave it as is @@ -332,6 +353,15 @@ async def _async_transform_recursive( return cast(object, data) inner_type = extract_type_arg(stripped_type, 0) + if _no_transform_needed(inner_type): + # for some types there is no need to transform anything, so we can get a small + # perf boost from skipping that work. + # + # but we still need to convert to a list to ensure the data is json-serializable + if is_list(data): + return data + return list(data) + return [await _async_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] if is_union_type(stripped_type): @@ -393,6 +423,11 @@ async def _async_transform_typeddict( result: dict[str, object] = {} annotations = get_type_hints(expected_type, include_extras=True) for key, value in data.items(): + if not is_given(value): + # we don't need to include `NotGiven` values here as they'll + # be stripped out before the request is sent anyway + continue + type_ = annotations.get(key) if type_ is None: # we do not have a type annotation for this field, leave it as is @@ -400,3 +435,13 @@ async def _async_transform_typeddict( else: result[_maybe_transform_key(key, type_)] = await _async_transform_recursive(value, annotation=type_) return result + + +@lru_cache(maxsize=8096) +def get_type_hints( + obj: Any, + globalns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, + include_extras: bool = False, +) -> dict[str, Any]: + return _get_type_hints(obj, globalns=globalns, localns=localns, include_extras=include_extras) diff --git a/src/contextual/_utils/_typing.py b/src/contextual/_utils/_typing.py index 278749b1..1bac9542 100644 --- a/src/contextual/_utils/_typing.py +++ b/src/contextual/_utils/_typing.py @@ -13,6 +13,7 @@ get_origin, ) +from ._utils import lru_cache from .._types import InheritsGeneric from .._compat import is_union as _is_union @@ -66,6 +67,7 @@ def is_type_alias_type(tp: Any, /) -> TypeIs[typing_extensions.TypeAliasType]: # Extracts T from Annotated[T, ...] or from Required[Annotated[T, ...]] +@lru_cache(maxsize=8096) def strip_annotated_type(typ: type) -> type: if is_required_type(typ) or is_annotated_type(typ): return strip_annotated_type(cast(type, get_args(typ)[0])) @@ -108,7 +110,7 @@ class MyResponse(Foo[_T]): ``` """ cls = cast(object, get_origin(typ) or typ) - if cls in generic_bases: + if cls in generic_bases: # pyright: ignore[reportUnnecessaryContains] # we're given the class directly return extract_type_arg(typ, index) diff --git a/src/contextual/_utils/_utils.py b/src/contextual/_utils/_utils.py index e5811bba..ea3cf3f2 100644 --- a/src/contextual/_utils/_utils.py +++ b/src/contextual/_utils/_utils.py @@ -72,8 +72,16 @@ def _extract_items( from .._files import assert_is_file_content # We have exhausted the path, return the entry we found. - assert_is_file_content(obj, key=flattened_key) assert flattened_key is not None + + if is_list(obj): + files: list[tuple[str, FileTypes]] = [] + for entry in obj: + assert_is_file_content(entry, key=flattened_key + "[]" if flattened_key else "") + files.append((flattened_key + "[]", cast(FileTypes, entry))) + return files + + assert_is_file_content(obj, key=flattened_key) return [(flattened_key, cast(FileTypes, obj))] index += 1 diff --git a/src/contextual/_version.py b/src/contextual/_version.py index 457f2de4..310e21a7 100644 --- a/src/contextual/_version.py +++ b/src/contextual/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "contextual" -__version__ = "0.5.1" # x-release-please-version +__version__ = "0.6.0" # x-release-please-version diff --git a/src/contextual/resources/__init__.py b/src/contextual/resources/__init__.py index 02594ebc..dc5e7f19 100644 --- a/src/contextual/resources/__init__.py +++ b/src/contextual/resources/__init__.py @@ -1,5 +1,13 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. +from .parse import ( + ParseResource, + AsyncParseResource, + ParseResourceWithRawResponse, + AsyncParseResourceWithRawResponse, + ParseResourceWithStreamingResponse, + AsyncParseResourceWithStreamingResponse, +) from .users import ( UsersResource, AsyncUsersResource, @@ -86,4 +94,10 @@ "AsyncGenerateResourceWithRawResponse", "GenerateResourceWithStreamingResponse", "AsyncGenerateResourceWithStreamingResponse", + "ParseResource", + "AsyncParseResource", + "ParseResourceWithRawResponse", + "AsyncParseResourceWithRawResponse", + "ParseResourceWithStreamingResponse", + "AsyncParseResourceWithStreamingResponse", ] diff --git a/src/contextual/resources/agents/agents.py b/src/contextual/resources/agents/agents.py index c55e149c..e823e2de 100644 --- a/src/contextual/resources/agents/agents.py +++ b/src/contextual/resources/agents/agents.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import List +from typing import Any, List, cast import httpx @@ -16,10 +16,7 @@ ) from ...types import agent_list_params, agent_create_params, agent_update_params from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import ( - maybe_transform, - async_maybe_transform, -) +from ..._utils import maybe_transform, async_maybe_transform from ..._compat import cached_property from .tune.tune import ( TuneResource, @@ -55,8 +52,9 @@ EvaluateResourceWithStreamingResponse, AsyncEvaluateResourceWithStreamingResponse, ) -from ...types.agent_metadata import AgentMetadata +from ...types.agent_configs_param import AgentConfigsParam from ...types.create_agent_output import CreateAgentOutput +from ...types.agent_metadata_response import AgentMetadataResponse __all__ = ["AgentsResource", "AsyncAgentsResource"] @@ -101,10 +99,11 @@ def create( self, *, name: str, - agent_configs: agent_create_params.AgentConfigs | NotGiven = NOT_GIVEN, + agent_configs: AgentConfigsParam | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, description: str | NotGiven = NOT_GIVEN, filter_prompt: str | NotGiven = NOT_GIVEN, + no_retrieval_system_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -144,6 +143,9 @@ def create( filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a given query and filters out irrelevant chunks. + no_retrieval_system_prompt: Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -169,6 +171,7 @@ def create( "datastore_ids": datastore_ids, "description": description, "filter_prompt": filter_prompt, + "no_retrieval_system_prompt": no_retrieval_system_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -184,10 +187,11 @@ def update( self, agent_id: str, *, - agent_configs: agent_update_params.AgentConfigs | NotGiven = NOT_GIVEN, + agent_configs: AgentConfigsParam | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, filter_prompt: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, + no_retrieval_system_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -216,6 +220,9 @@ def update( on which they were tuned. If no model is specified, the default model is used. Set to `default` to switch from a tuned model to the default model. + no_retrieval_system_prompt: Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -242,6 +249,7 @@ def update( "datastore_ids": datastore_ids, "filter_prompt": filter_prompt, "llm_model_id": llm_model_id, + "no_retrieval_system_prompt": no_retrieval_system_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -351,7 +359,7 @@ def metadata( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> AgentMetadata: + ) -> AgentMetadataResponse: """ Get metadata and configuration of a given `Agent`. @@ -368,12 +376,52 @@ def metadata( """ if not agent_id: raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") - return self._get( - f"/agents/{agent_id}/metadata", + return cast( + AgentMetadataResponse, + self._get( + f"/agents/{agent_id}/metadata", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=cast( + Any, AgentMetadataResponse + ), # Union types cannot be passed in as arguments in the type system + ), + ) + + def reset( + self, + agent_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """ + Reset a given `Agent` to default configuration. + + Args: + agent_id: ID of the agent to reset + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not agent_id: + raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") + return self._put( + f"/agents/{agent_id}/reset", options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=AgentMetadata, + cast_to=object, ) @@ -417,10 +465,11 @@ async def create( self, *, name: str, - agent_configs: agent_create_params.AgentConfigs | NotGiven = NOT_GIVEN, + agent_configs: AgentConfigsParam | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, description: str | NotGiven = NOT_GIVEN, filter_prompt: str | NotGiven = NOT_GIVEN, + no_retrieval_system_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -460,6 +509,9 @@ async def create( filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a given query and filters out irrelevant chunks. + no_retrieval_system_prompt: Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -485,6 +537,7 @@ async def create( "datastore_ids": datastore_ids, "description": description, "filter_prompt": filter_prompt, + "no_retrieval_system_prompt": no_retrieval_system_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -500,10 +553,11 @@ async def update( self, agent_id: str, *, - agent_configs: agent_update_params.AgentConfigs | NotGiven = NOT_GIVEN, + agent_configs: AgentConfigsParam | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, filter_prompt: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, + no_retrieval_system_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -532,6 +586,9 @@ async def update( on which they were tuned. If no model is specified, the default model is used. Set to `default` to switch from a tuned model to the default model. + no_retrieval_system_prompt: Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -558,6 +615,7 @@ async def update( "datastore_ids": datastore_ids, "filter_prompt": filter_prompt, "llm_model_id": llm_model_id, + "no_retrieval_system_prompt": no_retrieval_system_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -667,7 +725,7 @@ async def metadata( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> AgentMetadata: + ) -> AgentMetadataResponse: """ Get metadata and configuration of a given `Agent`. @@ -684,12 +742,52 @@ async def metadata( """ if not agent_id: raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") - return await self._get( - f"/agents/{agent_id}/metadata", + return cast( + AgentMetadataResponse, + await self._get( + f"/agents/{agent_id}/metadata", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=cast( + Any, AgentMetadataResponse + ), # Union types cannot be passed in as arguments in the type system + ), + ) + + async def reset( + self, + agent_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """ + Reset a given `Agent` to default configuration. + + Args: + agent_id: ID of the agent to reset + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not agent_id: + raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") + return await self._put( + f"/agents/{agent_id}/reset", options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=AgentMetadata, + cast_to=object, ) @@ -712,6 +810,9 @@ def __init__(self, agents: AgentsResource) -> None: self.metadata = to_raw_response_wrapper( agents.metadata, ) + self.reset = to_raw_response_wrapper( + agents.reset, + ) @cached_property def query(self) -> QueryResourceWithRawResponse: @@ -749,6 +850,9 @@ def __init__(self, agents: AsyncAgentsResource) -> None: self.metadata = async_to_raw_response_wrapper( agents.metadata, ) + self.reset = async_to_raw_response_wrapper( + agents.reset, + ) @cached_property def query(self) -> AsyncQueryResourceWithRawResponse: @@ -786,6 +890,9 @@ def __init__(self, agents: AgentsResource) -> None: self.metadata = to_streamed_response_wrapper( agents.metadata, ) + self.reset = to_streamed_response_wrapper( + agents.reset, + ) @cached_property def query(self) -> QueryResourceWithStreamingResponse: @@ -823,6 +930,9 @@ def __init__(self, agents: AsyncAgentsResource) -> None: self.metadata = async_to_streamed_response_wrapper( agents.metadata, ) + self.reset = async_to_streamed_response_wrapper( + agents.reset, + ) @cached_property def query(self) -> AsyncQueryResourceWithStreamingResponse: diff --git a/src/contextual/resources/agents/datasets/evaluate.py b/src/contextual/resources/agents/datasets/evaluate.py index 7eda6600..8ecaf1c8 100644 --- a/src/contextual/resources/agents/datasets/evaluate.py +++ b/src/contextual/resources/agents/datasets/evaluate.py @@ -8,12 +8,7 @@ import httpx from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes -from ...._utils import ( - extract_files, - maybe_transform, - deepcopy_minimal, - async_maybe_transform, -) +from ...._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import ( diff --git a/src/contextual/resources/agents/datasets/tune.py b/src/contextual/resources/agents/datasets/tune.py index 1a468bbb..e722664a 100644 --- a/src/contextual/resources/agents/datasets/tune.py +++ b/src/contextual/resources/agents/datasets/tune.py @@ -8,12 +8,7 @@ import httpx from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes -from ...._utils import ( - extract_files, - maybe_transform, - deepcopy_minimal, - async_maybe_transform, -) +from ...._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import ( diff --git a/src/contextual/resources/agents/evaluate/evaluate.py b/src/contextual/resources/agents/evaluate/evaluate.py index 11c00402..8eee69d3 100644 --- a/src/contextual/resources/agents/evaluate/evaluate.py +++ b/src/contextual/resources/agents/evaluate/evaluate.py @@ -16,12 +16,7 @@ AsyncJobsResourceWithStreamingResponse, ) from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes -from ...._utils import ( - extract_files, - maybe_transform, - deepcopy_minimal, - async_maybe_transform, -) +from ...._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import ( @@ -69,6 +64,8 @@ def create( evalset_file: FileTypes | NotGiven = NOT_GIVEN, evalset_name: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, + notes: str | NotGiven = NOT_GIVEN, + override_configuration: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -110,6 +107,11 @@ def create( llm_model_id: ID of the model to evaluate. Uses the default model if not specified. + notes: User notes for the evaluation job. + + override_configuration: Override the configuration for the query. This will override the configuration + for the agent during evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -126,6 +128,8 @@ def create( "evalset_file": evalset_file, "evalset_name": evalset_name, "llm_model_id": llm_model_id, + "notes": notes, + "override_configuration": override_configuration, } ) files = extract_files(cast(Mapping[str, object], body), paths=[["evalset_file"]]) @@ -176,6 +180,8 @@ async def create( evalset_file: FileTypes | NotGiven = NOT_GIVEN, evalset_name: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, + notes: str | NotGiven = NOT_GIVEN, + override_configuration: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -217,6 +223,11 @@ async def create( llm_model_id: ID of the model to evaluate. Uses the default model if not specified. + notes: User notes for the evaluation job. + + override_configuration: Override the configuration for the query. This will override the configuration + for the agent during evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -233,6 +244,8 @@ async def create( "evalset_file": evalset_file, "evalset_name": evalset_name, "llm_model_id": llm_model_id, + "notes": notes, + "override_configuration": override_configuration, } ) files = extract_files(cast(Mapping[str, object], body), paths=[["evalset_file"]]) diff --git a/src/contextual/resources/agents/query.py b/src/contextual/resources/agents/query.py index c02580a5..7e3f0c82 100644 --- a/src/contextual/resources/agents/query.py +++ b/src/contextual/resources/agents/query.py @@ -9,10 +9,7 @@ import httpx from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import ( - maybe_transform, - async_maybe_transform, -) +from ..._utils import maybe_transform, async_maybe_transform from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import ( @@ -66,6 +63,7 @@ def create( documents_filters: query_create_params.DocumentsFilters | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, stream: bool | NotGiven = NOT_GIVEN, + structured_output: query_create_params.StructuredOutput | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -138,6 +136,8 @@ def create( stream: Set to `true` to receive a streamed response + structured_output: Custom output structure format. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -157,6 +157,7 @@ def create( "documents_filters": documents_filters, "llm_model_id": llm_model_id, "stream": stream, + "structured_output": structured_output, }, query_create_params.QueryCreateParams, ), @@ -394,6 +395,7 @@ async def create( documents_filters: query_create_params.DocumentsFilters | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, stream: bool | NotGiven = NOT_GIVEN, + structured_output: query_create_params.StructuredOutput | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -466,6 +468,8 @@ async def create( stream: Set to `true` to receive a streamed response + structured_output: Custom output structure format. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -485,6 +489,7 @@ async def create( "documents_filters": documents_filters, "llm_model_id": llm_model_id, "stream": stream, + "structured_output": structured_output, }, query_create_params.QueryCreateParams, ), diff --git a/src/contextual/resources/agents/tune/tune.py b/src/contextual/resources/agents/tune/tune.py index 0b4e9ead..10bbb9d9 100644 --- a/src/contextual/resources/agents/tune/tune.py +++ b/src/contextual/resources/agents/tune/tune.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import Mapping, Optional, cast +from typing_extensions import Literal import httpx @@ -23,12 +24,7 @@ AsyncModelsResourceWithStreamingResponse, ) from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes -from ...._utils import ( - extract_files, - maybe_transform, - deepcopy_minimal, - async_maybe_transform, -) +from ...._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import ( @@ -76,6 +72,15 @@ def create( self, agent_id: str, *, + hyperparams_learning_rate: float | NotGiven = NOT_GIVEN, + hyperparams_lora_alpha: Literal[8, 16, 32, 64, 128] | NotGiven = NOT_GIVEN, + hyperparams_lora_dropout: float | NotGiven = NOT_GIVEN, + hyperparams_lora_rank: Literal[8, 16, 32, 64] | NotGiven = NOT_GIVEN, + hyperparams_num_epochs: int | NotGiven = NOT_GIVEN, + hyperparams_warmup_ratio: float | NotGiven = NOT_GIVEN, + metadata_file: FileTypes | NotGiven = NOT_GIVEN, + sdp_only: bool | NotGiven = NOT_GIVEN, + synth_data: bool | NotGiven = NOT_GIVEN, test_dataset_name: Optional[str] | NotGiven = NOT_GIVEN, test_file: Optional[FileTypes] | NotGiven = NOT_GIVEN, train_dataset_name: Optional[str] | NotGiven = NOT_GIVEN, @@ -117,6 +122,30 @@ def create( Args: agent_id: ID of the Agent to list tuning jobs for + hyperparams_learning_rate: Controls how quickly the model adapts to the training data. Must be greater than + 0 and less than or equal to 0.1. + + hyperparams_lora_alpha: Scaling factor that controls the magnitude of LoRA updates. Higher values lead + to stronger adaptation effects. The effective learning strength is determined by + the ratio of lora_alpha/lora_rank. Must be one of: 8, 16, 32, 64 or 128 + + hyperparams_lora_dropout: LoRA dropout randomly disables connections during training to prevent + overfitting and improve generalization when fine-tuning language models with + Low-Rank Adaptation. Must be between 0 and 1 (exclusive). + + hyperparams_lora_rank: Controls the capacity of the LoRA adapters. Must be one of: 8, 16, 32, or 64. + + hyperparams_num_epochs: Number of complete passes through the training dataset. + + hyperparams_warmup_ratio: Fraction of training steps used for learning rate warmup. Must be between 0 and + 1 (exclusive). + + metadata_file: Optional. Metadata file to use for synthetic data pipeline. + + sdp_only: Runs the SDP pipeline only if set to True. + + synth_data: Optional. Whether to generate synthetic data for training + test_dataset_name: Optional. `Dataset` to use for testing model checkpoints, created through the `/datasets/evaluate` API. @@ -173,13 +202,24 @@ def create( raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") body = deepcopy_minimal( { + "hyperparams_learning_rate": hyperparams_learning_rate, + "hyperparams_lora_alpha": hyperparams_lora_alpha, + "hyperparams_lora_dropout": hyperparams_lora_dropout, + "hyperparams_lora_rank": hyperparams_lora_rank, + "hyperparams_num_epochs": hyperparams_num_epochs, + "hyperparams_warmup_ratio": hyperparams_warmup_ratio, + "metadata_file": metadata_file, + "sdp_only": sdp_only, + "synth_data": synth_data, "test_dataset_name": test_dataset_name, "test_file": test_file, "train_dataset_name": train_dataset_name, "training_file": training_file, } ) - files = extract_files(cast(Mapping[str, object], body), paths=[["training_file"], ["test_file"]]) + files = extract_files( + cast(Mapping[str, object], body), paths=[["training_file"], ["test_file"], ["metadata_file"]] + ) # It should be noted that the actual Content-Type header that will be # sent to the server will contain a `boundary` parameter, e.g. # multipart/form-data; boundary=---abc-- @@ -227,6 +267,15 @@ async def create( self, agent_id: str, *, + hyperparams_learning_rate: float | NotGiven = NOT_GIVEN, + hyperparams_lora_alpha: Literal[8, 16, 32, 64, 128] | NotGiven = NOT_GIVEN, + hyperparams_lora_dropout: float | NotGiven = NOT_GIVEN, + hyperparams_lora_rank: Literal[8, 16, 32, 64] | NotGiven = NOT_GIVEN, + hyperparams_num_epochs: int | NotGiven = NOT_GIVEN, + hyperparams_warmup_ratio: float | NotGiven = NOT_GIVEN, + metadata_file: FileTypes | NotGiven = NOT_GIVEN, + sdp_only: bool | NotGiven = NOT_GIVEN, + synth_data: bool | NotGiven = NOT_GIVEN, test_dataset_name: Optional[str] | NotGiven = NOT_GIVEN, test_file: Optional[FileTypes] | NotGiven = NOT_GIVEN, train_dataset_name: Optional[str] | NotGiven = NOT_GIVEN, @@ -268,6 +317,30 @@ async def create( Args: agent_id: ID of the Agent to list tuning jobs for + hyperparams_learning_rate: Controls how quickly the model adapts to the training data. Must be greater than + 0 and less than or equal to 0.1. + + hyperparams_lora_alpha: Scaling factor that controls the magnitude of LoRA updates. Higher values lead + to stronger adaptation effects. The effective learning strength is determined by + the ratio of lora_alpha/lora_rank. Must be one of: 8, 16, 32, 64 or 128 + + hyperparams_lora_dropout: LoRA dropout randomly disables connections during training to prevent + overfitting and improve generalization when fine-tuning language models with + Low-Rank Adaptation. Must be between 0 and 1 (exclusive). + + hyperparams_lora_rank: Controls the capacity of the LoRA adapters. Must be one of: 8, 16, 32, or 64. + + hyperparams_num_epochs: Number of complete passes through the training dataset. + + hyperparams_warmup_ratio: Fraction of training steps used for learning rate warmup. Must be between 0 and + 1 (exclusive). + + metadata_file: Optional. Metadata file to use for synthetic data pipeline. + + sdp_only: Runs the SDP pipeline only if set to True. + + synth_data: Optional. Whether to generate synthetic data for training + test_dataset_name: Optional. `Dataset` to use for testing model checkpoints, created through the `/datasets/evaluate` API. @@ -324,13 +397,24 @@ async def create( raise ValueError(f"Expected a non-empty value for `agent_id` but received {agent_id!r}") body = deepcopy_minimal( { + "hyperparams_learning_rate": hyperparams_learning_rate, + "hyperparams_lora_alpha": hyperparams_lora_alpha, + "hyperparams_lora_dropout": hyperparams_lora_dropout, + "hyperparams_lora_rank": hyperparams_lora_rank, + "hyperparams_num_epochs": hyperparams_num_epochs, + "hyperparams_warmup_ratio": hyperparams_warmup_ratio, + "metadata_file": metadata_file, + "sdp_only": sdp_only, + "synth_data": synth_data, "test_dataset_name": test_dataset_name, "test_file": test_file, "train_dataset_name": train_dataset_name, "training_file": training_file, } ) - files = extract_files(cast(Mapping[str, object], body), paths=[["training_file"], ["test_file"]]) + files = extract_files( + cast(Mapping[str, object], body), paths=[["training_file"], ["test_file"], ["metadata_file"]] + ) # It should be noted that the actual Content-Type header that will be # sent to the server will contain a `boundary` parameter, e.g. # multipart/form-data; boundary=---abc-- diff --git a/src/contextual/resources/datastores/datastores.py b/src/contextual/resources/datastores/datastores.py index 549ba733..0eed851d 100644 --- a/src/contextual/resources/datastores/datastores.py +++ b/src/contextual/resources/datastores/datastores.py @@ -6,10 +6,7 @@ from ...types import datastore_list_params, datastore_create_params from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import ( - maybe_transform, - async_maybe_transform, -) +from ..._utils import maybe_transform, async_maybe_transform from ..._compat import cached_property from .documents import ( DocumentsResource, @@ -241,6 +238,43 @@ def metadata( cast_to=DatastoreMetadata, ) + def reset( + self, + datastore_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """Reset the give `Datastore`. + + This operation is irreversible and it deletes all + the documents associated with the datastore. + + Args: + datastore_id: ID of the datastore to edit + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not datastore_id: + raise ValueError(f"Expected a non-empty value for `datastore_id` but received {datastore_id!r}") + return self._put( + f"/datastores/{datastore_id}/reset", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=object, + ) + class AsyncDatastoresResource(AsyncAPIResource): @cached_property @@ -448,6 +482,43 @@ async def metadata( cast_to=DatastoreMetadata, ) + async def reset( + self, + datastore_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """Reset the give `Datastore`. + + This operation is irreversible and it deletes all + the documents associated with the datastore. + + Args: + datastore_id: ID of the datastore to edit + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not datastore_id: + raise ValueError(f"Expected a non-empty value for `datastore_id` but received {datastore_id!r}") + return await self._put( + f"/datastores/{datastore_id}/reset", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=object, + ) + class DatastoresResourceWithRawResponse: def __init__(self, datastores: DatastoresResource) -> None: @@ -465,6 +536,9 @@ def __init__(self, datastores: DatastoresResource) -> None: self.metadata = to_raw_response_wrapper( datastores.metadata, ) + self.reset = to_raw_response_wrapper( + datastores.reset, + ) @cached_property def documents(self) -> DocumentsResourceWithRawResponse: @@ -487,6 +561,9 @@ def __init__(self, datastores: AsyncDatastoresResource) -> None: self.metadata = async_to_raw_response_wrapper( datastores.metadata, ) + self.reset = async_to_raw_response_wrapper( + datastores.reset, + ) @cached_property def documents(self) -> AsyncDocumentsResourceWithRawResponse: @@ -509,6 +586,9 @@ def __init__(self, datastores: DatastoresResource) -> None: self.metadata = to_streamed_response_wrapper( datastores.metadata, ) + self.reset = to_streamed_response_wrapper( + datastores.reset, + ) @cached_property def documents(self) -> DocumentsResourceWithStreamingResponse: @@ -531,6 +611,9 @@ def __init__(self, datastores: AsyncDatastoresResource) -> None: self.metadata = async_to_streamed_response_wrapper( datastores.metadata, ) + self.reset = async_to_streamed_response_wrapper( + datastores.reset, + ) @cached_property def documents(self) -> AsyncDocumentsResourceWithStreamingResponse: diff --git a/src/contextual/resources/datastores/documents.py b/src/contextual/resources/datastores/documents.py index aab15af9..51294919 100644 --- a/src/contextual/resources/datastores/documents.py +++ b/src/contextual/resources/datastores/documents.py @@ -9,12 +9,7 @@ import httpx from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes -from ..._utils import ( - extract_files, - maybe_transform, - deepcopy_minimal, - async_maybe_transform, -) +from ..._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import ( @@ -57,7 +52,21 @@ def list( datastore_id: str, *, cursor: str | NotGiven = NOT_GIVEN, - ingestion_job_status: List[Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"]] + ingestion_job_status: List[ + Literal[ + "pending", + "processing", + "retrying", + "completed", + "failed", + "cancelled", + "failed_to_provision", + "generating_data", + "training_in_progress", + "failed_to_generate_data", + "provisioning", + ] + ] | NotGiven = NOT_GIVEN, limit: int | NotGiven = NOT_GIVEN, uploaded_after: Union[str, datetime] | NotGiven = NOT_GIVEN, @@ -209,13 +218,11 @@ def ingest( **Example Metadata JSON:** ```json - { - "metadata": { + metadata = { "custom_metadata": { - "customKey1": "value3", - "_filterKey": "filterValue3" - } - } + "field1": "value1", + "field2": "value2" + } } ``` @@ -363,7 +370,21 @@ def list( datastore_id: str, *, cursor: str | NotGiven = NOT_GIVEN, - ingestion_job_status: List[Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"]] + ingestion_job_status: List[ + Literal[ + "pending", + "processing", + "retrying", + "completed", + "failed", + "cancelled", + "failed_to_provision", + "generating_data", + "training_in_progress", + "failed_to_generate_data", + "provisioning", + ] + ] | NotGiven = NOT_GIVEN, limit: int | NotGiven = NOT_GIVEN, uploaded_after: Union[str, datetime] | NotGiven = NOT_GIVEN, @@ -515,13 +536,11 @@ async def ingest( **Example Metadata JSON:** ```json - { - "metadata": { + metadata = { "custom_metadata": { - "customKey1": "value3", - "_filterKey": "filterValue3" - } - } + "field1": "value1", + "field2": "value2" + } } ``` diff --git a/src/contextual/resources/generate.py b/src/contextual/resources/generate.py index 88a6009d..388f81da 100644 --- a/src/contextual/resources/generate.py +++ b/src/contextual/resources/generate.py @@ -8,10 +8,7 @@ from ..types import generate_create_params from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from .._utils import ( - maybe_transform, - async_maybe_transform, -) +from .._utils import maybe_transform, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import ( @@ -70,9 +67,11 @@ def create( parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation and agentic use cases. - The total request cannot exceed 32,000 tokens. See more details and code - examples in our - [our blog post](https://contextual.ai/blog/introducing-grounded-language-model/). + The total request cannot exceed 32,000 tokens. + + See our + [blog post](https://contextual.ai/blog/introducing-grounded-language-model/) and + [code examples](https://colab.research.google.com/github/ContextualAI/examples/blob/main/03-standalone-api/02-generate/generate.ipynb). Email [glm-feedback@contextual.ai](mailto:glm-feedback@contextual.ai) with any feedback or questions. @@ -176,9 +175,11 @@ async def create( parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation and agentic use cases. - The total request cannot exceed 32,000 tokens. See more details and code - examples in our - [our blog post](https://contextual.ai/blog/introducing-grounded-language-model/). + The total request cannot exceed 32,000 tokens. + + See our + [blog post](https://contextual.ai/blog/introducing-grounded-language-model/) and + [code examples](https://colab.research.google.com/github/ContextualAI/examples/blob/main/03-standalone-api/02-generate/generate.ipynb). Email [glm-feedback@contextual.ai](mailto:glm-feedback@contextual.ai) with any feedback or questions. diff --git a/src/contextual/resources/lmunit.py b/src/contextual/resources/lmunit.py index 23caecf8..ceb000cc 100644 --- a/src/contextual/resources/lmunit.py +++ b/src/contextual/resources/lmunit.py @@ -6,10 +6,7 @@ from ..types import lmunit_create_params from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from .._utils import ( - maybe_transform, - async_maybe_transform, -) +from .._utils import maybe_transform, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import ( diff --git a/src/contextual/resources/parse.py b/src/contextual/resources/parse.py new file mode 100644 index 00000000..4c80c491 --- /dev/null +++ b/src/contextual/resources/parse.py @@ -0,0 +1,570 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Mapping, cast +from datetime import datetime +from typing_extensions import Literal + +import httpx + +from ..types import parse_jobs_params, parse_create_params, parse_job_results_params +from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes +from .._utils import extract_files, maybe_transform, deepcopy_minimal, async_maybe_transform +from .._compat import cached_property +from .._resource import SyncAPIResource, AsyncAPIResource +from .._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from .._base_client import make_request_options +from ..types.parse_jobs_response import ParseJobsResponse +from ..types.parse_create_response import ParseCreateResponse +from ..types.parse_job_status_response import ParseJobStatusResponse +from ..types.parse_job_results_response import ParseJobResultsResponse + +__all__ = ["ParseResource", "AsyncParseResource"] + + +class ParseResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> ParseResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/ContextualAI/contextual-client-python#accessing-raw-response-data-eg-headers + """ + return ParseResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ParseResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/ContextualAI/contextual-client-python#with_streaming_response + """ + return ParseResourceWithStreamingResponse(self) + + def create( + self, + *, + raw_file: FileTypes, + enable_document_hierarchy: bool | NotGiven = NOT_GIVEN, + enable_split_tables: bool | NotGiven = NOT_GIVEN, + figure_caption_mode: Literal["concise", "detailed"] | NotGiven = NOT_GIVEN, + max_split_table_cells: int | NotGiven = NOT_GIVEN, + page_range: str | NotGiven = NOT_GIVEN, + parse_mode: Literal["basic", "standard"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseCreateResponse: + """Parse a file into a structured Markdown representation. + + The file size must be + less than 100MB and the number of pages must be less than 400. + + Email [parse-feedback@contextual.ai](mailto:parse-feedback@contextual.ai) with + any feedback or questions. + + Args: + raw_file: The file to be parsed. The file type must be PDF, DOC / DOCX, PPT / PPTX. + + enable_document_hierarchy: Controls parsing heading levels (e.g. H1, H2, H3) at higher quality. Adds a + table of contents to the output with the structure of the entire parsed + document. Not permitted in 'basic' parsing_mode, or if page_range is not + continuous and/or does not start from page zero. + + enable_split_tables: Controls whether tables are split into multiple tables by row with the headers + propagated. Use for improving LLM comprehension of very large tables. Not + permitted in 'basic' parsing_mode. + + figure_caption_mode: Controls how thorough figure captions are. 'concise' is short and minimizes + chances of hallucinations. 'detailed' is more thorough and can include + commentary. Not permitted in 'basic' parsing_mode. + + max_split_table_cells: Threshold number of table cells beyond which large tables are split if + `enable_split_tables` is True. Not permitted in 'basic' parsing_mode. + + page_range: Optional string representing page range to be parsed. Format: comma-separated + indexes (0-based) e.g. '0,1,2,5,6' or ranges (inclusive of both ends) e.g. + '0-2,5,6' + + parse_mode: The settings to use for parsing. 'basic' is for simple, text-only documents. + 'standard' is for complex documents with images, complex hierarchy, and/or no + natively encoded textual data (e.g. for scanned documents). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + body = deepcopy_minimal( + { + "raw_file": raw_file, + "enable_document_hierarchy": enable_document_hierarchy, + "enable_split_tables": enable_split_tables, + "figure_caption_mode": figure_caption_mode, + "max_split_table_cells": max_split_table_cells, + "page_range": page_range, + "parse_mode": parse_mode, + } + ) + files = extract_files(cast(Mapping[str, object], body), paths=[["raw_file"]]) + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} + return self._post( + "/parse", + body=maybe_transform(body, parse_create_params.ParseCreateParams), + files=files, + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParseCreateResponse, + ) + + def job_results( + self, + job_id: str, + *, + output_types: List[Literal["markdown-document", "markdown-per-page", "blocks-per-page"]] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobResultsResponse: + """ + Get the results of a parse job. + + Parse job results are retained for up to 30 days after job creation. Fetching + results for a parse job that is older than 30 days will return a 404 error. + + Args: + job_id: Unique ID of the parse job + + output_types: The desired output format(s) of the parsed file. Must be `markdown-document`, + `markdown-per-page`, and/or `blocks-per-page`. `markdown-document` parses the + whole document into a single concatenated markdown output. `markdown-per-page` + provides markdown output per page. `blocks-per-page` provides a structured JSON + representation of the content blocks on each page, sorted by reading order. + Specify multiple values to get multiple formats in the response. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return self._get( + f"/parse/jobs/{job_id}/results", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"output_types": output_types}, parse_job_results_params.ParseJobResultsParams), + ), + cast_to=ParseJobResultsResponse, + ) + + def job_status( + self, + job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobStatusResponse: + """ + Get the status of a parse job. + + Parse job results are retained for up to 30 days after job creation. Fetching a + status for a parse job that is older than 30 days will return a 404 error. + + Args: + job_id: Unique ID of the parse job + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return self._get( + f"/parse/jobs/{job_id}/status", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParseJobStatusResponse, + ) + + def jobs( + self, + *, + uploaded_after: Union[str, datetime] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobsResponse: + """ + Get list of parse jobs, sorted from most recent to oldest. + + Returns all jobs from the last 30 days, or since the optional `uploaded_after` + timestamp. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get( + "/parse/jobs", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"uploaded_after": uploaded_after}, parse_jobs_params.ParseJobsParams), + ), + cast_to=ParseJobsResponse, + ) + + +class AsyncParseResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncParseResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/ContextualAI/contextual-client-python#accessing-raw-response-data-eg-headers + """ + return AsyncParseResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncParseResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/ContextualAI/contextual-client-python#with_streaming_response + """ + return AsyncParseResourceWithStreamingResponse(self) + + async def create( + self, + *, + raw_file: FileTypes, + enable_document_hierarchy: bool | NotGiven = NOT_GIVEN, + enable_split_tables: bool | NotGiven = NOT_GIVEN, + figure_caption_mode: Literal["concise", "detailed"] | NotGiven = NOT_GIVEN, + max_split_table_cells: int | NotGiven = NOT_GIVEN, + page_range: str | NotGiven = NOT_GIVEN, + parse_mode: Literal["basic", "standard"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseCreateResponse: + """Parse a file into a structured Markdown representation. + + The file size must be + less than 100MB and the number of pages must be less than 400. + + Email [parse-feedback@contextual.ai](mailto:parse-feedback@contextual.ai) with + any feedback or questions. + + Args: + raw_file: The file to be parsed. The file type must be PDF, DOC / DOCX, PPT / PPTX. + + enable_document_hierarchy: Controls parsing heading levels (e.g. H1, H2, H3) at higher quality. Adds a + table of contents to the output with the structure of the entire parsed + document. Not permitted in 'basic' parsing_mode, or if page_range is not + continuous and/or does not start from page zero. + + enable_split_tables: Controls whether tables are split into multiple tables by row with the headers + propagated. Use for improving LLM comprehension of very large tables. Not + permitted in 'basic' parsing_mode. + + figure_caption_mode: Controls how thorough figure captions are. 'concise' is short and minimizes + chances of hallucinations. 'detailed' is more thorough and can include + commentary. Not permitted in 'basic' parsing_mode. + + max_split_table_cells: Threshold number of table cells beyond which large tables are split if + `enable_split_tables` is True. Not permitted in 'basic' parsing_mode. + + page_range: Optional string representing page range to be parsed. Format: comma-separated + indexes (0-based) e.g. '0,1,2,5,6' or ranges (inclusive of both ends) e.g. + '0-2,5,6' + + parse_mode: The settings to use for parsing. 'basic' is for simple, text-only documents. + 'standard' is for complex documents with images, complex hierarchy, and/or no + natively encoded textual data (e.g. for scanned documents). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + body = deepcopy_minimal( + { + "raw_file": raw_file, + "enable_document_hierarchy": enable_document_hierarchy, + "enable_split_tables": enable_split_tables, + "figure_caption_mode": figure_caption_mode, + "max_split_table_cells": max_split_table_cells, + "page_range": page_range, + "parse_mode": parse_mode, + } + ) + files = extract_files(cast(Mapping[str, object], body), paths=[["raw_file"]]) + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} + return await self._post( + "/parse", + body=await async_maybe_transform(body, parse_create_params.ParseCreateParams), + files=files, + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParseCreateResponse, + ) + + async def job_results( + self, + job_id: str, + *, + output_types: List[Literal["markdown-document", "markdown-per-page", "blocks-per-page"]] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobResultsResponse: + """ + Get the results of a parse job. + + Parse job results are retained for up to 30 days after job creation. Fetching + results for a parse job that is older than 30 days will return a 404 error. + + Args: + job_id: Unique ID of the parse job + + output_types: The desired output format(s) of the parsed file. Must be `markdown-document`, + `markdown-per-page`, and/or `blocks-per-page`. `markdown-document` parses the + whole document into a single concatenated markdown output. `markdown-per-page` + provides markdown output per page. `blocks-per-page` provides a structured JSON + representation of the content blocks on each page, sorted by reading order. + Specify multiple values to get multiple formats in the response. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return await self._get( + f"/parse/jobs/{job_id}/results", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + {"output_types": output_types}, parse_job_results_params.ParseJobResultsParams + ), + ), + cast_to=ParseJobResultsResponse, + ) + + async def job_status( + self, + job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobStatusResponse: + """ + Get the status of a parse job. + + Parse job results are retained for up to 30 days after job creation. Fetching a + status for a parse job that is older than 30 days will return a 404 error. + + Args: + job_id: Unique ID of the parse job + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return await self._get( + f"/parse/jobs/{job_id}/status", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParseJobStatusResponse, + ) + + async def jobs( + self, + *, + uploaded_after: Union[str, datetime] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ParseJobsResponse: + """ + Get list of parse jobs, sorted from most recent to oldest. + + Returns all jobs from the last 30 days, or since the optional `uploaded_after` + timestamp. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._get( + "/parse/jobs", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + {"uploaded_after": uploaded_after}, parse_jobs_params.ParseJobsParams + ), + ), + cast_to=ParseJobsResponse, + ) + + +class ParseResourceWithRawResponse: + def __init__(self, parse: ParseResource) -> None: + self._parse = parse + + self.create = to_raw_response_wrapper( + parse.create, + ) + self.job_results = to_raw_response_wrapper( + parse.job_results, + ) + self.job_status = to_raw_response_wrapper( + parse.job_status, + ) + self.jobs = to_raw_response_wrapper( + parse.jobs, + ) + + +class AsyncParseResourceWithRawResponse: + def __init__(self, parse: AsyncParseResource) -> None: + self._parse = parse + + self.create = async_to_raw_response_wrapper( + parse.create, + ) + self.job_results = async_to_raw_response_wrapper( + parse.job_results, + ) + self.job_status = async_to_raw_response_wrapper( + parse.job_status, + ) + self.jobs = async_to_raw_response_wrapper( + parse.jobs, + ) + + +class ParseResourceWithStreamingResponse: + def __init__(self, parse: ParseResource) -> None: + self._parse = parse + + self.create = to_streamed_response_wrapper( + parse.create, + ) + self.job_results = to_streamed_response_wrapper( + parse.job_results, + ) + self.job_status = to_streamed_response_wrapper( + parse.job_status, + ) + self.jobs = to_streamed_response_wrapper( + parse.jobs, + ) + + +class AsyncParseResourceWithStreamingResponse: + def __init__(self, parse: AsyncParseResource) -> None: + self._parse = parse + + self.create = async_to_streamed_response_wrapper( + parse.create, + ) + self.job_results = async_to_streamed_response_wrapper( + parse.job_results, + ) + self.job_status = async_to_streamed_response_wrapper( + parse.job_status, + ) + self.jobs = async_to_streamed_response_wrapper( + parse.jobs, + ) diff --git a/src/contextual/resources/rerank.py b/src/contextual/resources/rerank.py index c46e5e52..425fc35e 100644 --- a/src/contextual/resources/rerank.py +++ b/src/contextual/resources/rerank.py @@ -8,10 +8,7 @@ from ..types import rerank_create_params from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from .._utils import ( - maybe_transform, - async_maybe_transform, -) +from .._utils import maybe_transform, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import ( @@ -63,16 +60,21 @@ def create( timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RerankCreateResponse: """ - Rank a list of documents according to their relevance to a query and your custom - instructions about how to prioritize retrievals. We evaluated the model on - instructions for recency, document type, source, and metadata, and it can - generalize to other instructions as well. + Rank a list of documents according to their relevance to a query primarily and + your custom instructions secondarily. We evaluated the model on instructions for + recency, document type, source, and metadata, and it can generalize to other + instructions as well. The total request cannot exceed 400,000 tokens. The combined length of the query, instruction and any document with its metadata must not exceed 8,000 - tokens. Email - [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with any - feedback or questions. + tokens. + + See our + [blog post](https://contextual.ai/blog/introducing-instruction-following-reranker/) + and + [code examples](https://colab.research.google.com/github/ContextualAI/examples/blob/main/03-standalone-api/03-rerank/rerank.ipynb). + Email [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with + any feedback or questions. Args: documents: The texts to be reranked according to their relevance to the query and the @@ -83,15 +85,14 @@ def create( query: The string against which documents will be ranked for relevance - instruction: Instructions that the reranker references when ranking retrievals. We evaluated - the model on instructions for recency, document type, source, and metadata, and - it can generalize to other instructions as well. Note that we do not guarantee - that the reranker will follow these instructions exactly. Examples: "Prioritize - internal sales documents over market analysis reports. More recent documents - should be weighted higher. Enterprise portal content supersedes distributor - communications." and "Emphasize forecasts from top-tier investment banks. Recent - analysis should take precedence. Disregard aggregator sites and favor detailed - research notes over news summaries." + instruction: Instructions that the reranker references when ranking documents, after + considering relevance. We evaluated the model on instructions for recency, + document type, source, and metadata, and it can generalize to other instructions + as well. For instructions related to recency and timeframe, specify the + timeframe (e.g., instead of saying "this year") because the reranker doesn't + know the current date. Example: "Prioritize internal sales documents over market + analysis reports. More recent documents should be weighted higher. Enterprise + portal content supersedes distributor communications." metadata: Metadata for documents being passed to the reranker. Must be the same length as the documents list. If a document does not have metadata, add an empty string. @@ -163,16 +164,21 @@ async def create( timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RerankCreateResponse: """ - Rank a list of documents according to their relevance to a query and your custom - instructions about how to prioritize retrievals. We evaluated the model on - instructions for recency, document type, source, and metadata, and it can - generalize to other instructions as well. + Rank a list of documents according to their relevance to a query primarily and + your custom instructions secondarily. We evaluated the model on instructions for + recency, document type, source, and metadata, and it can generalize to other + instructions as well. The total request cannot exceed 400,000 tokens. The combined length of the query, instruction and any document with its metadata must not exceed 8,000 - tokens. Email - [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with any - feedback or questions. + tokens. + + See our + [blog post](https://contextual.ai/blog/introducing-instruction-following-reranker/) + and + [code examples](https://colab.research.google.com/github/ContextualAI/examples/blob/main/03-standalone-api/03-rerank/rerank.ipynb). + Email [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with + any feedback or questions. Args: documents: The texts to be reranked according to their relevance to the query and the @@ -183,15 +189,14 @@ async def create( query: The string against which documents will be ranked for relevance - instruction: Instructions that the reranker references when ranking retrievals. We evaluated - the model on instructions for recency, document type, source, and metadata, and - it can generalize to other instructions as well. Note that we do not guarantee - that the reranker will follow these instructions exactly. Examples: "Prioritize - internal sales documents over market analysis reports. More recent documents - should be weighted higher. Enterprise portal content supersedes distributor - communications." and "Emphasize forecasts from top-tier investment banks. Recent - analysis should take precedence. Disregard aggregator sites and favor detailed - research notes over news summaries." + instruction: Instructions that the reranker references when ranking documents, after + considering relevance. We evaluated the model on instructions for recency, + document type, source, and metadata, and it can generalize to other instructions + as well. For instructions related to recency and timeframe, specify the + timeframe (e.g., instead of saying "this year") because the reranker doesn't + know the current date. Example: "Prioritize internal sales documents over market + analysis reports. More recent documents should be weighted higher. Enterprise + portal content supersedes distributor communications." metadata: Metadata for documents being passed to the reranker. Must be the same length as the documents list. If a document does not have metadata, add an empty string. diff --git a/src/contextual/resources/users.py b/src/contextual/resources/users.py index d50f002e..74dd7404 100644 --- a/src/contextual/resources/users.py +++ b/src/contextual/resources/users.py @@ -9,10 +9,7 @@ from ..types import user_list_params, user_invite_params, user_update_params, user_deactivate_params from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from .._utils import ( - maybe_transform, - async_maybe_transform, -) +from .._utils import maybe_transform, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import ( @@ -55,8 +52,19 @@ def update( *, email: str, is_tenant_admin: bool | NotGiven = NOT_GIVEN, - per_agent_roles: Iterable[user_update_params.PerAgentRole] | NotGiven = NOT_GIVEN, - roles: List[Literal["AGENT_USER"]] | NotGiven = NOT_GIVEN, + roles: List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] + | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -74,10 +82,6 @@ def update( is_tenant_admin: Flag indicating if the user is a tenant admin - per_agent_roles: Per agent level roles for the user. If a user is granted any role under `roles`, - then the user has that role for all the agents. Only the roles that need to be - updated should be part of this. - roles: The user level roles of the user. extra_headers: Send extra headers @@ -94,7 +98,6 @@ def update( { "email": email, "is_tenant_admin": is_tenant_admin, - "per_agent_roles": per_agent_roles, "roles": roles, }, user_update_params.UserUpdateParams, @@ -266,8 +269,19 @@ async def update( *, email: str, is_tenant_admin: bool | NotGiven = NOT_GIVEN, - per_agent_roles: Iterable[user_update_params.PerAgentRole] | NotGiven = NOT_GIVEN, - roles: List[Literal["AGENT_USER"]] | NotGiven = NOT_GIVEN, + roles: List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] + | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -285,10 +299,6 @@ async def update( is_tenant_admin: Flag indicating if the user is a tenant admin - per_agent_roles: Per agent level roles for the user. If a user is granted any role under `roles`, - then the user has that role for all the agents. Only the roles that need to be - updated should be part of this. - roles: The user level roles of the user. extra_headers: Send extra headers @@ -305,7 +315,6 @@ async def update( { "email": email, "is_tenant_admin": is_tenant_admin, - "per_agent_roles": per_agent_roles, "roles": roles, }, user_update_params.UserUpdateParams, diff --git a/src/contextual/types/__init__.py b/src/contextual/types/__init__.py index d5285a29..a57d67be 100644 --- a/src/contextual/types/__init__.py +++ b/src/contextual/types/__init__.py @@ -4,27 +4,45 @@ from .agent import Agent as Agent from .datastore import Datastore as Datastore +from .agent_configs import AgentConfigs as AgentConfigs +from .global_config import GlobalConfig as GlobalConfig from .agent_metadata import AgentMetadata as AgentMetadata from .new_user_param import NewUserParam as NewUserParam +from .retrieval_config import RetrievalConfig as RetrievalConfig from .user_list_params import UserListParams as UserListParams from .agent_list_params import AgentListParams as AgentListParams +from .parse_jobs_params import ParseJobsParams as ParseJobsParams from .datastore_metadata import DatastoreMetadata as DatastoreMetadata from .user_invite_params import UserInviteParams as UserInviteParams from .user_update_params import UserUpdateParams as UserUpdateParams +from .agent_configs_param import AgentConfigsParam as AgentConfigsParam from .agent_create_params import AgentCreateParams as AgentCreateParams from .agent_update_params import AgentUpdateParams as AgentUpdateParams from .create_agent_output import CreateAgentOutput as CreateAgentOutput +from .global_config_param import GlobalConfigParam as GlobalConfigParam from .list_users_response import ListUsersResponse as ListUsersResponse +from .parse_create_params import ParseCreateParams as ParseCreateParams +from .parse_jobs_response import ParseJobsResponse as ParseJobsResponse from .list_agents_response import ListAgentsResponse as ListAgentsResponse from .lmunit_create_params import LMUnitCreateParams as LMUnitCreateParams from .rerank_create_params import RerankCreateParams as RerankCreateParams from .datastore_list_params import DatastoreListParams as DatastoreListParams from .invite_users_response import InviteUsersResponse as InviteUsersResponse +from .parse_create_response import ParseCreateResponse as ParseCreateResponse from .generate_create_params import GenerateCreateParams as GenerateCreateParams from .lmunit_create_response import LMUnitCreateResponse as LMUnitCreateResponse from .rerank_create_response import RerankCreateResponse as RerankCreateResponse +from .retrieval_config_param import RetrievalConfigParam as RetrievalConfigParam from .user_deactivate_params import UserDeactivateParams as UserDeactivateParams +from .agent_metadata_response import AgentMetadataResponse as AgentMetadataResponse from .datastore_create_params import DatastoreCreateParams as DatastoreCreateParams +from .filter_and_rerank_config import FilterAndRerankConfig as FilterAndRerankConfig from .generate_create_response import GenerateCreateResponse as GenerateCreateResponse +from .generate_response_config import GenerateResponseConfig as GenerateResponseConfig from .list_datastores_response import ListDatastoresResponse as ListDatastoresResponse +from .parse_job_results_params import ParseJobResultsParams as ParseJobResultsParams from .create_datastore_response import CreateDatastoreResponse as CreateDatastoreResponse +from .parse_job_status_response import ParseJobStatusResponse as ParseJobStatusResponse +from .parse_job_results_response import ParseJobResultsResponse as ParseJobResultsResponse +from .filter_and_rerank_config_param import FilterAndRerankConfigParam as FilterAndRerankConfigParam +from .generate_response_config_param import GenerateResponseConfigParam as GenerateResponseConfigParam diff --git a/src/contextual/types/agent.py b/src/contextual/types/agent.py index 61a39239..6a5f649e 100644 --- a/src/contextual/types/agent.py +++ b/src/contextual/types/agent.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from .._models import BaseModel __all__ = ["Agent"] diff --git a/src/contextual/types/agent_configs.py b/src/contextual/types/agent_configs.py new file mode 100644 index 00000000..fcc43976 --- /dev/null +++ b/src/contextual/types/agent_configs.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel +from .global_config import GlobalConfig +from .retrieval_config import RetrievalConfig +from .filter_and_rerank_config import FilterAndRerankConfig +from .generate_response_config import GenerateResponseConfig + +__all__ = ["AgentConfigs"] + + +class AgentConfigs(BaseModel): + filter_and_rerank_config: Optional[FilterAndRerankConfig] = None + """Parameters that affect filtering and reranking of retrieved knowledge""" + + generate_response_config: Optional[GenerateResponseConfig] = None + """Parameters that affect response generation""" + + global_config: Optional[GlobalConfig] = None + """Parameters that affect the agent's overall RAG workflow""" + + retrieval_config: Optional[RetrievalConfig] = None + """Parameters that affect how the agent retrieves from datastore(s)""" diff --git a/src/contextual/types/agent_configs_param.py b/src/contextual/types/agent_configs_param.py new file mode 100644 index 00000000..75b329ce --- /dev/null +++ b/src/contextual/types/agent_configs_param.py @@ -0,0 +1,26 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +from .global_config_param import GlobalConfigParam +from .retrieval_config_param import RetrievalConfigParam +from .filter_and_rerank_config_param import FilterAndRerankConfigParam +from .generate_response_config_param import GenerateResponseConfigParam + +__all__ = ["AgentConfigsParam"] + + +class AgentConfigsParam(TypedDict, total=False): + filter_and_rerank_config: FilterAndRerankConfigParam + """Parameters that affect filtering and reranking of retrieved knowledge""" + + generate_response_config: GenerateResponseConfigParam + """Parameters that affect response generation""" + + global_config: GlobalConfigParam + """Parameters that affect the agent's overall RAG workflow""" + + retrieval_config: RetrievalConfigParam + """Parameters that affect how the agent retrieves from datastore(s)""" diff --git a/src/contextual/types/agent_create_params.py b/src/contextual/types/agent_create_params.py index 2da638a3..b08ecfa1 100644 --- a/src/contextual/types/agent_create_params.py +++ b/src/contextual/types/agent_create_params.py @@ -5,21 +5,16 @@ from typing import List from typing_extensions import Required, TypedDict -__all__ = [ - "AgentCreateParams", - "AgentConfigs", - "AgentConfigsFilterAndRerankConfig", - "AgentConfigsGenerateResponseConfig", - "AgentConfigsGlobalConfig", - "AgentConfigsRetrievalConfig", -] +from .agent_configs_param import AgentConfigsParam + +__all__ = ["AgentCreateParams"] class AgentCreateParams(TypedDict, total=False): name: Required[str] """Name of the agent""" - agent_configs: AgentConfigs + agent_configs: AgentConfigsParam """The following advanced parameters are experimental and subject to change.""" datastore_ids: List[str] @@ -34,6 +29,12 @@ class AgentCreateParams(TypedDict, total=False): given query and filters out irrelevant chunks. """ + no_retrieval_system_prompt: str + """ + Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + """ + suggested_queries: List[str] """ These queries will show up as suggestions in the Contextual UI when users load @@ -48,76 +49,3 @@ class AgentCreateParams(TypedDict, total=False): Note that we do not guarantee that the system will follow these instructions exactly. """ - - -class AgentConfigsFilterAndRerankConfig(TypedDict, total=False): - top_k_reranked_chunks: int - """The number of highest ranked chunks after reranking to be used""" - - -class AgentConfigsGenerateResponseConfig(TypedDict, total=False): - calculate_groundedness: bool - """This parameter controls generation of groundedness scores.""" - - frequency_penalty: float - """ - This parameter adjusts how the model treats repeated tokens during text - generation. - """ - - max_new_tokens: int - """The maximum number of tokens the model can generate in a response.""" - - seed: int - """ - This parameter controls the randomness of how the model selects the next tokens - during text generation. - """ - - temperature: float - """The sampling temperature, which affects the randomness in the response.""" - - top_p: float - """ - A parameter for nucleus sampling, an alternative to `temperature` which also - affects the randomness of the response. - """ - - -class AgentConfigsGlobalConfig(TypedDict, total=False): - enable_filter: bool - """Enables filtering of retrieved chunks with a separate LLM""" - - enable_multi_turn: bool - """Enables multi-turn conversations. - - This feature is currently experimental and will be improved. - """ - - enable_rerank: bool - """Enables reranking of retrieved chunks""" - - -class AgentConfigsRetrievalConfig(TypedDict, total=False): - lexical_alpha: float - """The weight of lexical search during retrieval""" - - semantic_alpha: float - """The weight of semantic search during retrieval""" - - top_k_retrieved_chunks: int - """The maximum number of retrieved chunks from the datastore.""" - - -class AgentConfigs(TypedDict, total=False): - filter_and_rerank_config: AgentConfigsFilterAndRerankConfig - """Parameters that affect filtering and reranking of retrieved knowledge""" - - generate_response_config: AgentConfigsGenerateResponseConfig - """Parameters that affect response generation""" - - global_config: AgentConfigsGlobalConfig - """Parameters that affect the agent's overall RAG workflow""" - - retrieval_config: AgentConfigsRetrievalConfig - """Parameters that affect how the agent retrieves from datastore(s)""" diff --git a/src/contextual/types/agent_metadata.py b/src/contextual/types/agent_metadata.py index afd9230b..f85393df 100644 --- a/src/contextual/types/agent_metadata.py +++ b/src/contextual/types/agent_metadata.py @@ -3,88 +3,20 @@ from typing import List, Optional from .._models import BaseModel +from .agent_configs import AgentConfigs -__all__ = [ - "AgentMetadata", - "AgentConfigs", - "AgentConfigsFilterAndRerankConfig", - "AgentConfigsGenerateResponseConfig", - "AgentConfigsGlobalConfig", - "AgentConfigsRetrievalConfig", -] +__all__ = ["AgentMetadata", "AgentUsages"] -class AgentConfigsFilterAndRerankConfig(BaseModel): - top_k_reranked_chunks: Optional[int] = None - """The number of highest ranked chunks after reranking to be used""" +class AgentUsages(BaseModel): + eval: int + """eval request count""" + query: int + """query request count""" -class AgentConfigsGenerateResponseConfig(BaseModel): - calculate_groundedness: Optional[bool] = None - """This parameter controls generation of groundedness scores.""" - - frequency_penalty: Optional[float] = None - """ - This parameter adjusts how the model treats repeated tokens during text - generation. - """ - - max_new_tokens: Optional[int] = None - """The maximum number of tokens the model can generate in a response.""" - - seed: Optional[int] = None - """ - This parameter controls the randomness of how the model selects the next tokens - during text generation. - """ - - temperature: Optional[float] = None - """The sampling temperature, which affects the randomness in the response.""" - - top_p: Optional[float] = None - """ - A parameter for nucleus sampling, an alternative to `temperature` which also - affects the randomness of the response. - """ - - -class AgentConfigsGlobalConfig(BaseModel): - enable_filter: Optional[bool] = None - """Enables filtering of retrieved chunks with a separate LLM""" - - enable_multi_turn: Optional[bool] = None - """Enables multi-turn conversations. - - This feature is currently experimental and will be improved. - """ - - enable_rerank: Optional[bool] = None - """Enables reranking of retrieved chunks""" - - -class AgentConfigsRetrievalConfig(BaseModel): - lexical_alpha: Optional[float] = None - """The weight of lexical search during retrieval""" - - semantic_alpha: Optional[float] = None - """The weight of semantic search during retrieval""" - - top_k_retrieved_chunks: Optional[int] = None - """The maximum number of retrieved chunks from the datastore.""" - - -class AgentConfigs(BaseModel): - filter_and_rerank_config: Optional[AgentConfigsFilterAndRerankConfig] = None - """Parameters that affect filtering and reranking of retrieved knowledge""" - - generate_response_config: Optional[AgentConfigsGenerateResponseConfig] = None - """Parameters that affect response generation""" - - global_config: Optional[AgentConfigsGlobalConfig] = None - """Parameters that affect the agent's overall RAG workflow""" - - retrieval_config: Optional[AgentConfigsRetrievalConfig] = None - """Parameters that affect how the agent retrieves from datastore(s)""" + tune: int + """tune request count""" class AgentMetadata(BaseModel): @@ -97,6 +29,9 @@ class AgentMetadata(BaseModel): agent_configs: Optional[AgentConfigs] = None """The following advanced parameters are experimental and subject to change.""" + agent_usages: Optional[AgentUsages] = None + """Total API request counts for the agent.""" + description: Optional[str] = None """Description of the agent""" @@ -114,6 +49,12 @@ class AgentMetadata(BaseModel): tuned model to the default model. """ + no_retrieval_system_prompt: Optional[str] = None + """ + Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + """ + suggested_queries: Optional[List[str]] = None """ These queries will show up as suggestions in the Contextual UI when users load diff --git a/src/contextual/types/agent_metadata_response.py b/src/contextual/types/agent_metadata_response.py new file mode 100644 index 00000000..9ad80e86 --- /dev/null +++ b/src/contextual/types/agent_metadata_response.py @@ -0,0 +1,40 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import TypeAlias + +from .._models import BaseModel +from .agent_metadata import AgentMetadata + +__all__ = ["AgentMetadataResponse", "GetTwilightAgentResponse", "GetTwilightAgentResponseAgentUsages"] + + +class GetTwilightAgentResponseAgentUsages(BaseModel): + eval: int + """eval request count""" + + query: int + """query request count""" + + tune: int + """tune request count""" + + +class GetTwilightAgentResponse(BaseModel): + datastore_ids: List[str] + """The IDs of the datastore(s) associated with the agent""" + + name: str + """Name of the agent""" + + agent_configs: Optional[object] = None + """The following advanced parameters are experimental and subject to change.""" + + agent_usages: Optional[GetTwilightAgentResponseAgentUsages] = None + """Total API request counts for the agent.""" + + description: Optional[str] = None + """Description of the agent""" + + +AgentMetadataResponse: TypeAlias = Union[AgentMetadata, GetTwilightAgentResponse] diff --git a/src/contextual/types/agent_update_params.py b/src/contextual/types/agent_update_params.py index 791caf97..35b156a0 100644 --- a/src/contextual/types/agent_update_params.py +++ b/src/contextual/types/agent_update_params.py @@ -5,18 +5,13 @@ from typing import List from typing_extensions import TypedDict -__all__ = [ - "AgentUpdateParams", - "AgentConfigs", - "AgentConfigsFilterAndRerankConfig", - "AgentConfigsGenerateResponseConfig", - "AgentConfigsGlobalConfig", - "AgentConfigsRetrievalConfig", -] +from .agent_configs_param import AgentConfigsParam + +__all__ = ["AgentUpdateParams"] class AgentUpdateParams(TypedDict, total=False): - agent_configs: AgentConfigs + agent_configs: AgentConfigsParam """The following advanced parameters are experimental and subject to change.""" datastore_ids: List[str] @@ -36,6 +31,12 @@ class AgentUpdateParams(TypedDict, total=False): tuned model to the default model. """ + no_retrieval_system_prompt: str + """ + Instructions on how the agent should respond when there are no relevant + retrievals that can be used to answer a query. + """ + suggested_queries: List[str] """ These queries will show up as suggestions in the Contextual UI when users load @@ -50,76 +51,3 @@ class AgentUpdateParams(TypedDict, total=False): Note that we do not guarantee that the system will follow these instructions exactly. """ - - -class AgentConfigsFilterAndRerankConfig(TypedDict, total=False): - top_k_reranked_chunks: int - """The number of highest ranked chunks after reranking to be used""" - - -class AgentConfigsGenerateResponseConfig(TypedDict, total=False): - calculate_groundedness: bool - """This parameter controls generation of groundedness scores.""" - - frequency_penalty: float - """ - This parameter adjusts how the model treats repeated tokens during text - generation. - """ - - max_new_tokens: int - """The maximum number of tokens the model can generate in a response.""" - - seed: int - """ - This parameter controls the randomness of how the model selects the next tokens - during text generation. - """ - - temperature: float - """The sampling temperature, which affects the randomness in the response.""" - - top_p: float - """ - A parameter for nucleus sampling, an alternative to `temperature` which also - affects the randomness of the response. - """ - - -class AgentConfigsGlobalConfig(TypedDict, total=False): - enable_filter: bool - """Enables filtering of retrieved chunks with a separate LLM""" - - enable_multi_turn: bool - """Enables multi-turn conversations. - - This feature is currently experimental and will be improved. - """ - - enable_rerank: bool - """Enables reranking of retrieved chunks""" - - -class AgentConfigsRetrievalConfig(TypedDict, total=False): - lexical_alpha: float - """The weight of lexical search during retrieval""" - - semantic_alpha: float - """The weight of semantic search during retrieval""" - - top_k_retrieved_chunks: int - """The maximum number of retrieved chunks from the datastore.""" - - -class AgentConfigs(TypedDict, total=False): - filter_and_rerank_config: AgentConfigsFilterAndRerankConfig - """Parameters that affect filtering and reranking of retrieved knowledge""" - - generate_response_config: AgentConfigsGenerateResponseConfig - """Parameters that affect response generation""" - - global_config: AgentConfigsGlobalConfig - """Parameters that affect the agent's overall RAG workflow""" - - retrieval_config: AgentConfigsRetrievalConfig - """Parameters that affect how the agent retrieves from datastore(s)""" diff --git a/src/contextual/types/agents/create_evaluation_response.py b/src/contextual/types/agents/create_evaluation_response.py index db10df42..01175839 100644 --- a/src/contextual/types/agents/create_evaluation_response.py +++ b/src/contextual/types/agents/create_evaluation_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from ..._models import BaseModel __all__ = ["CreateEvaluationResponse"] diff --git a/src/contextual/types/agents/create_tune_response.py b/src/contextual/types/agents/create_tune_response.py index 542880fe..9ab67620 100644 --- a/src/contextual/types/agents/create_tune_response.py +++ b/src/contextual/types/agents/create_tune_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from ..._models import BaseModel __all__ = ["CreateTuneResponse"] diff --git a/src/contextual/types/agents/evaluate/evaluation_job_metadata.py b/src/contextual/types/agents/evaluate/evaluation_job_metadata.py index 9be981da..c0d97062 100644 --- a/src/contextual/types/agents/evaluate/evaluation_job_metadata.py +++ b/src/contextual/types/agents/evaluate/evaluation_job_metadata.py @@ -35,5 +35,17 @@ class EvaluationJobMetadata(BaseModel): metrics: object """Results of the evaluation round, grouped by each metric""" - status: Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"] + status: Literal[ + "pending", + "processing", + "retrying", + "completed", + "failed", + "cancelled", + "failed_to_provision", + "generating_data", + "training_in_progress", + "failed_to_generate_data", + "provisioning", + ] """Status of the evaluation round""" diff --git a/src/contextual/types/agents/evaluate/list_evaluation_jobs_response.py b/src/contextual/types/agents/evaluate/list_evaluation_jobs_response.py index 15477f04..7b07c377 100644 --- a/src/contextual/types/agents/evaluate/list_evaluation_jobs_response.py +++ b/src/contextual/types/agents/evaluate/list_evaluation_jobs_response.py @@ -16,7 +16,19 @@ class EvaluationRound(BaseModel): created_at: datetime """Timestamp indicating when the evaluation round was created""" - status: Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"] + status: Literal[ + "pending", + "processing", + "retrying", + "completed", + "failed", + "cancelled", + "failed_to_provision", + "generating_data", + "training_in_progress", + "failed_to_generate_data", + "provisioning", + ] """Status of the evaluation round""" user_email: str @@ -25,6 +37,9 @@ class EvaluationRound(BaseModel): finished_at: Optional[datetime] = None """Timestamp indicating when the evaluation round finished processing""" + notes: Optional[str] = None + """User notes for the evaluation job""" + num_failed_predictions: Optional[int] = None """Number of predictions that failed during the evaluation round""" diff --git a/src/contextual/types/agents/evaluate_create_params.py b/src/contextual/types/agents/evaluate_create_params.py index b84f2b31..42d3e8fb 100644 --- a/src/contextual/types/agents/evaluate_create_params.py +++ b/src/contextual/types/agents/evaluate_create_params.py @@ -30,3 +30,12 @@ class EvaluateCreateParams(TypedDict, total=False): llm_model_id: str """ID of the model to evaluate. Uses the default model if not specified.""" + + notes: str + """User notes for the evaluation job.""" + + override_configuration: str + """Override the configuration for the query. + + This will override the configuration for the agent during evaluation. + """ diff --git a/src/contextual/types/agents/query_create_params.py b/src/contextual/types/agents/query_create_params.py index ad2aceeb..71a023f9 100644 --- a/src/contextual/types/agents/query_create_params.py +++ b/src/contextual/types/agents/query_create_params.py @@ -5,7 +5,7 @@ from typing import List, Union, Iterable from typing_extensions import Literal, Required, TypeAlias, TypedDict -__all__ = ["QueryCreateParams", "Message", "DocumentsFilters", "DocumentsFiltersBaseMetadataFilter"] +__all__ = ["QueryCreateParams", "Message", "DocumentsFilters", "DocumentsFiltersBaseMetadataFilter", "StructuredOutput"] class QueryCreateParams(TypedDict, total=False): @@ -86,6 +86,9 @@ class QueryCreateParams(TypedDict, total=False): stream: bool """Set to `true` to receive a streamed response""" + structured_output: StructuredOutput + """Custom output structure format.""" + class Message(TypedDict, total=False): content: Required[str] @@ -113,4 +116,13 @@ class DocumentsFiltersBaseMetadataFilter(TypedDict, total=False): DocumentsFilters: TypeAlias = Union[DocumentsFiltersBaseMetadataFilter, "CompositeMetadataFilterParam"] + +class StructuredOutput(TypedDict, total=False): + json_schema: Required[object] + """The output json structure.""" + + type: Literal["JSON"] + """Type of the structured output. The default is JSON""" + + from ..datastores.composite_metadata_filter_param import CompositeMetadataFilterParam diff --git a/src/contextual/types/agents/query_response.py b/src/contextual/types/agents/query_response.py index 45aa5cde..2e45b095 100644 --- a/src/contextual/types/agents/query_response.py +++ b/src/contextual/types/agents/query_response.py @@ -39,6 +39,9 @@ class RetrievalContent(BaseModel): page: Optional[int] = None """Page number of the content in the document""" + score: Optional[float] = None + """Score of the retrieval, if applicable""" + url: Optional[str] = None """URL of the source content, if applicable""" diff --git a/src/contextual/types/agents/retrieval_info_response.py b/src/contextual/types/agents/retrieval_info_response.py index 6f7d96ab..56cb51d3 100644 --- a/src/contextual/types/agents/retrieval_info_response.py +++ b/src/contextual/types/agents/retrieval_info_response.py @@ -1,13 +1,20 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Optional +from typing import List, Union, Optional +from typing_extensions import Literal, Annotated, TypeAlias +from ..._utils import PropertyInfo from ..._models import BaseModel -__all__ = ["RetrievalInfoResponse", "ContentMetadata"] +__all__ = [ + "RetrievalInfoResponse", + "ContentMetadata", + "ContentMetadataUnstructuredContentMetadata", + "ContentMetadataStructuredContentMetadata", +] -class ContentMetadata(BaseModel): +class ContentMetadataUnstructuredContentMetadata(BaseModel): content_id: str """Id of the content.""" @@ -38,6 +45,24 @@ class ContentMetadata(BaseModel): y1: float """Y coordinate of the bottom right corner on the bounding box.""" + content_type: Optional[Literal["unstructured"]] = None + + +class ContentMetadataStructuredContentMetadata(BaseModel): + content_id: str + """Id of the content.""" + + content_text: object + """Text of the content.""" + + content_type: Optional[Literal["structured"]] = None + + +ContentMetadata: TypeAlias = Annotated[ + Union[ContentMetadataUnstructuredContentMetadata, ContentMetadataStructuredContentMetadata], + PropertyInfo(discriminator="content_type"), +] + class RetrievalInfoResponse(BaseModel): content_metadatas: Optional[List[ContentMetadata]] = None diff --git a/src/contextual/types/agents/tune_create_params.py b/src/contextual/types/agents/tune_create_params.py index ac1e1cdc..2c5dc144 100644 --- a/src/contextual/types/agents/tune_create_params.py +++ b/src/contextual/types/agents/tune_create_params.py @@ -3,14 +3,57 @@ from __future__ import annotations from typing import Optional -from typing_extensions import TypedDict +from typing_extensions import Literal, Annotated, TypedDict from ..._types import FileTypes +from ..._utils import PropertyInfo __all__ = ["TuneCreateParams"] class TuneCreateParams(TypedDict, total=False): + hyperparams_learning_rate: Annotated[float, PropertyInfo(alias="hyperparams[learning_rate]")] + """Controls how quickly the model adapts to the training data. + + Must be greater than 0 and less than or equal to 0.1. + """ + + hyperparams_lora_alpha: Annotated[Literal[8, 16, 32, 64, 128], PropertyInfo(alias="hyperparams[lora_alpha]")] + """Scaling factor that controls the magnitude of LoRA updates. + + Higher values lead to stronger adaptation effects. The effective learning + strength is determined by the ratio of lora_alpha/lora_rank. Must be one of: 8, + 16, 32, 64 or 128 + """ + + hyperparams_lora_dropout: Annotated[float, PropertyInfo(alias="hyperparams[lora_dropout]")] + """ + LoRA dropout randomly disables connections during training to prevent + overfitting and improve generalization when fine-tuning language models with + Low-Rank Adaptation. Must be between 0 and 1 (exclusive). + """ + + hyperparams_lora_rank: Annotated[Literal[8, 16, 32, 64], PropertyInfo(alias="hyperparams[lora_rank]")] + """Controls the capacity of the LoRA adapters. Must be one of: 8, 16, 32, or 64.""" + + hyperparams_num_epochs: Annotated[int, PropertyInfo(alias="hyperparams[num_epochs]")] + """Number of complete passes through the training dataset.""" + + hyperparams_warmup_ratio: Annotated[float, PropertyInfo(alias="hyperparams[warmup_ratio]")] + """Fraction of training steps used for learning rate warmup. + + Must be between 0 and 1 (exclusive). + """ + + metadata_file: FileTypes + """Optional. Metadata file to use for synthetic data pipeline.""" + + sdp_only: bool + """Runs the SDP pipeline only if set to True.""" + + synth_data: bool + """Optional. Whether to generate synthetic data for training""" + test_dataset_name: Optional[str] """Optional. diff --git a/src/contextual/types/create_datastore_response.py b/src/contextual/types/create_datastore_response.py index fcbbcf0d..a4f5ed22 100644 --- a/src/contextual/types/create_datastore_response.py +++ b/src/contextual/types/create_datastore_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from .._models import BaseModel __all__ = ["CreateDatastoreResponse"] diff --git a/src/contextual/types/datastore_metadata.py b/src/contextual/types/datastore_metadata.py index ae983d61..39208c03 100644 --- a/src/contextual/types/datastore_metadata.py +++ b/src/contextual/types/datastore_metadata.py @@ -1,11 +1,16 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List +from typing import List, Optional from datetime import datetime from .._models import BaseModel -__all__ = ["DatastoreMetadata"] +__all__ = ["DatastoreMetadata", "DatastoreUsages"] + + +class DatastoreUsages(BaseModel): + size_gb: float + """Actual size of the datastore in GB""" class DatastoreMetadata(BaseModel): @@ -17,3 +22,6 @@ class DatastoreMetadata(BaseModel): name: str """Name of the datastore""" + + datastore_usages: Optional[DatastoreUsages] = None + """Datastore usage""" diff --git a/src/contextual/types/datastores/document_ingest_params.py b/src/contextual/types/datastores/document_ingest_params.py index 8e98cc60..92b0ab41 100644 --- a/src/contextual/types/datastores/document_ingest_params.py +++ b/src/contextual/types/datastores/document_ingest_params.py @@ -25,13 +25,11 @@ class DocumentIngestParams(TypedDict, total=False): **Example Metadata JSON:** ```json - { - "metadata": { + metadata = { "custom_metadata": { - "customKey1": "value3", - "_filterKey": "filterValue3" - } - } + "field1": "value1", + "field2": "value2" + } } ``` """ diff --git a/src/contextual/types/datastores/document_list_params.py b/src/contextual/types/datastores/document_list_params.py index 268b5e46..596f4283 100644 --- a/src/contextual/types/datastores/document_list_params.py +++ b/src/contextual/types/datastores/document_list_params.py @@ -18,7 +18,21 @@ class DocumentListParams(TypedDict, total=False): of results """ - ingestion_job_status: List[Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"]] + ingestion_job_status: List[ + Literal[ + "pending", + "processing", + "retrying", + "completed", + "failed", + "cancelled", + "failed_to_provision", + "generating_data", + "training_in_progress", + "failed_to_generate_data", + "provisioning", + ] + ] """ Filters documents whose ingestion job status matches (one of) the provided status(es). diff --git a/src/contextual/types/datastores/ingestion_response.py b/src/contextual/types/datastores/ingestion_response.py index 8027f503..cd8558ff 100644 --- a/src/contextual/types/datastores/ingestion_response.py +++ b/src/contextual/types/datastores/ingestion_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from ..._models import BaseModel __all__ = ["IngestionResponse"] diff --git a/src/contextual/types/filter_and_rerank_config.py b/src/contextual/types/filter_and_rerank_config.py new file mode 100644 index 00000000..05071746 --- /dev/null +++ b/src/contextual/types/filter_and_rerank_config.py @@ -0,0 +1,31 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["FilterAndRerankConfig"] + + +class FilterAndRerankConfig(BaseModel): + rerank_instructions: Optional[str] = None + """Instructions that the reranker references when ranking retrievals. + + Note that we do not guarantee that the reranker will follow these instructions + exactly. Examples: "Prioritize internal sales documents over market analysis + reports. More recent documents should be weighted higher. Enterprise portal + content supersedes distributor communications." and "Emphasize forecasts from + top-tier investment banks. Recent analysis should take precedence. Disregard + aggregator sites and favor detailed research notes over news summaries." + """ + + reranker_score_filter_threshold: Optional[float] = None + """ + If the reranker relevance score associated with a chunk is below this threshold, + then the chunk will be filtered out and not used for generation. Scores are + between 0 and 1, with scores closer to 1 being more relevant. Set the value to 0 + to disable the reranker score filtering. + """ + + top_k_reranked_chunks: Optional[int] = None + """The number of highest ranked chunks after reranking to be used""" diff --git a/src/contextual/types/filter_and_rerank_config_param.py b/src/contextual/types/filter_and_rerank_config_param.py new file mode 100644 index 00000000..d06d6ce9 --- /dev/null +++ b/src/contextual/types/filter_and_rerank_config_param.py @@ -0,0 +1,31 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["FilterAndRerankConfigParam"] + + +class FilterAndRerankConfigParam(TypedDict, total=False): + rerank_instructions: str + """Instructions that the reranker references when ranking retrievals. + + Note that we do not guarantee that the reranker will follow these instructions + exactly. Examples: "Prioritize internal sales documents over market analysis + reports. More recent documents should be weighted higher. Enterprise portal + content supersedes distributor communications." and "Emphasize forecasts from + top-tier investment banks. Recent analysis should take precedence. Disregard + aggregator sites and favor detailed research notes over news summaries." + """ + + reranker_score_filter_threshold: float + """ + If the reranker relevance score associated with a chunk is below this threshold, + then the chunk will be filtered out and not used for generation. Scores are + between 0 and 1, with scores closer to 1 being more relevant. Set the value to 0 + to disable the reranker score filtering. + """ + + top_k_reranked_chunks: int + """The number of highest ranked chunks after reranking to be used""" diff --git a/src/contextual/types/generate_create_response.py b/src/contextual/types/generate_create_response.py index a786167b..25b08a79 100644 --- a/src/contextual/types/generate_create_response.py +++ b/src/contextual/types/generate_create_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from .._models import BaseModel __all__ = ["GenerateCreateResponse"] diff --git a/src/contextual/types/generate_response_config.py b/src/contextual/types/generate_response_config.py new file mode 100644 index 00000000..46357505 --- /dev/null +++ b/src/contextual/types/generate_response_config.py @@ -0,0 +1,45 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["GenerateResponseConfig"] + + +class GenerateResponseConfig(BaseModel): + avoid_commentary: Optional[bool] = None + """ + Flag to indicate whether the model should avoid providing additional commentary + in responses. Commentary is conversational in nature and does not contain + verifiable claims; therefore, commentary is not strictly grounded in available + context. However, commentary may provide useful context which improves the + helpfulness of responses. + """ + + calculate_groundedness: Optional[bool] = None + """This parameter controls generation of groundedness scores.""" + + frequency_penalty: Optional[float] = None + """ + This parameter adjusts how the model treats repeated tokens during text + generation. + """ + + max_new_tokens: Optional[int] = None + """The maximum number of tokens the model can generate in a response.""" + + seed: Optional[int] = None + """ + This parameter controls the randomness of how the model selects the next tokens + during text generation. + """ + + temperature: Optional[float] = None + """The sampling temperature, which affects the randomness in the response.""" + + top_p: Optional[float] = None + """ + A parameter for nucleus sampling, an alternative to `temperature` which also + affects the randomness of the response. + """ diff --git a/src/contextual/types/generate_response_config_param.py b/src/contextual/types/generate_response_config_param.py new file mode 100644 index 00000000..ea39181f --- /dev/null +++ b/src/contextual/types/generate_response_config_param.py @@ -0,0 +1,45 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["GenerateResponseConfigParam"] + + +class GenerateResponseConfigParam(TypedDict, total=False): + avoid_commentary: bool + """ + Flag to indicate whether the model should avoid providing additional commentary + in responses. Commentary is conversational in nature and does not contain + verifiable claims; therefore, commentary is not strictly grounded in available + context. However, commentary may provide useful context which improves the + helpfulness of responses. + """ + + calculate_groundedness: bool + """This parameter controls generation of groundedness scores.""" + + frequency_penalty: float + """ + This parameter adjusts how the model treats repeated tokens during text + generation. + """ + + max_new_tokens: int + """The maximum number of tokens the model can generate in a response.""" + + seed: int + """ + This parameter controls the randomness of how the model selects the next tokens + during text generation. + """ + + temperature: float + """The sampling temperature, which affects the randomness in the response.""" + + top_p: float + """ + A parameter for nucleus sampling, an alternative to `temperature` which also + affects the randomness of the response. + """ diff --git a/src/contextual/types/global_config.py b/src/contextual/types/global_config.py new file mode 100644 index 00000000..716eddef --- /dev/null +++ b/src/contextual/types/global_config.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["GlobalConfig"] + + +class GlobalConfig(BaseModel): + enable_filter: Optional[bool] = None + """Enables filtering of retrieved chunks with a separate LLM""" + + enable_multi_turn: Optional[bool] = None + """Enables multi-turn conversations. + + This feature is currently experimental and will be improved. + """ + + enable_rerank: Optional[bool] = None + """Enables reranking of retrieved chunks""" + + should_check_retrieval_need: Optional[bool] = None + """Enables checking if retrieval is needed for the query. + + This feature is currently experimental and will be improved. + """ diff --git a/src/contextual/types/global_config_param.py b/src/contextual/types/global_config_param.py new file mode 100644 index 00000000..e09fa3a9 --- /dev/null +++ b/src/contextual/types/global_config_param.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["GlobalConfigParam"] + + +class GlobalConfigParam(TypedDict, total=False): + enable_filter: bool + """Enables filtering of retrieved chunks with a separate LLM""" + + enable_multi_turn: bool + """Enables multi-turn conversations. + + This feature is currently experimental and will be improved. + """ + + enable_rerank: bool + """Enables reranking of retrieved chunks""" + + should_check_retrieval_need: bool + """Enables checking if retrieval is needed for the query. + + This feature is currently experimental and will be improved. + """ diff --git a/src/contextual/types/list_users_response.py b/src/contextual/types/list_users_response.py index fbe10474..80d8cb20 100644 --- a/src/contextual/types/list_users_response.py +++ b/src/contextual/types/list_users_response.py @@ -5,18 +5,7 @@ from .._models import BaseModel -__all__ = ["ListUsersResponse", "User", "UserPerAgentRole"] - - -class UserPerAgentRole(BaseModel): - agent_id: str - """ID of the agent on which to grant/revoke the role.""" - - grant: bool - """When set to true, the roles will be granted o/w revoked.""" - - roles: List[Literal["AGENT_USER"]] - """The roles that are granted/revoked""" +__all__ = ["ListUsersResponse", "User"] class User(BaseModel): @@ -25,17 +14,39 @@ class User(BaseModel): email: str """The email of the user""" + effective_roles: Optional[ + List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] + ] = None + """The effective roles of the user.""" + is_tenant_admin: Optional[bool] = None """Flag indicating if the user is a tenant admin""" - per_agent_roles: Optional[List[UserPerAgentRole]] = None - """Per agent level roles for the user. - - If a user is granted any role under `roles`, then the user has that role for all - the agents. Only the roles that need to be updated should be part of this. - """ - - roles: Optional[List[Literal["AGENT_USER"]]] = None + roles: Optional[ + List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] + ] = None """The user level roles of the user.""" diff --git a/src/contextual/types/lmunit_create_response.py b/src/contextual/types/lmunit_create_response.py index 069da3aa..37c1d6b1 100644 --- a/src/contextual/types/lmunit_create_response.py +++ b/src/contextual/types/lmunit_create_response.py @@ -1,6 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - from .._models import BaseModel __all__ = ["LMUnitCreateResponse"] diff --git a/src/contextual/types/new_user_param.py b/src/contextual/types/new_user_param.py index dd25792c..d809bb75 100644 --- a/src/contextual/types/new_user_param.py +++ b/src/contextual/types/new_user_param.py @@ -2,21 +2,10 @@ from __future__ import annotations -from typing import List, Iterable +from typing import List from typing_extensions import Literal, Required, TypedDict -__all__ = ["NewUserParam", "PerAgentRole"] - - -class PerAgentRole(TypedDict, total=False): - agent_id: Required[str] - """ID of the agent on which to grant/revoke the role.""" - - grant: Required[bool] - """When set to true, the roles will be granted o/w revoked.""" - - roles: Required[List[Literal["AGENT_USER"]]] - """The roles that are granted/revoked""" +__all__ = ["NewUserParam"] class NewUserParam(TypedDict, total=False): @@ -26,12 +15,16 @@ class NewUserParam(TypedDict, total=False): is_tenant_admin: bool """Flag indicating if the user is a tenant admin""" - per_agent_roles: Iterable[PerAgentRole] - """Per agent level roles for the user. - - If a user is granted any role under `roles`, then the user has that role for all - the agents. Only the roles that need to be updated should be part of this. - """ - - roles: List[Literal["AGENT_USER"]] + roles: List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] """The user level roles of the user.""" diff --git a/src/contextual/types/parse_create_params.py b/src/contextual/types/parse_create_params.py new file mode 100644 index 00000000..714bcf34 --- /dev/null +++ b/src/contextual/types/parse_create_params.py @@ -0,0 +1,57 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +from .._types import FileTypes + +__all__ = ["ParseCreateParams"] + + +class ParseCreateParams(TypedDict, total=False): + raw_file: Required[FileTypes] + """The file to be parsed. The file type must be PDF, DOC / DOCX, PPT / PPTX.""" + + enable_document_hierarchy: bool + """Controls parsing heading levels (e.g. + + H1, H2, H3) at higher quality. Adds a table of contents to the output with the + structure of the entire parsed document. Not permitted in 'basic' parsing_mode, + or if page_range is not continuous and/or does not start from page zero. + """ + + enable_split_tables: bool + """ + Controls whether tables are split into multiple tables by row with the headers + propagated. Use for improving LLM comprehension of very large tables. Not + permitted in 'basic' parsing_mode. + """ + + figure_caption_mode: Literal["concise", "detailed"] + """Controls how thorough figure captions are. + + 'concise' is short and minimizes chances of hallucinations. 'detailed' is more + thorough and can include commentary. Not permitted in 'basic' parsing_mode. + """ + + max_split_table_cells: int + """ + Threshold number of table cells beyond which large tables are split if + `enable_split_tables` is True. Not permitted in 'basic' parsing_mode. + """ + + page_range: str + """Optional string representing page range to be parsed. + + Format: comma-separated indexes (0-based) e.g. '0,1,2,5,6' or ranges (inclusive + of both ends) e.g. '0-2,5,6' + """ + + parse_mode: Literal["basic", "standard"] + """The settings to use for parsing. + + 'basic' is for simple, text-only documents. 'standard' is for complex documents + with images, complex hierarchy, and/or no natively encoded textual data (e.g. + for scanned documents). + """ diff --git a/src/contextual/types/parse_create_response.py b/src/contextual/types/parse_create_response.py new file mode 100644 index 00000000..168854b6 --- /dev/null +++ b/src/contextual/types/parse_create_response.py @@ -0,0 +1,10 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .._models import BaseModel + +__all__ = ["ParseCreateResponse"] + + +class ParseCreateResponse(BaseModel): + job_id: str + """Unique ID of the parse job""" diff --git a/src/contextual/types/parse_job_results_params.py b/src/contextual/types/parse_job_results_params.py new file mode 100644 index 00000000..33daa3c5 --- /dev/null +++ b/src/contextual/types/parse_job_results_params.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal, TypedDict + +__all__ = ["ParseJobResultsParams"] + + +class ParseJobResultsParams(TypedDict, total=False): + output_types: List[Literal["markdown-document", "markdown-per-page", "blocks-per-page"]] + """The desired output format(s) of the parsed file. + + Must be `markdown-document`, `markdown-per-page`, and/or `blocks-per-page`. + `markdown-document` parses the whole document into a single concatenated + markdown output. `markdown-per-page` provides markdown output per page. + `blocks-per-page` provides a structured JSON representation of the content + blocks on each page, sorted by reading order. Specify multiple values to get + multiple formats in the response. + """ diff --git a/src/contextual/types/parse_job_results_response.py b/src/contextual/types/parse_job_results_response.py new file mode 100644 index 00000000..2b3211b4 --- /dev/null +++ b/src/contextual/types/parse_job_results_response.py @@ -0,0 +1,185 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from .._models import BaseModel + +__all__ = [ + "ParseJobResultsResponse", + "Page", + "PageBlock", + "PageBlockBoundingBox", + "TableOfContents", + "TableOfContentsBlock", + "TableOfContentsBlockBoundingBox", +] + + +class PageBlockBoundingBox(BaseModel): + x0: float + """The x-coordinate of the top-left corner of the bounding box""" + + x1: float + """The x-coordinate of the bottom-right corner of the bounding box""" + + y0: float + """The y-coordinate of the top-left corner of the bounding box""" + + y1: float + """The y-coordinate of the bottom-right corner of the bounding box""" + + +class PageBlock(BaseModel): + id: str + """Unique ID of the block""" + + bounding_box: PageBlockBoundingBox + """ + The normalized bounding box of the block, as relative percentages of the page + width and height + """ + + markdown: str + """The Markdown representation of the block""" + + type: Literal["heading", "text", "table", "figure"] + """The type of the block""" + + confidence_level: Optional[Literal["low", "medium", "high"]] = None + """The confidence level of this block categorized as 'low', 'medium', or 'high'. + + Only available for blocks of type 'table' currently. + """ + + hierarchy_level: Optional[int] = None + """ + The level of the block in the document hierarchy, starting at 0 for the + root-level title block. Only present if `enable_document_hierarchy` was set to + true in the request. + """ + + page_index: Optional[int] = None + """The page (0-indexed) that this block belongs to. + + Only set for heading blocks that are returned in the table of contents. + """ + + parent_ids: Optional[List[str]] = None + """ + The IDs of the parent in the document hierarchy, sorted from root-level to + bottom. For root-level heading blocks, this will be an empty list. Only present + if `enable_document_hierarchy` was set to true in the request. + """ + + +class Page(BaseModel): + index: int + """The index of the parsed page (zero-indexed)""" + + blocks: Optional[List[PageBlock]] = None + """The parsed, structured blocks of this page. + + Present if `blocks-per-page` was among the requested output types. + """ + + markdown: Optional[str] = None + """The parsed, structured Markdown of this page. + + Present if `markdown-per-page` was among the requested output types. + """ + + +class TableOfContentsBlockBoundingBox(BaseModel): + x0: float + """The x-coordinate of the top-left corner of the bounding box""" + + x1: float + """The x-coordinate of the bottom-right corner of the bounding box""" + + y0: float + """The y-coordinate of the top-left corner of the bounding box""" + + y1: float + """The y-coordinate of the bottom-right corner of the bounding box""" + + +class TableOfContentsBlock(BaseModel): + id: str + """Unique ID of the block""" + + bounding_box: TableOfContentsBlockBoundingBox + """ + The normalized bounding box of the block, as relative percentages of the page + width and height + """ + + markdown: str + """The Markdown representation of the block""" + + type: Literal["heading", "text", "table", "figure"] + """The type of the block""" + + confidence_level: Optional[Literal["low", "medium", "high"]] = None + """The confidence level of this block categorized as 'low', 'medium', or 'high'. + + Only available for blocks of type 'table' currently. + """ + + hierarchy_level: Optional[int] = None + """ + The level of the block in the document hierarchy, starting at 0 for the + root-level title block. Only present if `enable_document_hierarchy` was set to + true in the request. + """ + + page_index: Optional[int] = None + """The page (0-indexed) that this block belongs to. + + Only set for heading blocks that are returned in the table of contents. + """ + + parent_ids: Optional[List[str]] = None + """ + The IDs of the parent in the document hierarchy, sorted from root-level to + bottom. For root-level heading blocks, this will be an empty list. Only present + if `enable_document_hierarchy` was set to true in the request. + """ + + +class TableOfContents(BaseModel): + blocks: Optional[List[TableOfContentsBlock]] = None + """Heading blocks that define the hierarchy of the document""" + + markdown: Optional[str] = None + """ + Markdown representation of the table of contents that can be pre-pended to the + markdown document. + """ + + +class ParseJobResultsResponse(BaseModel): + file_name: str + """The name of the file that was uploaded for parsing""" + + status: Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"] + """The current status of the parse job""" + + markdown_document: Optional[str] = None + """The parsed, structured Markdown of the input file. + + Only present if `markdown-document` was among the requested output types. + """ + + pages: Optional[List[Page]] = None + """ + Per-page parse results, containing per-page Markdown (if `markdown-per-page` was + requested) and/or per-page `ParsedBlock`s (if `blocks-per-page` was requested). + """ + + table_of_contents: Optional[TableOfContents] = None + """The table of contents representing the document's heading hierarchy. + + Only present if `enable_document_hierarchy` was set to true in the parse + request. + """ diff --git a/src/contextual/types/parse_job_status_response.py b/src/contextual/types/parse_job_status_response.py new file mode 100644 index 00000000..768ccfd0 --- /dev/null +++ b/src/contextual/types/parse_job_status_response.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from .._models import BaseModel + +__all__ = ["ParseJobStatusResponse"] + + +class ParseJobStatusResponse(BaseModel): + file_name: str + """The name of the file that was uploaded for parsing""" + + status: Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"] + """The current status of the parse job""" diff --git a/src/contextual/types/parse_jobs_params.py b/src/contextual/types/parse_jobs_params.py new file mode 100644 index 00000000..f44f3015 --- /dev/null +++ b/src/contextual/types/parse_jobs_params.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from datetime import datetime +from typing_extensions import Annotated, TypedDict + +from .._utils import PropertyInfo + +__all__ = ["ParseJobsParams"] + + +class ParseJobsParams(TypedDict, total=False): + uploaded_after: Annotated[Union[str, datetime], PropertyInfo(format="iso8601")] diff --git a/src/contextual/types/parse_jobs_response.py b/src/contextual/types/parse_jobs_response.py new file mode 100644 index 00000000..01b87c00 --- /dev/null +++ b/src/contextual/types/parse_jobs_response.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List +from typing_extensions import Literal + +from .._models import BaseModel + +__all__ = ["ParseJobsResponse", "Job"] + + +class Job(BaseModel): + id: str + """Unique ID of the parse job""" + + file_name: str + """The name of the file that was uploaded for parsing""" + + status: Literal["pending", "processing", "retrying", "completed", "failed", "cancelled"] + """The current status of the parse job""" + + +class ParseJobsResponse(BaseModel): + jobs: List[Job] + """List of parse jobs""" + + total_jobs: int + """Total number of parse jobs""" diff --git a/src/contextual/types/rerank_create_params.py b/src/contextual/types/rerank_create_params.py index 20dbe10c..cd218b3e 100644 --- a/src/contextual/types/rerank_create_params.py +++ b/src/contextual/types/rerank_create_params.py @@ -25,16 +25,15 @@ class RerankCreateParams(TypedDict, total=False): """The string against which documents will be ranked for relevance""" instruction: str - """Instructions that the reranker references when ranking retrievals. - - We evaluated the model on instructions for recency, document type, source, and - metadata, and it can generalize to other instructions as well. Note that we do - not guarantee that the reranker will follow these instructions exactly. - Examples: "Prioritize internal sales documents over market analysis reports. - More recent documents should be weighted higher. Enterprise portal content - supersedes distributor communications." and "Emphasize forecasts from top-tier - investment banks. Recent analysis should take precedence. Disregard aggregator - sites and favor detailed research notes over news summaries." + """ + Instructions that the reranker references when ranking documents, after + considering relevance. We evaluated the model on instructions for recency, + document type, source, and metadata, and it can generalize to other instructions + as well. For instructions related to recency and timeframe, specify the + timeframe (e.g., instead of saying "this year") because the reranker doesn't + know the current date. Example: "Prioritize internal sales documents over market + analysis reports. More recent documents should be weighted higher. Enterprise + portal content supersedes distributor communications." """ metadata: List[str] diff --git a/src/contextual/types/retrieval_config.py b/src/contextual/types/retrieval_config.py new file mode 100644 index 00000000..663748f1 --- /dev/null +++ b/src/contextual/types/retrieval_config.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["RetrievalConfig"] + + +class RetrievalConfig(BaseModel): + lexical_alpha: Optional[float] = None + """The weight of lexical search during retrieval. + + Must sum to 1 with semantic_alpha. + """ + + semantic_alpha: Optional[float] = None + """The weight of semantic search during retrieval. + + Must sum to 1 with lexical_alpha. + """ + + top_k_retrieved_chunks: Optional[int] = None + """The maximum number of retrieved chunks from the datastore.""" diff --git a/src/contextual/types/retrieval_config_param.py b/src/contextual/types/retrieval_config_param.py new file mode 100644 index 00000000..9fa0b207 --- /dev/null +++ b/src/contextual/types/retrieval_config_param.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["RetrievalConfigParam"] + + +class RetrievalConfigParam(TypedDict, total=False): + lexical_alpha: float + """The weight of lexical search during retrieval. + + Must sum to 1 with semantic_alpha. + """ + + semantic_alpha: float + """The weight of semantic search during retrieval. + + Must sum to 1 with lexical_alpha. + """ + + top_k_retrieved_chunks: int + """The maximum number of retrieved chunks from the datastore.""" diff --git a/src/contextual/types/user_update_params.py b/src/contextual/types/user_update_params.py index f41a6adf..be6ee2ed 100644 --- a/src/contextual/types/user_update_params.py +++ b/src/contextual/types/user_update_params.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import List, Iterable +from typing import List from typing_extensions import Literal, Required, TypedDict -__all__ = ["UserUpdateParams", "PerAgentRole"] +__all__ = ["UserUpdateParams"] class UserUpdateParams(TypedDict, total=False): @@ -15,23 +15,16 @@ class UserUpdateParams(TypedDict, total=False): is_tenant_admin: bool """Flag indicating if the user is a tenant admin""" - per_agent_roles: Iterable[PerAgentRole] - """Per agent level roles for the user. - - If a user is granted any role under `roles`, then the user has that role for all - the agents. Only the roles that need to be updated should be part of this. - """ - - roles: List[Literal["AGENT_USER"]] + roles: List[ + Literal[ + "VISITOR", + "AGENT_USER", + "CUSTOMER_INTERNAL_USER", + "CONTEXTUAL_STAFF_USER", + "CONTEXTUAL_EXTERNAL_STAFF_USER", + "CONTEXTUAL_INTERNAL_STAFF_USER", + "TENANT_ADMIN", + "SUPER_ADMIN", + ] + ] """The user level roles of the user.""" - - -class PerAgentRole(TypedDict, total=False): - agent_id: Required[str] - """ID of the agent on which to grant/revoke the role.""" - - grant: Required[bool] - """When set to true, the roles will be granted o/w revoked.""" - - roles: Required[List[Literal["AGENT_USER"]]] - """The roles that are granted/revoked""" diff --git a/tests/api_resources/agents/test_evaluate.py b/tests/api_resources/agents/test_evaluate.py index 512abf8b..ab28d729 100644 --- a/tests/api_resources/agents/test_evaluate.py +++ b/tests/api_resources/agents/test_evaluate.py @@ -33,6 +33,8 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: evalset_file=b"raw file contents", evalset_name="evalset_name", llm_model_id="llm_model_id", + notes="notes", + override_configuration="override_configuration", ) assert_matches_type(CreateEvaluationResponse, evaluate, path=["response"]) @@ -90,6 +92,8 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual evalset_file=b"raw file contents", evalset_name="evalset_name", llm_model_id="llm_model_id", + notes="notes", + override_configuration="override_configuration", ) assert_matches_type(CreateEvaluationResponse, evaluate, path=["response"]) diff --git a/tests/api_resources/agents/test_query.py b/tests/api_resources/agents/test_query.py index cf5ea0ca..26b5b1d0 100644 --- a/tests/api_resources/agents/test_query.py +++ b/tests/api_resources/agents/test_query.py @@ -49,12 +49,21 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: retrievals_only=True, conversation_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", documents_filters={ - "field": "field", - "operator": "equals", - "value": "string", + "filters": [ + { + "field": "field1", + "operator": "equals", + "value": "value1", + } + ], + "operator": "AND", }, llm_model_id="llm_model_id", stream=True, + structured_output={ + "json_schema": {}, + "type": "JSON", + }, ) assert_matches_type(QueryResponse, query, path=["response"]) @@ -299,12 +308,21 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual retrievals_only=True, conversation_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", documents_filters={ - "field": "field", - "operator": "equals", - "value": "string", + "filters": [ + { + "field": "field1", + "operator": "equals", + "value": "value1", + } + ], + "operator": "AND", }, llm_model_id="llm_model_id", stream=True, + structured_output={ + "json_schema": {}, + "type": "JSON", + }, ) assert_matches_type(QueryResponse, query, path=["response"]) diff --git a/tests/api_resources/agents/test_tune.py b/tests/api_resources/agents/test_tune.py index 18871a25..83f66694 100644 --- a/tests/api_resources/agents/test_tune.py +++ b/tests/api_resources/agents/test_tune.py @@ -28,6 +28,15 @@ def test_method_create(self, client: ContextualAI) -> None: def test_method_create_with_all_params(self, client: ContextualAI) -> None: tune = client.agents.tune.create( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + hyperparams_learning_rate=0.001, + hyperparams_lora_alpha=8, + hyperparams_lora_dropout=0, + hyperparams_lora_rank=8, + hyperparams_num_epochs=1, + hyperparams_warmup_ratio=0, + metadata_file=b"raw file contents", + sdp_only=True, + synth_data=True, test_dataset_name="test_dataset_name", test_file=b"raw file contents", train_dataset_name="train_dataset_name", @@ -81,6 +90,15 @@ async def test_method_create(self, async_client: AsyncContextualAI) -> None: async def test_method_create_with_all_params(self, async_client: AsyncContextualAI) -> None: tune = await async_client.agents.tune.create( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + hyperparams_learning_rate=0.001, + hyperparams_lora_alpha=8, + hyperparams_lora_dropout=0, + hyperparams_lora_rank=8, + hyperparams_num_epochs=1, + hyperparams_warmup_ratio=0, + metadata_file=b"raw file contents", + sdp_only=True, + synth_data=True, test_dataset_name="test_dataset_name", test_file=b"raw file contents", train_dataset_name="train_dataset_name", diff --git a/tests/api_resources/datastores/test_documents.py b/tests/api_resources/datastores/test_documents.py index e1af38ea..525988ac 100644 --- a/tests/api_resources/datastores/test_documents.py +++ b/tests/api_resources/datastores/test_documents.py @@ -133,7 +133,7 @@ def test_method_ingest_with_all_params(self, client: ContextualAI) -> None: document = client.datastores.documents.ingest( datastore_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", file=b"raw file contents", - metadata="metadata", + metadata='{"field1": "value1", "field2": "value2"}}', ) assert_matches_type(IngestionResponse, document, path=["response"]) @@ -391,7 +391,7 @@ async def test_method_ingest_with_all_params(self, async_client: AsyncContextual document = await async_client.datastores.documents.ingest( datastore_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", file=b"raw file contents", - metadata="metadata", + metadata='{"field1": "value1", "field2": "value2"}}', ) assert_matches_type(IngestionResponse, document, path=["response"]) diff --git a/tests/api_resources/test_agents.py b/tests/api_resources/test_agents.py index efcdd219..9009cee8 100644 --- a/tests/api_resources/test_agents.py +++ b/tests/api_resources/test_agents.py @@ -11,8 +11,8 @@ from tests.utils import assert_matches_type from contextual.types import ( Agent, - AgentMetadata, CreateAgentOutput, + AgentMetadataResponse, ) from contextual.pagination import SyncPage, AsyncPage @@ -34,8 +34,13 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: agent = client.agents.create( name="xxx", agent_configs={ - "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, + "filter_and_rerank_config": { + "rerank_instructions": "rerank_instructions", + "reranker_score_filter_threshold": 0, + "top_k_reranked_chunks": 0, + }, "generate_response_config": { + "avoid_commentary": True, "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, @@ -47,6 +52,7 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: "enable_filter": True, "enable_multi_turn": True, "enable_rerank": True, + "should_check_retrieval_need": True, }, "retrieval_config": { "lexical_alpha": 0, @@ -55,8 +61,9 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: }, }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], - description="xxx", + description="description", filter_prompt="filter_prompt", + no_retrieval_system_prompt="no_retrieval_system_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -98,8 +105,13 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: agent = client.agents.update( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", agent_configs={ - "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, + "filter_and_rerank_config": { + "rerank_instructions": "rerank_instructions", + "reranker_score_filter_threshold": 0, + "top_k_reranked_chunks": 0, + }, "generate_response_config": { + "avoid_commentary": True, "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, @@ -111,6 +123,7 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: "enable_filter": True, "enable_multi_turn": True, "enable_rerank": True, + "should_check_retrieval_need": True, }, "retrieval_config": { "lexical_alpha": 0, @@ -121,6 +134,7 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], filter_prompt="filter_prompt", llm_model_id="llm_model_id", + no_retrieval_system_prompt="no_retrieval_system_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -233,7 +247,7 @@ def test_method_metadata(self, client: ContextualAI) -> None: agent = client.agents.metadata( "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", ) - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) @parametrize def test_raw_response_metadata(self, client: ContextualAI) -> None: @@ -244,7 +258,7 @@ def test_raw_response_metadata(self, client: ContextualAI) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" agent = response.parse() - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) @parametrize def test_streaming_response_metadata(self, client: ContextualAI) -> None: @@ -255,7 +269,7 @@ def test_streaming_response_metadata(self, client: ContextualAI) -> None: assert response.http_request.headers.get("X-Stainless-Lang") == "python" agent = response.parse() - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) assert cast(Any, response.is_closed) is True @@ -266,6 +280,44 @@ def test_path_params_metadata(self, client: ContextualAI) -> None: "", ) + @parametrize + def test_method_reset(self, client: ContextualAI) -> None: + agent = client.agents.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(object, agent, path=["response"]) + + @parametrize + def test_raw_response_reset(self, client: ContextualAI) -> None: + response = client.agents.with_raw_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + agent = response.parse() + assert_matches_type(object, agent, path=["response"]) + + @parametrize + def test_streaming_response_reset(self, client: ContextualAI) -> None: + with client.agents.with_streaming_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + agent = response.parse() + assert_matches_type(object, agent, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_reset(self, client: ContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `agent_id` but received ''"): + client.agents.with_raw_response.reset( + "", + ) + class TestAsyncAgents: parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) @@ -282,8 +334,13 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual agent = await async_client.agents.create( name="xxx", agent_configs={ - "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, + "filter_and_rerank_config": { + "rerank_instructions": "rerank_instructions", + "reranker_score_filter_threshold": 0, + "top_k_reranked_chunks": 0, + }, "generate_response_config": { + "avoid_commentary": True, "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, @@ -295,6 +352,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual "enable_filter": True, "enable_multi_turn": True, "enable_rerank": True, + "should_check_retrieval_need": True, }, "retrieval_config": { "lexical_alpha": 0, @@ -303,8 +361,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual }, }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], - description="xxx", + description="description", filter_prompt="filter_prompt", + no_retrieval_system_prompt="no_retrieval_system_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -346,8 +405,13 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual agent = await async_client.agents.update( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", agent_configs={ - "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, + "filter_and_rerank_config": { + "rerank_instructions": "rerank_instructions", + "reranker_score_filter_threshold": 0, + "top_k_reranked_chunks": 0, + }, "generate_response_config": { + "avoid_commentary": True, "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, @@ -359,6 +423,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual "enable_filter": True, "enable_multi_turn": True, "enable_rerank": True, + "should_check_retrieval_need": True, }, "retrieval_config": { "lexical_alpha": 0, @@ -369,6 +434,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], filter_prompt="filter_prompt", llm_model_id="llm_model_id", + no_retrieval_system_prompt="no_retrieval_system_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -481,7 +547,7 @@ async def test_method_metadata(self, async_client: AsyncContextualAI) -> None: agent = await async_client.agents.metadata( "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", ) - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) @parametrize async def test_raw_response_metadata(self, async_client: AsyncContextualAI) -> None: @@ -492,7 +558,7 @@ async def test_raw_response_metadata(self, async_client: AsyncContextualAI) -> N assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" agent = await response.parse() - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) @parametrize async def test_streaming_response_metadata(self, async_client: AsyncContextualAI) -> None: @@ -503,7 +569,7 @@ async def test_streaming_response_metadata(self, async_client: AsyncContextualAI assert response.http_request.headers.get("X-Stainless-Lang") == "python" agent = await response.parse() - assert_matches_type(AgentMetadata, agent, path=["response"]) + assert_matches_type(AgentMetadataResponse, agent, path=["response"]) assert cast(Any, response.is_closed) is True @@ -513,3 +579,41 @@ async def test_path_params_metadata(self, async_client: AsyncContextualAI) -> No await async_client.agents.with_raw_response.metadata( "", ) + + @parametrize + async def test_method_reset(self, async_client: AsyncContextualAI) -> None: + agent = await async_client.agents.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(object, agent, path=["response"]) + + @parametrize + async def test_raw_response_reset(self, async_client: AsyncContextualAI) -> None: + response = await async_client.agents.with_raw_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + agent = await response.parse() + assert_matches_type(object, agent, path=["response"]) + + @parametrize + async def test_streaming_response_reset(self, async_client: AsyncContextualAI) -> None: + async with async_client.agents.with_streaming_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + agent = await response.parse() + assert_matches_type(object, agent, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_reset(self, async_client: AsyncContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `agent_id` but received ''"): + await async_client.agents.with_raw_response.reset( + "", + ) diff --git a/tests/api_resources/test_datastores.py b/tests/api_resources/test_datastores.py index 9d1a6b1e..fe997240 100644 --- a/tests/api_resources/test_datastores.py +++ b/tests/api_resources/test_datastores.py @@ -163,6 +163,44 @@ def test_path_params_metadata(self, client: ContextualAI) -> None: "", ) + @parametrize + def test_method_reset(self, client: ContextualAI) -> None: + datastore = client.datastores.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(object, datastore, path=["response"]) + + @parametrize + def test_raw_response_reset(self, client: ContextualAI) -> None: + response = client.datastores.with_raw_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + datastore = response.parse() + assert_matches_type(object, datastore, path=["response"]) + + @parametrize + def test_streaming_response_reset(self, client: ContextualAI) -> None: + with client.datastores.with_streaming_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + datastore = response.parse() + assert_matches_type(object, datastore, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_reset(self, client: ContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `datastore_id` but received ''"): + client.datastores.with_raw_response.reset( + "", + ) + class TestAsyncDatastores: parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) @@ -307,3 +345,41 @@ async def test_path_params_metadata(self, async_client: AsyncContextualAI) -> No await async_client.datastores.with_raw_response.metadata( "", ) + + @parametrize + async def test_method_reset(self, async_client: AsyncContextualAI) -> None: + datastore = await async_client.datastores.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(object, datastore, path=["response"]) + + @parametrize + async def test_raw_response_reset(self, async_client: AsyncContextualAI) -> None: + response = await async_client.datastores.with_raw_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + datastore = await response.parse() + assert_matches_type(object, datastore, path=["response"]) + + @parametrize + async def test_streaming_response_reset(self, async_client: AsyncContextualAI) -> None: + async with async_client.datastores.with_streaming_response.reset( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + datastore = await response.parse() + assert_matches_type(object, datastore, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_reset(self, async_client: AsyncContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `datastore_id` but received ''"): + await async_client.datastores.with_raw_response.reset( + "", + ) diff --git a/tests/api_resources/test_parse.py b/tests/api_resources/test_parse.py new file mode 100644 index 00000000..08898d90 --- /dev/null +++ b/tests/api_resources/test_parse.py @@ -0,0 +1,348 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from contextual import ContextualAI, AsyncContextualAI +from tests.utils import assert_matches_type +from contextual.types import ( + ParseJobsResponse, + ParseCreateResponse, + ParseJobStatusResponse, + ParseJobResultsResponse, +) +from contextual._utils import parse_datetime + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestParse: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_create(self, client: ContextualAI) -> None: + parse = client.parse.create( + raw_file=b"raw file contents", + ) + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + def test_method_create_with_all_params(self, client: ContextualAI) -> None: + parse = client.parse.create( + raw_file=b"raw file contents", + enable_document_hierarchy=True, + enable_split_tables=True, + figure_caption_mode="concise", + max_split_table_cells=0, + page_range="page_range", + parse_mode="standard", + ) + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + def test_raw_response_create(self, client: ContextualAI) -> None: + response = client.parse.with_raw_response.create( + raw_file=b"raw file contents", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = response.parse() + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + def test_streaming_response_create(self, client: ContextualAI) -> None: + with client.parse.with_streaming_response.create( + raw_file=b"raw file contents", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = response.parse() + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_job_results(self, client: ContextualAI) -> None: + parse = client.parse.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + def test_method_job_results_with_all_params(self, client: ContextualAI) -> None: + parse = client.parse.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + output_types=["markdown-document"], + ) + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + def test_raw_response_job_results(self, client: ContextualAI) -> None: + response = client.parse.with_raw_response.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = response.parse() + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + def test_streaming_response_job_results(self, client: ContextualAI) -> None: + with client.parse.with_streaming_response.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = response.parse() + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_job_results(self, client: ContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + client.parse.with_raw_response.job_results( + job_id="", + ) + + @parametrize + def test_method_job_status(self, client: ContextualAI) -> None: + parse = client.parse.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + @parametrize + def test_raw_response_job_status(self, client: ContextualAI) -> None: + response = client.parse.with_raw_response.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = response.parse() + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + @parametrize + def test_streaming_response_job_status(self, client: ContextualAI) -> None: + with client.parse.with_streaming_response.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = response.parse() + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_job_status(self, client: ContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + client.parse.with_raw_response.job_status( + "", + ) + + @parametrize + def test_method_jobs(self, client: ContextualAI) -> None: + parse = client.parse.jobs() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + def test_method_jobs_with_all_params(self, client: ContextualAI) -> None: + parse = client.parse.jobs( + uploaded_after=parse_datetime("2019-12-27T18:11:19.117Z"), + ) + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + def test_raw_response_jobs(self, client: ContextualAI) -> None: + response = client.parse.with_raw_response.jobs() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = response.parse() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + def test_streaming_response_jobs(self, client: ContextualAI) -> None: + with client.parse.with_streaming_response.jobs() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = response.parse() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + +class TestAsyncParse: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_create(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.create( + raw_file=b"raw file contents", + ) + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.create( + raw_file=b"raw file contents", + enable_document_hierarchy=True, + enable_split_tables=True, + figure_caption_mode="concise", + max_split_table_cells=0, + page_range="page_range", + parse_mode="standard", + ) + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + async def test_raw_response_create(self, async_client: AsyncContextualAI) -> None: + response = await async_client.parse.with_raw_response.create( + raw_file=b"raw file contents", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = await response.parse() + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + @parametrize + async def test_streaming_response_create(self, async_client: AsyncContextualAI) -> None: + async with async_client.parse.with_streaming_response.create( + raw_file=b"raw file contents", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = await response.parse() + assert_matches_type(ParseCreateResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_job_results(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + async def test_method_job_results_with_all_params(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + output_types=["markdown-document"], + ) + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + async def test_raw_response_job_results(self, async_client: AsyncContextualAI) -> None: + response = await async_client.parse.with_raw_response.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = await response.parse() + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + @parametrize + async def test_streaming_response_job_results(self, async_client: AsyncContextualAI) -> None: + async with async_client.parse.with_streaming_response.job_results( + job_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = await response.parse() + assert_matches_type(ParseJobResultsResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_job_results(self, async_client: AsyncContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + await async_client.parse.with_raw_response.job_results( + job_id="", + ) + + @parametrize + async def test_method_job_status(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + @parametrize + async def test_raw_response_job_status(self, async_client: AsyncContextualAI) -> None: + response = await async_client.parse.with_raw_response.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = await response.parse() + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + @parametrize + async def test_streaming_response_job_status(self, async_client: AsyncContextualAI) -> None: + async with async_client.parse.with_streaming_response.job_status( + "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = await response.parse() + assert_matches_type(ParseJobStatusResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_job_status(self, async_client: AsyncContextualAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + await async_client.parse.with_raw_response.job_status( + "", + ) + + @parametrize + async def test_method_jobs(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.jobs() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + async def test_method_jobs_with_all_params(self, async_client: AsyncContextualAI) -> None: + parse = await async_client.parse.jobs( + uploaded_after=parse_datetime("2019-12-27T18:11:19.117Z"), + ) + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + async def test_raw_response_jobs(self, async_client: AsyncContextualAI) -> None: + response = await async_client.parse.with_raw_response.jobs() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + parse = await response.parse() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + @parametrize + async def test_streaming_response_jobs(self, async_client: AsyncContextualAI) -> None: + async with async_client.parse.with_streaming_response.jobs() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + parse = await response.parse() + assert_matches_type(ParseJobsResponse, parse, path=["response"]) + + assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_users.py b/tests/api_resources/test_users.py index 90652e1a..acd99a9e 100644 --- a/tests/api_resources/test_users.py +++ b/tests/api_resources/test_users.py @@ -33,14 +33,7 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: user = client.users.update( email="email", is_tenant_admin=True, - per_agent_roles=[ - { - "agent_id": "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", - "grant": True, - "roles": ["AGENT_USER"], - } - ], - roles=["AGENT_USER"], + roles=["VISITOR"], ) assert_matches_type(object, user, path=["response"]) @@ -184,14 +177,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual user = await async_client.users.update( email="email", is_tenant_admin=True, - per_agent_roles=[ - { - "agent_id": "182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", - "grant": True, - "roles": ["AGENT_USER"], - } - ], - roles=["AGENT_USER"], + roles=["VISITOR"], ) assert_matches_type(object, user, path=["response"]) diff --git a/tests/conftest.py b/tests/conftest.py index 310afee3..aa99261a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ from contextual import ContextualAI, AsyncContextualAI if TYPE_CHECKING: - from _pytest.fixtures import FixtureRequest + from _pytest.fixtures import FixtureRequest # pyright: ignore[reportPrivateImportUsage] pytest.register_assert_rewrite("tests.utils") diff --git a/tests/test_client.py b/tests/test_client.py index 40052e12..4617f677 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1624,7 +1624,7 @@ def test_get_platform(self) -> None: import threading from contextual._utils import asyncify - from contextual._base_client import get_platform + from contextual._base_client import get_platform async def test_main() -> None: result = await asyncify(get_platform)() diff --git a/tests/test_models.py b/tests/test_models.py index 5538e6d6..5adfed97 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -492,12 +492,15 @@ class Model(BaseModel): resource_id: Optional[str] = None m = Model.construct() + assert m.resource_id is None assert "resource_id" not in m.model_fields_set m = Model.construct(resource_id=None) + assert m.resource_id is None assert "resource_id" in m.model_fields_set m = Model.construct(resource_id="foo") + assert m.resource_id == "foo" assert "resource_id" in m.model_fields_set @@ -832,7 +835,7 @@ class B(BaseModel): @pytest.mark.skipif(not PYDANTIC_V2, reason="TypeAliasType is not supported in Pydantic v1") def test_type_alias_type() -> None: - Alias = TypeAliasType("Alias", str) + Alias = TypeAliasType("Alias", str) # pyright: ignore class Model(BaseModel): alias: Alias @@ -854,3 +857,35 @@ class Model(BaseModel): m = construct_type(value={"cls": "foo"}, type_=Model) assert isinstance(m, Model) assert isinstance(m.cls, str) + + +def test_discriminated_union_case() -> None: + class A(BaseModel): + type: Literal["a"] + + data: bool + + class B(BaseModel): + type: Literal["b"] + + data: List[Union[A, object]] + + class ModelA(BaseModel): + type: Literal["modelA"] + + data: int + + class ModelB(BaseModel): + type: Literal["modelB"] + + required: str + + data: Union[A, B] + + # when constructing ModelA | ModelB, value data doesn't match ModelB exactly - missing `required` + m = construct_type( + value={"type": "modelB", "data": {"type": "a", "data": True}}, + type_=cast(Any, Annotated[Union[ModelA, ModelB], PropertyInfo(discriminator="type")]), + ) + + assert isinstance(m, ModelB) diff --git a/tests/test_transform.py b/tests/test_transform.py index 0566b45d..6fe1ce3f 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -8,7 +8,7 @@ import pytest -from contextual._types import Base64FileInput +from contextual._types import NOT_GIVEN, Base64FileInput from contextual._utils import ( PropertyInfo, transform as _transform, @@ -432,3 +432,22 @@ async def test_base64_file_input(use_async: bool) -> None: assert await transform({"foo": io.BytesIO(b"Hello, world!")}, TypedDictBase64Input, use_async) == { "foo": "SGVsbG8sIHdvcmxkIQ==" } # type: ignore[comparison-overlap] + + +@parametrize +@pytest.mark.asyncio +async def test_transform_skipping(use_async: bool) -> None: + # lists of ints are left as-is + data = [1, 2, 3] + assert await transform(data, List[int], use_async) is data + + # iterables of ints are converted to a list + data = iter([1, 2, 3]) + assert await transform(data, Iterable[int], use_async) == [1, 2, 3] + + +@parametrize +@pytest.mark.asyncio +async def test_strips_notgiven(use_async: bool) -> None: + assert await transform({"foo_bar": "bar"}, Foo1, use_async) == {"fooBar": "bar"} + assert await transform({"foo_bar": NOT_GIVEN}, Foo1, use_async) == {}