Extends base fern client with eval utilities (#18)

* Release 0.8.6b0

* Evaluation utils for the Humanloop SDK.

---------

Co-authored-by: fern-api <115122769+fern-api[bot]@users.noreply.github.com>

Author: peadaroh

.fernignore

@@ -1,2 +1,6 @@
 # Specify files that shouldn't be modified by Fern
 
+src/humanloop/eval_utils.py
+src/humanloop/client.py
+mypy.ini
+

README.md

@@ -41,7 +41,7 @@ client.prompts.log(
     messages=[{"role": "user", "content": "What really happened at Roswell?"}],
     inputs={"person": "Trump"},
     created_at=datetime.datetime.fromisoformat(
-        "2024-07-19 00:29:35.178000+00:00",
+        "2024-07-18 23:29:35.178000+00:00",
     ),
     provider_latency=6.5931549072265625,
     output_message={
@@ -88,7 +88,7 @@ async def main() -> None:
         ],
         inputs={"person": "Trump"},
         created_at=datetime.datetime.fromisoformat(
-            "2024-07-19 00:29:35.178000+00:00",
+            "2024-07-18 23:29:35.178000+00:00",
         ),
         provider_latency=6.5931549072265625,
         output_message={

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+exclude = ^src/humanloop/eval_utils\.py$

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "humanloop"
-version = "0.8.5"
+version = "0.8.6b0"
 description = ""
 readme = "README.md"
 authors = []

reference.md

@@ -56,7 +56,7 @@ client.prompts.log(
     messages=[{"role": "user", "content": "What really happened at Roswell?"}],
     inputs={"person": "Trump"},
     created_at=datetime.datetime.fromisoformat(
-        "2024-07-19 00:29:35.178000+00:00",
+        "2024-07-18 23:29:35.178000+00:00",
     ),
     provider_latency=6.5931549072265625,
     output_message={
@@ -6258,10 +6258,10 @@ client.flows.log(
     output="The patient is likely experiencing a myocardial infarction. Immediate medical attention is required.",
     trace_status="incomplete",
     start_time=datetime.datetime.fromisoformat(
-        "2024-07-08 22:40:35+00:00",
+        "2024-07-08 21:40:35+00:00",
     ),
     end_time=datetime.datetime.fromisoformat(
-        "2024-07-08 22:40:39+00:00",
+        "2024-07-08 21:40:39+00:00",
     ),
 )
 

src/humanloop/base_client.py

@@ -0,0 +1,182 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+from .environment import HumanloopEnvironment
+import os
+import httpx
+from .core.api_error import ApiError
+from .core.client_wrapper import SyncClientWrapper
+from .prompts.client import PromptsClient
+from .tools.client import ToolsClient
+from .datasets.client import DatasetsClient
+from .evaluators.client import EvaluatorsClient
+from .flows.client import FlowsClient
+from .directories.client import DirectoriesClient
+from .files.client import FilesClient
+from .evaluations.client import EvaluationsClient
+from .logs.client import LogsClient
+from .core.client_wrapper import AsyncClientWrapper
+from .prompts.client import AsyncPromptsClient
+from .tools.client import AsyncToolsClient
+from .datasets.client import AsyncDatasetsClient
+from .evaluators.client import AsyncEvaluatorsClient
+from .flows.client import AsyncFlowsClient
+from .directories.client import AsyncDirectoriesClient
+from .files.client import AsyncFilesClient
+from .evaluations.client import AsyncEvaluationsClient
+from .logs.client import AsyncLogsClient
+
+
+class BaseHumanloop:
+    """
+    Use this class to access the different functions within the SDK. You can instantiate any number of clients with different configuration that will propagate to these functions.
+
+    Parameters
+    ----------
+    base_url : typing.Optional[str]
+        The base url to use for requests from the client.
+
+    environment : HumanloopEnvironment
+        The environment to use for requests from the client. from .environment import HumanloopEnvironment
+
+
+
+        Defaults to HumanloopEnvironment.DEFAULT
+
+
+
+    api_key : typing.Optional[str]
+    timeout : typing.Optional[float]
+        The timeout to be used, in seconds, for requests. By default the timeout is 60 seconds, unless a custom httpx client is used, in which case this default is not enforced.
+
+    follow_redirects : typing.Optional[bool]
+        Whether the default httpx client follows redirects or not, this is irrelevant if a custom httpx client is passed in.
+
+    httpx_client : typing.Optional[httpx.Client]
+        The httpx client to use for making requests, a preconfigured client is used by default, however this is useful should you want to pass in any custom httpx configuration.
+
+    Examples
+    --------
+    from humanloop import Humanloop
+
+    client = Humanloop(
+        api_key="YOUR_API_KEY",
+    )
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: typing.Optional[str] = None,
+        environment: HumanloopEnvironment = HumanloopEnvironment.DEFAULT,
+        api_key: typing.Optional[str] = os.getenv("HUMANLOOP_API_KEY"),
+        timeout: typing.Optional[float] = None,
+        follow_redirects: typing.Optional[bool] = True,
+        httpx_client: typing.Optional[httpx.Client] = None,
+    ):
+        _defaulted_timeout = timeout if timeout is not None else 60 if httpx_client is None else None
+        if api_key is None:
+            raise ApiError(
+                body="The client must be instantiated be either passing in api_key or setting HUMANLOOP_API_KEY"
+            )
+        self._client_wrapper = SyncClientWrapper(
+            base_url=_get_base_url(base_url=base_url, environment=environment),
+            api_key=api_key,
+            httpx_client=httpx_client
+            if httpx_client is not None
+            else httpx.Client(timeout=_defaulted_timeout, follow_redirects=follow_redirects)
+            if follow_redirects is not None
+            else httpx.Client(timeout=_defaulted_timeout),
+            timeout=_defaulted_timeout,
+        )
+        self.prompts = PromptsClient(client_wrapper=self._client_wrapper)
+        self.tools = ToolsClient(client_wrapper=self._client_wrapper)
+        self.datasets = DatasetsClient(client_wrapper=self._client_wrapper)
+        self.evaluators = EvaluatorsClient(client_wrapper=self._client_wrapper)
+        self.flows = FlowsClient(client_wrapper=self._client_wrapper)
+        self.directories = DirectoriesClient(client_wrapper=self._client_wrapper)
+        self.files = FilesClient(client_wrapper=self._client_wrapper)
+        self.evaluations = EvaluationsClient(client_wrapper=self._client_wrapper)
+        self.logs = LogsClient(client_wrapper=self._client_wrapper)
+
+
+class AsyncBaseHumanloop:
+    """
+    Use this class to access the different functions within the SDK. You can instantiate any number of clients with different configuration that will propagate to these functions.
+
+    Parameters
+    ----------
+    base_url : typing.Optional[str]
+        The base url to use for requests from the client.
+
+    environment : HumanloopEnvironment
+        The environment to use for requests from the client. from .environment import HumanloopEnvironment
+
+
+
+        Defaults to HumanloopEnvironment.DEFAULT
+
+
+
+    api_key : typing.Optional[str]
+    timeout : typing.Optional[float]
+        The timeout to be used, in seconds, for requests. By default the timeout is 60 seconds, unless a custom httpx client is used, in which case this default is not enforced.
+
+    follow_redirects : typing.Optional[bool]
+        Whether the default httpx client follows redirects or not, this is irrelevant if a custom httpx client is passed in.
+
+    httpx_client : typing.Optional[httpx.AsyncClient]
+        The httpx client to use for making requests, a preconfigured client is used by default, however this is useful should you want to pass in any custom httpx configuration.
+
+    Examples
+    --------
+    from humanloop import AsyncHumanloop
+
+    client = AsyncHumanloop(
+        api_key="YOUR_API_KEY",
+    )
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: typing.Optional[str] = None,
+        environment: HumanloopEnvironment = HumanloopEnvironment.DEFAULT,
+        api_key: typing.Optional[str] = os.getenv("HUMANLOOP_API_KEY"),
+        timeout: typing.Optional[float] = None,
+        follow_redirects: typing.Optional[bool] = True,
+        httpx_client: typing.Optional[httpx.AsyncClient] = None,
+    ):
+        _defaulted_timeout = timeout if timeout is not None else 60 if httpx_client is None else None
+        if api_key is None:
+            raise ApiError(
+                body="The client must be instantiated be either passing in api_key or setting HUMANLOOP_API_KEY"
+            )
+        self._client_wrapper = AsyncClientWrapper(
+            base_url=_get_base_url(base_url=base_url, environment=environment),
+            api_key=api_key,
+            httpx_client=httpx_client
+            if httpx_client is not None
+            else httpx.AsyncClient(timeout=_defaulted_timeout, follow_redirects=follow_redirects)
+            if follow_redirects is not None
+            else httpx.AsyncClient(timeout=_defaulted_timeout),
+            timeout=_defaulted_timeout,
+        )
+        self.prompts = AsyncPromptsClient(client_wrapper=self._client_wrapper)
+        self.tools = AsyncToolsClient(client_wrapper=self._client_wrapper)
+        self.datasets = AsyncDatasetsClient(client_wrapper=self._client_wrapper)
+        self.evaluators = AsyncEvaluatorsClient(client_wrapper=self._client_wrapper)
+        self.flows = AsyncFlowsClient(client_wrapper=self._client_wrapper)
+        self.directories = AsyncDirectoriesClient(client_wrapper=self._client_wrapper)
+        self.files = AsyncFilesClient(client_wrapper=self._client_wrapper)
+        self.evaluations = AsyncEvaluationsClient(client_wrapper=self._client_wrapper)
+        self.logs = AsyncLogsClient(client_wrapper=self._client_wrapper)
+
+
+def _get_base_url(*, base_url: typing.Optional[str] = None, environment: HumanloopEnvironment) -> str:
+    if base_url is not None:
+        return base_url
+    elif environment is not None:
+        return environment.value
+    else:
+        raise Exception("Please pass in either base_url or environment to construct the client")