Release 0.8.0b7

Author: fern-api[bot]

.mock/definition/__package__.yml

@@ -176,7 +176,7 @@ service:
                   question: Who wrote Hamlet?
                 target:
                   answer: William Shakespeare
-            action: add
+            action: set
             commit_message: Add two new questions and answers
           response:
             body:
@@ -196,24 +196,53 @@ service:
             datapoints:
               - messages:
                   - role: user
-                    content: >-
-                      Hi Humanloop support team, I'm having trouble
-                      understanding how to use the evaluations feature in your
-                      software. Can you provide a step-by-step guide or any
-                      resources to help me get started?
+                    content: |
+                      How do i manage my organizations API keys?
                 target:
-                  feature: evaluations
-                  issue: needs step-by-step guide
+                  response: >-
+                    Hey, thanks for your questions. Here are steps for how to
+                    achieve: 1. Log in to the Humanloop Dashboard 
+
+
+                    2. Click on "Organization Settings."
+                     If you do not see this option, you might need to contact your organization admin to gain the necessary permissions.
+
+                    3. Within the settings or organization settings, select the
+                    option labeled "API Keys" on the left. Here you will be able
+                    to view and manage your API keys.
+
+
+                    4. You will see a list of existing API keys. You can perform
+                    various actions, such as:
+                         - **Generate New API Key:** Click on the "Generate New Key" button if you need a new API key.
+                         - **Revoke an API Key:** If you need to disable an existing key, find the key in the list and click the "Revoke" or "Delete" button.
+                         - **Copy an API Key:** If you need to use an existing key, you can copy it to your clipboard by clicking the "Copy" button next to the key.
+
+                    5. **Save and Secure API Keys:** Make sure to securely store
+                    any new or existing API keys you are using. Treat them like
+                    passwords and do not share them publicly.
+
+
+                    If you encounter any issues or need further assistance, it
+                    might be helpful to engage with an engineer or your IT
+                    department to ensure you have the necessary permissions and
+                    support.
+
+
+                    Would you need help with anything else?
               - messages:
                   - role: user
                     content: >-
-                      Hi there, I'm interested in fine-tuning a language model
-                      using your software. Can you explain the process and
-                      provide any best practices or guidelines?
+                      Hey, can do I use my code evaluator for monitoring my
+                      legal-copilot prompt?
                 target:
-                  feature: fine-tuning
-                  issue: process explanation and best practices
-            action: add
+                  response: >-
+                    Hey, thanks for your questions. Here are steps for how to
+                    achieve: 1. Navigate to your Prompt dashboard. 
+                     2. Select the `Monitoring` button on the top right of the Prompt dashboard 
+                     3. Within the model select the Version of the Evaluator you want to turn on for monitoring. 
+
+                    Would you need help with anything else?
             commit_message: Add two new questions and answers
           response:
             body:
@@ -770,6 +799,8 @@ service:
                       evaluator_version_id: evaluator_version_id
                       created_at: '2024-01-15T09:30:00Z'
                       updated_at: '2024-01-15T09:30:00Z'
+  source:
+    openapi: openapi/openapi.auto.json
   display-name: Datasets
 docs: >+
   Datasets are collections of input-output pairs that you can use within

.mock/definition/datasets.yml

@@ -125,7 +125,9 @@ service:
                           arguments_type: target_required
                           return_type: number
                           evaluator_type: python
-                          code: def evaluate(answer, target):\n    return 0.5
+                          code: |-
+                            def evaluate(answer, target):
+                                return 0.5
                         version_logs_count: 1
                         total_logs_count: 1
                         inputs:
@@ -258,7 +260,9 @@ service:
                       arguments_type: target_required
                       return_type: number
                       evaluator_type: python
-                      code: def evaluate(answer, target):\n    return 0.5
+                      code: |-
+                        def evaluate(answer, target):
+                            return 0.5
                     version_logs_count: 1
                     total_logs_count: 1
                     inputs:
@@ -362,7 +366,9 @@ service:
                       arguments_type: target_required
                       return_type: number
                       evaluator_type: python
-                      code: def evaluate(answer, target):\n    return 0.5
+                      code: |-
+                        def evaluate(answer, target):
+                            return 0.5
                     version_logs_count: 1
                     total_logs_count: 1
                     inputs:
@@ -507,7 +513,9 @@ service:
                       arguments_type: target_required
                       return_type: number
                       evaluator_type: python
-                      code: def evaluate(answer, target):\n    return 0.5
+                      code: |-
+                        def evaluate(answer, target):
+                            return 0.5
                     version_logs_count: 1
                     total_logs_count: 1
                     inputs:
@@ -596,6 +604,7 @@ service:
                     spec:
                       arguments_type: target_free
                       return_type: boolean
+                      evaluator_type: llm
                     name: name
                     version_id: version_id
                     created_at: '2024-01-15T09:30:00Z'
@@ -614,6 +623,7 @@ service:
                 email_address: email_address
                 full_name: full_name
               updated_at: '2024-01-15T09:30:00Z'
+              url: url
     getStats:
       path: /evaluations/{id}/stats
       method: GET
@@ -734,6 +744,26 @@ service:
                       inputs:
                         - name: name
                     id: id
+                    evaluator_logs:
+                      - id: id
+                        evaluator_logs: []
+                        evaluator:
+                          path: path
+                          id: id
+                          spec:
+                            arguments_type: target_free
+                            return_type: boolean
+                            evaluator_type: llm
+                          name: name
+                          version_id: version_id
+                          created_at: '2024-01-15T09:30:00Z'
+                          updated_at: '2024-01-15T09:30:00Z'
+                          status: uncommitted
+                          last_used_at: '2024-01-15T09:30:00Z'
+                          version_logs_count: 1
+                          total_logs_count: 1
+                          inputs:
+                            - name: name
                   evaluator_logs:
                     - prompt:
                         path: path
@@ -750,9 +780,31 @@ service:
                         inputs:
                           - name: name
                       id: id
+                      evaluator_logs:
+                        - id: id
+                          evaluator_logs: []
+                          evaluator:
+                            path: path
+                            id: id
+                            spec:
+                              arguments_type: target_free
+                              return_type: boolean
+                              evaluator_type: llm
+                            name: name
+                            version_id: version_id
+                            created_at: '2024-01-15T09:30:00Z'
+                            updated_at: '2024-01-15T09:30:00Z'
+                            status: uncommitted
+                            last_used_at: '2024-01-15T09:30:00Z'
+                            version_logs_count: 1
+                            total_logs_count: 1
+                            inputs:
+                              - name: name
               page: 1
               size: 1
               total: 1
+  source:
+    openapi: openapi/openapi.auto.json
   display-name: Evaluations
 docs: >+
   Evaluations help you measure the performance of your Prompts, Tools and LLM

.mock/definition/evaluations.yml

@@ -61,7 +61,9 @@ service:
                     arguments_type: target_required
                     return_type: number
                     evaluator_type: python
-                    code: def evaluate(answer, target):\n    return 0.5
+                    code: |-
+                      def evaluate(answer, target):
+                          return 0.5
                   version_logs_count: 1
                   total_logs_count: 1
                   inputs:
@@ -119,7 +121,9 @@ service:
               arguments_type: target_required
               return_type: number
               evaluator_type: python
-              code: def evaluate(answer, target):\n    return 0.5
+              code: |-
+                def evaluate(answer, target):
+                    return 0.5
             commit_message: Initial commit
           response:
             body:
@@ -136,7 +140,9 @@ service:
                 arguments_type: target_required
                 return_type: number
                 evaluator_type: python
-                code: def evaluate(answer, target):\n    return 0.5
+                code: |-
+                  def evaluate(answer, target):
+                      return 0.5
               version_logs_count: 1
               total_logs_count: 1
               inputs:
@@ -192,7 +198,9 @@ service:
                 arguments_type: target_required
                 return_type: number
                 evaluator_type: python
-                code: def evaluate(answer, target):\n    return 0.5
+                code: |-
+                  def evaluate(answer, target):
+                      return 0.5
               version_logs_count: 1
               total_logs_count: 1
               inputs:
@@ -261,7 +269,9 @@ service:
                 arguments_type: target_required
                 return_type: number
                 evaluator_type: python
-                code: def evaluate(answer, target):\n    return 0.5
+                code: |-
+                  def evaluate(answer, target):
+                      return 0.5
               version_logs_count: 1
               total_logs_count: 1
               inputs:
@@ -314,7 +324,9 @@ service:
                     arguments_type: target_required
                     return_type: number
                     evaluator_type: python
-                    code: def evaluate(answer, target):\n    return 0.5
+                    code: |-
+                      def evaluate(answer, target):
+                          return 0.5
                   version_logs_count: 1
                   total_logs_count: 1
                   inputs:
@@ -364,7 +376,9 @@ service:
                 arguments_type: target_required
                 return_type: number
                 evaluator_type: python
-                code: def evaluate(answer, target):\n    return 0.5
+                code: |-
+                  def evaluate(answer, target):
+                      return 0.5
               version_logs_count: 1
               total_logs_count: 1
               inputs:
@@ -419,7 +433,9 @@ service:
                 arguments_type: target_required
                 return_type: number
                 evaluator_type: python
-                code: def evaluate(answer, target):\n    return 0.5
+                code: |-
+                  def evaluate(answer, target):
+                      return 0.5
               version_logs_count: 1
               total_logs_count: 1
               inputs:
@@ -491,7 +507,9 @@ service:
                     arguments_type: target_required
                     return_type: number
                     evaluator_type: python
-                    code: def evaluate(answer, target):\n    return 0.5
+                    code: |-
+                      def evaluate(answer, target):
+                          return 0.5
                   version_logs_count: 1
                   total_logs_count: 1
                   inputs:
@@ -501,8 +519,11 @@ service:
       method: POST
       auth: true
       docs: >-
-        Submit evalutor judgment for an existing Log. Creates a new Log and
-        makes evaluated one its parent.
+        Submit Evaluator judgment for an existing Log.
+
+
+        Creates a new Log. The evaluated Log will be set as the parent of the
+        created Log.
       display-name: Log
       request:
         name: CreateEvaluatorLogRequest
@@ -538,6 +559,9 @@ service:
             provider_latency:
               type: optional<double>
               docs: Duration of the logged event in seconds.
+            stdout:
+              type: optional<string>
+              docs: Captured log and debug statements.
             provider_request:
               type: optional<map<string, unknown>>
               docs: >-
@@ -554,7 +578,7 @@ service:
                 Unique identifier for the Session to associate the Log to.
                 Allows you to record multiple Logs to a Session (using an ID
                 kept by your internal systems) by passing the same `session_id`
-                in subsequent log requests.
+                in subsequent log requests. 
             parent_id:
               type: string
               docs: >-
@@ -595,7 +619,9 @@ service:
               type: optional<string>
               docs: The name of the Environment the Log is associated to.
               name: createEvaluatorLogRequestEnvironment
-            judgment: optional<unknown>
+            judgment:
+              type: optional<CreateEvaluatorLogRequestJudgment>
+              docs: Evaluator assessment of the Log.
             spec: optional<CreateEvaluatorLogRequestSpec>
       response:
         docs: Successful Response
@@ -611,6 +637,8 @@ service:
               parent_id: parent_id
               session_id: session_id
               version_id: version_id
+  source:
+    openapi: openapi/openapi.auto.json
 types:
   SrcExternalAppModelsV5EvaluatorsEvaluatorRequestSpec:
     discriminated: false
@@ -619,10 +647,24 @@ types:
       - root.CodeEvaluatorRequest
       - root.HumanEvaluatorRequest
       - root.ExternalEvaluatorRequest
+    source:
+      openapi: openapi/openapi.auto.json
+  CreateEvaluatorLogRequestJudgment:
+    discriminated: false
+    docs: Evaluator assessment of the Log.
+    union:
+      - boolean
+      - string
+      - list<string>
+      - double
+    source:
+      openapi: openapi/openapi.auto.json
   CreateEvaluatorLogRequestSpec:
     discriminated: false
     union:
       - root.LlmEvaluatorRequest
       - root.CodeEvaluatorRequest
       - root.HumanEvaluatorRequest
       - root.ExternalEvaluatorRequest
+    source:
+      openapi: openapi/openapi.auto.json