You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
@@ -79,9 +80,52 @@ export function overloadLog<T extends Flows | Prompts>(client: T): T {
79
80
sourceDatapointId: sourceDatapointId,
80
81
};
81
82
}
83
+
if(clientinstanceofFlows){
84
+
request={
85
+
...request,
86
+
traceStatus: "complete",
87
+
};
88
+
}
82
89
83
-
// @ts-ignore
84
-
response=awaitoriginalLog(request,options);
90
+
if("flow"inrequest){
91
+
if(
92
+
JSON.stringify(state!.evaluatedVersion)!==
93
+
JSON.stringify(request.flow)
94
+
){
95
+
response=awaitoriginalLog(
96
+
{
97
+
...request,
98
+
// @ts-ignore Log under the version expected by evaluation, not
99
+
// one determined by decorators. Otherwise the evaluation will stale
100
+
flow: state?.evaluatedVersion,
101
+
output: undefined,
102
+
error: `The version of the evaluated Flow must match the version of the callable. Expected: ${JSON.stringify(state!.evaluatedVersion)}, got: ${JSON.stringify(request.flow)}`,
103
+
},
104
+
options,
105
+
);
106
+
}
107
+
}
108
+
109
+
if("prompt"inrequest){
110
+
if(
111
+
JSON.stringify(state!.evaluatedVersion)!==
112
+
JSON.stringify(request.prompt)
113
+
){
114
+
response=awaitoriginalLog({
115
+
...request,
116
+
// @ts-ignore Log under the version expected by evaluation, not
117
+
// one determined by decorators. Otherwise the evaluation will stale
118
+
prompt: state?.evaluatedVersion,
119
+
output: undefined,
120
+
error: `The version of the evaluated Prompt must match the version of the callable. Expected: ${JSON.stringify(state!.evaluatedVersion)}, got: ${JSON.stringify(request.prompt)}`,
121
+
});
122
+
}
123
+
}
124
+
125
+
if(response===undefined){
126
+
// Version validation passed, make a normal request
127
+
response=awaitoriginalLog(request,options);
128
+
}
85
129
86
130
// @ts-ignore
87
131
uploadCallback(response.id);
@@ -111,6 +155,28 @@ export async function runEval(
111
155
thrownewError("You must provide a path or id in your `file`.");
112
156
}
113
157
158
+
if(file.callable&&"path"infile.callable){
159
+
if(file.path!==file.callable.path){
160
+
thrownewError(
161
+
"The path of the evaluated `file` must match the path of your decorated `callable`. Expected path: "+
162
+
file.path+
163
+
", got: "+
164
+
file.callable.path,
165
+
);
166
+
}
167
+
}
168
+
169
+
if(file.callable&&"version"infile.callable){
170
+
if(file.version!==file.callable.version){
171
+
thrownewError(
172
+
"The version of the evaluated `file` must match the version of your decorated `callable`. Expected version: "+
173
+
JSON.stringify(file.version)+
174
+
", got: "+
175
+
JSON.stringify(file.callable.version),
176
+
);
177
+
}
178
+
}
179
+
114
180
lettype: FileType;
115
181
if(file.type){
116
182
type=file.type;
@@ -308,7 +374,9 @@ export async function runEval(
/** The type of arguments the Evaluator expects - only required for local Evaluators. */
93
-
argsType?: EvaluatorArgumentsType;
94
99
/** The type of return value the Evaluator produces - only required for local Evaluators. */
95
100
returnType?: EvaluatorReturnTypeEnum;
96
-
/** The function to run on the logs to produce the judgment - only required for local Evaluators. */
97
-
callable?: (...args: any[])=>any;// TODO define explicitly the args and return type
98
-
/**
99
-
* Optional function that logs the output judgment from your Evaluator to Humanloop.
100
-
* If provided, it will be called as follows:
101
-
*
102
-
* ```typescript
103
-
* judgment = callable(log);
104
-
* log = custom_logger(client, judgment);
105
-
* ```
106
-
*
107
-
* Inside the custom_logger, you can use the Humanloop client to log the judgment to Humanloop.
108
-
* If not provided, your function must return a single string, and by default, the code will be used to inform the version of the external Evaluator on Humanloop.
109
-
*/
110
101
/**The threshold to check the Evaluator against. If the aggregate value of the Evaluator is below this threshold, the check will fail.*/
0 commit comments