Run grader
fine_tuning.alpha.graders.run(GraderRunParams**kwargs) -> GraderRunResponse
POST/fine_tuning/alpha/graders/run
Run a grader.
Parameters
model_sample: str
The model sample to be evaluated. This value will be used to populate
the sample namespace. See the guide for more details.
The output_json variable will be populated if the model sample is a
valid JSON string.
Returns
Run grader
import os
from openai import OpenAI
client = OpenAI(
api_key=os.environ.get("OPENAI_API_KEY"), # This is the default and can be omitted
)
response = client.fine_tuning.alpha.graders.run(
grader={
"input": "input",
"name": "name",
"operation": "eq",
"reference": "reference",
"type": "string_check",
},
model_sample="model_sample",
)
print(response.metadata){
"metadata": {
"errors": {
"formula_parse_error": true,
"invalid_variable_error": true,
"model_grader_parse_error": true,
"model_grader_refusal_error": true,
"model_grader_server_error": true,
"model_grader_server_error_details": "model_grader_server_error_details",
"other_error": true,
"python_grader_runtime_error": true,
"python_grader_runtime_error_details": "python_grader_runtime_error_details",
"python_grader_server_error": true,
"python_grader_server_error_type": "python_grader_server_error_type",
"sample_parse_error": true,
"truncated_observation_error": true,
"unresponsive_reward_error": true
},
"execution_time": 0,
"name": "name",
"sampled_model_name": "sampled_model_name",
"scores": {
"foo": "bar"
},
"token_usage": 0,
"type": "type"
},
"model_grader_token_usage_per_model": {
"foo": "bar"
},
"reward": 0,
"sub_rewards": {
"foo": "bar"
}
}Returns Examples
{
"metadata": {
"errors": {
"formula_parse_error": true,
"invalid_variable_error": true,
"model_grader_parse_error": true,
"model_grader_refusal_error": true,
"model_grader_server_error": true,
"model_grader_server_error_details": "model_grader_server_error_details",
"other_error": true,
"python_grader_runtime_error": true,
"python_grader_runtime_error_details": "python_grader_runtime_error_details",
"python_grader_server_error": true,
"python_grader_server_error_type": "python_grader_server_error_type",
"sample_parse_error": true,
"truncated_observation_error": true,
"unresponsive_reward_error": true
},
"execution_time": 0,
"name": "name",
"sampled_model_name": "sampled_model_name",
"scores": {
"foo": "bar"
},
"token_usage": 0,
"type": "type"
},
"model_grader_token_usage_per_model": {
"foo": "bar"
},
"reward": 0,
"sub_rewards": {
"foo": "bar"
}
}