Create eval run
RunCreateResponse evals().runs().create(RunCreateParamsparams, RequestOptionsrequestOptions = RequestOptions.none())
POST/evals/{eval_id}/runs
Kicks off a new run for a given evaluation, specifying the data source, and what model configuration to use to test. The datasource will be validated against the schema specified in the config of the evaluation.
Create eval run
package com.openai.example;
import com.openai.client.OpenAIClient;
import com.openai.client.okhttp.OpenAIOkHttpClient;
import com.openai.core.JsonValue;
import com.openai.models.evals.runs.CreateEvalJsonlRunDataSource;
import com.openai.models.evals.runs.RunCreateParams;
import com.openai.models.evals.runs.RunCreateResponse;
import java.util.List;
public final class Main {
private Main() {}
public static void main(String[] args) {
OpenAIClient client = OpenAIOkHttpClient.fromEnv();
RunCreateParams params = RunCreateParams.builder()
.evalId("eval_id")
.dataSource(CreateEvalJsonlRunDataSource.builder()
.fileContentSource(List.of(CreateEvalJsonlRunDataSource.Source.FileContent.Content.builder()
.item(CreateEvalJsonlRunDataSource.Source.FileContent.Content.Item.builder()
.putAdditionalProperty("foo", JsonValue.from("bar"))
.build())
.build()))
.build())
.build();
RunCreateResponse run = client.evals().runs().create(params);
}
}{
"object": "eval.run",
"id": "evalrun_67e57965b480819094274e3a32235e4c",
"eval_id": "eval_67e579652b548190aaa83ada4b125f47",
"report_url": "https://platform.openai.com/evaluations/eval_67e579652b548190aaa83ada4b125f47&run_id=evalrun_67e57965b480819094274e3a32235e4c",
"status": "queued",
"model": "gpt-4o-mini",
"name": "gpt-4o-mini",
"created_at": 1743092069,
"result_counts": {
"total": 0,
"errored": 0,
"failed": 0,
"passed": 0
},
"per_model_usage": null,
"per_testing_criteria_results": null,
"data_source": {
"type": "completions",
"source": {
"type": "file_content",
"content": [
{
"item": {
"input": "Tech Company Launches Advanced Artificial Intelligence Platform",
"ground_truth": "Technology"
}
}
]
},
"input_messages": {
"type": "template",
"template": [
{
"type": "message",
"role": "developer",
"content": {
"type": "input_text",
"text": "Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\n\n# Steps\n\n1. Analyze the content of the news headline to understand its primary focus.\n2. Extract the subject matter, identifying any key indicators or keywords.\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\n4. Ensure only one category is selected per headline.\n\n# Output Format\n\nRespond with the chosen category as a single word. For instance: \"Technology\", \"Markets\", \"World\", \"Business\", or \"Sports\".\n\n# Examples\n\n**Input**: \"Apple Unveils New iPhone Model, Featuring Advanced AI Features\" \n**Output**: \"Technology\"\n\n**Input**: \"Global Stocks Mixed as Investors Await Central Bank Decisions\" \n**Output**: \"Markets\"\n\n**Input**: \"War in Ukraine: Latest Updates on Negotiation Status\" \n**Output**: \"World\"\n\n**Input**: \"Microsoft in Talks to Acquire Gaming Company for $2 Billion\" \n**Output**: \"Business\"\n\n**Input**: \"Manchester United Secures Win in Premier League Football Match\" \n**Output**: \"Sports\" \n\n# Notes\n\n- If the headline appears to fit into more than one category, choose the most dominant theme.\n- Keywords or phrases such as \"stocks\", \"company acquisition\", \"match\", or technological brands can be good indicators for classification.\n"
}
},
{
"type": "message",
"role": "user",
"content": {
"type": "input_text",
"text": "{{item.input}}"
}
}
]
},
"model": "gpt-4o-mini",
"sampling_params": {
"seed": 42,
"temperature": 1.0,
"top_p": 1.0,
"max_completions_tokens": 2048
}
},
"error": null,
"metadata": {}
}
Returns Examples
{
"object": "eval.run",
"id": "evalrun_67e57965b480819094274e3a32235e4c",
"eval_id": "eval_67e579652b548190aaa83ada4b125f47",
"report_url": "https://platform.openai.com/evaluations/eval_67e579652b548190aaa83ada4b125f47&run_id=evalrun_67e57965b480819094274e3a32235e4c",
"status": "queued",
"model": "gpt-4o-mini",
"name": "gpt-4o-mini",
"created_at": 1743092069,
"result_counts": {
"total": 0,
"errored": 0,
"failed": 0,
"passed": 0
},
"per_model_usage": null,
"per_testing_criteria_results": null,
"data_source": {
"type": "completions",
"source": {
"type": "file_content",
"content": [
{
"item": {
"input": "Tech Company Launches Advanced Artificial Intelligence Platform",
"ground_truth": "Technology"
}
}
]
},
"input_messages": {
"type": "template",
"template": [
{
"type": "message",
"role": "developer",
"content": {
"type": "input_text",
"text": "Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\n\n# Steps\n\n1. Analyze the content of the news headline to understand its primary focus.\n2. Extract the subject matter, identifying any key indicators or keywords.\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\n4. Ensure only one category is selected per headline.\n\n# Output Format\n\nRespond with the chosen category as a single word. For instance: \"Technology\", \"Markets\", \"World\", \"Business\", or \"Sports\".\n\n# Examples\n\n**Input**: \"Apple Unveils New iPhone Model, Featuring Advanced AI Features\" \n**Output**: \"Technology\"\n\n**Input**: \"Global Stocks Mixed as Investors Await Central Bank Decisions\" \n**Output**: \"Markets\"\n\n**Input**: \"War in Ukraine: Latest Updates on Negotiation Status\" \n**Output**: \"World\"\n\n**Input**: \"Microsoft in Talks to Acquire Gaming Company for $2 Billion\" \n**Output**: \"Business\"\n\n**Input**: \"Manchester United Secures Win in Premier League Football Match\" \n**Output**: \"Sports\" \n\n# Notes\n\n- If the headline appears to fit into more than one category, choose the most dominant theme.\n- Keywords or phrases such as \"stocks\", \"company acquisition\", \"match\", or technological brands can be good indicators for classification.\n"
}
},
{
"type": "message",
"role": "user",
"content": {
"type": "input_text",
"text": "{{item.input}}"
}
}
]
},
"model": "gpt-4o-mini",
"sampling_params": {
"seed": 42,
"temperature": 1.0,
"top_p": 1.0,
"max_completions_tokens": 2048
}
},
"error": null,
"metadata": {}
}