你可以在 Realtime 工作階段中加入工具,讓模型在即時對話期間查詢資料、執行動作或呼叫服務。無論用戶端使用 WebRTC 資料通道 或 WebSocket,都可以透過相同的事件設定工具。
如果需要由應用程式執行工具並傳回結果,請使用函式工具。如果需要由 Realtime API 代為連線至遠端工具伺服器,請使用 MCP 工具。
| 工具類型 |
適用情境 |
執行者 |
function |
應用程式負責處理業務邏輯、檢查核准狀態或存取私有系統。 |
用戶端或伺服器接收函式呼叫並傳回 function_call_output。 |
mcp 搭配 server_url |
你想讓模型呼叫遠端 MCP 伺服器提供的工具。 |
Realtime API 呼叫遠端 MCP 伺服器。 |
mcp 搭配 connector_id |
你使用的既有模型搭配舊版內建連接器。 |
Realtime API 使用你提供的授權呼叫連接器。 |
你可以選擇下列其中一個層級加入工具:
如果工具需要在應用程式中執行,建議優先使用函式工具。模型會輸出函式呼叫所需的引數,接著由你的程式碼執行動作,再透過 function_call_output 項目傳回結果。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27const event = {
type: "session.update",
session: {
type: "realtime",
model: "gpt-realtime-2.1",
tools: [
{
type: "function",
name: "lookup_order",
description: "Look up an order by its order number.",
parameters: {
type: "object",
properties: {
order_number: {
type: "string",
description: "The customer-facing order number.",
},
},
required: ["order_number"],
},
},
],
tool_choice: "auto",
},
};
ws.send(JSON.stringify(event));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27event = {
"type": "session.update",
"session": {
"type": "realtime",
"model": "gpt-realtime-2.1",
"tools": [
{
"type": "function",
"name": "lookup_order",
"description": "Look up an order by its order number.",
"parameters": {
"type": "object",
"properties": {
"order_number": {
"type": "string",
"description": "The customer-facing order number.",
}
},
"required": ["order_number"],
},
}
],
"tool_choice": "auto",
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{
Model: "gpt-realtime-2.1",
ToolChoice: realtime.RealtimeToolChoiceConfigUnionParam{
OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("auto"),
},
Tools: realtime.RealtimeToolsConfigParam{
realtime.RealtimeToolsConfigUnionParam{
OfFunction: &realtime.RealtimeFunctionToolParam{
Description: openai.String("Look up an order by its order number."),
Name: openai.String("lookup_order"),
Parameters: map[string]any{
"properties": map[string]any{
"order_number": map[string]any{
"description": "The customer-facing order number.",
"type": "string",
},
},
"required": []any{"order_number"},
"type": "object",
},
Type: "function",
},
},
},
})); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.core.JsonValue;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import java.util.List;
import java.util.Map;
connection.send(
RealtimeClientEvent.ofSessionUpdate(
SessionUpdateEvent.builder()
.session(
RealtimeSessionCreateRequest.builder()
.model("gpt-realtime-2.1")
.addTool(
RealtimeFunctionTool.builder()
.type(RealtimeFunctionTool.Type.FUNCTION)
.name("lookup_order")
.description("Look up an order by its order number.")
.parameters(
JsonValue.from(
Map.of(
"type",
"object",
"properties",
Map.of(
"order_number",
Map.of(
"type",
"string",
"description",
"The customer-facing order number.")),
"required",
List.of("order_number"))))
.build())
.toolChoice(com.openai.models.responses.ToolChoiceOptions.AUTO)
.build())
.build()));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17using OpenAI.Realtime;
#pragma warning disable OPENAI002
await session.SendCommandAsync(new RealtimeClientCommandSessionUpdate(new RealtimeConversationSessionOptions
{
Model = "gpt-realtime-2.1",
Tools =
{
new RealtimeFunctionTool("lookup_order")
{
FunctionDescription = "Look up an order by its order number.",
FunctionParameters = BinaryData.FromObjectAsJson(new { type = "object", properties = new { order_number = new { type = "string", description = "The customer-facing order number." } }, required = (string[])["order_number"] })
}
},
ToolChoice = RealtimeDefaultToolChoice.Auto
}), timeout.Token);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29connection.session.update(
type: :realtime,
model: "gpt-realtime-2.1",
tools: [
{
type: :function,
name: "lookup_order",
description: "Look up an order by its order number.",
parameters: {
type: "object",
properties: {
order_number: {
type: "string",
description: "The customer-facing order number."
}
},
required: ["order_number"]
}
}
],
tool_choice: :auto
)
updated = connection.find do |event|
raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent)
event.is_a?(OpenAI::Realtime::SessionUpdatedEvent)
end
raise "Connection closed before the session update was acknowledged" unless updated
模型呼叫函式時,請監聽函式呼叫項目,執行應用程式邏輯,再傳回輸出:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19const event = {
type: "conversation.item.create",
item: {
type: "function_call_output",
call_id: functionCall.call_id,
output: JSON.stringify({
status: "shipped",
delivery_date: "2026-05-09",
}),
},
};
ws.send(JSON.stringify(event));
ws.send(
JSON.stringify({
type: "response.create",
response: { output_modalities: ["text"], tool_choice: "none" },
})
);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23event = {
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": function_call["call_id"],
"output": json.dumps(
{
"status": "shipped",
"delivery_date": "2026-05-09",
}
),
},
}
ws.send(json.dumps(event))
ws.send(
json.dumps(
{
"type": "response.create",
"response": {"output_modalities": ["text"], "tool_choice": "none"},
}
)
)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20// callID comes from the model's function_call item.
if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeConversationItemFunctionCallOutputParam{
CallID: callID,
Output: "{\"status\":\"shipped\",\"delivery_date\":\"2026-05-09\"}",
})); err != nil {
return err
}
if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{
OfResponseCreate: &realtime.ResponseCreateEventParam{
Response: realtime.RealtimeResponseCreateParams{
OutputModalities: []string{"text"},
ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{
OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("none"),
},
},
},
}); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.core.JsonMissing;
import com.openai.core.JsonValue;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import com.openai.models.responses.ToolChoiceFunction;
import com.openai.models.responses.ToolChoiceOptions;
import java.util.Map;
static void sendFunctionCallOutput(RealtimeConnection connection, String callId)
throws Exception {
send(
connection,
RealtimeClientEvent.ofConversationItemCreate(
ConversationItemCreateEvent.builder()
.item(
RealtimeConversationItemFunctionCallOutput.builder()
.object_(JsonMissing.of())
.callId(callId)
.output("{\"status\":\"shipped\",\"delivery_date\":\"2026-05-09\"}")
.build())
.build()));
send(
connection,
RealtimeClientEvent.ofResponseCreate(
ResponseCreateEvent.builder()
.response(
RealtimeResponseCreateParams.builder()
.metadata(
RealtimeResponseCreateParams.Metadata.builder()
.putAdditionalProperty(
"topic", JsonValue.from("lookup_order_followup"))
.build())
.addOutputModality(RealtimeResponseCreateParams.OutputModality.TEXT)
.toolChoice(ToolChoiceOptions.NONE)
.build())
.build()));
}
// Retry only explicit admission rejections; these guarantee nothing was sent.
private static void send(RealtimeConnection connection, RealtimeClientEvent event)
throws Exception {
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5);
while (true) {
try {
connection.send(event);
return;
} catch (com.openai.core.http.WebSocketWriteNotAttempted.Busy busy) {
if (System.nanoTime() >= deadline) throw busy;
Thread.sleep(10);
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18using OpenAI.Realtime;
#pragma warning disable OPENAI002
internal static async Task SendFunctionCallOutputAsync(RealtimeSessionClient session, string callId, CancellationToken cancellationToken)
{
string output = System.Text.Json.JsonSerializer.Serialize(new { status = "shipped", delivery_date = "2026-05-09" });
await session.SendCommandAsync(new RealtimeClientCommandConversationItemCreate(new RealtimeFunctionCallOutputItem(callId, output)), cancellationToken);
await session.SendCommandAsync(new RealtimeClientCommandResponseCreate
{
ResponseOptions = new()
{
OutputModalities = { RealtimeOutputModality.Text },
Metadata = new Dictionary<string, BinaryData> { ["topic"] = BinaryData.FromObjectAsJson("lookup_order_followup") },
ToolChoice = RealtimeDefaultToolChoice.None
}
}, cancellationToken);
}
1
2
3
4
5
6connection.conversation.items.create(
type: :function_call_output,
call_id: call_id,
output: JSON.generate(status: "shipped", delivery_date: "2026-05-09")
)
connection.response.create(output_modalities: [:text], tool_choice: :none)
如需瞭解函式呼叫的完整流程及各步驟涉及的事件,請參閱管理對話。
如果遠端 MCP 伺服器已提供所需的工具,或既有模型使用舊版內建連接器,就適合使用 MCP 工具。與函式工具不同,MCP 工具由 Realtime API 直接執行。
Realtime 中的 MCP 工具定義包含下列欄位:
type: "mcp"
server_label
server_url 或 connector_id,擇一使用
- 選用欄位
authorization 和 headers
- 選用欄位
allowed_tools
- 選用欄位
require_approval
- 選用欄位
server_description
以下範例讓整個工作階段都能使用提供文件的 MCP 伺服器:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19const event = {
type: "session.update",
session: {
type: "realtime",
model: "gpt-realtime-2.1",
output_modalities: ["text"],
tools: [
{
type: "mcp",
server_label: "openai_docs",
server_url: "https://developers.openai.com/mcp",
allowed_tools: ["search_openai_docs", "fetch_openai_doc"],
require_approval: "never",
},
],
},
};
ws.send(JSON.stringify(event));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19event = {
"type": "session.update",
"session": {
"type": "realtime",
"model": "gpt-realtime-2.1",
"output_modalities": ["text"],
"tools": [
{
"type": "mcp",
"server_label": "openai_docs",
"server_url": "https://developers.openai.com/mcp",
"allowed_tools": ["search_openai_docs", "fetch_openai_doc"],
"require_approval": "never",
}
],
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{
Model: "gpt-realtime-2.1",
OutputModalities: []string{"text"},
Tools: realtime.RealtimeToolsConfigParam{
{
OfMcp: &realtime.RealtimeToolsConfigUnionMcpParam{
ServerLabel: "openai_docs",
ServerURL: openai.String("https://developers.openai.com/mcp"),
AllowedTools: realtime.RealtimeToolsConfigUnionMcpAllowedToolsParam{
OfMcpAllowedTools: []string{"search_openai_docs", "fetch_openai_doc"},
},
RequireApproval: realtime.RealtimeToolsConfigUnionMcpRequireApprovalParam{
OfMcpToolApprovalSetting: openai.String("never"),
},
},
},
},
})); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import java.util.List;
connection.send(
RealtimeClientEvent.ofSessionUpdate(
SessionUpdateEvent.builder()
.session(
RealtimeSessionCreateRequest.builder()
.model("gpt-realtime-2.1")
.addOutputModality(RealtimeSessionCreateRequest.OutputModality.TEXT)
.addTool(
RealtimeToolsConfigUnion.Mcp.builder()
.serverLabel("openai_docs")
.serverUrl("https://developers.openai.com/mcp")
.allowedToolsOfMcp(
List.of("search_openai_docs", "fetch_openai_doc"))
.requireApproval(
RealtimeToolsConfigUnion.Mcp.RequireApproval
.McpToolApprovalSetting.NEVER)
.build())
.build())
.build()));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27using OpenAI.Realtime;
#pragma warning disable OPENAI002
await session.SendCommandAsync(new RealtimeClientCommandSessionUpdate(new RealtimeConversationSessionOptions
{
Model = "gpt-realtime-2.1",
OutputModalities =
{
RealtimeOutputModality.Text
},
Tools =
{
new RealtimeMcpTool("openai_docs", new Uri("https://developers.openai.com/mcp"))
{
AllowedTools = new()
{
ToolNames =
{
"search_openai_docs",
"fetch_openai_doc"
}
},
ToolCallApprovalPolicy = RealtimeDefaultMcpToolCallApprovalPolicy.NeverRequireApproval
}
}
}), timeout.Token);
1
2
3
4
5
6
7
8
9
10
11
12
13
14connection.session.update(
type: :realtime,
model: "gpt-realtime-2.1",
output_modalities: [:text],
tools: [
{
type: :mcp,
server_label: "openai_docs",
server_url: "https://developers.openai.com/mcp",
allowed_tools: ["search_openai_docs", "fetch_openai_doc"],
require_approval: :never
}
]
)
connector_id 在 2026 年 9 月 1 日之後發布的模型中已棄用。請使用 server_url 連線至遠端 MCP 伺服器,或使用
tunnel_id 透過
安全 MCP 通道連線至本機 MCP 伺服器。既有模型仍支援連接器。以下範例使用的
gpt-realtime-1.5 於上述日期之前發布。
內建連接器的 MCP 工具定義格式相同,但要傳入 connector_id 來取代 server_url。例如,Google 日曆使用 connector_googlecalendar。在 Realtime 中,請使用這些內建連接器執行讀取操作,例如搜尋或讀取日曆活動或電子郵件。請在 authorization 中傳入使用者的 OAuth 存取權杖,並盡可能透過 allowed_tools 限制可用的工具:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20const event = {
type: "session.update",
session: {
type: "realtime",
model: "gpt-realtime-1.5",
output_modalities: ["text"],
tools: [
{
type: "mcp",
server_label: "google_calendar",
connector_id: "connector_googlecalendar",
authorization: "<google-oauth-access-token>",
allowed_tools: ["search_events", "read_event"],
require_approval: "never",
},
],
},
};
ws.send(JSON.stringify(event));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24import os
connector_authorization = os.environ["OPENAI_CONNECTOR_AUTHORIZATION"]
event = {
"type": "session.update",
"session": {
"type": "realtime",
"model": "gpt-realtime-1.5",
"output_modalities": ["text"],
"tools": [
{
"type": "mcp",
"server_label": "google_calendar",
"connector_id": "connector_googlecalendar",
"authorization": connector_authorization,
"allowed_tools": ["search_events", "read_event"],
"require_approval": "never",
}
],
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26accessToken := os.Getenv("OPENAI_MCP_ACCESS_TOKEN")
if accessToken == "" {
return fmt.Errorf("OPENAI_MCP_ACCESS_TOKEN is required")
}
if err := conn.Send(ctx, realtime.RealtimeClientEventParamOfSessionUpdate(realtime.RealtimeSessionCreateRequestParam{
Model: "gpt-realtime-1.5",
OutputModalities: []string{"text"},
Tools: realtime.RealtimeToolsConfigParam{
{
OfMcp: &realtime.RealtimeToolsConfigUnionMcpParam{
ServerLabel: "google_calendar",
ConnectorID: "connector_googlecalendar",
Authorization: param.NewOpt(accessToken),
AllowedTools: realtime.RealtimeToolsConfigUnionMcpAllowedToolsParam{
OfMcpAllowedTools: []string{"search_events", "read_event"},
},
RequireApproval: realtime.RealtimeToolsConfigUnionMcpRequireApprovalParam{
OfMcpToolApprovalSetting: openai.String("never"),
},
},
},
},
})); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import java.util.List;
connection.send(
RealtimeClientEvent.ofSessionUpdate(
SessionUpdateEvent.builder()
.session(
RealtimeSessionCreateRequest.builder()
.model("gpt-realtime-1.5")
.addOutputModality(RealtimeSessionCreateRequest.OutputModality.TEXT)
.addTool(
RealtimeToolsConfigUnion.Mcp.builder()
.serverLabel("google_calendar")
.connectorId(
RealtimeToolsConfigUnion.Mcp.ConnectorId
.CONNECTOR_GOOGLECALENDAR)
.authorization(System.getenv("OPENAI_CONNECTOR_AUTHORIZATION"))
.allowedToolsOfMcp(List.of("search_events", "read_event"))
.requireApproval(
RealtimeToolsConfigUnion.Mcp.RequireApproval
.McpToolApprovalSetting.NEVER)
.build())
.build())
.build()));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28using OpenAI.Realtime;
#pragma warning disable OPENAI002
await session.SendCommandAsync(new RealtimeClientCommandSessionUpdate(new RealtimeConversationSessionOptions
{
Model = "gpt-realtime-1.5",
OutputModalities =
{
RealtimeOutputModality.Text
},
Tools =
{
new RealtimeMcpTool("google_calendar", RealtimeMcpToolConnectorId.GoogleCalendar)
{
AuthorizationToken = Environment.GetEnvironmentVariable("OPENAI_CONNECTOR_AUTHORIZATION")!,
AllowedTools = new()
{
ToolNames =
{
"search_events",
"read_event"
}
},
ToolCallApprovalPolicy = RealtimeDefaultMcpToolCallApprovalPolicy.NeverRequireApproval
}
}
}), timeout.Token);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17access_token = ENV.fetch("OPENAI_MCP_ACCESS_TOKEN")
connection.session.update(
type: :realtime,
model: "gpt-realtime-1.5",
output_modalities: [:text],
tools: [
{
type: :mcp,
server_label: "google_calendar",
connector_id: "connector_googlecalendar",
authorization: access_token,
allowed_tools: ["search_events", "read_event"],
require_approval: :never
}
]
)
遠端 MCP 伺服器
不會自動收到完整的對話上下文,但可以看到模型在工具呼叫中傳送的所有資料。
請限制可用工具的範圍,做法是設定 allowed_tools,並針對任何不宜自動執行的動作要求核准。
與 Realtime 的 function 工具不同,遠端 MCP 工具由 Realtime API 直接執行。用戶端不會執行遠端工具,也不會傳回 function_call_output。用戶端負責設定存取權及監聽 MCP 生命週期事件,並可在伺服器要求核准時傳送核准回覆。
典型流程如下:
- 傳送
session.update 或 response.create,其中包含一個 tools 項目,並將其 type 設為 mcp。
- 伺服器開始匯入工具,並發出
mcp_list_tools.in_progress。
- 在伺服器列出工具的過程中,模型無法呼叫尚未載入的工具。如果接下來的一輪對話需要這些工具,且你想等工具載入後再開始,請監聽
mcp_list_tools.completed。在 conversation.item.done 事件中,若 item.type 為 mcp_list_tools,該事件就會列出實際匯入的工具名稱。如果匯入失敗,你會收到 mcp_list_tools.failed。
- 使用者說話或傳送文字後,用戶端會建立回應,或由系統依工作階段設定自動建立回應。
- 如果模型選擇 MCP 工具,你會看到
response.mcp_call_arguments.delta 和 response.mcp_call_arguments.done。
- 如果需要核准,伺服器會加入一個對話項目,其
item.type 為 mcp_approval_request。用戶端必須以 mcp_approval_response 項目回覆。
- 工具開始執行時,你會看到
response.mcp_call.in_progress。如果執行成功,你隨後會收到 response.output_item.done 事件,其 item.type 為 mcp_call。如果執行失敗,你會收到 response.mcp_call.failed。
response.done 可能會在該回應的 MCP 呼叫完成前送達。等到回應結束,且該回應的所有 MCP 呼叫都完成後,請再傳送一個 response.create 事件,讓模型使用這些結果繼續對話。如果模型再次呼叫 MCP 工具,請重複這個步驟。Realtime API 不會自動建立這些後續回應。
以下事件處理常式會記錄主要的 MCP 生命週期事件,但不會管理後續回應:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90import type WebSocket from "ws";
import type {
ConversationItem,
RealtimeServerEvent,
} from "openai/resources/realtime/realtime";
function parseRealtimeEvent(
rawMessage: string | WebSocket.RawData | MessageEvent<string>
): RealtimeServerEvent {
if (typeof rawMessage === "string") {
return JSON.parse(rawMessage);
}
if ("data" in rawMessage && typeof rawMessage.data === "string") {
return JSON.parse(rawMessage.data);
}
return JSON.parse(rawMessage.toString());
}
function getOutputText(item: ConversationItem) {
if (item.type !== "message") return "";
return (item.content ?? [])
.filter((part) => part.type === "output_text")
.map((part) => part.text)
.join("");
}
ws.on("message", (rawMessage) => {
const event = parseRealtimeEvent(rawMessage);
switch (event.type) {
case "mcp_list_tools.in_progress":
console.log("Listing MCP tools for item:", event.item_id);
break;
case "mcp_list_tools.completed":
console.log("MCP tool listing complete for item:", event.item_id);
break;
case "mcp_list_tools.failed":
console.error("MCP tool listing failed for item:", event.item_id);
break;
case "conversation.item.done":
if (event.item.type === "mcp_list_tools") {
const names = event.item.tools.map((tool) => tool.name).join(", ");
console.log(`MCP tools ready on ${event.item.server_label}: ${names}`);
}
if (event.item.type === "mcp_approval_request") {
console.log(
"Approval required for:",
event.item.name,
event.item.arguments
);
}
break;
case "response.mcp_call_arguments.done":
console.log("Final MCP call arguments:", event.arguments);
break;
case "response.mcp_call.in_progress":
console.log("Running MCP tool for item:", event.item_id);
break;
case "response.mcp_call.failed":
console.error("MCP tool call failed for item:", event.item_id);
break;
case "response.output_item.done":
if (event.item.type === "mcp_call") {
console.log(
`MCP output from ${event.item.server_label}.${event.item.name}:`,
event.item.output
);
}
if (event.item.type === "message") {
console.log("Assistant:", getOutputText(event.item));
}
break;
case "response.done":
console.log("Realtime turn complete.");
break;
}
});
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61def on_message(ws, message):
event = json.loads(message)
event_type = event["type"]
if event_type == "mcp_list_tools.in_progress":
print("Listing MCP tools for item:", event["item_id"])
return
if event_type == "mcp_list_tools.completed":
print("MCP tool listing complete for item:", event["item_id"])
return
if event_type == "mcp_list_tools.failed":
print("MCP tool listing failed for item:", event["item_id"])
return
if event_type == "conversation.item.done":
item = event["item"]
if item["type"] == "mcp_list_tools":
names = ", ".join(tool["name"] for tool in item["tools"])
print(f"MCP tools ready on {item['server_label']}: {names}")
return
if item["type"] == "mcp_approval_request":
print("Approval required for:", item["name"], item["arguments"])
return
if event_type == "response.mcp_call_arguments.done":
print("Final MCP call arguments:", event["arguments"])
return
if event_type == "response.mcp_call.in_progress":
print("Running MCP tool for item:", event["item_id"])
return
if event_type == "response.mcp_call.failed":
print("MCP tool call failed for item:", event["item_id"])
return
if event_type == "response.output_item.done":
item = event["item"]
if item["type"] == "mcp_call":
print(
f"MCP output from {item['server_label']}.{item['name']}:",
item.get("output"),
)
return
if item["type"] == "message":
text_parts = [
part["text"]
for part in item.get("content", [])
if part["type"] == "output_text"
]
print("Assistant:", "".join(text_parts))
return
if event_type == "response.done":
print("Realtime turn complete.")
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18// Call for each received event. Tool results can arrive after response.done.
func logMCPEvent(event realtime.RealtimeServerEventUnion) error {
switch e := event.AsAny().(type) {
case realtime.RealtimeErrorEvent:
return fmt.Errorf("Realtime API error: %s", e.Error.Message)
case realtime.McpListToolsFailed, realtime.ResponseMcpCallFailed:
return fmt.Errorf("MCP failed: %s", event.Type)
case realtime.ResponseOutputItemDoneEvent:
item := e.Item.OfRealtimeMcpToolCall
if item.Type == "mcp_call" {
if item.JSON.Error.Valid() || !item.JSON.Output.Valid() {
return fmt.Errorf("MCP call failed or has no output")
}
fmt.Fprintln(log.Writer(), "Tool output:", item.Output)
}
}
return nil
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77completed_events = {}
expected_calls = []
completed_calls = {}
complete = connection.find do |event|
case event
when OpenAI::Realtime::McpListToolsInProgress
puts("Listing MCP tools for item: #{event.item_id}")
when OpenAI::Realtime::McpListToolsFailed
raise "MCP tool listing failed for item: #{event.item_id}"
when OpenAI::Realtime::McpListToolsCompleted
puts("MCP tools ready for item: #{event.item_id}")
connection.response.create(
output_modalities: [:text],
input: [
{
type: :message,
role: :user,
content: [
{
type: :input_text,
text: "Which Realtime API transport should browser clients use?"
}
]
}
],
tool_choice: :required,
parallel_tool_calls: false
)
when OpenAI::Realtime::ConversationItemDone
item = event.item
case item
when OpenAI::Realtime::RealtimeMcpListTools
names = item.tools.map(&:name).join(", ")
puts("MCP tools ready on #{item.server_label}: #{names}")
when OpenAI::Realtime::RealtimeMcpApprovalRequest
puts("Approval required for: #{item.name} #{item.arguments}")
end
when OpenAI::Realtime::ResponseMcpCallArgumentsDone
puts("Final MCP call arguments: #{event.arguments}")
when OpenAI::Realtime::ResponseMcpCallInProgress
puts("Running MCP tool for item: #{event.item_id}")
when OpenAI::Realtime::ResponseMcpCallCompleted
puts("MCP tool call completed: #{event.item_id}")
when OpenAI::Realtime::ResponseMcpCallFailed
raise "MCP tool call failed: #{event.item_id}"
when OpenAI::Realtime::ResponseOutputItemDoneEvent
item = event.item
case item
when OpenAI::Realtime::RealtimeMcpToolCall
raise "MCP tool call failed" if item.error || item.output.nil?
completed_calls[item.id] = true
puts("MCP output from #{item.server_label}.#{item.name}: #{item.output}")
when OpenAI::Realtime::RealtimeConversationItemAssistantMessage
text = item.content.filter_map do |content|
content.text if content.type == :output_text
end.join
puts("Assistant: #{text}")
end
when OpenAI::Realtime::RealtimeErrorEvent
raise "Realtime API error: #{event.error.message}"
when OpenAI::Realtime::ResponseDoneEvent
raise "Realtime response did not complete" unless event.response.status == :completed
expected_calls.replace(Array(event.response.output).grep(OpenAI::Realtime::RealtimeMcpToolCall).map(&:id))
raise "The response did not call an MCP tool" if expected_calls.empty?
completed_events[:response] = true
end
if completed_events[:response] && expected_calls.all? { |id| completed_calls[id] }
puts("Realtime turn complete.")
true
else
false
end
end
raise "Connection closed before the MCP tool result arrived" unless complete
mcp_list_tools.failed:Realtime API 無法從遠端伺服器或連接器匯入工具。請檢查 server_url 或 connector_id、驗證設定、伺服器連線狀態,以及你在 allowed_tools 中指定的工具名稱。
response.mcp_call.failed:模型選擇了工具,但工具呼叫未完成。請檢查事件承載資料及後續的 mcp_call 項目,確認是否發生 MCP 通訊協定、執行或傳輸錯誤。
mcp_approval_request 沒有對應的 mcp_approval_response:用戶端必須明確核准或拒絕,工具呼叫才能繼續處理。
- 新一輪對話開始時,
mcp_list_tools.in_progress 仍在進行中:這一輪只能使用已載入完成的工具。
- 回應使用
tool_choice: "required",但目前沒有可用工具:模型沒有符合條件的工具可呼叫。請等到收到 mcp_list_tools.completed,並確認至少已匯入一個工具,或針對不需要工具的對話輪次使用其他 tool_choice 設定。
- MCP 工具定義在開始匯入前未通過驗證:常見原因包括同一個
server_label 在同一個 tools 陣列中出現多次、同時設定 server_url 和 connector_id、在最初建立工作階段的請求中省略這兩個欄位、使用無效的 connector_id,或同時傳送 authorization 和 headers.Authorization。使用連接器時,一律不要傳送 headers.Authorization。
如果工具需要核准,Realtime API 會在對話中插入一個 mcp_approval_request 項目。若要繼續,請傳送新的 conversation.item.create 事件,並將其 item.type 設為 mcp_approval_response。
1
2
3
4
5
6
7
8
9
10
11
12function approveMcpRequest(approvalRequestId: string) {
const event = {
type: "conversation.item.create",
item: {
type: "mcp_approval_response",
approval_request_id: approvalRequestId,
approve: true,
},
};
ws.send(JSON.stringify(event));
}
1
2
3
4
5
6
7
8
9
10
11
12# Use the ID from the received MCP approval-request item.
def approve_mcp_request(ws, approval_request_id):
event = {
"type": "conversation.item.create",
"item": {
"type": "mcp_approval_response",
"approval_request_id": approval_request_id,
"approve": True,
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18// Call with the approval request received in conversation.item.done.
func approveDocumentationSearch(
ctx context.Context,
conn *realtime.RealtimeConnection,
request realtime.RealtimeMcpApprovalRequest,
) error {
// This example permits only a read-only search of the OpenAI documentation.
if request.ID == "" || request.ServerLabel != "openai_docs" || request.Name != "search_openai_docs" {
return fmt.Errorf("unapproved tool requested")
}
// Derive a distinct item ID within the API's 32-character limit.
approvalID := sha256.Sum256([]byte(request.ID))
return conn.Send(ctx, realtime.RealtimeClientEventParamOfConversationItemCreate(realtime.RealtimeMcpApprovalResponseParam{
ID: fmt.Sprintf("%x", approvalID[:16]),
ApprovalRequestID: request.ID,
Approve: true,
}))
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.core.JsonMissing;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import com.openai.models.responses.ToolChoiceOptions;
// Replace https://mcp.example.com/mcp with your company's MCP server URL.
static void approveMcpRequest(RealtimeConnection connection, String approvalRequestId)
throws Exception {
send(
connection,
RealtimeClientEvent.ofConversationItemCreate(
ConversationItemCreateEvent.builder()
.item(
RealtimeMcpApprovalResponse.builder()
.id(java.util.UUID.randomUUID().toString().replace("-", ""))
.approvalRequestId(approvalRequestId)
.approve(true)
.build())
.build()));
}
// Retry only explicit admission rejections; these guarantee nothing was sent.
private static void send(RealtimeConnection connection, RealtimeClientEvent event)
throws Exception {
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5);
while (true) {
try {
connection.send(event);
return;
} catch (com.openai.core.http.WebSocketWriteNotAttempted.Busy busy) {
if (System.nanoTime() >= deadline) throw busy;
Thread.sleep(10);
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13using System.Text.Json;
using OpenAI.Realtime;
#pragma warning disable OPENAI002
// Connect to the official OpenAI documentation MCP server.
internal static async Task ApproveMcpRequestAsync(RealtimeSessionClient session, string approvalRequestId, CancellationToken cancellationToken)
{
await session.SendCommandAsync(new RealtimeClientCommandConversationItemCreate(new RealtimeMcpToolCallApprovalResponseItem(approvalRequestId, true)
{
Id = Guid.NewGuid().ToString("N")
}), cancellationToken);
}
1
2
3
4
5
6
7
8
9# Use the ID from the received MCP approval-request item.
approval_request_id = item.id
connection.conversation.items.create(
type: :mcp_approval_response,
id: "approval_#{SecureRandom.hex(8)}",
approval_request_id: approval_request_id,
approve: true
)
若要拒絕請求,請將 approve 設為 false,也可以在 reason 中說明原因。
如果 MCP 只應在一輪對話中可用,請將同一個 MCP 工具物件加入 response.tools,而非 session.tools:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31const event = {
type: "response.create",
response: {
output_modalities: ["text"],
tool_choice: "required",
parallel_tool_calls: false,
input: [
{
type: "message",
role: "user",
content: [
{
type: "input_text",
text: "Which transport should I use for browser clients in the Realtime API?",
},
],
},
],
tools: [
{
type: "mcp",
server_label: "openai_docs",
server_url: "https://developers.openai.com/mcp",
allowed_tools: ["search_openai_docs", "fetch_openai_doc"],
require_approval: "never",
},
],
},
};
ws.send(JSON.stringify(event));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31event = {
"type": "response.create",
"response": {
"output_modalities": ["text"],
"tool_choice": "required",
"parallel_tool_calls": False,
"input": [
{
"type": "message",
"role": "user",
"content": [
{
"type": "input_text",
"text": "Which transport should I use for browser clients in the Realtime API?",
}
],
}
],
"tools": [
{
"type": "mcp",
"server_label": "openai_docs",
"server_url": "https://developers.openai.com/mcp",
"allowed_tools": ["search_openai_docs", "fetch_openai_doc"],
"require_approval": "never",
}
],
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29// input contains the conversation items for this response.
if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{
OfResponseCreate: &realtime.ResponseCreateEventParam{
Response: realtime.RealtimeResponseCreateParams{
ParallelToolCalls: openai.Bool(false),
Input: input,
OutputModalities: []string{"text"},
ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{
OfToolChoiceMode: param.NewOpt[responses.ToolChoiceOptions]("required"),
},
Tools: []realtime.RealtimeResponseCreateParamsToolUnion{
{
OfMcp: &realtime.RealtimeResponseCreateMcpToolParam{
ServerLabel: "openai_docs",
ServerURL: openai.String("https://developers.openai.com/mcp"),
AllowedTools: realtime.RealtimeResponseCreateMcpToolAllowedToolsUnionParam{
OfMcpAllowedTools: []string{"search_openai_docs", "fetch_openai_doc"},
},
RequireApproval: realtime.RealtimeResponseCreateMcpToolRequireApprovalUnionParam{
OfMcpToolApprovalSetting: openai.String("never"),
},
},
},
},
},
},
}); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.core.JsonMissing;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import java.util.List;
connection.send(
RealtimeClientEvent.ofResponseCreate(
ResponseCreateEvent.builder()
.response(
RealtimeResponseCreateParams.builder()
.toolChoice(com.openai.models.responses.ToolChoiceOptions.REQUIRED)
.parallelToolCalls(false)
.metadata(
RealtimeResponseCreateParams.Metadata.builder()
.putAdditionalProperty(
"topic", com.openai.core.JsonValue.from("mcp_initial"))
.build())
.addOutputModality(RealtimeResponseCreateParams.OutputModality.TEXT)
.addInput(
RealtimeConversationItemUserMessage.builder()
.object_(JsonMissing.of())
.addContent(
RealtimeConversationItemUserMessage.Content.builder()
.type(
RealtimeConversationItemUserMessage.Content.Type
.INPUT_TEXT)
.text(
"Which transport should I use for browser clients in the Realtime API?")
.build())
.build())
.addTool(
RealtimeResponseCreateMcpTool.builder()
.serverLabel("openai_docs")
.serverUrl("https://developers.openai.com/mcp")
.allowedToolsOfMcp(
List.of("search_openai_docs", "fetch_openai_doc"))
.requireApproval(
RealtimeResponseCreateMcpTool.RequireApproval
.McpToolApprovalSetting.NEVER)
.build())
.build())
.build()));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36using System.Text.Json;
using OpenAI.Realtime;
#pragma warning disable OPENAI002
await session.SendCommandAsync(new RealtimeClientCommandResponseCreate
{
ResponseOptions = new()
{
Metadata = new Dictionary<string, BinaryData> { ["topic"] = BinaryData.FromObjectAsJson("mcp_initial") },
ToolChoice = RealtimeDefaultToolChoice.Required,
OutputModalities =
{
RealtimeOutputModality.Text
},
InputItems =
{
RealtimeItem.CreateUserMessageItem("Which transport should I use for browser clients in the Realtime API?")
},
Tools =
{
new RealtimeMcpTool("openai_docs", new Uri("https://developers.openai.com/mcp"))
{
AllowedTools = new()
{
ToolNames =
{
"search_openai_docs",
"fetch_openai_doc"
}
},
ToolCallApprovalPolicy = RealtimeDefaultMcpToolCallApprovalPolicy.NeverRequireApproval
}
}
}
}, timeout.Token);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26connection.response.create(
output_modalities: [:text],
tool_choice: :required,
parallel_tool_calls: false,
input: [
{
type: :message,
role: :user,
content: [
{
type: :input_text,
text: "Which Realtime API transport should browser clients use?"
}
]
}
],
tools: [
{
type: :mcp,
server_label: "openai_docs",
server_url: "https://developers.openai.com/mcp",
allowed_tools: ["search_openai_docs", "fetch_openai_doc"],
require_approval: :never
}
]
)
如果只有一個回應需要外部上下文,或不同輪次的對話需要使用不同的 MCP 伺服器,就適合採用這種做法。
server_label 是目前 Realtime 工作階段中用來識別工具定義的固定標籤。使用 server_label 搭配 server_url 或 connector_id 定義伺服器或連接器後,後續的 session.update 或 response.create 事件只需參照同一個 server_label。Realtime API 會沿用先前的定義,你不必再次傳送完整的工具物件。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27const event = {
type: "response.create",
response: {
output_modalities: ["text"],
input: [
{
type: "message",
role: "user",
content: [
{
type: "input_text",
text: "Check my schedule for this afternoon.",
},
],
},
],
// Reuses the google_calendar connector defined earlier in this session.
tools: [
{
type: "mcp",
server_label: "google_calendar",
},
],
},
};
ws.send(JSON.stringify(event));
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27event = {
"type": "response.create",
"response": {
"output_modalities": ["text"],
"input": [
{
"type": "message",
"role": "user",
"content": [
{
"type": "input_text",
"text": "Check my schedule for this afternoon.",
}
],
}
],
# Reuses the google_calendar connector defined earlier in this session.
"tools": [
{
"type": "mcp",
"server_label": "google_calendar",
}
],
},
}
ws.send(json.dumps(event))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21// input contains the conversation items for this response.
if err := conn.Send(ctx, realtime.RealtimeClientEventUnionParam{
OfResponseCreate: &realtime.ResponseCreateEventParam{
Response: realtime.RealtimeResponseCreateParams{
OutputModalities: []string{"text"},
Input: input,
Tools: []realtime.RealtimeResponseCreateParamsToolUnion{
{
OfMcp: &realtime.RealtimeResponseCreateMcpToolParam{
ServerLabel: "google_calendar",
},
},
},
ToolChoice: realtime.RealtimeResponseCreateParamsToolChoiceUnion{
OfToolChoiceMode: param.NewOpt(responses.ToolChoiceOptions("required")),
},
},
},
}); err != nil {
return err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50import com.openai.client.okhttp.OkHttpClient;
import com.openai.core.ClientOptions;
import com.openai.helpers.RealtimeConnection;
import com.openai.helpers.RealtimeWebSocketOptions;
import com.openai.models.realtime.*;
import java.util.List;
send(
connection,
RealtimeClientEvent.ofResponseCreate(
ResponseCreateEvent.builder()
.response(
RealtimeResponseCreateParams.builder()
.metadata(
RealtimeResponseCreateParams.Metadata.builder()
.putAdditionalProperty(
"topic", com.openai.core.JsonValue.from("mcp_initial"))
.build())
.addOutputModality(RealtimeResponseCreateParams.OutputModality.TEXT)
.addInput(
RealtimeConversationItemUserMessage.builder()
.addContent(
RealtimeConversationItemUserMessage.Content.builder()
.type(
RealtimeConversationItemUserMessage.Content.Type
.INPUT_TEXT)
.text("Check my schedule for this afternoon.")
.build())
.build())
.addTool(
RealtimeResponseCreateMcpTool.builder()
.serverLabel("google_calendar")
.build())
.build())
.build()));
// Retry only explicit admission rejections; these guarantee nothing was sent.
private static void send(RealtimeConnection connection, RealtimeClientEvent event)
throws Exception {
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5);
while (true) {
try {
connection.send(event);
return;
} catch (com.openai.core.http.WebSocketWriteNotAttempted.Busy busy) {
if (System.nanoTime() >= deadline) throw busy;
Thread.sleep(10);
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26using OpenAI.Realtime;
#pragma warning disable OPENAI002
await session.SendCommandAsync(new RealtimeClientCommandResponseCreate
{
ResponseOptions = new()
{
Metadata = new Dictionary<string, BinaryData> { ["topic"] = BinaryData.FromObjectAsJson("mcp_initial") },
OutputModalities =
{
RealtimeOutputModality.Text
},
InputItems =
{
RealtimeItem.CreateUserMessageItem("Check my schedule for this afternoon.")
},
Tools =
{
new RealtimeMcpTool
{
ServerLabel = "google_calendar"
}
}
}
}, timeout.Token);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21connection.response.create(
output_modalities: [:text],
input: [
{
type: :message,
role: :user,
content: [
{
type: :input_text,
text: "Check my schedule this afternoon."
}
]
}
],
tools: [
{
type: :mcp,
server_label: "google_calendar"
}
]
)
只有在同一個工作階段中才能沿用定義。如果你開始新的 Realtime 工作階段,請再次傳送完整的 MCP 定義,讓伺服器匯入工具清單。