Computer use

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72import json
import uuid

from openai import OpenAI
from openai.types.responses import (
    FunctionToolParam,
    ResponseInputParam,
)

def run_computer_use(endpoint, prompt, model="gpt-6-astra"):
    client = OpenAI()
    session_id = str(uuid.uuid4())
    tools: list[FunctionToolParam] = [
        {
            "type": "function",
            "name": "exec_py",
            "description": (
                "Run Python in a persistent desktop. Variables persist across calls. "
                "PyAutoGUI operations are synchronous. Available: pyautogui, time, "
                "log(value), and display(PIL_image). Inspect the screen with "
                "display(pyautogui.screenshot()) before acting. Use screenshot "
                "coordinates and check the screen after a short group of actions. "
                "Keep screenshots in memory and PyAutoGUI's fail-safe enabled."
            ),
            "parameters": {
                "type": "object",
                "properties": {"code": {"type": "string"}},
                "required": ["code"],
                "additionalProperties": False,
            },
            "strict": True,
        }
    ]
    next_input: ResponseInputParam = [{"role": "user", "content": prompt}]
    previous_response_id = None

    for turn in range(20):
        response = client.responses.create(
            model=model,
            tools=tools,
            input=next_input,
            previous_response_id=previous_response_id,
        )
        if response.status != "completed":
            raise RuntimeError(f"Response stopped with status: {response.status}")

        calls = [item for item in response.output if item.type == "function_call"]
        if not calls and any(
            item.type == "message" and item.phase != "commentary"
            for item in response.output
        ):
            print(response.output_text)
            return
        if turn == 19:
            raise RuntimeError(
                "The task reached the 20-response limit. Inspect the last result."
            )

        next_input = []
        for call in calls:
            if call.name != "exec_py":
                raise ValueError(f"Unexpected tool: {call.name}")
            code = json.loads(call.arguments)["code"]
            output = execute_in_sandbox(code, session_id, endpoint)
            next_input.append(
                {
                    "type": "function_call_output",
                    "call_id": call.call_id,
                    "output": output,
                }
            )
        previous_response_id = response.id

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74import { randomUUID } from "node:crypto";
import OpenAI from "openai";

async function runComputerUse(endpoint, prompt, model = "gpt-6-astra") {
  const client = new OpenAI();
  const sessionId = randomUUID();

  const tools = [
    {
      type: "function",
      name: "exec_js",
      description: `Run JavaScript in a persistent browser. Available: Playwright's
browser, context, and page objects; console.log(value); and display(base64Image).
Save reusable variables on globalThis. Inspect a screenshot before acting and
check the screen after a short group of actions. Keep screenshots in memory.
Use top-level await for async operations. Return images with display() and concise
text with console.log(). The context viewport is 1440x900.`,
      parameters: {
        type: "object",
        properties: { code: { type: "string" } },
        required: ["code"],
        additionalProperties: false,
      },
      strict: true,
    },
  ];

  let nextInput = [{ role: "user", content: prompt }];
  let previousResponseId;

  for (let turn = 0; turn < 20; turn++) {
    const response = await client.responses.create({
      model,
      tools,
      input: nextInput,
      previous_response_id: previousResponseId,
      reasoning: { effort: "low" },
    });
    if (response.status !== "completed") {
      throw new Error(`Response stopped with status: ${response.status}`);
    }
    const calls = response.output.filter(
      (item) => item.type === "function_call"
    );
    if (
      calls.length === 0 &&
      response.output.some(
        (item) => item.type === "message" && item.phase !== "commentary"
      )
    ) {
      console.log(response.output_text);
      return;
    }
    if (turn === 19) {
      throw new Error(
        "The task reached the 20-response limit. Inspect the last result."
      );
    }

    nextInput = [];
    for (const call of calls) {
      if (call.name !== "exec_js")
        throw new Error(`Unexpected tool: ${call.name}`);
      const { code } = JSON.parse(call.arguments);
      const output = await executeInSandbox(code, sessionId, endpoint);
      nextInput.push({
        type: "function_call_output",
        call_id: call.call_id,
        output,
      });
    }
    previousResponseId = response.id;
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57require "json"
require "openai"
require "securerandom"

def run_computer_use(endpoint, prompt)
  client = OpenAI::Client.new
  session_id = SecureRandom.uuid
  tools = [
    {
      type: :function,
      name: "exec_py",
      description: "Run Python in a persistent desktop. Variables persist across calls. PyAutoGUI operations are synchronous. Available: pyautogui, time, log(value), and display(PIL_image). Inspect the screen with display(pyautogui.screenshot()) before acting. Use screenshot coordinates and check the screen after a short group of actions. Keep screenshots in memory and PyAutoGUI's fail-safe enabled.",
      parameters: {
        type: :object,
        properties: { code: { type: :string } },
        required: ["code"],
        additionalProperties: false
      },
      strict: true
    }
  ]
  next_input = []
  next_input << {
    role: :user,
    content: prompt
  }
  history = {}
  20.times do |turn|
    response = client.responses.create(
      model: "gpt-6-astra", tools: tools, input: next_input, previous_response_id: history[:id]
    )
    raise "Response stopped with status: #{response.status}" unless response.status == OpenAI::Responses::ResponseStatus::COMPLETED

    calls = response.output.grep(OpenAI::Responses::ResponseFunctionToolCall)
    if calls.empty? && response.output.any? { |item| item.is_a?(OpenAI::Responses::ResponseOutputMessage) && item.phase != :commentary }
      puts(response.output_text)
      return response
    end
    raise "The task reached the 20-response limit" if turn == 19

    next_input.clear
    calls.each do |call|
      raise "Unexpected tool: #{call.name}" unless call.name == "exec_py"

      code = JSON.parse(call.arguments).fetch("code")
      raise "Expected Python source text" unless code.is_a?(String)

      output = execute_in_sandbox(code, session_id, endpoint)
      next_input << {
        type: :function_call_output,
        call_id: call.call_id,
        output: output
      }
    end
    history[:id] = response.id
  end
end

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75func runComputerUse(ctx context.Context, client openai.Client, endpoint, prompt string, terminal *bufio.Scanner, output io.Writer) error {
	session := make([]byte, 16)
	if _, err := rand.Read(session); err != nil {
		return err
	}
	sessionID := hex.EncodeToString(session)
	tool := responses.ToolParamOfFunction("exec_py", map[string]any{
		"type": "object", "properties": map[string]any{"code": map[string]any{"type": "string"}},
		"required": []string{"code"}, "additionalProperties": false,
	}, true)
	tool.OfFunction.Description = openai.String("Run Python in a persistent desktop. Variables persist across calls. " +
		"PyAutoGUI operations are synchronous. Available: pyautogui, time, " +
		"log(value), and display(PIL_image). Inspect the screen with " +
		"display(pyautogui.screenshot()) before acting. Use screenshot " +
		"coordinates and check the screen after a short group of actions. " +
		"Keep screenshots in memory and PyAutoGUI's fail-safe enabled.")
	params := responses.ResponseNewParams{
		Model: "gpt-6-astra", Tools: []responses.ToolUnionParam{tool},
		Input: responses.ResponseNewParamsInputUnion{OfString: openai.String(prompt)},
	}
	for turn := 0; turn < 20; turn++ {
		response, err := client.Responses.New(ctx, params)
		if err != nil {
			return err
		}
		if response.Status != "completed" {
			return fmt.Errorf("response stopped with status: %s", response.Status)
		}
		var calls []responses.ResponseFunctionToolCall
		finalMessage := false
		for _, item := range response.Output {
			if item.Type == "function_call" {
				calls = append(calls, item.AsFunctionCall())
			}
			if item.Type == "message" && item.AsMessage().Phase != "commentary" {
				finalMessage = true
			}
		}
		if len(calls) == 0 && finalMessage {
			fmt.Fprintln(output, response.OutputText())
			return nil
		}
		if turn == 19 {
			return fmt.Errorf("task reached the 20-response limit; inspect the last result")
		}
		nextInput := responses.ResponseInputParam{}
		for _, call := range calls {
			if call.Name != "exec_py" {
				return fmt.Errorf("unexpected tool: %s", call.Name)
			}
			var arguments struct {
				Code *string `json:"code"`
			}
			if err := json.Unmarshal([]byte(call.Arguments), &arguments); err != nil {
				return err
			}
			if arguments.Code == nil {
				return fmt.Errorf("exec_py requires code")
			}
			observations, err := executeInSandbox(ctx, endpoint, sessionID, *arguments.Code, terminal, output)
			if err != nil {
				return err
			}
			nextInput = append(nextInput, responses.ResponseInputItemUnionParam{
				OfFunctionCallOutput: &responses.ResponseInputItemFunctionCallOutputParam{
					CallID: openai.String(call.CallID),
					Output: responses.ResponseInputItemFunctionCallOutputOutputUnionParam{OfResponseFunctionCallOutputItemArray: observations},
				},
			})
		}
		params.Input = responses.ResponseNewParamsInputUnion{OfInputItemList: nextInput}
		params.PreviousResponseID = openai.String(response.ID)
	}
	return nil
}