コンピューターの使用を組み込むための実装例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54def normalize_key(key):
    """Map model-emitted key names to the names Playwright expects."""
    key_map = {
        "ENTER": "Enter",
        "RETURN": "Enter",
        "ESC": "Escape",
        "ESCAPE": "Escape",
        "TAB": "Tab",
        "SPACE": "Space",
        "BACKSPACE": "Backspace",
        "DELETE": "Delete",
        "DEL": "Delete",
        "HOME": "Home",
        "END": "End",
        "PAGEUP": "PageUp",
        "PAGEDOWN": "PageDown",
        "UP": "ArrowUp",
        "DOWN": "ArrowDown",
        "LEFT": "ArrowLeft",
        "RIGHT": "ArrowRight",
        "ARROWUP": "ArrowUp",
        "ARROWDOWN": "ArrowDown",
        "ARROWLEFT": "ArrowLeft",
        "ARROWRIGHT": "ArrowRight",
        "CTRL": "Control",
        "CONTROL": "Control",
        "SHIFT": "Shift",
        "OPTION": "Alt",
        "ALT": "Alt",
        "META": "Meta",
        "CMD": "Meta",
        "COMMAND": "Meta",
    }
    return key_map.get(key, key)


def normalize_playwright_button(button="left"):
    """Translate API button names to Playwright's supported button names."""
    button_map = {
        "left": "left",
        "right": "right",
        "wheel": "middle",
    }
    if button not in button_map:
        raise ValueError(
            f"Unsupported Playwright mouse button: {button}. "
            "The back and forward buttons are not supported."
        )
    return button_map[button]


def normalize_drag_path(path):
    """Convert the Python SDK's drag-path points to coordinate pairs."""
    return [(point.x, point.y) for point in path]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68def normalize_xdotool_key(key):
    """Map model-emitted key names to the names xdotool expects."""
    key_map = {
        "ENTER": "Return",
        "RETURN": "Return",
        "ESC": "Escape",
        "ESCAPE": "Escape",
        "TAB": "Tab",
        "SPACE": "space",
        "BACKSPACE": "BackSpace",
        "DELETE": "Delete",
        "DEL": "Delete",
        "HOME": "Home",
        "END": "End",
        "PAGEUP": "Page_Up",
        "PAGEDOWN": "Page_Down",
        "UP": "Up",
        "DOWN": "Down",
        "LEFT": "Left",
        "RIGHT": "Right",
        "ARROWUP": "Up",
        "ARROWDOWN": "Down",
        "ARROWLEFT": "Left",
        "ARROWRIGHT": "Right",
        "CTRL": "ctrl",
        "CONTROL": "ctrl",
        "SHIFT": "shift",
        "OPTION": "alt",
        "ALT": "alt",
        "META": "super",
        "CMD": "super",
        "COMMAND": "super",
    }
    return key_map.get(key, key)


def normalize_xdotool_button(button="left"):
    """Translate API button names to X11 button numbers."""
    button_map = {
        "left": 1,
        "wheel": 2,
        "right": 3,
        "back": 8,
        "forward": 9,
    }
    if button not in button_map:
        raise ValueError(f"Unsupported xdotool mouse button: {button}")
    return button_map[button]


def get_xdotool_scroll_buttons(scroll_x, scroll_y):
    """Translate API scroll deltas to vertical and horizontal X11 wheel clicks."""
    buttons = []
    for delta, negative_button, positive_button in (
        (scroll_y, 4, 5),
        (scroll_x, 6, 7),
    ):
        if not delta:
            continue
        button = negative_button if delta < 0 else positive_button
        clicks = max(1, abs(round(delta / 100)))
        buttons.extend([button] * clicks)
    return buttons


def normalize_drag_path(path):
    """Convert the Python SDK's drag-path points to coordinate pairs."""
    return [(point.x, point.y) for point in path]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90import time

# Reuse normalize_key from the helper above.
# Reuse normalize_playwright_button from the helper above.
# Reuse normalize_drag_path from the helper above.


def with_modifiers(page, keys, callback):
    normalized_keys = [normalize_key(key) for key in (keys or [])]
    pressed_keys = []

    try:
        for key in normalized_keys:
            page.keyboard.down(key)
            pressed_keys.append(key)

        callback()
    finally:
        for key in reversed(pressed_keys):
            page.keyboard.up(key)


def handle_computer_actions(page, actions):
    for action in actions:
        match action.type:
            case "click":
                with_modifiers(
                    page,
                    getattr(action, "keys", None),
                    lambda: page.mouse.click(
                        action.x,
                        action.y,
                        button=normalize_playwright_button(
                            getattr(action, "button", "left")
                        ),
                    ),
                )
            case "double_click":
                with_modifiers(
                    page,
                    getattr(action, "keys", None),
                    lambda: page.mouse.dblclick(action.x, action.y),
                )
            case "drag":
                path = normalize_drag_path(action.path)
                if len(path) < 2:
                    raise ValueError("drag action requires at least two path points")

                def do_drag():
                    start_x, start_y = path[0]
                    page.mouse.move(start_x, start_y)
                    page.mouse.down()
                    for x, y in path[1:]:
                        page.mouse.move(x, y)
                    page.mouse.up()

                with_modifiers(
                    page,
                    getattr(action, "keys", None),
                    do_drag,
                )
            case "move":
                with_modifiers(
                    page,
                    getattr(action, "keys", None),
                    lambda: page.mouse.move(action.x, action.y),
                )
            case "scroll":
                with_modifiers(
                    page,
                    getattr(action, "keys", None),
                    lambda: (
                        page.mouse.move(action.x, action.y),
                        page.mouse.wheel(
                            action.scroll_x,
                            action.scroll_y,
                        ),
                    ),
                )
            case "keypress":
                page.keyboard.press("+".join(normalize_key(key) for key in action.keys))
            case "type":
                page.keyboard.type(action.text)
            case "wait":
                time.sleep(2)
            case "screenshot":
                # The caller captures a screenshot after every action.
                continue
            case _:
                raise ValueError(f"Unsupported action: {action.type}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141import time

# Reuse normalize_xdotool_key from the helper above.
# Reuse normalize_xdotool_button and get_xdotool_scroll_buttons from the helper above.
# Reuse normalize_drag_path from the helper above.


def with_modifiers(vm, keys, callback):
    normalized_keys = [normalize_xdotool_key(key) for key in (keys or [])]
    pressed_keys = []

    try:
        for key in normalized_keys:
            docker_exec(
                vm.container_name,
                "xdotool",
                ["keydown", key],
                env={"DISPLAY": vm.display},
            )
            pressed_keys.append(key)

        callback()
    finally:
        for key in reversed(pressed_keys):
            docker_exec(
                vm.container_name,
                "xdotool",
                ["keyup", key],
                env={"DISPLAY": vm.display},
            )


def handle_computer_actions(vm, actions):
    for action in actions:
        match action.type:
            case "click":
                button = normalize_xdotool_button(getattr(action, "button", "left"))
                with_modifiers(
                    vm,
                    getattr(action, "keys", None),
                    lambda: docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mousemove", action.x, action.y, "click", button],
                        env={"DISPLAY": vm.display},
                    ),
                )
            case "double_click":
                with_modifiers(
                    vm,
                    getattr(action, "keys", None),
                    lambda: docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mousemove", action.x, action.y, "click", "--repeat", 2, 1],
                        env={"DISPLAY": vm.display},
                    ),
                )
            case "drag":
                path = normalize_drag_path(action.path)
                if len(path) < 2:
                    raise ValueError("drag action requires at least two path points")

                def do_drag():
                    start_x, start_y = path[0]
                    docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mousemove", start_x, start_y, "mousedown", 1],
                        env={"DISPLAY": vm.display},
                    )
                    for x, y in path[1:]:
                        docker_exec(
                            vm.container_name,
                            "xdotool",
                            ["mousemove", x, y],
                            env={"DISPLAY": vm.display},
                        )
                    docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mouseup", 1],
                        env={"DISPLAY": vm.display},
                    )

                with_modifiers(vm, getattr(action, "keys", None), do_drag)
            case "move":
                with_modifiers(
                    vm,
                    getattr(action, "keys", None),
                    lambda: docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mousemove", action.x, action.y],
                        env={"DISPLAY": vm.display},
                    ),
                )
            case "scroll":
                buttons = get_xdotool_scroll_buttons(
                    action.scroll_x,
                    action.scroll_y,
                )

                def do_scroll():
                    docker_exec(
                        vm.container_name,
                        "xdotool",
                        ["mousemove", action.x, action.y],
                        env={"DISPLAY": vm.display},
                    )
                    for button in buttons:
                        docker_exec(
                            vm.container_name,
                            "xdotool",
                            ["click", button],
                            env={"DISPLAY": vm.display},
                        )

                with_modifiers(vm, getattr(action, "keys", None), do_scroll)
            case "keypress":
                keys = "+".join(normalize_xdotool_key(key) for key in action.keys)
                docker_exec(
                    vm.container_name,
                    "xdotool",
                    ["key", keys],
                    env={"DISPLAY": vm.display},
                )
            case "type":
                docker_exec(
                    vm.container_name,
                    "xdotool",
                    ["type", "--delay", 0, "--", action.text],
                    env={"DISPLAY": vm.display},
                )
            case "wait":
                time.sleep(2)
            case "screenshot":
                # The caller captures a screenshot after every action.
                continue
            case _:
                raise ValueError(f"Unsupported action: {action.type}")