计算机使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import time

# Reuse normalize_key from the helper above.
# Reuse normalize_drag_path from the helper above.


def handle_computer_actions(page, actions):
    for action in actions:
        match action.type:
            case "click":
                page.mouse.click(
                    action.x,
                    action.y,
                    button=getattr(action, "button", "left"),
                )
            case "double_click":
                page.mouse.dblclick(
                    action.x,
                    action.y,
                    button=getattr(action, "button", "left"),
                )
            case "drag":
                path = normalize_drag_path(action.path)
                if len(path) < 2:
                    raise ValueError("drag action requires at least two path points")
                start_x, start_y = path[0]
                page.mouse.move(start_x, start_y)
                page.mouse.down()
                for x, y in path[1:]:
                    page.mouse.move(x, y)
                page.mouse.up()
            case "move":
                page.mouse.move(action.x, action.y)
            case "scroll":
                page.mouse.move(action.x, action.y)
                page.mouse.wheel(
                    getattr(action, "scrollX", 0),
                    getattr(action, "scrollY", 0),
                )
            case "keypress":
                for key in action.keys:
                    page.keyboard.press(normalize_key(key))
            case "type":
                page.keyboard.type(action.text)
            case "wait":
                time.sleep(2)
            case "screenshot":
                pass
            case _:
                raise ValueError(f"Unsupported action: {action.type}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import time

# Reuse normalize_xdotool_key from the helper above.
# Reuse normalize_drag_path from the helper above.


def handle_computer_actions(vm, actions):
    button_map = {"left": 1, "middle": 2, "right": 3}

    for action in actions:
        match action.type:
            case "click":
                button = button_map.get(getattr(action, "button", "left"), 1)
                docker_exec(
                    f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click {button}",
                    vm.container_name,
                )
            case "double_click":
                button = button_map.get(getattr(action, "button", "left"), 1)
                docker_exec(
                    f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y} click --repeat 2 {button}",
                    vm.container_name,
                )
            case "drag":
                path = normalize_drag_path(action.path)
                if len(path) < 2:
                    raise ValueError("drag action requires at least two path points")
                start_x, start_y = path[0]
                docker_exec(
                    f"DISPLAY={vm.display} xdotool mousemove {start_x} {start_y} mousedown 1",
                    vm.container_name,
                )
                for x, y in path[1:]:
                    docker_exec(
                        f"DISPLAY={vm.display} xdotool mousemove {x} {y}",
                        vm.container_name,
                    )
                docker_exec(
                    f"DISPLAY={vm.display} xdotool mouseup 1",
                    vm.container_name,
                )
            case "move":
                docker_exec(
                    f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",
                    vm.container_name,
                )
            case "scroll":
                button = 4 if getattr(action, "scrollY", 0) < 0 else 5
                clicks = max(1, abs(round(getattr(action, "scrollY", 0) / 100)))

                docker_exec(
                    f"DISPLAY={vm.display} xdotool mousemove {action.x} {action.y}",
                    vm.container_name,
                )
                for _ in range(clicks):
                    docker_exec(
                        f"DISPLAY={vm.display} xdotool click {button}",
                        vm.container_name,
                    )
            case "keypress":
                for key in action.keys:
                    docker_exec(
                        f"DISPLAY={vm.display} xdotool key '{normalize_xdotool_key(key)}'",
                        vm.container_name,
                    )
            case "type":
                docker_exec(
                    f"DISPLAY={vm.display} xdotool type --delay 0 '{action.text}'",
                    vm.container_name,
                )
            case "wait":
                time.sleep(2)
            case "screenshot":
                pass
            case _:
                raise ValueError(f"Unsupported action: {action.type}")

def capture_screenshot(page):
    return page.screenshot(type="png")

1
2
3
4
5
6
def capture_screenshot(vm):
    return docker_exec(
        f"export DISPLAY={vm.display} && import -window root png:-",
        vm.container_name,
        decode=False,
    )

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
// Run with:
//   bun run -i cua_code_mode.ts
// Override the user prompt with:
//   bun run -i cua_code_mode.ts --prompt "Go to example.com and summarize the page."
// Note: this script intentionally leaves the Playwright browser open after the
// model reaches a final answer. Because the browser/context are not closed,
// Bun stays alive until you close the browser or stop the process manually.

import OpenAI from "openai";
import readline from "node:readline/promises";
import vm from "node:vm";
import { chromium } from "playwright";
import util from "node:util";

async function main(
  prompt: string = "Go to Hacker News, click on the most interesting link (be prepared to justify your choice), take a screenshot, and give me a critique of the visual layout.",
  max_steps: number = 50,
  model: string = "gpt-5.5"
) {
  type Phase = null | "commentary" | "final_answer";
  const client = new OpenAI();
  const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout,
  });
  const browser = await chromium.launch({
    headless: false,
    args: ["--window-size=1440,900"],
  });
  const context = await browser.newContext({
    viewport: { width: 1440, height: 900 },
  });
  const page = await context.newPage();

  const conversation: any[] = [];
  const js_output: any[] = [];
  const sandbox: Record<string, any> = {
    console: {
      log: (...xs: any[]) => {
        js_output.push({
          type: "input_text",
          text: util.formatWithOptions(
            { showHidden: false, getters: false, maxStringLength: 2000 },
            ...xs
          ),
        });
      },
    },
    browser: browser,
    context: context,
    page: page,
    display: (base64_image: string) => {
      js_output.push({
        type: "input_image",
        image_url: `data:image/png;base64,${base64_image}`,
        detail: "original",
      });
    },
  };
  const ctx = vm.createContext(sandbox);

  conversation.push({
    role: "user",
    content: prompt,
  });

  for (let i = 0; i < max_steps; i++) {
    const resp = await client.responses.create({
      model,
      tools: [
        {
          type: "function" as const,
          name: "exec_js",
          description:
            "Execute provided interactive JavaScript in a persistent REPL context.",
          parameters: {
            type: "object",
            properties: {
              code: {
                type: "string",
                description: `
JavaScript to execute. Write small snippets of interactive code. To persist variables or functions across tool calls, you must save them to globalThis. Code is executed in an async node:vm context, so you can use await. You have access to ONLY the following:
- console.log(x): Use this to read contents back to you. But be minimal: otherwise the output may be too long. Avoid using console.log() for large base64 payloads like screenshots or buffer. If you create an image or screenshot, pass the base64 string to display().
- display(base64_image_string): Use this to view a base64-encoded image.
- Do not write screenshots or image data to temporary files or disk just to pass them back. Keep image data in memory and send it directly to display().
- Do not assume package globals like Bun.file are available unless they are explicitly provided.
- browser: A playwright chromium browser instance.
- context: A playwright browser context with viewport 1440x900.
- page: A playwright page already created in that context.
`,
              },
            },
            required: ["code"],
            additionalProperties: false,
          },
        },
        {
          type: "function" as const,
          name: "ask_user",
          description:
            "Ask the user a clarification question and wait for their response.",
          parameters: {
            type: "object",
            properties: {
              question: {
                type: "string",
                description:
                  "The exact question to show the human. Use this instead of answering with a freeform clarifying question in a final answer.",
              },
            },
            required: ["question"],
            additionalProperties: false,
          },
        },
      ],
      input: conversation,
      reasoning: {
        effort: "low",
      },
    });

    // Save model outputs into the running conversation
    conversation.push(...resp.output);

    let hadToolCall = false;
    let latestPhase: Phase = null;

    // Handle tool calls
    for (const item of resp.output) {
      if (item.type === "function_call" && item.name === "exec_js") {
        hadToolCall = true;
        const parsed = JSON.parse(item.arguments ?? "{}") as {
          code?: string;
        };
        const code = parsed.code ?? "";
        console.log(code);
        console.log("----");
        const wrappedCode = `
                (async () => {
                    ${code}
                })();
            `;

        try {
          await new vm.Script(wrappedCode, {
            filename: "exec_js.js",
          }).runInContext(ctx);
        } catch (e: any) {
          sandbox.console.log(e, e?.message, e?.stack);
        }

        // Send tool output back to the model, keyed by call_id
        conversation.push({
          type: "function_call_output",
          call_id: item.call_id,
          output: js_output.slice(),
        });

        for (const out of js_output) {
          if (out.type === "input_text") {
            console.log("JS LOG:", out.text);
          } else if (out.type === "input_image") {
            console.log("JS IMAGE: [base64 string omitted]");
          }
        }
        console.log("=====");

        js_output.length = 0;
      } else if (item.type === "function_call" && item.name === "ask_user") {
        hadToolCall = true;
        const parsed = JSON.parse(item.arguments ?? "{}") as {
          question?: string;
        };
        const question = parsed.question ?? "Please provide more information.";
        console.log(`MODEL QUESTION: ${question}`);
        const answer = await rl.question("> ");
        conversation.push({
          type: "function_call_output",
          call_id: item.call_id,
          output: answer,
        });
      } else if (item.type === "message") {
        console.log(item.content[0]?.text ?? item.content);
        if ("phase" in item) {
          latestPhase = (item.phase as Phase) ?? null;
        }
      } else if (item.type === "output_item.done" && "phase" in item) {
        latestPhase = (item.phase as Phase) ?? null;
      }
    }

    // Stop only when the model explicitly marks the turn as a final answer
    // and there were no tool calls in the same turn.
    if (!hadToolCall && latestPhase === "final_answer") return;
  }
}

function getCliPrompt(): string | undefined {
  const args = Bun.argv.slice(2);
  for (let i = 0; i < args.length; i++) {
    if (args[i] === "--prompt") {
      return args[i + 1];
    }
  }
  return undefined;
}

main(getCliPrompt());