OpenAI computer use

/** * OpenAI computer use with Boxline: your own loop with OpenAI's computer tool (Responses API, `{type: "computer"}`). * The model looks at screenshots and answers with `computer_call`s; each action goes to the session unchanged * (POST /v1/sessions/:id/browser/computer), and the screen after it goes back to the model. * *   npm install && npx tsx index.ts        (BOXLINE_API_KEY and OPENAI_API_KEY in the environment; CUA_MODEL, TASK) * * The tool sees only the page (there is no address bar): the loop opens the start page first. When OpenAI asks for a * safety check, a person confirms in the terminal; the loop never acknowledges one by itself. Writes output/result.json. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { createInterface } from "node:readline/promises";import { Boxline, type ComputerAction, type ComputerResult } from "@boxline/sdk";import OpenAI from "openai";const out = process.env.OUTPUT_DIR ?? "output";const start = process.env.START_URL ?? "https://books.toscrape.com/";const task = process.env.TASK ?? "Open the Poetry category and tell me the title and price of the cheapest book in it.";const model = process.env.CUA_MODEL ?? "gpt-6-luna";const MAX_WIDTH = 1024; // screenshots are this wide; the model's coordinates are read in these pixelsconst openai = new OpenAI();const bx = new Boxline();const session = await bx.sessions.create({ timeout: 900, userMetadata: { example: "integrations/openai-computer-use" } });console.log(`Session: ${session.id}`);const actions: unknown[] = [];let tokens = 0;let answer = "";try {  await session.goto(start);  const first = await session.computer({ type: "screenshot" }, { maxWidth: MAX_WIDTH });  const image = (s: ComputerResult) => `data:${s.mimeType};base64,${s.screenshot}`;  let input: OpenAI.Responses.ResponseInput = [    { role: "user", content: [{ type: "input_text", text: `${task} (The browser shows ${start}.)` }, { type: "input_image", image_url: image(first), detail: "auto" }] },  ];  let previous: string | undefined;  for (let turn = 0; turn < 30; turn++) {    const res = await openai.responses.create({ model, tools: [{ type: "computer" }], input, max_output_tokens: 4000, ...(previous ? { previous_response_id: previous } : {}) });    previous = res.id;    tokens += (res.usage?.input_tokens ?? 0) + (res.usage?.output_tokens ?? 0);    const calls = res.output.filter((o): o is OpenAI.Responses.ResponseComputerToolCall => o.type === "computer_call");    if (!calls.length) {      answer = res.output_text;      break;    }    input = [];    for (const call of calls) {      const checks = call.pending_safety_checks ?? [];      if (checks.length) {        const rl = createInterface({ input: process.stdin, output: process.stdout });        const ok = await rl.question(`OpenAI asks a person to confirm: ${checks.map((c) => c.message ?? c.code).join("; ")}. Go on? (yes/no) `);        rl.close();        if (!/^y(es)?$/i.test(ok.trim())) throw new Error("stopped: the safety check was not confirmed");      }      // One computer_call can carry several actions; only the last needs a screenshot.      const list = call.actions?.length ? call.actions : call.action ? [call.action] : [];      let screen: ComputerResult | null = null;      for (const [i, action] of list.entries()) {        screen = await session.computer(action as unknown as ComputerAction, { maxWidth: MAX_WIDTH, screenshot: i === list.length - 1 });        actions.push(action);        console.log(`→ ${action.type}: ${screen.text}`);      }      if (!screen?.screenshot) screen = await session.computer({ type: "screenshot" }, { maxWidth: MAX_WIDTH });      input.push({        type: "computer_call_output",        call_id: call.call_id,        output: { type: "computer_screenshot", image_url: image(screen) },        ...(checks.length ? { acknowledged_safety_checks: checks.map(({ id, code, message }) => ({ id, code, message })) } : {}),      });    }  }  console.log(`\n${answer}`);  mkdirSync(out, { recursive: true });  writeFileSync(join(out, "result.json"), JSON.stringify({ model, task, start, answer, actions, usage: { ownModelTokens: tokens } }, null, 2));} finally {  await session.stop();}

"""OpenAI computer use with Boxline: your own loop with OpenAI's computer tool (Responses API, {"type": "computer"}).The model looks at screenshots and answers with computer_calls; each action goes to the session unchanged(POST /v1/sessions/:id/browser/computer), and the screen after it goes back to the model.    pip install -r requirements.txt && python main.py     (BOXLINE_API_KEY and OPENAI_API_KEY; CUA_MODEL, TASK)The tool sees only the page (there is no address bar): the loop opens the start page first. When OpenAI asks for asafety check, a person confirms in the terminal; the loop never acknowledges one by itself. Writes output/result.json."""import jsonimport osfrom pathlib import Pathfrom openai import OpenAIfrom boxline import Boxlineout = Path(os.environ.get("OUTPUT_DIR", "output"))start = os.environ.get("START_URL", "https://books.toscrape.com/")task = os.environ.get("TASK", "Open the Poetry category and tell me the title and price of the cheapest book in it.")model = os.environ.get("CUA_MODEL", "gpt-6-luna")MAX_WIDTH = 1024  # screenshots are this wide; the model's coordinates are read in these pixelsclient = OpenAI()bx = Boxline()def image(screen: dict) -> str:    return f"data:{screen['mimeType']};base64,{screen['screenshot']}"with bx.sessions.create(timeout=900, user_metadata={"example": "integrations/openai-computer-use"}) as session:    print(f"Session: {session.id}", flush=True)    session.goto(start)    first = session.computer({"type": "screenshot"}, max_width=MAX_WIDTH)    items = [{"role": "user", "content": [{"type": "input_text", "text": f"{task} (The browser shows {start}.)"},                                          {"type": "input_image", "image_url": image(first), "detail": "auto"}]}]    previous, actions, tokens, answer = None, [], 0, ""    for _ in range(30):        res = client.responses.create(model=model, tools=[{"type": "computer"}], input=items, max_output_tokens=4000,                                      **({"previous_response_id": previous} if previous else {}))        previous = res.id        tokens += (res.usage.input_tokens + res.usage.output_tokens) if res.usage else 0        calls = [o for o in res.output if o.type == "computer_call"]        if not calls:            answer = res.output_text            break        items = []        for call in calls:            checks = call.pending_safety_checks or []            if checks:                ok = input(f"OpenAI asks a person to confirm: {'; '.join(c.message or c.code or '' for c in checks)}. Go on? (yes/no) ")                if ok.strip().lower() not in ("y", "yes"):                    raise SystemExit("stopped: the safety check was not confirmed")            # One computer_call can carry several actions; only the last needs a screenshot.            batch = [a.model_dump(exclude_none=True) for a in (call.actions or ([call.action] if call.action else []))]            screen = None            for i, action in enumerate(batch):                screen = session.computer(action, max_width=MAX_WIDTH, screenshot=i == len(batch) - 1)                actions.append(action)                print(f"-> {action['type']}: {screen['text']}")            if not screen or not screen.get("screenshot"):                screen = session.computer({"type": "screenshot"}, max_width=MAX_WIDTH)            items.append({                "type": "computer_call_output",                "call_id": call.call_id,                "output": {"type": "computer_screenshot", "image_url": image(screen)},                **({"acknowledged_safety_checks": [{"id": c.id, "code": c.code, "message": c.message} for c in checks]} if checks else {}),            })    print(f"\n{answer}")    out.mkdir(parents=True, exist_ok=True)    (out / "result.json").write_text(json.dumps({"model": model, "task": task, "start": start, "answer": answer, "actions": actions,                                                 "usage": {"ownModelTokens": tokens}}, indent=2, ensure_ascii=False))

# Node 22 or newerexport BOXLINE_API_KEY=…        # your Boxline API keyexport OPENAI_API_KEY=…         # your own OpenAI keynpm install @boxline/sdk openai@7.25.0 tsx@4.23.15npx tsx index.ts

# Python 3.10 or newerexport BOXLINE_API_KEY=…        # your Boxline API keyexport OPENAI_API_KEY=…         # your own OpenAI keypip install boxline-sdk openai==3.22.0python main.py