Computer use

curl -X POST https://api.boxline.dev/v1/sessions/$SESSION_ID/browser/computer \  -H "x-api-key: $BOXLINE_API_KEY" \  -H "content-type: application/json" \  -d '{"action": "left_click", "coordinate": [320, 240], "maxWidth": 1024}'curl -X POST https://api.boxline.dev/v1/sessions/$SESSION_ID/browser/computer \  -H "x-api-key: $BOXLINE_API_KEY" \  -H "content-type: application/json" \  -d '{"type": "click", "x": 320, "y": 240, "button": "left", "maxWidth": 1024}'

import { Boxline } from "@boxline/sdk";const bx = new Boxline(); // reads BOXLINE_API_KEY and BOXLINE_API_URL// Anthropic's shape: the input of a tool_use block of Claude's computer tool.const screen = await bx.sessions.computer("SESSION_ID", { action: "left_click", coordinate: [320, 240] }, { maxWidth: 1024 });console.log(screen.ok ? screen.text : screen.error, screen.cursor, screen.scale);// OpenAI's shape: one item of a computer_call's actions. No screenshot for this one.await bx.sessions.computer("SESSION_ID", { type: "keypress", keys: ["CTRL", "A"] }, { maxWidth: 1024, screenshot: false });

from boxline import Boxlinebx = Boxline()  # reads BOXLINE_API_KEY and BOXLINE_API_URL# Anthropic's shape: the input of a tool_use block of Claude's computer tool.screen = bx.sessions.computer("SESSION_ID", {"action": "left_click", "coordinate": [320, 240]}, max_width=1024)print(screen["text"] if screen["ok"] else screen["error"], screen["cursor"], screen["scale"])# OpenAI's shape: one item of a computer_call's actions. No screenshot for this one.bx.sessions.computer("SESSION_ID", {"type": "keypress", "keys": ["CTRL", "A"]}, max_width=1024, screenshot=False)

/** * Claude computer use with Boxline: your own loop with Claude's computer toolset (Messages API). Claude looks at * screenshots and answers with one `tool_use` block per action, named after the action (`left_click`, `type`, …); each * goes to the session as `{action: <name>, ...input}` (POST /v1/sessions/:id/browser/computer), and the result goes * back to Claude with `toolset_name: "computer"`. * *   npm install && npx tsx index.ts        (BOXLINE_API_KEY and ANTHROPIC_API_KEY in the environment; CUA_MODEL, TASK) * * The toolset sees only the page (there is no address bar): the loop opens the start page first. Writes output/result.json. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import Anthropic from "@anthropic-ai/sdk";import { Boxline, type ComputerAction } from "@boxline/sdk";const out = process.env.OUTPUT_DIR ?? "output";const start = process.env.START_URL ?? "https://books.toscrape.com/";const task = process.env.TASK ?? "Open the Poetry category and tell me the title and price of the cheapest book in it.";const model = process.env.CUA_MODEL ?? "claude-sonnet-5-5"; // any Claude 5.5 model: they take computer_toolset_20260801const MAX_WIDTH = 1024; // screenshots are this wide; Claude's coordinates are read in these pixelsconst anthropic = new Anthropic();const bx = new Boxline();const session = await bx.sessions.create({ timeout: 900, userMetadata: { example: "integrations/claude-computer-use" } });console.log(`Session: ${session.id}`);const actions: unknown[] = [];let tokens = 0;let answer = "";try {  await session.goto(start);  // No screen size: Claude's coordinates are in the pixels of the screenshots it gets (all MAX_WIDTH wide).  const tools: Anthropic.ToolUnion[] = [{ type: "computer_toolset_20260801" }];  const messages: Anthropic.MessageParam[] = [{ role: "user", content: `${task} (The browser shows ${start}.)` }];  for (let turn = 0; turn < 30; turn++) {    // Streamed: a large max_tokens needs it. The conversation is only ever appended to (Claude's thinking is bound to it).    const res = await anthropic.messages.stream({ model, max_tokens: 64000, tools, messages }).finalMessage();    tokens += res.usage.input_tokens + res.usage.output_tokens;    messages.push({ role: "assistant", content: res.content });    const uses = res.content.filter((b): b is Anthropic.ToolUseBlock => b.type === "tool_use");    if (!uses.length) {      answer = res.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("\n");      break;    }    // A turn's actions run in order; after a failed one the rest are not run.    const results: Anthropic.ToolResultBlockParam[] = [];    let failed = false;    for (const [i, use] of uses.entries()) {      if (failed) {        results.push({ type: "tool_result", tool_use_id: use.id, toolset_name: "computer", is_error: true, content: "Not executed: an earlier computer action in this turn failed." });        continue;      }      const input = { ...(use.input as object), action: use.name } as ComputerAction;      const after = await session.computer(input, { maxWidth: MAX_WIDTH });      actions.push(input);      console.log(`→ ${use.name}: ${after.text}`);      failed = !after.ok;      // The screen goes back with screenshot and zoom, a failed action and the turn's last action; others say what they did.      const showScreen = use.name === "screenshot" || use.name === "zoom" || failed || i === uses.length - 1;      results.push({        type: "tool_result",        tool_use_id: use.id,        toolset_name: "computer",        is_error: failed,        content: [          { type: "text", text: after.ok ? after.text || "Done." : (after.error ?? "the action failed") },          ...(showScreen && after.screenshot ? [{ type: "image" as const, source: { type: "base64" as const, media_type: after.mimeType as "image/png" | "image/jpeg", data: after.screenshot } }] : []),        ],      });    }    messages.push({ role: "user", content: results });  }  console.log(`\n${answer}`);  mkdirSync(out, { recursive: true });  writeFileSync(join(out, "result.json"), JSON.stringify({ model, task, start, answer, actions, usage: { ownModelTokens: tokens } }, null, 2));} finally {  await session.stop();}

"""Claude computer use with Boxline: your own loop with Claude's computer toolset (Messages API). Claude looks atscreenshots and answers with one tool_use block per action, named after the action (left_click, type, ...); each goes tothe session as {"action": <name>, **input} (POST /v1/sessions/:id/browser/computer), and the result goes back to Claudewith toolset_name "computer".    pip install -r requirements.txt && python main.py     (BOXLINE_API_KEY and ANTHROPIC_API_KEY; CUA_MODEL, TASK)The toolset sees only the page (there is no address bar): the loop opens the start page first. Writes output/result.json."""import jsonimport osfrom pathlib import Pathimport anthropicfrom boxline import Boxlineout = Path(os.environ.get("OUTPUT_DIR", "output"))start = os.environ.get("START_URL", "https://books.toscrape.com/")task = os.environ.get("TASK", "Open the Poetry category and tell me the title and price of the cheapest book in it.")model = os.environ.get("CUA_MODEL", "claude-sonnet-5-5")  # any Claude 5.5 model: they take computer_toolset_20260801MAX_WIDTH = 1024  # screenshots are this wide; Claude's coordinates are read in these pixelsclient = anthropic.Anthropic()bx = Boxline()with bx.sessions.create(timeout=900, user_metadata={"example": "integrations/claude-computer-use"}) as session:    print(f"Session: {session.id}", flush=True)    session.goto(start)    # No screen size: Claude's coordinates are in the pixels of the screenshots it gets (all MAX_WIDTH wide).    tools = [{"type": "computer_toolset_20260801"}]    messages = [{"role": "user", "content": f"{task} (The browser shows {start}.)"}]    actions, tokens, answer = [], 0, ""    for _ in range(30):        # Streamed: a large max_tokens needs it. The conversation is only ever appended to (Claude's thinking is bound to it).        with client.messages.stream(model=model, max_tokens=64000, tools=tools, messages=messages) as stream:            res = stream.get_final_message()        tokens += res.usage.input_tokens + res.usage.output_tokens        messages.append({"role": "assistant", "content": res.content})        uses = [b for b in res.content if b.type == "tool_use"]        if not uses:            answer = "\n".join(b.text for b in res.content if b.type == "text")            break        # A turn's actions run in order; after a failed one the rest are not run.        results, failed = [], False        for i, use in enumerate(uses):            if failed:                results.append({"type": "tool_result", "tool_use_id": use.id, "toolset_name": "computer", "is_error": True,                                "content": "Not executed: an earlier computer action in this turn failed."})                continue            action = {**use.input, "action": use.name}            after = session.computer(action, max_width=MAX_WIDTH)            actions.append(action)            print(f"-> {use.name}: {after['text']}")            failed = not after["ok"]            # The screen goes back with screenshot and zoom, a failed action and the turn's last action; others say what they did.            show_screen = use.name in ("screenshot", "zoom") or failed or i == len(uses) - 1            content = [{"type": "text", "text": (after.get("text") or "Done.") if after["ok"] else (after.get("error") or "the action failed")}]            if show_screen and after.get("screenshot"):                content.append({"type": "image", "source": {"type": "base64", "media_type": after["mimeType"], "data": after["screenshot"]}})            results.append({"type": "tool_result", "tool_use_id": use.id, "toolset_name": "computer", "is_error": failed, "content": content})        messages.append({"role": "user", "content": results})    print(f"\n{answer}")    out.mkdir(parents=True, exist_ok=True)    (out / "result.json").write_text(json.dumps({"model": model, "task": task, "start": start, "answer": answer, "actions": actions,                                                 "usage": {"ownModelTokens": tokens}}, indent=2, ensure_ascii=False))

/** * OpenAI computer use with Boxline: your own loop with OpenAI's computer tool (Responses API, `{type: "computer"}`). * The model looks at screenshots and answers with `computer_call`s; each action goes to the session unchanged * (POST /v1/sessions/:id/browser/computer), and the screen after it goes back to the model. * *   npm install && npx tsx index.ts        (BOXLINE_API_KEY and OPENAI_API_KEY in the environment; CUA_MODEL, TASK) * * The tool sees only the page (there is no address bar): the loop opens the start page first. When OpenAI asks for a * safety check, a person confirms in the terminal; the loop never acknowledges one by itself. Writes output/result.json. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { createInterface } from "node:readline/promises";import { Boxline, type ComputerAction, type ComputerResult } from "@boxline/sdk";import OpenAI from "openai";const out = process.env.OUTPUT_DIR ?? "output";const start = process.env.START_URL ?? "https://books.toscrape.com/";const task = process.env.TASK ?? "Open the Poetry category and tell me the title and price of the cheapest book in it.";const model = process.env.CUA_MODEL ?? "gpt-6-luna";const MAX_WIDTH = 1024; // screenshots are this wide; the model's coordinates are read in these pixelsconst openai = new OpenAI();const bx = new Boxline();const session = await bx.sessions.create({ timeout: 900, userMetadata: { example: "integrations/openai-computer-use" } });console.log(`Session: ${session.id}`);const actions: unknown[] = [];let tokens = 0;let answer = "";try {  await session.goto(start);  const first = await session.computer({ type: "screenshot" }, { maxWidth: MAX_WIDTH });  const image = (s: ComputerResult) => `data:${s.mimeType};base64,${s.screenshot}`;  let input: OpenAI.Responses.ResponseInput = [    { role: "user", content: [{ type: "input_text", text: `${task} (The browser shows ${start}.)` }, { type: "input_image", image_url: image(first), detail: "auto" }] },  ];  let previous: string | undefined;  for (let turn = 0; turn < 30; turn++) {    const res = await openai.responses.create({ model, tools: [{ type: "computer" }], input, max_output_tokens: 4000, ...(previous ? { previous_response_id: previous } : {}) });    previous = res.id;    tokens += (res.usage?.input_tokens ?? 0) + (res.usage?.output_tokens ?? 0);    const calls = res.output.filter((o): o is OpenAI.Responses.ResponseComputerToolCall => o.type === "computer_call");    if (!calls.length) {      answer = res.output_text;      break;    }    input = [];    for (const call of calls) {      const checks = call.pending_safety_checks ?? [];      if (checks.length) {        const rl = createInterface({ input: process.stdin, output: process.stdout });        const ok = await rl.question(`OpenAI asks a person to confirm: ${checks.map((c) => c.message ?? c.code).join("; ")}. Go on? (yes/no) `);        rl.close();        if (!/^y(es)?$/i.test(ok.trim())) throw new Error("stopped: the safety check was not confirmed");      }      // One computer_call can carry several actions; only the last needs a screenshot.      const list = call.actions?.length ? call.actions : call.action ? [call.action] : [];      let screen: ComputerResult | null = null;      for (const [i, action] of list.entries()) {        screen = await session.computer(action as unknown as ComputerAction, { maxWidth: MAX_WIDTH, screenshot: i === list.length - 1 });        actions.push(action);        console.log(`→ ${action.type}: ${screen.text}`);      }      if (!screen?.screenshot) screen = await session.computer({ type: "screenshot" }, { maxWidth: MAX_WIDTH });      input.push({        type: "computer_call_output",        call_id: call.call_id,        output: { type: "computer_screenshot", image_url: image(screen) },        ...(checks.length ? { acknowledged_safety_checks: checks.map(({ id, code, message }) => ({ id, code, message })) } : {}),      });    }  }  console.log(`\n${answer}`);  mkdirSync(out, { recursive: true });  writeFileSync(join(out, "result.json"), JSON.stringify({ model, task, start, answer, actions, usage: { ownModelTokens: tokens } }, null, 2));} finally {  await session.stop();}

"""OpenAI computer use with Boxline: your own loop with OpenAI's computer tool (Responses API, {"type": "computer"}).The model looks at screenshots and answers with computer_calls; each action goes to the session unchanged(POST /v1/sessions/:id/browser/computer), and the screen after it goes back to the model.    pip install -r requirements.txt && python main.py     (BOXLINE_API_KEY and OPENAI_API_KEY; CUA_MODEL, TASK)The tool sees only the page (there is no address bar): the loop opens the start page first. When OpenAI asks for asafety check, a person confirms in the terminal; the loop never acknowledges one by itself. Writes output/result.json."""import jsonimport osfrom pathlib import Pathfrom openai import OpenAIfrom boxline import Boxlineout = Path(os.environ.get("OUTPUT_DIR", "output"))start = os.environ.get("START_URL", "https://books.toscrape.com/")task = os.environ.get("TASK", "Open the Poetry category and tell me the title and price of the cheapest book in it.")model = os.environ.get("CUA_MODEL", "gpt-6-luna")MAX_WIDTH = 1024  # screenshots are this wide; the model's coordinates are read in these pixelsclient = OpenAI()bx = Boxline()def image(screen: dict) -> str:    return f"data:{screen['mimeType']};base64,{screen['screenshot']}"with bx.sessions.create(timeout=900, user_metadata={"example": "integrations/openai-computer-use"}) as session:    print(f"Session: {session.id}", flush=True)    session.goto(start)    first = session.computer({"type": "screenshot"}, max_width=MAX_WIDTH)    items = [{"role": "user", "content": [{"type": "input_text", "text": f"{task} (The browser shows {start}.)"},                                          {"type": "input_image", "image_url": image(first), "detail": "auto"}]}]    previous, actions, tokens, answer = None, [], 0, ""    for _ in range(30):        res = client.responses.create(model=model, tools=[{"type": "computer"}], input=items, max_output_tokens=4000,                                      **({"previous_response_id": previous} if previous else {}))        previous = res.id        tokens += (res.usage.input_tokens + res.usage.output_tokens) if res.usage else 0        calls = [o for o in res.output if o.type == "computer_call"]        if not calls:            answer = res.output_text            break        items = []        for call in calls:            checks = call.pending_safety_checks or []            if checks:                ok = input(f"OpenAI asks a person to confirm: {'; '.join(c.message or c.code or '' for c in checks)}. Go on? (yes/no) ")                if ok.strip().lower() not in ("y", "yes"):                    raise SystemExit("stopped: the safety check was not confirmed")            # One computer_call can carry several actions; only the last needs a screenshot.            batch = [a.model_dump(exclude_none=True) for a in (call.actions or ([call.action] if call.action else []))]            screen = None            for i, action in enumerate(batch):                screen = session.computer(action, max_width=MAX_WIDTH, screenshot=i == len(batch) - 1)                actions.append(action)                print(f"-> {action['type']}: {screen['text']}")            if not screen or not screen.get("screenshot"):                screen = session.computer({"type": "screenshot"}, max_width=MAX_WIDTH)            items.append({                "type": "computer_call_output",                "call_id": call.call_id,                "output": {"type": "computer_screenshot", "image_url": image(screen)},                **({"acknowledged_safety_checks": [{"id": c.id, "code": c.code, "message": c.message} for c in checks]} if checks else {}),            })    print(f"\n{answer}")    out.mkdir(parents=True, exist_ok=True)    (out / "result.json").write_text(json.dumps({"model": model, "task": task, "start": start, "answer": answer, "actions": actions,                                                 "usage": {"ownModelTokens": tokens}}, indent=2, ensure_ascii=False))

curl -X POST https://api.boxline.dev/v1/agent/runs \  -H "x-api-key: $BOXLINE_API_KEY" \  -H "content-type: application/json" \  -H "Idempotency-Key: $(uuidgen)" \  -d '{    "task": "Open https://books.toscrape.com, find the cheapest book in the Poetry category, and tell me its title and price.",    "mode": "computer",    "model": "claude-sonnet-5-5"  }'# Then follow it: its tool steps are name "computer", with the model's own action as input.curl https://api.boxline.dev/v1/agent/runs/$RUN_ID \  -H "x-api-key: $BOXLINE_API_KEY"

import { Boxline } from "@boxline/sdk";const bx = new Boxline(); // reads BOXLINE_API_KEY and BOXLINE_API_URLconst run = await bx.agent.run({  task: "Open https://books.toscrape.com, find the cheapest book in the Poetry category, and tell me its title and price.",  mode: "computer",  model: "claude-sonnet-5-5",});for await (const event of bx.agent.stream(run.id)) {  if (event.type === "thought") console.log("…", event.text); // what it is about to do  if (event.type === "tool") console.log("→", event.name, JSON.stringify(event.input)); // name "computer"  if (event.type === "done") console.log(event.status, event.result);}

from boxline import Boxlinebx = Boxline()  # reads BOXLINE_API_KEY and BOXLINE_API_URLrun = bx.agent.run(    "Open https://books.toscrape.com, find the cheapest book in the Poetry category, and tell me its title and price.",    mode="computer",    model="claude-sonnet-5-5",)for event in bx.agent.stream(run["id"]):    if event["type"] == "thought":        print("…", event["text"])  # what it is about to do    elif event["type"] == "tool":        print("→", event["name"], event.get("input"))  # name "computer"    elif event["type"] == "done":        print(event["status"], event.get("result"))