Compare models

/** * Compare models: a small benchmark. The same browser tasks, each with its right answer written down beforehand * (tasks.json), are run by two or more models side by side; the example scores every run itself (no model judges) and * prints each model's pass rate, time, steps and cost. * *   npx tsx node/index.ts            (BOXLINE_API_KEY; MODELS, TRIALS, TASKS_FILE) * * A run that did not pass is sorted, never guessed: model_failure (a wrong answer, no answer, a step, cost or time * limit), web_failure (it ended on a bot wall or CAPTCHA page), platform_failure (its session or the server failed) or * unclassified. Writes output/result.json, output/runs.jsonl (one line per run, with its steps) and output/summary.md. */import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";import { dirname, join } from "node:path";import { fileURLToPath } from "node:url";import { Boxline, type AgentProvider, type AgentRun } from "@boxline/sdk";interface Condition {  answerContains?: string;  answerMatches?: string;  urlReached?: string;}interface Task {  id: string;  startUrl: string;  instruction: string;  pass: Condition[];}const out = process.env.OUTPUT_DIR ?? "output";const tasksFile = process.env.TASKS_FILE ?? join(dirname(fileURLToPath(import.meta.url)), "..", "tasks.json");const trials = Number(process.env.TRIALS ?? 1);const timeLimitMs = 3 * 60_000; // per runconst bx = new Boxline();const { tasks } = JSON.parse(readFileSync(tasksFile, "utf8")) as { tasks: Task[] };const catalog = await bx.agent.models();const all = catalog.providers.filter((p) => p.available).flatMap((p) => p.models.map((m) => ({ provider: p.id as AgentProvider, ...m })));const wanted = (process.env.MODELS ?? "").split(",").map((s) => s.trim()).filter(Boolean);let models = wanted.map((id) => {  const m = all.find((x) => x.id === id);  if (!m) throw new Error(`${id} is not a configured model (see bx.agent.models())`);  return m;});if (!models.length) {  const same = all.filter((m) => m.provider === catalog.default.provider);  const first = same.find((m) => m.id === catalog.default.model) ?? same[0]!;  const second = same.filter((m) => m.id !== first.id).sort((a, b) => a.pricePerMTok.input - b.pricePerMTok.input)[0] ?? first;  models = [first, second];}// Pages that are a wall in front of the site: the run failed on the web, not because of the model.const WALL = ["captcha", "verify you are human", "are you a robot", "unusual traffic", "access denied", "attention required", "just a moment", "enable javascript and cookies to continue"];const MODEL_LIMITS = ["max_steps", "max_cost", "too_many_errors", "no_progress", "output_invalid"];const PLATFORM = ["server_restarted", "session_timeout", "session_ended", "internal"];const bare = (u: string) => u.replace(/[#?].*$/, "").replace(/\/$/, "");function score(task: Task, run: AgentRun, timedOut: boolean) {  const answer = run.status === "completed" ? String(run.result ?? "") : "";  const text = JSON.stringify(run.steps.map((s) => [s.input, s.output]));  const visited = [...new Set((text.match(/https?:\/\/[^\s"'\\)]+/g) ?? []).map(bare))];  const unmet = task.pass.filter((c) =>    c.answerContains !== undefined      ? !answer.toLowerCase().includes(c.answerContains.toLowerCase())      : c.answerMatches !== undefined        ? !new RegExp(c.answerMatches, "i").test(answer)        : !visited.includes(bare(c.urlReached ?? "")),  );  if (run.status === "completed" && !unmet.length) return { outcome: "pass", detail: null, visited };  const lastPages = run.steps.filter((s) => s.type === "tool").slice(-3).map((s) => String(s.output ?? "")).join("\n").toLowerCase();  if (WALL.some((m) => lastPages.includes(m))) return { outcome: "web_failure", detail: "it ended on a bot wall or CAPTCHA page", visited };  if (timedOut) return { outcome: "model_failure", detail: `not done in ${timeLimitMs / 60_000} minutes`, visited };  if (run.status === "completed") return { outcome: "model_failure", detail: answer ? `wrong answer (unmet: ${unmet.map((c) => JSON.stringify(c)).join(", ")})` : "no answer", visited };  const code = run.errorCode ?? "";  if (MODEL_LIMITS.includes(code)) return { outcome: "model_failure", detail: `${code}: ${run.error}`, visited };  if (PLATFORM.includes(code)) return { outcome: "platform_failure", detail: `${code}: ${run.error}`, visited };  return { outcome: "unclassified", detail: `${run.status} ${code}: ${run.error ?? ""}`.trim(), visited };}async function trial(task: Task, m: (typeof models)[number], n: number) {  const t0 = Date.now();  const started = await bx.agent.run({ task: `Start at ${task.startUrl}. ${task.instruction} Give the answer in your final message.`, provider: m.provider, model: m.id, maxSteps: 20 });  console.log(`  ${m.id}: run ${started.id} · Session: ${started.sessionId}`);  let timedOut = false;  let run: AgentRun;  try {    run = await bx.agent.wait(started.id, { timeoutMs: timeLimitMs });  } catch {    timedOut = true;    await bx.agent.cancel(started.id).catch(() => {});    run = await bx.agent.wait(started.id, { timeoutMs: 60_000 });  }  const s = score(task, run, timedOut);  return {    task: task.id,    trial: n,    model: m.id,    provider: m.provider,    runId: run.id,    status: run.status,    pass: s.outcome === "pass",    outcome: s.outcome,    detail: s.detail,    seconds: Math.round((Date.now() - t0) / 100) / 10,    toolSteps: run.steps.filter((x) => x.type === "tool").length,    inputTokens: run.usage.inputTokens,    outputTokens: run.usage.outputTokens,    costUsd: run.usage.costUsd ?? 0,    answer: run.status === "completed" ? run.result : run.error,    visited: s.visited,    steps: run.steps.filter((x) => x.type === "tool").map((x) => ({ name: x.name, input: x.input, ms: x.ms, isError: x.isError ?? false })),  };}mkdirSync(out, { recursive: true });writeFileSync(join(out, "runs.jsonl"), "");const rows: Awaited<ReturnType<typeof trial>>[] = [];// Task by task, the models side by side: each task's runs see the same web at the same time.for (const task of tasks) {  for (let n = 1; n <= trials; n++) {    console.log(`${task.id}${trials > 1 ? ` (trial ${n})` : ""}`);    for (const row of await Promise.all(models.map((m) => trial(task, m, n)))) {      rows.push(row);      appendFileSync(join(out, "runs.jsonl"), `${JSON.stringify(row)}\n`);      console.log(`    ${row.model}: ${row.outcome}${row.detail ? ` (${row.detail})` : ""}, ${row.seconds} s, ${row.toolSteps} steps, $${row.costUsd.toFixed(4)}`);    }  }}const median = (xs: number[]) => {  const s = [...xs].sort((a, b) => a - b);  return s.length ? (s.length % 2 ? s[(s.length - 1) / 2]! : (s[s.length / 2 - 1]! + s[s.length / 2]!) / 2) : 0;};const summary = models.map((m) => {  const mine = rows.filter((r) => r.model === m.id);  const outcomes: Record<string, number> = {};  for (const r of mine) outcomes[r.outcome] = (outcomes[r.outcome] ?? 0) + 1;  return {    model: m.id,    provider: m.provider,    passed: mine.filter((r) => r.pass).length,    runs: mine.length,    medianSeconds: median(mine.map((r) => r.seconds)),    medianSteps: median(mine.map((r) => r.toolSteps)),    costUsd: Math.round(mine.reduce((a, r) => a + r.costUsd, 0) * 10_000) / 10_000,    outcomes,  };});console.table(summary.map(({ model, passed, runs, medianSeconds, medianSteps, costUsd }) => ({ model, passed: `${passed}/${runs}`, medianSeconds, medianSteps, costUsd })));const md = [  `# ${tasks.length} tasks × ${models.length} models × ${trials} trial${trials > 1 ? "s" : ""}`,  "",  "| Model | Passed | Median time | Median steps | Cost | Outcomes |",  "|---|---|---|---|---|---|",  ...summary.map((s) => `| ${s.model} | ${s.passed}/${s.runs} | ${s.medianSeconds} s | ${s.medianSteps} | $${s.costUsd.toFixed(4)} | ${Object.entries(s.outcomes).map(([k, v]) => `${k} ${v}`).join(", ")} |`),  "",  "| Task | " + models.map((m) => m.id).join(" | ") + " |",  "|---|" + models.map(() => "---|").join(""),  ...tasks.map((t) => `| ${t.id} | ${models.map((m) => rows.filter((r) => r.task === t.id && r.model === m.id).map((r) => (r.pass ? "pass" : r.outcome)).join(", ")).join(" | ")} |`),];writeFileSync(join(out, "summary.md"), `${md.join("\n")}\n`);writeFileSync(  join(out, "result.json"),  JSON.stringify({ tasksFile, tasks, models: models.map((m) => ({ id: m.id, provider: m.provider })), trials, summary, rows: rows.map(({ steps, ...r }) => r), agentRuns: rows.map((r) => ({ id: r.runId })) }, null, 2),);

"""Compare models: a small benchmark. The same browser tasks, each with its right answer written down beforehand(tasks.json), are run by two or more models side by side; the example scores every run itself (no model judges) andprints each model's pass rate, time, steps and cost.    python python/main.py            (BOXLINE_API_KEY; MODELS, TRIALS, TASKS_FILE)A run that did not pass is sorted, never guessed: model_failure (a wrong answer, no answer, a step, cost or timelimit), web_failure (it ended on a bot wall or CAPTCHA page), platform_failure (its session or the server failed) orunclassified. Writes output/result.json, output/runs.jsonl (one line per run, with its steps) and output/summary.md."""import jsonimport osimport reimport statisticsimport timefrom concurrent.futures import ThreadPoolExecutorfrom pathlib import Pathfrom boxline import Boxline, BoxlineTimeoutErrorhere = Path(__file__).resolve().parentout = Path(os.environ.get("OUTPUT_DIR", "output"))tasks_file = os.environ.get("TASKS_FILE", str(here.parent / "tasks.json"))trials = int(os.environ.get("TRIALS", "1"))time_limit_s = 3 * 60  # per runbx = Boxline()tasks = json.loads(Path(tasks_file).read_text())["tasks"]catalog = bx.agent.models()available = [{"provider": p["id"], **m} for p in catalog["providers"] if p["available"] for m in p["models"]]wanted = [s.strip() for s in os.environ.get("MODELS", "").split(",") if s.strip()]if wanted:    models = []    for model_id in wanted:        match = next((m for m in available if m["id"] == model_id), None)        if not match:            raise SystemExit(f"{model_id} is not a configured model")        models.append(match)else:    same = [m for m in available if m["provider"] == catalog["default"]["provider"]]    first = next((m for m in same if m["id"] == catalog["default"]["model"]), same[0])    others = sorted((m for m in same if m["id"] != first["id"]), key=lambda m: m["pricePerMTok"]["input"])    models = [first, others[0] if others else first]# Pages that are a wall in front of the site: the run failed on the web, not because of the model.WALL = ["captcha", "verify you are human", "are you a robot", "unusual traffic", "access denied", "attention required", "just a moment", "enable javascript and cookies to continue"]MODEL_LIMITS = {"max_steps", "max_cost", "too_many_errors", "no_progress", "output_invalid"}PLATFORM = {"server_restarted", "session_timeout", "session_ended", "internal"}def bare(u: str) -> str:    return re.sub(r"[#?].*$", "", u).rstrip("/")def unmet_condition(c: dict, answer: str, visited: list) -> bool:    if "answerContains" in c:        return c["answerContains"].lower() not in answer.lower()    if "answerMatches" in c:        return not re.search(c["answerMatches"], answer, re.I)    return bare(c.get("urlReached", "")) not in visiteddef score(task: dict, run: dict, timed_out: bool) -> dict:    answer = str(run["result"] or "") if run["status"] == "completed" else ""    text = json.dumps([[s.get("input"), s.get("output")] for s in run["steps"]])    visited = list(dict.fromkeys(bare(u) for u in re.findall(r"https?://[^\s\"'\\)]+", text)))    unmet = [c for c in task["pass"] if unmet_condition(c, answer, visited)]    if run["status"] == "completed" and not unmet:        return {"outcome": "pass", "detail": None, "visited": visited}    last_pages = "\n".join(str(s.get("output") or "") for s in [s for s in run["steps"] if s["type"] == "tool"][-3:]).lower()    if any(m in last_pages for m in WALL):        return {"outcome": "web_failure", "detail": "it ended on a bot wall or CAPTCHA page", "visited": visited}    if timed_out:        return {"outcome": "model_failure", "detail": f"not done in {time_limit_s // 60} minutes", "visited": visited}    if run["status"] == "completed":        detail = f"wrong answer (unmet: {', '.join(json.dumps(c) for c in unmet)})" if answer else "no answer"        return {"outcome": "model_failure", "detail": detail, "visited": visited}    code = run.get("errorCode") or ""    if code in MODEL_LIMITS:        return {"outcome": "model_failure", "detail": f"{code}: {run.get('error')}", "visited": visited}    if code in PLATFORM:        return {"outcome": "platform_failure", "detail": f"{code}: {run.get('error')}", "visited": visited}    return {"outcome": "unclassified", "detail": f"{run['status']} {code}: {run.get('error') or ''}".strip(), "visited": visited}def trial(task: dict, m: dict, n: int) -> dict:    t0 = time.monotonic()    started = bx.agent.run(f"Start at {task['startUrl']}. {task['instruction']} Give the answer in your final message.", provider=m["provider"], model=m["id"], max_steps=20)    print(f"  {m['id']}: run {started['id']} · Session: {started['sessionId']}", flush=True)    timed_out = False    try:        run = bx.agent.wait(started["id"], timeout=time_limit_s)    except BoxlineTimeoutError:        timed_out = True        try:            bx.agent.cancel(started["id"])        except Exception:            pass        run = bx.agent.wait(started["id"], timeout=60)    s = score(task, run, timed_out)    tools = [x for x in run["steps"] if x["type"] == "tool"]    return {        "task": task["id"],        "trial": n,        "model": m["id"],        "provider": m["provider"],        "runId": run["id"],        "status": run["status"],        "pass": s["outcome"] == "pass",        "outcome": s["outcome"],        "detail": s["detail"],        "seconds": round(time.monotonic() - t0, 1),        "toolSteps": len(tools),        "inputTokens": run["usage"]["inputTokens"],        "outputTokens": run["usage"]["outputTokens"],        "costUsd": run["usage"]["costUsd"],        "answer": run["result"] if run["status"] == "completed" else run.get("error"),        "visited": s["visited"],        "steps": [{"name": x.get("name"), "input": x.get("input"), "ms": x.get("ms"), "isError": x.get("isError", False)} for x in tools],    }out.mkdir(parents=True, exist_ok=True)runs_file = out / "runs.jsonl"runs_file.write_text("")rows = []# Task by task, the models side by side: each task's runs see the same web at the same time.with ThreadPoolExecutor(len(models)) as pool:    for task in tasks:        for n in range(1, trials + 1):            print(f"{task['id']}{f' (trial {n})' if trials > 1 else ''}", flush=True)            for row in pool.map(lambda m: trial(task, m, n), models):                rows.append(row)                detail = f" ({row['detail']})" if row["detail"] else ""                with runs_file.open("a") as f:                    f.write(json.dumps(row, ensure_ascii=False) + "\n")                print(f"    {row['model']}: {row['outcome']}{detail}, {row['seconds']} s, {row['toolSteps']} steps, ${row['costUsd']:.4f}", flush=True)summary = []for m in models:    mine = [r for r in rows if r["model"] == m["id"]]    outcomes: dict = {}    for r in mine:        outcomes[r["outcome"]] = outcomes.get(r["outcome"], 0) + 1    summary.append({        "model": m["id"],        "provider": m["provider"],        "passed": sum(r["pass"] for r in mine),        "runs": len(mine),        "medianSeconds": statistics.median(r["seconds"] for r in mine),        "medianSteps": statistics.median(r["toolSteps"] for r in mine),        "costUsd": round(sum(r["costUsd"] for r in mine), 4),        "outcomes": outcomes,    })print(f"\n{'model':<20} {'passed':>7} {'median s':>9} {'steps':>6} {'cost':>9}")for s in summary:    print(f"{s['model']:<20} {str(s['passed']) + '/' + str(s['runs']):>7} {s['medianSeconds']:>9} {s['medianSteps']:>6} {'$%.4f' % s['costUsd']:>9}")md = [    f"# {len(tasks)} tasks × {len(models)} models × {trials} trial{'s' if trials > 1 else ''}",    "",    "| Model | Passed | Median time | Median steps | Cost | Outcomes |",    "|---|---|---|---|---|---|",    *(f"| {s['model']} | {s['passed']}/{s['runs']} | {s['medianSeconds']} s | {s['medianSteps']} | ${s['costUsd']:.4f} | {', '.join(f'{k} {v}' for k, v in s['outcomes'].items())} |" for s in summary),    "",    "| Task | " + " | ".join(m["id"] for m in models) + " |",    "|---|" + "---|" * len(models),    *(f"| {t['id']} | " + " | ".join(", ".join("pass" if r["pass"] else r["outcome"] for r in rows if r["task"] == t["id"] and r["model"] == m["id"]) for m in models) + " |" for t in tasks),](out / "summary.md").write_text("\n".join(md) + "\n")result = {"tasksFile": tasks_file, "tasks": tasks, "models": [{"id": m["id"], "provider": m["provider"]} for m in models], "trials": trials, "summary": summary,          "rows": [{k: v for k, v in r.items() if k != "steps"} for r in rows], "agentRuns": [{"id": r["runId"]} for r in rows]}(out / "result.json").write_text(json.dumps(result, indent=2, ensure_ascii=False))

# Node 18 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keynpm install @boxline/sdk tsxnpx tsx node/index.ts

# Python 3.9 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keypip install boxline-sdkpython python/main.py