Page resources

/** * Page resources: open a page in a cloud browser and list everything it loads, from the session's own network log: * requests and bytes by type and by domain, the third parties, what failed, and the slowest and heaviest requests. * *   npx tsx node/index.ts            (BOXLINE_API_KEY; PAGE_URL) * * Writes output/result.json, output/report.md and output/page.png. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { Boxline, type SessionEvent } from "@boxline/sdk";const out = process.env.OUTPUT_DIR ?? "output";const url = process.env.PAGE_URL ?? "https://books.toscrape.com/";const bx = new Boxline();/** The site a host belongs to, roughly (its last two labels; three for "co.uk"-style endings). */const site = (host: string) => {  const parts = host.split(".");  return parts.slice(/^(co|com|org|net|gov|ac|edu)$/.test(parts.at(-2) ?? "") && parts.length > 2 ? -3 : -2).join(".");};const session = await bx.sessions.create({ timeout: 300, userMetadata: { example: "page-resources" } });console.log(`Session: ${session.id}`);let requests: SessionEvent[] = [];try {  // 1. Load the page until the network is quiet, and keep a picture of it.  const t0 = Date.now();  const opened = await session.goto(url, { waitUntil: "networkidle" });  const loadMs = Date.now() - t0;  const shot = await session.screenshot({ maxWidth: 1280 });  mkdirSync(out, { recursive: true });  writeFileSync(join(out, "page.png"), Buffer.from(shot.data, "base64"));  console.log(`${opened.url} (${opened.status}) loaded in ${(loadMs / 1000).toFixed(1)} s`);  // 2. Its network log: one event per finished (or failed) request. The log is written as requests end: wait a moment.  await new Promise((r) => setTimeout(r, 1500));  for await (const e of session.events({ types: ["network"], limit: 2000 })) requests.push(e);  const mine = site(new URL(opened.url).hostname);  const rows = requests.map((e) => {    const host = (() => {      try {        return new URL(e.url!).hostname;      } catch {        return "?";      }    })();    return { url: e.url!, host, type: e.resourceType ?? "Other", status: e.status ?? null, ms: e.durationMs ?? 0, bytes: Number(e.data?.bytes ?? 0), failed: e.level === "error" || (e.status ?? 0) >= 400, error: e.text ?? null, thirdParty: site(host) !== mine };  });  // 3. Grouped and ranked.  const group = (key: (r: (typeof rows)[number]) => string) => {    const m = new Map<string, { requests: number; bytes: number }>();    for (const r of rows) {      const g = m.get(key(r)) ?? { requests: 0, bytes: 0 };      g.requests++;      g.bytes += r.bytes;      m.set(key(r), g);    }    return [...m].map(([name, g]) => ({ name, ...g })).sort((a, b) => b.bytes - a.bytes);  };  const byType = group((r) => r.type);  const byDomain = group((r) => r.host);  const thirdParties = [...new Set(rows.filter((r) => r.thirdParty).map((r) => r.host))];  const failed = rows.filter((r) => r.failed).map((r) => ({ url: r.url, status: r.status, error: r.error }));  const slowest = [...rows].sort((a, b) => b.ms - a.ms).slice(0, 5).map((r) => ({ url: r.url, ms: r.ms }));  const heaviest = [...rows].sort((a, b) => b.bytes - a.bytes).slice(0, 5).map((r) => ({ url: r.url, kb: Math.round(r.bytes / 102.4) / 10 }));  const totalKb = Math.round(rows.reduce((n, r) => n + r.bytes, 0) / 102.4) / 10;  const kb = (b: number) => `${Math.round(b / 102.4) / 10} KB`;  console.log(`${rows.length} requests, ${totalKb} KB, ${thirdParties.length} third-party hosts, ${failed.length} failed`);  for (const t of byType) console.log(`  ${t.name.padEnd(12)} ${String(t.requests).padStart(3)}  ${kb(t.bytes)}`);  if (failed.length) console.log(`Failed: ${failed.map((f) => `${f.url} (${f.status ?? f.error})`).join(", ")}`);  console.log(`Slowest: ${slowest.slice(0, 3).map((s) => `${s.url} ${s.ms} ms`).join(", ")}`);  const md = [    `# ${opened.url}`,    "",    `${rows.length} requests, ${totalKb} KB transferred, loaded (network quiet) in ${(loadMs / 1000).toFixed(1)} s.`,    "",    "## By type",    "",    "| Type | Requests | Transferred |",    "|---|---|---|",    ...byType.map((t) => `| ${t.name} | ${t.requests} | ${kb(t.bytes)} |`),    "",    "## By domain",    "",    "| Domain | Requests | Transferred | Third party |",    "|---|---|---|---|",    ...byDomain.map((d) => `| ${d.name} | ${d.requests} | ${kb(d.bytes)} | ${thirdParties.includes(d.name) ? "yes" : ""} |`),    "",    `## Failed\n\n${failed.map((f) => `- ${f.url}: ${f.status ?? f.error}`).join("\n") || "None."}`,    "",    `## Slowest\n\n${slowest.map((s) => `- ${s.url}: ${s.ms} ms`).join("\n")}`,    "",    `## Heaviest\n\n${heaviest.map((h) => `- ${h.url}: ${h.kb} KB`).join("\n")}`,    "",    "![The page](page.png)",    "",  ].join("\n");  writeFileSync(join(out, "report.md"), md);  writeFileSync(join(out, "result.json"), JSON.stringify({ url, finalUrl: opened.url, status: opened.status, loadMs, requests: rows.length, totalKb, byType, byDomain, thirdParties, failed, slowest, heaviest }, null, 2));} finally {  await session.stop();}

"""Page resources: open a page in a cloud browser and list everything it loads, from the session's own network log:requests and bytes by type and by domain, the third parties, what failed, and the slowest and heaviest requests.    python python/main.py            (BOXLINE_API_KEY; PAGE_URL)Writes output/result.json, output/report.md and output/page.png."""import jsonimport osimport reimport timefrom pathlib import Pathfrom urllib.parse import urlparsefrom boxline import Boxlineout = Path(os.environ.get("OUTPUT_DIR", "output"))url = os.environ.get("PAGE_URL", "https://books.toscrape.com/")bx = Boxline()def site(host: str) -> str:    """The site a host belongs to, roughly (its last two labels; three for "co.uk"-style endings)."""    parts = host.split(".")    return ".".join(parts[-3:] if len(parts) > 2 and re.fullmatch(r"co|com|org|net|gov|ac|edu", parts[-2]) else parts[-2:])def kb(b: float) -> str:    return f"{round(b / 1024, 1)} KB"with bx.sessions.create(timeout=300, user_metadata={"example": "page-resources"}) as session:    print(f"Session: {session.id}", flush=True)    # 1. Load the page until the network is quiet, and keep a picture of it.    t0 = time.monotonic()    opened = session.goto(url, wait_until="networkidle")    load_ms = round((time.monotonic() - t0) * 1000)    out.mkdir(parents=True, exist_ok=True)    (out / "page.png").write_bytes(session.screenshot(max_width=1280))  # PNG bytes    print(f"{opened['url']} ({opened['status']}) loaded in {load_ms / 1000:.1f} s")    # 2. Its network log: one event per finished (or failed) request. The log is written as requests end: wait a moment.    time.sleep(1.5)    events = list(session.events(types=["network"], limit=2000))mine = site(urlparse(opened["url"]).hostname or "")rows = []for e in events:    host = urlparse(e.get("url") or "").hostname or "?"    status = e.get("status")    rows.append({"url": e.get("url"), "host": host, "type": e.get("resourceType") or "Other", "status": status, "ms": e.get("durationMs") or 0, "bytes": int((e.get("data") or {}).get("bytes") or 0), "failed": e.get("level") == "error" or (status or 0) >= 400, "error": e.get("text"), "thirdParty": site(host) != mine})# 3. Grouped and ranked.def group(key):    g = {}    for r in rows:        x = g.setdefault(key(r), {"name": key(r), "requests": 0, "bytes": 0})        x["requests"] += 1        x["bytes"] += r["bytes"]    return sorted(g.values(), key=lambda x: -x["bytes"])by_type = group(lambda r: r["type"])by_domain = group(lambda r: r["host"])third_parties = sorted({r["host"] for r in rows if r["thirdParty"]})failed = [{"url": r["url"], "status": r["status"], "error": r["error"]} for r in rows if r["failed"]]slowest = [{"url": r["url"], "ms": r["ms"]} for r in sorted(rows, key=lambda r: -r["ms"])[:5]]heaviest = [{"url": r["url"], "kb": round(r["bytes"] / 1024, 1)} for r in sorted(rows, key=lambda r: -r["bytes"])[:5]]total_kb = round(sum(r["bytes"] for r in rows) / 1024, 1)print(f"{len(rows)} requests, {total_kb} KB, {len(third_parties)} third-party hosts, {len(failed)} failed")for t in by_type:    print(f"  {t['name']:<12} {t['requests']:>3}  {kb(t['bytes'])}")if failed:    print("Failed: " + ", ".join(f"{f['url']} ({f['status'] or f['error']})" for f in failed))print("Slowest: " + ", ".join(f"{s['url']} {s['ms']} ms" for s in slowest[:3]))md = [f"# {opened['url']}", "", f"{len(rows)} requests, {total_kb} KB transferred, loaded (network quiet) in {load_ms / 1000:.1f} s.", ""]md += ["## By type", "", "| Type | Requests | Transferred |", "|---|---|---|", *[f"| {t['name']} | {t['requests']} | {kb(t['bytes'])} |" for t in by_type], ""]md += ["## By domain", "", "| Domain | Requests | Transferred | Third party |", "|---|---|---|---|", *[f"| {d['name']} | {d['requests']} | {kb(d['bytes'])} | {'yes' if d['name'] in third_parties else ''} |" for d in by_domain], ""]md += ["## Failed", "", "\n".join(f"- {f['url']}: {f['status'] or f['error']}" for f in failed) or "None.", ""]md += ["## Slowest", "", "\n".join(f"- {s['url']}: {s['ms']} ms" for s in slowest), "", "## Heaviest", "", "\n".join(f"- {h['url']}: {h['kb']} KB" for h in heaviest), "", "![The page](page.png)", ""](out / "report.md").write_text("\n".join(md))result = {"url": url, "finalUrl": opened["url"], "status": opened["status"], "loadMs": load_ms, "requests": len(rows), "totalKb": total_kb, "byType": by_type, "byDomain": by_domain, "thirdParties": third_parties, "failed": failed, "slowest": slowest, "heaviest": heaviest}(out / "result.json").write_text(json.dumps(result, indent=2, ensure_ascii=False))

# Node 18 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keynpm install @boxline/sdk tsxnpx tsx node/index.ts

# Python 3.9 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keypip install boxline-sdkpython python/main.py