Link checker

/** * Link checker: crawl the pages a page links to and report the broken ones and the slow ones. * *   npx tsx node/index.ts            (BOXLINE_API_KEY; SITE_URL for your own page, MAX_PAGES, SLOW_MS) * * Writes output/result.json: every page checked, the broken ones (HTTP 400+ or not loading) and the slow ones. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { Boxline, BoxlineError } from "@boxline/sdk";const out = process.env.OUTPUT_DIR ?? "output";const start = process.env.SITE_URL ?? "https://books.toscrape.com/";const maxPages = Number(process.env.MAX_PAGES ?? 20);const slowMs = Number(process.env.SLOW_MS ?? 2000);const maxExternal = Number(process.env.MAX_EXTERNAL ?? 15);const bx = new Boxline();// 1. Crawl the page and the pages it links to on the same site (depth 1): each one's HTTP status, or why it failed.const job = await bx.crawl.start({ url: start, maxPages, maxDepth: 1, format: "text" });const crawl = await bx.crawl.wait(job.id, { pollMs: 1500, timeoutMs: 10 * 60_000 });console.log(`Crawled ${crawl.data.length} pages from ${start}`);const broken: { url: string; status: number | null; error: string | null }[] = [];const timed: { url: string; status: number | null; ms: number }[] = [];for (const page of crawl.data) {  if (page.error || (page.status ?? 0) >= 400) {    broken.push({ url: page.url, status: page.status, error: page.error });    continue;  }  // 2. Time each page that loaded, one at a time to be polite to the site (fetch reports how long the page took).  try {    const r = await bx.fetch(page.url, { format: "text", timeoutMs: 30_000 });    timed.push({ url: page.url, status: r.status, ms: r.ms });  } catch (err) {    broken.push({ url: page.url, status: null, error: err instanceof BoxlineError ? err.code : String(err) });  }}const slow = timed.filter((t) => t.ms > slowMs).sort((a, b) => b.ms - a.ms);// 3. Links to other sites from the start page, opened in a sandboxed browser three at a time. "Blocked" is a site that//    refuses automated visitors (401, 403, 429 or a CAPTCHA): the link may well work for people, so check it by hand.const home = await bx.fetch(start, { format: "text", links: true });const host = new URL(home.finalUrl).host;const outbound = [...new Set((home.links ?? []).filter((l) => new URL(l).host !== host).map((l) => l.replace(/#.*$/, "")))].slice(0, maxExternal);const external: { url: string; verdict: "working" | "broken" | "blocked"; status: number | null; why: string }[] = [];for (let i = 0; i < outbound.length; i += 3) {  external.push(    ...(await Promise.all(      outbound.slice(i, i + 3).map(async (url) => {        try {          const p = await bx.fetch(url, { format: "text", timeoutMs: 30_000 });          const status = p.status;          if (p.captcha || [401, 403, 429].includes(status ?? 0)) return { url, verdict: "blocked" as const, status, why: p.captcha ? `a ${p.captcha} CAPTCHA` : `HTTP ${status}` };          if ((status ?? 0) >= 400) return { url, verdict: "broken" as const, status, why: `HTTP ${status}` };          return { url, verdict: "working" as const, status, why: `HTTP ${status}` };        } catch (err) {          return { url, verdict: "broken" as const, status: null, why: err instanceof BoxlineError ? err.message : String(err) };        }      }),    )),  );}for (const b of broken) console.log(`BROKEN  ${b.status ?? b.error}  ${b.url}`);for (const s of slow) console.log(`SLOW    ${(s.ms / 1000).toFixed(1)} s  ${s.url}`);for (const e of external.filter((x) => x.verdict !== "working")) console.log(`${e.verdict.toUpperCase().padEnd(7)} ${e.why}  ${e.url}`);console.log(`${crawl.data.length} checked: ${broken.length} broken, ${slow.length} slower than ${slowMs / 1000} s`);console.log(`${external.length} links to other sites: ${external.filter((x) => x.verdict === "working").length} working, ${external.filter((x) => x.verdict === "broken").length} broken, ${external.filter((x) => x.verdict === "blocked").length} blocked`);mkdirSync(out, { recursive: true });writeFileSync(join(out, "result.json"), JSON.stringify({ start, slowMs, checked: crawl.data.length, broken, slow, pages: timed, external }, null, 2));

"""Link checker: crawl the pages a page links to and report the broken ones and the slow ones.    python python/main.py            (BOXLINE_API_KEY; SITE_URL for your own page, MAX_PAGES, SLOW_MS)Writes output/result.json: every page checked, the broken ones (HTTP 400+ or not loading) and the slow ones."""import jsonimport osfrom concurrent.futures import ThreadPoolExecutorfrom pathlib import Pathfrom urllib.parse import urlparsefrom boxline import Boxline, BoxlineErrorout = Path(os.environ.get("OUTPUT_DIR", "output"))start = os.environ.get("SITE_URL", "https://books.toscrape.com/")max_pages = int(os.environ.get("MAX_PAGES", "20"))slow_ms = int(os.environ.get("SLOW_MS", "2000"))max_external = int(os.environ.get("MAX_EXTERNAL", "15"))bx = Boxline()# 1. Crawl the page and the pages it links to on the same site (depth 1): each one's HTTP status, or why it failed.job = bx.crawl.start(start, max_pages=max_pages, max_depth=1, format="text")crawl = bx.crawl.wait(job["id"], poll=1.5, timeout=600)print(f"Crawled {len(crawl['data'])} pages from {start}")broken, timed = [], []for page in crawl["data"]:    if page.get("error") or (page.get("status") or 0) >= 400:        broken.append({"url": page["url"], "status": page.get("status"), "error": page.get("error")})        continue    # 2. Time each page that loaded, one at a time to be polite to the site (fetch reports how long the page took).    try:        r = bx.fetch(page["url"], format="text", timeout_ms=30_000)        timed.append({"url": page["url"], "status": r["status"], "ms": r["ms"]})    except BoxlineError as e:        broken.append({"url": page["url"], "status": None, "error": e.code})slow = sorted((t for t in timed if t["ms"] > slow_ms), key=lambda t: -t["ms"])# 3. Links to other sites from the start page, opened in a sandboxed browser three at a time. "Blocked" is a site that#    refuses automated visitors (401, 403, 429 or a CAPTCHA): the link may well work for people, so check it by hand.def judge(url):    try:        p = bx.fetch(url, format="text", timeout_ms=30_000)    except BoxlineError as e:        return {"url": url, "verdict": "broken", "status": None, "why": str(e)}    status = p["status"]    if p.get("captcha") or status in (401, 403, 429):        return {"url": url, "verdict": "blocked", "status": status, "why": f"a {p['captcha']} CAPTCHA" if p.get("captcha") else f"HTTP {status}"}    if (status or 0) >= 400:        return {"url": url, "verdict": "broken", "status": status, "why": f"HTTP {status}"}    return {"url": url, "verdict": "working", "status": status, "why": f"HTTP {status}"}home = bx.fetch(start, format="text", links=True)host = urlparse(home["finalUrl"]).netlocoutbound = list(dict.fromkeys(l.split("#")[0] for l in home.get("links") or [] if urlparse(l).netloc != host))[:max_external]with ThreadPoolExecutor(3) as pool:    external = list(pool.map(judge, outbound))for b in broken:    print(f"BROKEN  {b['status'] or b['error']}  {b['url']}")for s in slow:    print(f"SLOW    {s['ms'] / 1000:.1f} s  {s['url']}")for e in external:    if e["verdict"] != "working":        print(f"{e['verdict'].upper():<7} {e['why']}  {e['url']}")print(f"{len(crawl['data'])} checked: {len(broken)} broken, {len(slow)} slower than {slow_ms / 1000:g} s")count = lambda v: sum(1 for x in external if x["verdict"] == v)  # noqa: E731print(f"{len(external)} links to other sites: {count('working')} working, {count('broken')} broken, {count('blocked')} blocked")out.mkdir(parents=True, exist_ok=True)result = {"start": start, "slowMs": slow_ms, "checked": len(crawl["data"]), "broken": broken, "slow": slow, "pages": timed, "external": external}(out / "result.json").write_text(json.dumps(result, indent=2, ensure_ascii=False))

# Node 18 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keynpm install @boxline/sdk tsxnpx tsx node/index.ts

# Python 3.9 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keypip install boxline-sdkpython python/main.py