SEO and metadata audit

/** * SEO and metadata audit: titles, descriptions, headings, Open Graph and canonical tags, page by page, with a * suggested description (written by a model) for pages that have none. * *   npx tsx node/index.ts            (BOXLINE_API_KEY; SITE_URL for your own site, MAX_PAGES) * * Writes output/report.json: each page's tags and what to fix. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { Boxline } from "@boxline/sdk";const out = process.env.OUTPUT_DIR ?? "output";const start = process.env.SITE_URL ?? "https://books.toscrape.com/";const maxPages = Number(process.env.MAX_PAGES ?? 10);const bx = new Boxline();// Every tag with its attributes, e.g. <meta property="og:title" content="…">.function tags(html: string, name: string): Record<string, string>[] {  return [...html.matchAll(new RegExp(`<${name}\\b([^>]*)>`, "gi"))].map((m) =>    Object.fromEntries([...m[1]!.matchAll(/([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g)].map((a) => [a[1]!.toLowerCase(), a[2] ?? a[3] ?? a[4] ?? ""])),  );}const metaContent = (html: string, key: string) => tags(html, "meta").find((t) => t.name === key || t.property === key)?.content?.trim() ?? "";function audit(url: string, html: string) {  const title = (/<title[^>]*>([\s\S]*?)<\/title>/i.exec(html)?.[1] ?? "").replace(/\s+/g, " ").trim();  const page = {    url,    title,    description: metaContent(html, "description"),    h1: (html.match(/<h1\b/gi) ?? []).length,    ogTitle: metaContent(html, "og:title"),    ogDescription: metaContent(html, "og:description"),    // How the page looks when shared (Open Graph image, Twitter/X card), whether it may be listed, and its language.    ogImage: metaContent(html, "og:image"),    twitterCard: metaContent(html, "twitter:card"),    robots: metaContent(html, "robots"),    lang: tags(html, "html")[0]?.lang ?? "",    canonical: tags(html, "link").find((t) => (t.rel ?? "").split(/\s+/).includes("canonical"))?.href ?? "",    imagesWithoutAlt: tags(html, "img").filter((t) => !("alt" in t)).length,  };  const fixes: string[] = [];  if (!page.title) fixes.push("title missing");  else if (page.title.length > 60) fixes.push(`title over 60 characters (${page.title.length})`);  if (!page.description) fixes.push("no meta description");  else if (page.description.length > 160) fixes.push(`meta description over 160 characters (${page.description.length})`);  if (page.h1 !== 1) fixes.push(`${page.h1} H1 headings (want 1)`);  if (!page.ogTitle) fixes.push("no og:title");  if (!page.ogDescription) fixes.push("no og:description");  if (!page.ogImage) fixes.push("no og:image (shared links show no picture)");  if (!page.twitterCard) fixes.push("no twitter:card");  if (/noindex/i.test(page.robots)) fixes.push("noindex: the page asks search engines not to list it");  if (!page.lang) fixes.push("no lang attribute on <html>");  if (!page.canonical) fixes.push("no canonical link");  if (page.imagesWithoutAlt) fixes.push(`${page.imagesWithoutAlt} images without alt text`);  return { ...page, fixes, suggestedDescription: null as string | null };}// 1. Crawl the site's pages as HTML (same host, the start page and the pages it links to).const job = await bx.crawl.start({ url: start, maxPages, maxDepth: 1, format: "html" });const crawl = await bx.crawl.wait(job.id, { pollMs: 1500, timeoutMs: 10 * 60_000 });const report = crawl.data.filter((p) => p.content && (p.status ?? 0) < 400).map((p) => audit(p.finalUrl ?? p.url, p.content!));// 2. A model suggests a description for up to 3 pages that have none.let modelUsd = 0;for (const page of report.filter((p) => !p.description).slice(0, 3)) {  const r = await bx.extract<{ description: string }>({    url: page.url,    prompt: "Write a meta description for this page: one plain, specific sentence of at most 155 characters.",    schema: { type: "object", properties: { description: { type: "string" } }, required: ["description"] },  });  page.suggestedDescription = r.data.description;  modelUsd += r.usage.costUsd;}for (const p of report) {  console.log(`${p.url}\n  ${p.fixes.length ? p.fixes.join("; ") : "nothing to fix"}${p.suggestedDescription ? `\n  suggested description: ${p.suggestedDescription}` : ""}`);}mkdirSync(out, { recursive: true });writeFileSync(join(out, "report.json"), JSON.stringify(report, null, 2));writeFileSync(join(out, "result.json"), JSON.stringify({ start, pages: report.length, report, usage: { modelUsd } }, null, 2));

"""SEO and metadata audit: titles, descriptions, headings, Open Graph and canonical tags, page by page, with asuggested description (written by a model) for pages that have none.    python python/main.py            (BOXLINE_API_KEY; SITE_URL for your own site, MAX_PAGES)Writes output/report.json: each page's tags and what to fix."""import jsonimport osfrom html.parser import HTMLParserfrom pathlib import Pathfrom boxline import Boxlineout = Path(os.environ.get("OUTPUT_DIR", "output"))start = os.environ.get("SITE_URL", "https://books.toscrape.com/")max_pages = int(os.environ.get("MAX_PAGES", "10"))bx = Boxline()class Tags(HTMLParser):    """Collects the tags an audit looks at (Python's own HTML parser: nothing to install)."""    def __init__(self) -> None:        super().__init__()        self.title, self.in_title = "", False        self.meta: dict = {}        self.h1 = 0        self.canonical = ""        self.images_without_alt = 0        self.lang = ""    def handle_starttag(self, tag, attrs):        a = {k.lower(): (v or "") for k, v in attrs}        if tag == "html":            self.lang = a.get("lang", "")        elif tag == "title":            self.in_title = True        elif tag == "meta" and (a.get("name") or a.get("property")):            self.meta.setdefault(a.get("name") or a.get("property"), a.get("content", "").strip())        elif tag == "h1":            self.h1 += 1        elif tag == "link" and "canonical" in a.get("rel", "").split():            self.canonical = a.get("href", "")        elif tag == "img" and "alt" not in a:            self.images_without_alt += 1    def handle_endtag(self, tag):        if tag == "title":            self.in_title = False    def handle_data(self, data):        if self.in_title:            self.title += datadef audit(url: str, html: str) -> dict:    t = Tags()    t.feed(html)    page = {        "url": url,        "title": " ".join(t.title.split()),        "description": t.meta.get("description", ""),        "h1": t.h1,        "ogTitle": t.meta.get("og:title", ""),        "ogDescription": t.meta.get("og:description", ""),        # How the page looks when shared (Open Graph image, Twitter/X card), whether it may be listed, and its language.        "ogImage": t.meta.get("og:image", ""),        "twitterCard": t.meta.get("twitter:card", ""),        "robots": t.meta.get("robots", ""),        "lang": t.lang,        "canonical": t.canonical,        "imagesWithoutAlt": t.images_without_alt,    }    fixes = []    if not page["title"]:        fixes.append("title missing")    elif len(page["title"]) > 60:        fixes.append(f"title over 60 characters ({len(page['title'])})")    if not page["description"]:        fixes.append("no meta description")    elif len(page["description"]) > 160:        fixes.append(f"meta description over 160 characters ({len(page['description'])})")    if page["h1"] != 1:        fixes.append(f"{page['h1']} H1 headings (want 1)")    if not page["ogTitle"]:        fixes.append("no og:title")    if not page["ogDescription"]:        fixes.append("no og:description")    if not page["ogImage"]:        fixes.append("no og:image (shared links show no picture)")    if not page["twitterCard"]:        fixes.append("no twitter:card")    if "noindex" in page["robots"].lower():        fixes.append("noindex: the page asks search engines not to list it")    if not page["lang"]:        fixes.append("no lang attribute on <html>")    if not page["canonical"]:        fixes.append("no canonical link")    if page["imagesWithoutAlt"]:        fixes.append(f"{page['imagesWithoutAlt']} images without alt text")    return {**page, "fixes": fixes, "suggestedDescription": None}# 1. Crawl the site's pages as HTML (same host, the start page and the pages it links to).job = bx.crawl.start(start, max_pages=max_pages, max_depth=1, format="html")crawl = bx.crawl.wait(job["id"], poll=1.5, timeout=600)report = [audit(p.get("finalUrl") or p["url"], p["content"]) for p in crawl["data"] if p.get("content") and (p.get("status") or 0) < 400]# 2. A model suggests a description for up to 3 pages that have none.model_usd = 0.0for page in [p for p in report if not p["description"]][:3]:    r = bx.extract(        url=page["url"],        prompt="Write a meta description for this page: one plain, specific sentence of at most 155 characters.",        schema={"type": "object", "properties": {"description": {"type": "string"}}, "required": ["description"]},    )    page["suggestedDescription"] = r["data"]["description"]    model_usd += r["usage"]["costUsd"]for p in report:    print(p["url"])    print("  " + ("; ".join(p["fixes"]) or "nothing to fix"))    if p["suggestedDescription"]:        print(f"  suggested description: {p['suggestedDescription']}")out.mkdir(parents=True, exist_ok=True)(out / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False))(out / "result.json").write_text(json.dumps({"start": start, "pages": len(report), "report": report, "usage": {"modelUsd": model_usd}}, indent=2, ensure_ascii=False))

# Node 18 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keynpm install @boxline/sdk tsxnpx tsx node/index.ts

# Python 3.9 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keypip install boxline-sdkpython python/main.py