Context pack

/** * Context pack: squeeze a few pages into a small, cited context for a model's prompt. Each source gets a tag ([S1], * [S2], …), a summary, its key facts (each carrying its tag, each backed by a quote checked against the page) and its * main links, and the pack is cut to a token budget, least important facts first. * *   npx tsx node/index.ts            (BOXLINE_API_KEY; URLS comma-separated, QUESTION to focus on, BUDGET in tokens) * * Writes output/pack.md (paste it into a prompt), output/pack.json and output/result.json. */import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { Boxline } from "@boxline/sdk";const out = process.env.OUTPUT_DIR ?? "output";const urls = (process.env.URLS ?? "https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Caching,https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Cache-Control")  .split(",")  .map((u) => u.trim())  .filter(Boolean)  .slice(0, 10);const question = process.env.QUESTION ?? "";const budget = Number(process.env.BUDGET ?? 1500);const bx = new Boxline();const tokens = (s: string) => Math.ceil(s.length / 4);const norm = (s: string) => s.toLowerCase().replace(/[*_`#>\[\]()"“”'’]/g, " ").replace(/\s+/g, " ").trim();type Fact = { fact: string; quote: string; importance: 1 | 2 | 3 };// 1. The pages' own text (to check quotes against) and their links.const pages = await Promise.all(urls.map((u) => bx.fetch(u, { format: "markdown", links: true }).catch((e: Error) => ({ url: u, error: e.message }))));const read = pages.filter((p): p is Exclude<typeof p, { error: string }> => !("error" in p) && (p.status ?? 0) < 400);for (const p of pages) if ("error" in p) console.log(`  skipped ${p.url}: ${p.error}`);if (!read.length) throw new Error("no page could be read");// 2. Summary and facts per source, in one call.const r = await bx.extract<{ sources: { url: string; summary: string; facts: Fact[] }[] }>({  urls: read.map((p) => p.finalUrl),  prompt:    `For each page: summary, two sentences; facts: the 5 to 8 facts that matter most${question ? ` for this question: "${question}"` : ""}, each one short ` +    `sentence, with quote (words copied exactly from the page that state it) and importance (3 essential, 2 useful, 1 detail). url: the page's address as given.`,  schema: {    type: "object",    properties: {      sources: {        type: "array",        items: {          type: "object",          properties: {            url: { type: "string" },            summary: { type: "string" },            facts: { type: "array", items: { type: "object", properties: { fact: { type: "string" }, quote: { type: "string" }, importance: { type: "integer", enum: [1, 2, 3] } }, required: ["fact", "quote", "importance"] } },          },          required: ["url", "summary", "facts"],        },      },    },    required: ["sources"],  },});// 3. Tag the sources, keep only facts whose quote is on their page, and pick each page's main links.let unverified = 0;const sources = read.map((p, i) => {  const got = r.data.sources.find((s) => s.url === p.finalUrl || s.url === p.url) ?? r.data.sources[i];  const text = norm(p.content);  const facts = (got?.facts ?? []).filter((f) => {    const ok = f.quote.trim().length > 0 && text.includes(norm(f.quote));    if (!ok) unverified++;    return ok;  });  const host = new URL(p.finalUrl).host;  const links = (p.links ?? []).filter((l) => new URL(l).host === host && !l.includes("#") && l !== p.finalUrl).slice(0, 5);  return { id: `S${i + 1}`, url: p.finalUrl, title: p.title, summary: got?.summary ?? "", facts: facts.sort((a, b) => b.importance - a.importance), links };});// 4. The pack, cut to the budget: details first, then useful facts, then links, from the last source back.const render = () =>  [    `# Context (${sources.length} sources${question ? `; question: ${question}` : ""})`,    "",    "Cite facts with their tags. Sources:",    ...sources.map((s) => `[${s.id}] ${s.title} (${s.url})`),    "",    ...sources.flatMap((s) => [`## [${s.id}] ${s.title}`, "", s.summary, "", ...s.facts.map((f) => `- ${f.fact} [${s.id}]`), ...(s.links.length ? ["", `Links: ${s.links.join(" ")}`] : []), ""]),  ].join("\n");let dropped = 0;for (const level of [1, 2, "links", 3] as const) {  for (const s of [...sources].reverse()) {    while (tokens(render()) > budget) {      if (level === "links") {        if (!s.links.length) break;        s.links.pop();      } else {        const i = s.facts.map((f) => f.importance).lastIndexOf(level);        if (i < 0 || s.facts.length <= 1) break; // every source keeps at least its top fact        s.facts.splice(i, 1);        dropped++;      }    }  }}const pack = render();console.log(`${sources.length} sources, ${sources.reduce((n, s) => n + s.facts.length, 0)} facts kept (${unverified} without a quote on the page, ${dropped} cut for the budget); ${tokens(pack)} of ${budget} tokens\n`);console.log(pack);mkdirSync(out, { recursive: true });writeFileSync(join(out, "pack.md"), pack);writeFileSync(join(out, "pack.json"), JSON.stringify({ question: question || null, sources }, null, 2));writeFileSync(join(out, "result.json"), JSON.stringify({ urls, question: question || null, budget, tokens: tokens(pack), sources: sources.map((s) => ({ id: s.id, url: s.url, facts: s.facts.length })), unverified, dropped, usage: { modelUsd: r.usage.costUsd } }, null, 2));

"""Context pack: squeeze a few pages into a small, cited context for a model's prompt. Each source gets a tag ([S1],[S2], …), a summary, its key facts (each carrying its tag, each backed by a quote checked against the page) and itsmain links, and the pack is cut to a token budget, least important facts first.    python python/main.py            (BOXLINE_API_KEY; URLS comma-separated, QUESTION to focus on, BUDGET in tokens)Writes output/pack.md (paste it into a prompt), output/pack.json and output/result.json."""import jsonimport mathimport osimport refrom concurrent.futures import ThreadPoolExecutorfrom pathlib import Pathfrom urllib.parse import urlparsefrom boxline import Boxline, BoxlineErrorout = Path(os.environ.get("OUTPUT_DIR", "output"))urls = [u.strip() for u in os.environ.get("URLS", "https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Caching,https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Cache-Control").split(",") if u.strip()][:10]question = os.environ.get("QUESTION", "")budget = int(os.environ.get("BUDGET", "1500"))bx = Boxline()def tokens(s: str) -> int:    return math.ceil(len(s) / 4)def norm(s: str) -> str:    return re.sub(r"\s+", " ", re.sub(r"[*_`#>\[\]()\"“”'’]", " ", s.lower())).strip()# 1. The pages' own text (to check quotes against) and their links.def get(u):    try:        return bx.fetch(u, format="markdown", links=True)    except BoxlineError as err:        print(f"  skipped {u}: {err}")        return Nonewith ThreadPoolExecutor(4) as pool:    read = [p for p in pool.map(get, urls) if p and (p["status"] or 0) < 400]if not read:    raise SystemExit("no page could be read")# 2. Summary and facts per source, in one call.focus = f' for this question: "{question}"' if question else ""r = bx.extract(    urls=[p["finalUrl"] for p in read],    prompt=(        f"For each page: summary, two sentences; facts: the 5 to 8 facts that matter most{focus}, each one short "        "sentence, with quote (words copied exactly from the page that state it) and importance (3 essential, 2 useful, 1 detail). url: the page's address as given."    ),    schema={        "type": "object",        "properties": {            "sources": {                "type": "array",                "items": {                    "type": "object",                    "properties": {                        "url": {"type": "string"},                        "summary": {"type": "string"},                        "facts": {"type": "array", "items": {"type": "object", "properties": {"fact": {"type": "string"}, "quote": {"type": "string"}, "importance": {"type": "integer", "enum": [1, 2, 3]}}, "required": ["fact", "quote", "importance"]}},                    },                    "required": ["url", "summary", "facts"],                },            }        },        "required": ["sources"],    },)# 3. Tag the sources, keep only facts whose quote is on their page, and pick each page's main links.unverified = 0sources = []for i, p in enumerate(read):    got = next((s for s in r["data"]["sources"] if s["url"] in (p["finalUrl"], p["url"])), None) or (r["data"]["sources"][i] if i < len(r["data"]["sources"]) else None)    text = norm(p["content"])    facts = []    for f in (got or {}).get("facts", []):        if f["quote"].strip() and norm(f["quote"]) in text:            facts.append(f)        else:            unverified += 1    host = urlparse(p["finalUrl"]).netloc    links = [l for l in p.get("links") or [] if urlparse(l).netloc == host and "#" not in l and l != p["finalUrl"]][:5]    sources.append({"id": f"S{i + 1}", "url": p["finalUrl"], "title": p["title"], "summary": (got or {}).get("summary", ""), "facts": sorted(facts, key=lambda f: -f["importance"]), "links": links})# 4. The pack, cut to the budget: details first, then useful facts, then links, from the last source back.def render() -> str:    lines = [f"# Context ({len(sources)} sources" + (f"; question: {question}" if question else "") + ")", "", "Cite facts with their tags. Sources:"]    lines += [f"[{s['id']}] {s['title']} ({s['url']})" for s in sources] + [""]    for s in sources:        lines += [f"## [{s['id']}] {s['title']}", "", s["summary"], ""] + [f"- {f['fact']} [{s['id']}]" for f in s["facts"]]        if s["links"]:            lines += ["", "Links: " + " ".join(s["links"])]        lines.append("")    return "\n".join(lines)dropped = 0for level in (1, 2, "links", 3):    for s in reversed(sources):        while tokens(render()) > budget:            if level == "links":                if not s["links"]:                    break                s["links"].pop()            else:                idx = [k for k, f in enumerate(s["facts"]) if f["importance"] == level]                if not idx or len(s["facts"]) <= 1:                    break  # every source keeps at least its top fact                s["facts"].pop(idx[-1])                dropped += 1pack = render()kept = sum(len(s["facts"]) for s in sources)print(f"{len(sources)} sources, {kept} facts kept ({unverified} without a quote on the page, {dropped} cut for the budget); {tokens(pack)} of {budget} tokens\n")print(pack)out.mkdir(parents=True, exist_ok=True)(out / "pack.md").write_text(pack)(out / "pack.json").write_text(json.dumps({"question": question or None, "sources": sources}, indent=2, ensure_ascii=False))result = {"urls": urls, "question": question or None, "budget": budget, "tokens": tokens(pack), "sources": [{"id": s["id"], "url": s["url"], "facts": len(s["facts"])} for s in sources], "unverified": unverified, "dropped": dropped, "usage": {"modelUsd": r["usage"]["costUsd"]}}(out / "result.json").write_text(json.dumps(result, indent=2, ensure_ascii=False))

# Node 18 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keynpm install @boxline/sdk tsxnpx tsx node/index.ts

# Python 3.9 or newer, in the example's folderexport BOXLINE_API_KEY=…   # your API keypip install boxline-sdkpython python/main.py