Context Window Planner — JavaScript source

/**
 * Context Window Planner - plan labeled prompt sections against a model's
 * context window.
 *
 * Language:   JavaScript (ES2020+, ES module; runs unmodified in Node 16+
 *             and modern browsers)
 * Source:     CosmoDev polyglot showcase port of the Context Window Planner
 *             tool, ported from src/lib/contextPlanner.ts (the canonical
 *             TypeScript implementation).
 * Live page:  https://dev.cosmolabs.org/tools/context-window-planner
 * License:    display source - part of CosmoDev's tool pages.
 *
 * Design goals:
 *   - Pure + deterministic; never throws.
 *   - Functionally equivalent to the TS reference: same inputs -> same outputs.
 *   - Self-contained: no imports, no snapshot fetch.
 *
 * Port notes: the TS lib delegates to two siblings — `estimateTokens` from
 * src/lib/tokenEstimator.ts and `fitsWindow` from src/lib/ai/models.ts (which
 * defaults to the bundled pricing snapshot, src/data/ai-models.json). A
 * dependency-free port cannot load that file, so:
 *   - the estimator is inlined below in the exact form the planner uses it
 *     (`estimateTokens(text).tokens`, auto content type — the full heuristic
 *     lives in the token-estimator port), and
 *   - window math is inlined from fitsWindow() and `models` is an explicit
 *     parameter (defaulting to an empty list), never re-derived.
 */

/**
 * One labeled block of the prompt (system / docs / history / ...).
 *
 * @typedef {Object} PlanSection
 * @property {string} label
 * @property {string} text
 */

/**
 * The subset of the TS `AiModel` record the planner reads. Production code
 * passes the full snapshot entry; only these fields influence the plan.
 *
 * @typedef {Object} Model
 * @property {string} id
 * @property {number} contextWindow  Total context window in tokens.
 * @property {number} maxOutput      The model's output cap (informational).
 */

/**
 * Result of planWindow. Field-for-field twin of the TS `WindowPlan`.
 *
 * @typedef {Object} WindowPlan
 * @property {string}  id              The model id planned against.
 * @property {number}  inputTokens     Sum of per-section token estimates.
 * @property {number}  contextWindow   The model's context window.
 * @property {number}  free            Context tokens left; negative on overflow.
 * @property {boolean} fits            Raw fit: free >= 0.
 * @property {boolean} outputReserveOk Room for the reserve: free >= reserve.
 * @property {number}  maxOutput       The model's output cap (informational).
 */

/**
 * Result of the fit check. Mirrors `WindowFit` from src/lib/ai/models.ts.
 *
 * @typedef {Object} WindowFit
 * @property {Model}   model
 * @property {number}  contextWindow
 * @property {number}  tokens
 * @property {number}  free
 * @property {boolean} fits
 */

/** Average characters per token, by content type. Mirrors CHARS_PER_TOKEN
 *  in src/lib/tokenEstimator.ts. */
const CHARS_PER_TOKEN = { prose: 4, code: 3.5, json: 3, cjk: 1.5 };

const CJK_RE = /[一-鿿぀-ヿ가-힯]/;
const CODE_SYMBOL_RE = /[{}();=<>\[\]#]/g;

/** Sample table for standalone use (mirrors the shared test fixtures).
 *  Production code passes the model snapshot instead. */
export const SAMPLE_MODELS = [
  { id: 'alpha-mini', contextWindow: 200_000, maxOutput: 10_000 },
  { id: 'beta-pro', contextWindow: 1_000_000, maxOutput: 10_000 },
  { id: 'gamma-open', contextWindow: 100_000, maxOutput: 10_000 },
];

/**
 * Classify a single line by its shape. Order: json, cjk, code, prose.
 * Inlined from detectLineType() in src/lib/tokenEstimator.ts.
 *
 * @param {string} line
 * @returns {'prose' | 'code' | 'json' | 'cjk'}
 */
function detectLineType(line) {
  const trimmed = line.trim();
  // JSON-ish: opens like a JSON fragment AND carries a separator.
  if ((trimmed.startsWith('{') || trimmed.startsWith('}') || trimmed.startsWith('[') || trimmed.startsWith('"')) && (line.includes(':') || line.includes(','))) {
    return 'json';
  }
  // CJK ideographs / kana / Hangul pack roughly one token per 1.5 chars.
  if (CJK_RE.test(line)) return 'cjk';
  // Code: symbol-dense, or a statement terminator / block opener at EOL.
  const density = (line.match(CODE_SYMBOL_RE) ?? []).length / line.length;
  if (density > 0.08 || trimmed.endsWith(';') || trimmed.endsWith('{') || trimmed.endsWith('}')) {
    return 'code';
  }
  return 'prose';
}

/** Whole-text JSON gate (isValidJson in the estimator lib). */
function isValidJson(text) {
  if (!text.trim()) return false;
  try {
    JSON.parse(text);
    return true;
  } catch {
    return false;
  }
}

/**
 * Token count of `text` under auto content detection — exactly the slice of
 * estimateTokens() the planner consumes (`.tokens`). Per non-empty line:
 * max(1, round(utf16Length / CHARS_PER_TOKEN[type])); framing tokens are the
 * caller's job.
 *
 * @param {string} text
 * @returns {number}
 */
export function estimateTokens(text) {
  // AUTO + whole-text JSON: a document that parses as JSON is json all the
  // way down — json's 3 chars/token rate applies to every line.
  const wholeTextJson = isValidJson(text);
  const nonEmpty = text.split(/\r?\n/).filter((line) => line.trim() !== '');
  let tokens = 0;
  for (const line of nonEmpty) {
    const type = wholeTextJson ? 'json' : detectLineType(line);
    tokens += Math.max(1, Math.round(line.length / CHARS_PER_TOKEN[type]));
  }
  return tokens;
}

/**
 * Sum of per-section token estimates (framing tokens are the caller's job).
 * Mirrors inputTokenTotal() in the TS lib.
 *
 * @param {PlanSection[]} sections
 * @returns {number}
 */
export function inputTokenTotal(sections) {
  return sections.reduce((n, s) => n + estimateTokens(s.text), 0);
}

/**
 * Fit check for one request against a model's context window. Inlined from
 * fitsWindow() in src/lib/ai/models.ts — returns undefined for an unknown id.
 *
 * @param {string} id
 * @param {number} tokens
 * @param {Model[]} [models=[]]
 * @returns {WindowFit | undefined}
 */
export function fitsWindow(id, tokens, models = []) {
  const m = models.find((m) => m.id === id);
  if (m === undefined) return undefined;
  const free = m.contextWindow - tokens;
  return { model: m, contextWindow: m.contextWindow, tokens, free, fits: free >= 0 };
}

/**
 * Plan one section set against one model's context window. Returns undefined
 * for an unknown model id (window math is fitsWindow's, never re-derived).
 * Mirrors planWindow() in the TS lib.
 *
 * @param {PlanSection[]} sections
 * @param {string} modelId
 * @param {number} [outputReserve=0]
 * @param {Model[]} [models=[]]
 * @returns {WindowPlan | undefined}
 */
export function planWindow(sections, modelId, outputReserve = 0, models = []) {
  const inputTokens = inputTokenTotal(sections);
  const fit = fitsWindow(modelId, inputTokens, models);
  if (fit === undefined) return undefined;
  return {
    id: modelId,
    inputTokens,
    contextWindow: fit.contextWindow,
    free: fit.free,
    fits: fit.fits,
    outputReserveOk: fit.free >= outputReserve,
    maxOutput: fit.model.maxOutput,
  };
}

/**
 * Plan against several models; unknown ids are dropped from the result.
 * Mirrors planAll() in the TS lib.
 *
 * @param {PlanSection[]} sections
 * @param {string[]} modelIds
 * @param {number} [outputReserve=0]
 * @param {Model[]} [models=[]]
 * @returns {WindowPlan[]}
 */
export function planAll(sections, modelIds, outputReserve = 0, models = []) {
  return modelIds
    .map((id) => planWindow(sections, id, outputReserve, models))
    .filter((p) => p !== undefined);
}