Context Window Planner — C++ source
Paste your system prompt, docs, and history — see how they fill any model's context window, with overflow warnings and output headroom.
This is the C++ implementation — the same logic the interactive tool runs, in a shareable, citable form.
// Context Window Planner — plan labeled prompt sections against a model's
// context window.
//
// Language: C++ (C++17, standard library only)
// Source: CosmoDev polyglot showcase port of the Context Window Planner
// tool, ported from src/lib/contextPlanner.ts (the canonical
// TypeScript implementation).
// Live at: https://dev.cosmolabs.org/tools/context-window-planner
// License: display source — part of CosmoDev's polyglot tool pages.
//
// Design goals:
// - Pure + deterministic; never throws (public API returns plain values).
// - Functionally equivalent to the TS reference: same inputs -> same outputs.
// - Self-contained: std only (no rapidjson, no PCRE — the JSON validator is
// a small strict recursive-descent parser).
//
// Port notes: the TS lib delegates to two siblings — `estimateTokens` from
// src/lib/tokenEstimator.ts and `fitsWindow` from src/lib/ai/models.ts (which
// defaults to the bundled pricing snapshot, src/data/ai-models.json). A
// dependency-free port cannot load that file, so the estimator is inlined
// below in the exact form the planner uses it (`estimateTokens(text).tokens`,
// auto content type — the full heuristic lives in the token-estimator port),
// window math is inlined from `fitsWindow` and `models` is an explicit
// parameter, never re-derived.
//
// Faithfulness notes (the places C++'s std silently differs from JS):
// - Length: TS's `String.length` counts UTF-16 code units (an astral-plane
// character — emoji, rare CJK ext-B ideographs — counts as 2).
// `std::string` holds UTF-8 bytes, so line arithmetic goes through
// `utf16_len()`, which decodes and counts the same unit.
// - JSON: std has no JSON parser, so this port ships a small strict
// recursive-descent validator (`is_valid_json`) implementing exactly the
// grammar `JSON.parse` accepts. (Multibyte UTF-8 inside strings never
// contains an ASCII byte, so byte-level scanning is safe.)
// - Rounding: `js_round()` is `floor(x + 0.5)` — JS `Math.round` rounds
// halfway cases up; `std::round` rounds away from zero (identical on the
// non-negative numbers used here, but the helper states the contract).
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
namespace context_window_planner {
/// One labeled block of the prompt (system / docs / history / ...).
/// Mirrors the TS `PlanSection` interface.
struct PlanSection {
std::string label; ///< Section label, e.g. "system" or "docs".
std::string text; ///< The section's raw text.
};
/// Convenience constructor mirroring the TS object literal `{ label, text }`.
PlanSection sec(std::string_view label, std::string_view text) {
return PlanSection{std::string(label), std::string(text)};
}
/// The subset of the TS `AiModel` record the planner reads. Production code
/// passes the full snapshot entry; only these fields influence the plan.
struct Model {
std::string id; ///< Model id, e.g. "beta-pro".
long long context_window; ///< Total context window in tokens.
long long max_output; ///< The model's output cap (informational).
};
/// Sample table for standalone use (mirrors the shared test fixtures).
/// Production code passes the model snapshot instead.
const std::vector<Model>& sample_models() {
static const std::vector<Model> table = {
{"alpha-mini", 200'000, 10'000},
{"beta-pro", 1'000'000, 10'000},
{"gamma-open", 100'000, 10'000},
};
return table;
}
/// Result of `plan_window`. Field-for-field twin of the TS `WindowPlan`
/// interface.
struct WindowPlan {
std::string id; ///< The model id planned against.
long long input_tokens; ///< Sum of per-section token estimates.
long long context_window; ///< The model's context window.
long long free; ///< Window tokens left; negative on overflow.
bool fits; ///< Raw fit: free >= 0.
bool output_reserve_ok; ///< Room for the output reserve: free >= reserve.
long long max_output; ///< The model's output cap (informational).
};
/// Content classification of a single line. The planner only needs each
/// type's chars-per-token rate (mirrors `CHARS_PER_TOKEN` in
/// src/lib/tokenEstimator.ts: prose 4, code 3.5, json 3, cjk 1.5).
enum class ContentType { Prose, Code, Json, Cjk };
double chars_per_token(ContentType t) {
switch (t) {
case ContentType::Prose: return 4.0;
case ContentType::Code: return 3.5;
case ContentType::Json: return 3.0;
case ContentType::Cjk: return 1.5;
}
return 4.0; // unreachable
}
/// Length of `s` in UTF-16 code units — the unit TS's `String.length`
/// counts. BMP code points are one unit, astral-plane ones two. Decodes the
/// UTF-8 bytes; an invalid lead byte counts as one unit.
long long utf16_len(std::string_view s) {
long long units = 0;
for (std::size_t i = 0; i < s.size();) {
unsigned char b = static_cast<unsigned char>(s[i]);
std::uint32_t cp;
std::size_t step;
if (b < 0x80) { cp = b; step = 1; }
else if ((b & 0xE0) == 0xC0) { cp = b & 0x1F; step = 2; }
else if ((b & 0xF0) == 0xE0) { cp = b & 0x0F; step = 3; }
else if ((b & 0xF8) == 0xF0) { cp = b & 0x07; step = 4; }
else { cp = b; step = 1; } // invalid byte
if (i + step <= s.size()) {
for (std::size_t k = 1; k < step; ++k)
cp = (cp << 6) |
(static_cast<unsigned char>(s[i + k]) & 0x3F);
}
units += cp > 0xFFFF ? 2 : 1;
i += step;
}
return units;
}
/// Reports whether `s` contains a CJK ideograph (U+4E00–U+9FFF), kana
/// (U+3040–U+30FF), or a Hangul syllable (U+AC00–U+D7AF). Mirrors `CJK_RE`
/// in the TS lib.
bool has_cjk(std::string_view s) {
for (std::size_t i = 0; i < s.size();) {
unsigned char b = static_cast<unsigned char>(s[i]);
std::uint32_t cp;
std::size_t step;
if (b < 0x80) { cp = b; step = 1; }
else if ((b & 0xE0) == 0xC0) { cp = b & 0x1F; step = 2; }
else if ((b & 0xF0) == 0xE0) { cp = b & 0x0F; step = 3; }
else if ((b & 0xF8) == 0xF0) { cp = b & 0x07; step = 4; }
else { cp = b; step = 1; }
if (i + step <= s.size()) {
for (std::size_t k = 1; k < step; ++k)
cp = (cp << 6) |
(static_cast<unsigned char>(s[i + k]) & 0x3F);
}
if ((cp >= 0x4E00 && cp <= 0x9FFF) ||
(cp >= 0x3040 && cp <= 0x30FF) ||
(cp >= 0xAC00 && cp <= 0xD7AF))
return true;
i += step;
}
return false;
}
/// Reports whether `c` is one of the code-flavored symbols counted by
/// `CODE_SYMBOL_RE` (`{}();=<>[]#`).
bool is_code_symbol(char c) {
return c == '{' || c == '}' || c == '(' || c == ')' || c == ';' ||
c == '=' || c == '<' || c == '>' || c == '[' || c == ']' || c == '#';
}
bool is_space(char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
/// JS `Math.round`: halfway cases round up (`floor(x + 0.5)`).
long long js_round(double x) {
return static_cast<long long>(x + 0.5);
}
/// Splits `text` on LF or CRLF, mirroring `text.split(/\r?\n/)`: strip the
/// optional CR that belongs to the newline, then split on LF. A lone CR is
/// NOT a line break.
std::vector<std::string_view> split_lines(std::string_view text) {
std::vector<std::string_view> lines;
std::size_t start = 0;
while (start <= text.size()) {
std::size_t nl = text.find('\n', start);
std::string_view line = nl == std::string_view::npos
? text.substr(start)
: text.substr(start, nl - start);
if (!line.empty() && line.back() == '\r') line.remove_suffix(1);
lines.push_back(line);
if (nl == std::string_view::npos) break;
start = nl + 1;
}
return lines;
}
std::string_view trim_view(std::string_view s) {
while (!s.empty() && is_space(s.front())) s.remove_prefix(1);
while (!s.empty() && is_space(s.back())) s.remove_suffix(1);
return s;
}
/// Classifies a single line by its shape. Order: json, cjk, code, prose.
/// Inlined from `detectLineType()` in src/lib/tokenEstimator.ts.
ContentType detect_line_type(std::string_view line) {
std::string_view trimmed = trim_view(line);
// JSON-ish: opens like a JSON fragment AND carries a separator.
bool starts_jsonish = !trimmed.empty() &&
(trimmed.front() == '{' || trimmed.front() == '}' ||
trimmed.front() == '[' || trimmed.front() == '"');
if (starts_jsonish &&
(line.find(':') != std::string_view::npos ||
line.find(',') != std::string_view::npos)) {
return ContentType::Json;
}
// CJK ideographs / kana / Hangul pack roughly one token per 1.5 chars.
if (has_cjk(line)) {
return ContentType::Cjk;
}
// Code: symbol-dense, or a statement terminator / block opener at EOL.
long long length = utf16_len(line);
long long symbols = 0;
for (char c : line)
if (is_code_symbol(c)) symbols++;
double density = length > 0 ? static_cast<double>(symbols) /
static_cast<double>(length)
: 0.0;
if (density > 0.08 || (!trimmed.empty() &&
(trimmed.back() == ';' || trimmed.back() == '{' ||
trimmed.back() == '}'))) {
return ContentType::Code;
}
return ContentType::Prose;
}
/// A strict JSON syntax validator — the exact grammar `JSON.parse` accepts,
/// walked with a byte cursor. (Multibyte UTF-8 inside strings never contains
/// an ASCII byte, so byte-level scanning is safe.)
class JsonParser {
public:
explicit JsonParser(std::string_view text)
: bytes_(reinterpret_cast<const unsigned char*>(text.data())),
len_(text.size()) {}
/// value := ws* (object | array | string | number | 'true' | 'false' | 'null') ws*
bool value() {
skip_ws();
switch (peek()) {
case '{': return object();
case '[': return array();
case '"': return string();
case '-': case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return number();
case 't': return literal("true");
case 'f': return literal("false");
case 'n': return literal("null");
default: return false;
}
}
bool at_end() {
skip_ws();
return pos_ == len_; // reject trailing garbage
}
private:
void skip_ws() {
while (pos_ < len_) {
unsigned char b = bytes_[pos_];
if (b == ' ' || b == '\t' || b == '\n' || b == '\r') ++pos_;
else break;
}
}
unsigned char peek() const { return pos_ < len_ ? bytes_[pos_] : 0; }
bool eat(unsigned char b) {
if (pos_ < len_ && bytes_[pos_] == b) { ++pos_; return true; }
return false;
}
bool literal(const char* lit) {
std::size_t n = std::strlen(lit);
if (len_ - pos_ >= n && std::memcmp(bytes_ + pos_, lit, n) == 0) {
pos_ += n;
return true;
}
return false;
}
/// object := '{' ws* (string ws* ':' value (ws* ',' ...)*)? ws* '}'
bool object() {
if (!eat('{')) return false;
skip_ws();
if (eat('}')) return true;
for (;;) {
if (!string()) return false;
skip_ws();
if (!eat(':')) return false;
if (!value()) return false;
skip_ws();
if (eat(',')) skip_ws();
else return eat('}');
}
}
/// array := '[' ws* (value (ws* ',' ws* value)*)? ws* ']'
bool array() {
if (!eat('[')) return false;
skip_ws();
if (eat(']')) return true;
for (;;) {
if (!value()) return false;
skip_ws();
if (eat(',')) skip_ws();
else return eat(']');
}
}
/// string := '"' (escape | any byte >= 0x20)* '"'
/// escape := '\' ("\"" | "/" | "\" | 'b' | 'f' | 'n' | 'r' | 't' | 'u' hex4)
bool string() {
if (!eat('"')) return false;
while (pos_ < len_) {
unsigned char b = bytes_[pos_];
if (b == '"') { ++pos_; return true; }
if (b == '\\') {
++pos_;
if (pos_ >= len_) return false;
unsigned char esc = bytes_[pos_++];
switch (esc) {
case '"': case '/': case '\\': case 'b':
case 'f': case 'n': case 'r': case 't':
break;
case 'u':
for (int i = 0; i < 4; ++i) {
unsigned char h = peek();
bool hex = (h >= '0' && h <= '9') ||
(h >= 'a' && h <= 'f') ||
(h >= 'A' && h <= 'F');
if (!hex) return false;
++pos_;
}
break;
default:
return false;
}
} else if (b < 0x20) {
return false; // raw control characters not allowed in strings
} else {
++pos_;
}
}
return false; // unterminated string
}
/// number := '-'? int frac? exp? — no leading zeros, like JSON.parse.
bool number() {
eat('-');
if (peek() == '0') {
++pos_;
} else if (peek() >= '1' && peek() <= '9') {
while (peek() >= '0' && peek() <= '9') ++pos_;
} else {
return false;
}
if (peek() == '.') {
++pos_;
int digits = 0;
while (peek() >= '0' && peek() <= '9') { ++pos_; ++digits; }
if (digits == 0) return false;
}
if (peek() == 'e' || peek() == 'E') {
++pos_;
if (peek() == '+' || peek() == '-') ++pos_;
int digits = 0;
while (peek() >= '0' && peek() <= '9') { ++pos_; ++digits; }
if (digits == 0) return false;
}
return true;
}
const unsigned char* bytes_;
std::size_t len_;
std::size_t pos_ = 0;
};
/// Whole-text JSON gate: a document that parses as JSON is json all the way
/// down. Mirrors `isValidJson()` (`JSON.parse` in a try/catch);
/// empty/whitespace text is not.
bool is_valid_json(std::string_view text) {
if (trim_view(text).empty()) return false;
JsonParser p(text);
return p.value() && p.at_end();
}
/// Token count of `text` under auto content detection — exactly the slice of
/// `estimateTokens()` the planner consumes (`.tokens`): per non-empty line,
/// `max(1, round(utf16_len / chars_per_token))`. Framing tokens are the
/// caller's job.
long long estimate_tokens(std::string_view text) {
// AUTO + whole-text JSON: json's 3 chars/token rate applies to every
// line, not just the reported content type.
bool whole_text_json = is_valid_json(text);
long long tokens = 0;
for (std::string_view line : split_lines(text)) {
if (trim_view(line).empty()) continue;
ContentType t = whole_text_json ? ContentType::Json
: detect_line_type(line);
long long line_tokens = js_round(static_cast<double>(utf16_len(line)) /
chars_per_token(t));
tokens += line_tokens >= 1 ? line_tokens : 1;
}
return tokens;
}
/// Sum of per-section token estimates (framing tokens are the caller's job).
/// Mirrors `inputTokenTotal()` in the TS lib.
long long input_token_total(const std::vector<PlanSection>& sections) {
long long total = 0;
for (const PlanSection& s : sections) total += estimate_tokens(s.text);
return total;
}
/// Plan one section set against one model's context window. Returns
/// `std::nullopt` for an unknown model id (window math is `fitsWindow`'s,
/// never re-derived). Mirrors `planWindow()` in the TS lib.
std::optional<WindowPlan> plan_window(const std::vector<PlanSection>& sections,
std::string_view model_id,
long long output_reserve,
const std::vector<Model>& models) {
long long input_tokens = input_token_total(sections);
// Fit check inlined from fitsWindow() in src/lib/ai/models.ts.
const Model* m = nullptr;
for (const Model& cand : models)
if (cand.id == model_id) { m = &cand; break; }
if (!m) return std::nullopt;
long long free = m->context_window - input_tokens;
return WindowPlan{
std::string(model_id),
input_tokens,
m->context_window,
free,
free >= 0,
free >= output_reserve,
m->max_output,
};
}
/// Plan against several models; unknown ids are dropped from the result.
/// Mirrors `planAll()` in the TS lib.
std::vector<WindowPlan> plan_all(const std::vector<PlanSection>& sections,
const std::vector<std::string_view>& model_ids,
long long output_reserve,
const std::vector<Model>& models) {
std::vector<WindowPlan> plans;
for (std::string_view id : model_ids) {
auto plan = plan_window(sections, id, output_reserve, models);
if (plan) plans.push_back(*plan);
}
return plans;
}
} // namespace context_window_planner
// ---------- tests (showcase-only; the canonical suite lives in src/lib) ----------
#ifdef CWP_TEST
#include <cassert>
#include <cstdio>
#include <string>
using namespace context_window_planner;
int main() {
// A 1600-char single line of 'a' is pure prose: 1600 / 4 = 400 tokens.
const std::string line_a(1600, 'a');
const std::vector<PlanSection> two_sections = {
sec("sys", line_a), sec("docs", line_a)};
// input totals
assert(input_token_total(two_sections) == 800);
assert(input_token_total({}) == 0);
assert(input_token_total({sec("sys", "")}) == 0);
// plans two 400-token sections against beta-pro
auto p = plan_window(two_sections, "beta-pro", 0, sample_models()).value();
assert(p.id == "beta-pro");
assert(p.input_tokens == 800);
assert(p.context_window == 1'000'000);
assert(p.free == 999'200);
assert(p.fits);
assert(p.output_reserve_ok);
assert(p.max_output == 10'000);
// reserve larger than free leaves raw fit true
p = plan_window(two_sections, "beta-pro", 1'000'000, sample_models()).value();
assert(p.fits);
assert(!p.output_reserve_ok);
// reserve exactly equal to free is ok
p = plan_window(two_sections, "beta-pro", 999'200, sample_models()).value();
assert(p.output_reserve_ok);
// smaller window leaves 199,200 free
p = plan_window(two_sections, "alpha-mini", 0, sample_models()).value();
assert(p.context_window == 200'000);
assert(p.free == 199'200);
assert(p.fits);
// unknown model id returns nullopt
assert(!plan_window(two_sections, "ghost", 0, sample_models()).has_value());
// no sections: full window free
p = plan_window({}, "beta-pro", 0, sample_models()).value();
assert(p.input_tokens == 0);
assert(p.free == 1'000'000);
assert(p.fits);
// overflow: fits false, reserve false
const std::vector<PlanSection> big = {sec("big", std::string(4'400'000, 'z'))};
p = plan_window(big, "beta-pro", 0, sample_models()).value();
assert(p.input_tokens == 1'100'000);
assert(p.free == -100'000);
assert(!p.fits);
assert(!p.output_reserve_ok);
// plan_all drops unknown ids and keeps order
auto plans = plan_all(two_sections,
{"beta-pro", "alpha-mini", "ghost"}, 0, sample_models());
assert(plans.size() == 2);
assert(plans[0].id == "beta-pro");
assert(plans[1].id == "alpha-mini");
assert(plans[1].free == 199'200);
assert(plan_all(two_sections, {}, 0, sample_models()).empty());
std::printf("context-window-planner (C++): all tests passed\n");
return 0;
}
#endif // CWP_TEST
Also available in 13 other languages
Every CosmoDev tool ships its pure logic in TypeScript (web) and Go (CLI), with authored implementations in a dozen-plus languages — the same contract, ported. Compare all languages side by side →