robots.txt Generator — C++ source

// robots-txt-generator — C++ port: standards-compliant robots.txt generator + parser.
//
// Self-contained display port of the CosmoDev Robots.txt Generator tool — same
// contract as cli/robots-txt-generator/robots-txt-generator.go (the live Go
// twin) and src/lib/robotsTxt.ts (canonical TypeScript), matching the
// dependency-free Rust port in this directory. Standard library only. A group
// carries a LIST of user-agents (the REP spec's stacked User-agent lines);
// parsing aggregates repeated blocks per agent, then re-merges agents whose
// accumulated rules are identical into one stacked group.
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <optional>
#include <string>
#include <utility>
#include <vector>

namespace robots_txt {

struct RuleGroup {
    std::vector<std::string> userAgents; // {"*"} or {"Googlebot", "Bingbot"}
    std::vector<std::string> disallow;   // "" renders as a bare "Disallow:"
    std::vector<std::string> allow;
    std::optional<double> crawlDelay;    // nullopt = unset; non-finite never emitted
};

struct RobotsConfig {
    std::vector<RuleGroup> groups;
    std::vector<std::string> sitemaps;
};

static std::string trim(const std::string& s) {
    size_t b = 0, e = s.size();
    while (b < e && std::isspace(static_cast<unsigned char>(s[b]))) b++;
    while (e > b && std::isspace(static_cast<unsigned char>(s[e - 1]))) e--;
    return s.substr(b, e - b);
}

// cleanAgents in the TS lib: trim + drop blanks. A non-empty list that trims
// away entirely degrades to {"*"}; an empty list means "no group yet".
static std::vector<std::string> cleanAgents(const std::vector<std::string>& list) {
    std::vector<std::string> out;
    for (const auto& u : list) {
        auto t = trim(u);
        if (!t.empty()) out.push_back(std::move(t));
    }
    if (out.empty() && !list.empty()) out.emplace_back("*");
    return out;
}

// JS-style number rendering for Crawl-delay: 10.0 -> "10", 1.5 -> "1.5".
static std::string numStr(double f) {
    if (f == std::floor(f)) return std::to_string(static_cast<long long>(f));
    char buf[32];
    std::snprintf(buf, sizeof buf, "%g", f);
    return buf;
}

// Fold 3+ consecutive newlines to exactly two, strip trailing whitespace and
// guarantee a single terminating newline (mirrors TS /\n{3,}/g + trimEnd).
static std::string finish(std::string s) {
    std::string out;
    out.reserve(s.size() + 1);
    int run = 0;
    for (char c : s) {
        if (c == '\n') { if (++run <= 2) out += c; }
        else { run = 0; out += c; }
    }
    size_t end = out.size();
    while (end > 0 && std::isspace(static_cast<unsigned char>(out[end - 1]))) end--;
    return out.substr(0, end) + "\n";
}

std::string generateRobots(const RobotsConfig& cfg) {
    std::vector<std::string> out;
    for (const auto& g : cfg.groups) {
        auto uas = cleanAgents(g.userAgents);
        if (uas.empty()) continue;
        for (const auto& ua : uas) out.push_back("User-agent: " + ua);
        for (const auto& a : g.allow) {
            auto p = trim(a);
            if (!p.empty()) out.push_back("Allow: " + p); // blanks carry no meaning
        }
        if (g.disallow.empty()) {
            out.push_back("Disallow:"); // no entries -> the allow-all marker
        } else {
            for (const auto& d : g.disallow) out.push_back("Disallow: " + d); // "" kept verbatim
        }
        if (g.crawlDelay && std::isfinite(*g.crawlDelay)) out.push_back("Crawl-delay: " + numStr(*g.crawlDelay));
        out.push_back(""); // blank line separates groups
    }
    for (const auto& s : cfg.sitemaps) {
        auto url = trim(s);
        if (!url.empty()) out.push_back("Sitemap: " + url);
    }
    std::string joined;
    for (size_t i = 0; i < out.size(); i++) {
        if (i) joined += '\n';
        joined += out[i];
    }
    return finish(std::move(joined));
}

struct Acc {
    std::string ua;
    std::vector<std::string> disallow, allow;
    std::optional<double> crawlDelay;
};

// ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
// (values may contain colons), unknown directives ignored. Garbage
// Crawl-delay values coerce to NaN like JS Number() — never re-emitted.
RobotsConfig parseRobots(const std::string& text) {
    RobotsConfig cfg;
    std::vector<Acc> perUA;              // insertion-ordered per-agent buckets
    std::vector<std::string> stack;      // pending User-agent stack
    std::vector<size_t> current;         // perUA indices rules attach to

    auto flush = [&] { // a rule line applies to every agent of the pending stack
        if (stack.empty()) return;
        current.clear();
        for (const auto& ua : stack) {
            size_t i = 0;
            for (; i < perUA.size(); i++)
                if (perUA[i].ua == ua) break;
            if (i == perUA.size()) perUA.push_back({ua, {}, {}, {}});
            current.push_back(i);
        }
        stack.clear();
    };

    size_t pos = 0;
    while (pos <= text.size()) {
        size_t nl = text.find('\n', pos);
        std::string raw = text.substr(pos, (nl == std::string::npos ? text.size() : nl) - pos);
        pos = nl == std::string::npos ? text.size() + 1 : nl + 1;

        if (size_t hash = raw.find('#'); hash != std::string::npos) raw.resize(hash);
        std::string line = trim(raw);
        if (line.empty()) continue;
        size_t colon = line.find(':');
        if (colon == std::string::npos) continue;
        std::string field = trim(line.substr(0, colon));
        std::string value = trim(line.substr(colon + 1));
        std::transform(field.begin(), field.end(), field.begin(),
                       [](unsigned char c) { return static_cast<char>(std::tolower(c)); });

        if (field == "user-agent") {
            std::string ua = value.empty() ? "*" : value;
            if (std::find(stack.begin(), stack.end(), ua) == stack.end()) stack.push_back(std::move(ua));
        } else if (field == "disallow") {
            flush();
            for (size_t i : current) perUA[i].disallow.push_back(value);
        } else if (field == "allow") {
            flush();
            for (size_t i : current) perUA[i].allow.push_back(value);
        } else if (field == "crawl-delay") {
            flush();
            char* endp = nullptr;
            double v = std::strtod(value.c_str(), &endp);
            std::optional<double> cd = (endp && *endp == '\0') ? v : std::optional<double>(std::nan(""));
            for (size_t i : current) perUA[i].crawlDelay = cd;
        } else if (field == "sitemap") {
            flush();
            cfg.sitemaps.push_back(value);
        }
    }

    // Merge agents with identical rule sets into one stacked group.
    for (auto& acc : perUA) {
        RuleGroup* twin = nullptr;
        for (auto& g : cfg.groups)
            if (g.disallow == acc.disallow && g.allow == acc.allow && g.crawlDelay == acc.crawlDelay) { twin = &g; break; }
        if (twin) twin->userAgents.push_back(acc.ua);
        else cfg.groups.push_back({{acc.ua}, acc.disallow, acc.allow, acc.crawlDelay});
    }
    return cfg;
}

} // namespace robots_txt

int main() {
    // Sample config -> generated body -> parse round-trip check (tool page behavior).
    using robots_txt::RobotsConfig;
    RobotsConfig cfg{
        {
            {{"*"}, {"/private/", "/admin/"}, {"/admin/public/"}, {}},
            {{"GPTBot", "CCBot"}, {"/"}, {}, 10.0},
        },
        {"https://example.com/sitemap.xml"},
    };
    std::string body = robots_txt::generateRobots(cfg);
    std::printf("%s", body.c_str());
    std::string again = robots_txt::generateRobots(robots_txt::parseRobots(body));
    std::printf("round-trip stable: %s\n", again == body ? "true" : "false");
}