robots.txt Generator — C source

/* robots-txt-generator — C port: standards-compliant robots.txt generator + parser.
 *
 * Self-contained display port of the CosmoDev Robots.txt Generator tool — same
 * contract as cli/robots-txt-generator/robots-txt-generator.go (the live Go
 * twin) and src/lib/robotsTxt.ts (canonical TypeScript), matching the
 * dependency-free Rust port in this directory. A group carries a LIST of
 * user-agents (the REP spec's stacked User-agent lines); parsing aggregates
 * repeated blocks per agent, then re-merges agents whose accumulated rules
 * are identical into one stacked group. Fixed-capacity demo storage: 8 groups,
 * 12 agents, 16 rules/group, 8 KB body. The parser's scratch is a
 * function-local static (single-threaded demo, keeps the stack shallow).
 */
#include <ctype.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define MAX_AGENTS 12
#define MAX_RULES  16
#define MAX_GROUPS 8
#define UA_LEN   64
#define PATH_LEN 128
#define URL_LEN  256

typedef struct {
    char userAgents[MAX_AGENTS][UA_LEN]; int nUserAgents; /* {"*"} or stacked agents */
    char disallow[MAX_RULES][PATH_LEN];  int nDisallow;   /* "" renders as a bare "Disallow:" */
    char allow[MAX_RULES][PATH_LEN];     int nAllow;
    double crawlDelay; int hasCrawlDelay;                 /* NaN = unset, never emitted */
} RuleGroup;

typedef struct {
    RuleGroup groups[MAX_GROUPS]; int nGroups;
    char sitemaps[MAX_RULES][URL_LEN]; int nSitemaps;
} RobotsConfig;

static void rg_init(RuleGroup *g) { memset(g, 0, sizeof *g); }
static void rg_agent(RuleGroup *g, const char *ua) {
    if (g->nUserAgents < MAX_AGENTS) snprintf(g->userAgents[g->nUserAgents++], UA_LEN, "%s", ua);
}
static void rg_disallow(RuleGroup *g, const char *d) {
    if (g->nDisallow < MAX_RULES) snprintf(g->disallow[g->nDisallow++], PATH_LEN, "%s", d);
}
static void rg_allow(RuleGroup *g, const char *a) {
    if (g->nAllow < MAX_RULES) snprintf(g->allow[g->nAllow++], PATH_LEN, "%s", a);
}

static int visible(const char *s) { /* any non-whitespace char? */
    for (; *s; s++)
        if (!isspace((unsigned char)*s)) return 1;
    return 0;
}

static char *trim_in_place(char *s) {
    while (isspace((unsigned char)*s)) s++;
    char *end = s + strlen(s);
    while (end > s && isspace((unsigned char)end[-1])) end--;
    *end = '\0';
    return s;
}

/* Append prefix + trim(v) + '\n' to out; a NULL v emits the bare prefix. */
static void append_trimmed(char *out, size_t cap, const char *prefix, const char *v) {
    size_t used = strnlen(out, cap);
    if (used + 1 >= cap) return;
    int n = snprintf(out + used, cap - used, "%s", prefix);
    if (n < 0) return;
    used += (size_t)n;
    if (used >= cap) return;
    if (v) {
        while (isspace((unsigned char)*v)) v++;
        const char *end = v + strlen(v);
        while (end > v && isspace((unsigned char)end[-1])) end--;
        n = snprintf(out + used, cap - used, "%.*s", (int)(end - v), v);
        if (n < 0) return;
        used += (size_t)n;
        if (used >= cap) return;
    }
    snprintf(out + used, cap - used, "\n");
}

/* Fold 3+ consecutive newlines to two, strip trailing whitespace, guarantee a
 * single terminating newline (mirrors TS /\n{3,}/g + trimEnd). */
static void finish(char *out) {
    size_t w = 0, r = 0, len = strlen(out);
    int run = 0;
    for (; r < len; r++) {
        if (out[r] == '\n') { if (++run <= 2) out[w++] = '\n'; }
        else { run = 0; out[w++] = out[r]; }
    }
    while (w > 0 && isspace((unsigned char)out[w - 1])) w--;
    out[w] = '\n';
    out[w + 1] = '\0';
}

void generate_robots(const RobotsConfig *cfg, char *out, size_t cap) {
    out[0] = '\0';
    for (int i = 0; i < cfg->nGroups; i++) {
        const RuleGroup *g = &cfg->groups[i];
        /* cleanAgents in the TS lib: trim + drop blanks. A non-empty list that
         * trims away entirely degrades to "*"; an empty list means "no group". */
        int emitted = 0;
        for (int a = 0; a < g->nUserAgents; a++) {
            if (!visible(g->userAgents[a])) continue;
            append_trimmed(out, cap, "User-agent: ", g->userAgents[a]);
            emitted = 1;
        }
        if (!emitted) {
            if (g->nUserAgents > 0) append_trimmed(out, cap, "User-agent: ", "*");
            else continue;
        }
        for (int a = 0; a < g->nAllow; a++) /* blanks carry no meaning */
            if (visible(g->allow[a])) append_trimmed(out, cap, "Allow: ", g->allow[a]);
        if (g->nDisallow == 0) {
            append_trimmed(out, cap, "Disallow:", NULL); /* the allow-all marker */
        } else {
            for (int d = 0; d < g->nDisallow; d++)
                append_trimmed(out, cap, "Disallow: ", g->disallow[d]); /* "" kept verbatim */
        }
        if (g->hasCrawlDelay && isfinite(g->crawlDelay)) { /* JS-style number: 10.0 -> "10" */
            char line[64];
            if (g->crawlDelay == floor(g->crawlDelay))
                snprintf(line, sizeof line, "Crawl-delay: %lld", (long long)g->crawlDelay);
            else
                snprintf(line, sizeof line, "Crawl-delay: %g", g->crawlDelay);
            append_trimmed(out, cap, line, NULL);
        }
        append_trimmed(out, cap, "", NULL); /* blank line separates groups */
    }
    for (int s = 0; s < cfg->nSitemaps; s++)
        if (visible(cfg->sitemaps[s])) append_trimmed(out, cap, "Sitemap: ", cfg->sitemaps[s]);
    finish(out);
}

/* Move the pending User-agent stack into per-agent rule buckets (Google's
 * merge semantics: a rule line applies to every agent of the immediately
 * preceding consecutive User-agent stack). */
static void flush_stack(char stack[][UA_LEN], int *nStack, RuleGroup *perUA, int *nUA,
                        int *curIdx, int *nCur) {
    if (*nStack == 0) return;
    *nCur = 0;
    for (int s = 0; s < *nStack; s++) {
        int i = 0;
        for (; i < *nUA; i++)
            if (strcmp(perUA[i].userAgents[0], stack[s]) == 0) break;
        if (i == *nUA) {
            rg_init(&perUA[*nUA]);
            rg_agent(&perUA[*nUA], stack[s]);
            (*nUA)++;
        }
        curIdx[(*nCur)++] = i;
    }
    *nStack = 0;
}

static int paths_equal(int nA, const char a[][PATH_LEN], int nB, const char b[][PATH_LEN]) {
    if (nA != nB) return 0;
    for (int i = 0; i < nA; i++)
        if (strcmp(a[i], b[i]) != 0) return 0;
    return 1;
}

static int cd_equal(const RuleGroup *a, const RuleGroup *b) { /* non-finite merges as unset, like TS */
    int ha = a->hasCrawlDelay && isfinite(a->crawlDelay);
    int hb = b->hasCrawlDelay && isfinite(b->crawlDelay);
    return ha && hb ? a->crawlDelay == b->crawlDelay : ha == hb;
}

/* ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
 * (values may contain colons), unknown directives ignored. Garbage
 * Crawl-delay values coerce to NaN like JS Number() — never re-emitted. */
void parse_robots(const char *text, RobotsConfig *cfg) {
    static RuleGroup perUA[MAX_AGENTS]; /* fixed-capacity demo scratch */
    memset(perUA, 0, sizeof perUA);
    memset(cfg, 0, sizeof *cfg);

    char stack[MAX_AGENTS][UA_LEN];
    int nStack = 0, nUA = 0, nCur = 0;
    int curIdx[MAX_AGENTS];
    char line[512];

    const char *p = text;
    while (p && *p) {
        const char *nl = strchr(p, '\n');
        size_t len = nl ? (size_t)(nl - p) : strlen(p);
        if (len >= sizeof line) len = sizeof line - 1;
        memcpy(line, p, len);
        line[len] = '\0';
        p = nl ? nl + 1 : NULL;

        char *hash = strchr(line, '#');
        if (hash) *hash = '\0';
        char *t = trim_in_place(line);
        if (!*t) continue;
        char *colon = strchr(t, ':');
        if (!colon) continue;
        *colon = '\0';
        char *field = trim_in_place(t);
        char *value = trim_in_place(colon + 1);
        for (char *f = field; *f; f++) *f = (char)tolower((unsigned char)*f);

        if (strcmp(field, "user-agent") == 0) {
            const char *ua = *value ? value : "*";
            int dup = 0;
            for (int i = 0; i < nStack; i++)
                if (strcmp(stack[i], ua) == 0) { dup = 1; break; }
            if (!dup && nStack < MAX_AGENTS) snprintf(stack[nStack++], UA_LEN, "%s", ua);
        } else if (strcmp(field, "sitemap") == 0) {
            flush_stack(stack, &nStack, perUA, &nUA, curIdx, &nCur);
            if (cfg->nSitemaps < MAX_RULES) snprintf(cfg->sitemaps[cfg->nSitemaps++], URL_LEN, "%s", value);
        } else if (strcmp(field, "disallow") == 0 || strcmp(field, "allow") == 0 ||
                   strcmp(field, "crawl-delay") == 0) {
            flush_stack(stack, &nStack, perUA, &nUA, curIdx, &nCur);
            for (int i = 0; i < nCur; i++) {
                RuleGroup *acc = &perUA[curIdx[i]];
                if (strcmp(field, "disallow") == 0) rg_disallow(acc, value);
                else if (strcmp(field, "allow") == 0) rg_allow(acc, value);
                else {
                    char *endp = NULL;
                    double v = strtod(value, &endp);
                    acc->hasCrawlDelay = 1;
                    acc->crawlDelay = (endp && *endp == '\0') ? v : NAN;
                }
            }
        }
    }

    /* Merge agents with identical rule sets into one stacked group. */
    for (int i = 0; i < nUA; i++) {
        RuleGroup *acc = &perUA[i];
        int k = -1;
        for (int j = 0; j < cfg->nGroups; j++) {
            RuleGroup *g = &cfg->groups[j];
            if (paths_equal(g->nDisallow, g->disallow, acc->nDisallow, acc->disallow) &&
                paths_equal(g->nAllow, g->allow, acc->nAllow, acc->allow) && cd_equal(g, acc)) { k = j; break; }
        }
        if (k >= 0) {
            rg_agent(&cfg->groups[k], acc->userAgents[0]);
        } else if (cfg->nGroups < MAX_GROUPS) {
            RuleGroup *g = &cfg->groups[cfg->nGroups++];
            rg_agent(g, acc->userAgents[0]);
            for (int d = 0; d < acc->nDisallow; d++) rg_disallow(g, acc->disallow[d]);
            for (int a = 0; a < acc->nAllow; a++) rg_allow(g, acc->allow[a]);
            g->hasCrawlDelay = acc->hasCrawlDelay;
            g->crawlDelay = acc->crawlDelay;
        }
    }
}

int main(void) {
    /* Sample config -> generated body -> parse round-trip check (tool page behavior). */
    static RobotsConfig cfg, parsed;
    static char body[8192], again[8192];

    RuleGroup *g1 = &cfg.groups[cfg.nGroups++];
    rg_agent(g1, "*");
    rg_disallow(g1, "/private/");
    rg_disallow(g1, "/admin/");
    rg_allow(g1, "/admin/public/");

    RuleGroup *g2 = &cfg.groups[cfg.nGroups++];
    rg_agent(g2, "GPTBot");
    rg_agent(g2, "CCBot");
    rg_disallow(g2, "/");
    g2->hasCrawlDelay = 1;
    g2->crawlDelay = 10.0;

    snprintf(cfg.sitemaps[cfg.nSitemaps++], URL_LEN, "%s", "https://example.com/sitemap.xml");

    generate_robots(&cfg, body, sizeof body);
    fputs(body, stdout);

    parse_robots(body, &parsed);
    generate_robots(&parsed, again, sizeof again);
    printf("round-trip stable: %s\n", strcmp(again, body) == 0 ? "true" : "false");
}