robots.txt Generator — C# source

// robots-txt-generator — C# port: standards-compliant robots.txt generator + parser.
//
// Display port of the CosmoDev Robots.txt Generator tool — same contract as
// cli/robots-txt-generator/robots-txt-generator.go (the live Go twin) and
// src/lib/robotsTxt.ts (canonical TypeScript). Pure + deterministic; BCL only.
// A group carries a LIST of user-agents (the REP spec's stacked User-agent
// lines); parsing aggregates repeated blocks per agent, then re-merges agents
// whose accumulated rules are identical into one stacked group, like the TS lib.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;

namespace RobotsTxtGenerator;

/// <summary>One REP group — mirrors <c>RuleGroup</c> in the Go twin.</summary>
public sealed record RuleGroup(
    IReadOnlyList<string> UserAgents,          // ["*"] or ["Googlebot", "Bingbot"]
    IReadOnlyList<string> Disallow,            // "" renders as a bare "Disallow:"
    IReadOnlyList<string> Allow,
    double? CrawlDelay = null);                // null = unset; non-finite never emitted

public sealed record RobotsConfig(
    IReadOnlyList<RuleGroup> Groups,
    IReadOnlyList<string> Sitemaps);

public static class RobotsTxt
{
    // cleanAgents in the TS lib: trim + drop blanks. A non-empty list that
    // trims away entirely degrades to ["*"]; an empty list means "no group yet".
    private static List<string> CleanAgents(IReadOnlyList<string> list)
    {
        var trimmed = list.Select(u => u.Trim()).Where(u => u.Length > 0).ToList();
        return trimmed.Count == 0 && list.Count > 0 ? new List<string> { "*" } : trimmed;
    }

    // JS-style number rendering for Crawl-delay: 10.0 -> "10", 1.5 -> "1.5".
    private static string NumStr(double f) => f == Math.Floor(f) ? ((long)f).ToString() : f.ToString();

    public static string GenerateRobots(RobotsConfig config)
    {
        var lines = new List<string>();
        foreach (var g in config.Groups)
        {
            var uas = CleanAgents(g.UserAgents);
            if (uas.Count == 0) continue;
            uas.ForEach(ua => lines.Add($"User-agent: {ua}"));
            foreach (var a in g.Allow)
            {
                var p = a.Trim();
                if (p.Length > 0) lines.Add($"Allow: {p}");
            }
            if (g.Disallow.Count == 0)
            {
                lines.Add("Disallow:"); // no entries -> the allow-all marker
            }
            else
            {
                foreach (var d in g.Disallow) lines.Add($"Disallow: {d}"); // "" kept verbatim
            }
            if (g.CrawlDelay is double cd && double.IsFinite(cd))
                lines.Add($"Crawl-delay: {NumStr(cd)}");
            lines.Add(""); // blank line separates groups
        }
        foreach (var s in config.Sitemaps)
        {
            var url = s.Trim();
            if (url.Length > 0) lines.Add($"Sitemap: {url}");
        }
        return Regex.Replace(string.Join('\n', lines), @"\n{3,}", "\n\n").TrimEnd() + "\n";
    }

    private sealed class Acc
    {
        public readonly string Ua;
        public readonly List<string> Disallow = new(), Allow = new();
        public double? CrawlDelay;
        public Acc(string ua) { Ua = ua; }
    }

    // ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
    // (values may contain colons), unknown directives ignored. TryParse yields
    // null for garbage where JS Number() yields NaN — both are never emitted.
    public static RobotsConfig ParseRobots(string? text)
    {
        var perUA = new List<Acc>(); // insertion-ordered
        var sitemaps = new List<string>();
        var stack = new List<string>();
        var current = new List<Acc>();

        void Flush() // a rule line applies to every agent of the pending stack
        {
            if (stack.Count == 0) return;
            current = stack.Select(ua =>
            {
                var hit = perUA.Find(a => a.Ua == ua);
                if (hit is null) { hit = new Acc(ua); perUA.Add(hit); }
                return hit;
            }).ToList();
            stack.Clear();
        }

        foreach (var raw in (text ?? "").Split('\n'))
        {
            var line = Regex.Replace(raw, @"#.*$", "").Trim();
            if (line.Length == 0) continue;
            var colon = line.IndexOf(':');
            if (colon < 0) continue;
            var field = line[..colon].Trim().ToLowerInvariant();
            var value = line[(colon + 1)..].Trim();

            switch (field)
            {
                case "user-agent":
                    var ua = value.Length > 0 ? value : "*";
                    if (!stack.Contains(ua)) stack.Add(ua);
                    break;
                case "disallow":
                    Flush(); current.ForEach(a => a.Disallow.Add(value)); break;
                case "allow":
                    Flush(); current.ForEach(a => a.Allow.Add(value)); break;
                case "crawl-delay":
                    Flush(); // Number() semantics: garbage -> NaN (never emitted)
                    current.ForEach(a => a.CrawlDelay = double.TryParse(value, out var v) ? v : double.NaN);
                    break;
                case "sitemap":
                    Flush(); sitemaps.Add(value); break;
            }
        }

        // Merge agents with identical rule sets into one stacked group.
        var groups = new List<RuleGroup>();
        foreach (var acc in perUA)
        {
            var i = groups.FindIndex(g => g.Disallow.SequenceEqual(acc.Disallow)
                                       && g.Allow.SequenceEqual(acc.Allow) && g.CrawlDelay == acc.CrawlDelay);
            if (i >= 0) groups[i] = groups[i] with { UserAgents = groups[i].UserAgents.Append(acc.Ua).ToList() };
            else groups.Add(new RuleGroup(new[] { acc.Ua }, acc.Disallow, acc.Allow, acc.CrawlDelay));
        }
        return new RobotsConfig(groups, sitemaps);
    }
}

public static class Program
{
    public static void Main()
    {
        // Sample config -> generated body -> parse round-trip check (tool page behavior).
        var config = new RobotsConfig(
            new RuleGroup[]
            {
                new(new[] { "*" }, new[] { "/private/", "/admin/" }, new[] { "/admin/public/" }),
                new(new[] { "GPTBot", "CCBot" }, new[] { "/" }, Array.Empty<string>(), 10.0),
            },
            new[] { "https://example.com/sitemap.xml" });
        var body = RobotsTxt.GenerateRobots(config);
        Console.Write(body);
        Console.WriteLine($"round-trip stable: {RobotsTxt.GenerateRobots(RobotsTxt.ParseRobots(body)) == body}");
    }
}