robots.txt Generator — Swift source

// robots-txt-generator — Swift port: standards-compliant robots.txt generator + parser.
//
// Display port of the CosmoDev Robots.txt Generator tool — same contract as
// cli/robots-txt-generator/robots-txt-generator.go (the live Go twin) and
// src/lib/robotsTxt.ts (canonical TypeScript). Pure + deterministic; stdlib
// only. A group carries a LIST of user-agents (the REP spec's stacked
// User-agent lines); parsing aggregates repeated blocks per agent, then
// re-merges agents whose accumulated rules are identical into one stacked
// group, exactly like the TS lib.
import Foundation

public struct RuleGroup: Equatable {
    /// `["*"]` or `["Googlebot", "Bingbot"]` — one REP group.
    public var userAgents: [String]
    /// Paths to disallow; `""` renders as a bare `Disallow:`.
    public var disallow: [String]
    /// Paths to allow.
    public var allow: [String]
    /// Seconds between requests; `nil` = unset, non-finite never emitted.
    public var crawlDelay: Double?

    public init(userAgents: [String], disallow: [String] = [], allow: [String] = [], crawlDelay: Double? = nil) {
        self.userAgents = userAgents
        self.disallow = disallow
        self.allow = allow
        self.crawlDelay = crawlDelay
    }
}

public struct RobotsConfig: Equatable {
    /// Ordered rule groups.
    public var groups: [RuleGroup]
    /// Sitemap URLs, emitted after the groups.
    public var sitemaps: [String]

    public init(groups: [RuleGroup] = [], sitemaps: [String] = []) {
        self.groups = groups
        self.sitemaps = sitemaps
    }
}

public enum RobotsTxt {
    // cleanAgents in the TS lib: trim + drop blanks. A non-empty list that
    // trims away entirely degrades to ["*"]; an empty list means "no group yet".
    private static func cleanAgents(_ list: [String]) -> [String] {
        let trimmed = list.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
        return trimmed.isEmpty && !list.isEmpty ? ["*"] : trimmed
    }

    // JS-style number rendering for Crawl-delay: 10.0 -> "10", 1.5 -> "1.5".
    private static func numStr(_ f: Double) -> String {
        f == f.rounded() ? String(Int64(f)) : String(f)
    }

    public static func generateRobots(_ config: RobotsConfig) -> String {
        var out: [String] = []
        for g in config.groups {
            let uas = cleanAgents(g.userAgents)
            if uas.isEmpty { continue }
            uas.forEach { out.append("User-agent: \($0)") }
            for a in g.allow {
                let p = a.trimmingCharacters(in: .whitespacesAndNewlines)
                if !p.isEmpty { out.append("Allow: \(p)") }
            }
            if g.disallow.isEmpty {
                out.append("Disallow:") // no entries -> the allow-all marker
            } else {
                g.disallow.forEach { out.append("Disallow: \($0)") } // "" kept verbatim
            }
            if let cd = g.crawlDelay, cd.isFinite {
                out.append("Crawl-delay: \(numStr(cd))")
            }
            out.append("") // blank line separates groups
        }
        for s in config.sitemaps {
            let url = s.trimmingCharacters(in: .whitespacesAndNewlines)
            if !url.isEmpty { out.append("Sitemap: \(url)") }
        }
        let joined = out.joined(separator: "\n")
        return joined.replacingOccurrences(of: #"\n{3,}"#, with: "\n\n", options: .regularExpression)
            .trimmingCharacters(in: .whitespacesAndNewlines) + "\n"
    }

    private final class Acc {
        let ua: String
        var disallow: [String] = []
        var allow: [String] = []
        var crawlDelay: Double?
        init(_ ua: String) { self.ua = ua }
    }

    // ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
    // (values may contain colons), unknown directives ignored. Double(value)
    // yields nil for garbage where JS Number() yields NaN — both are never
    // emitted (the TS lib drops non-finite delays).
    public static func parseRobots(_ text: String?) -> RobotsConfig {
        var perUA: [(ua: String, acc: Acc)] = [] // insertion-ordered
        var sitemaps: [String] = []
        var stack: [String] = []
        var current: [Acc] = []

        func flush() { // a rule line applies to every agent of the pending stack
            guard !stack.isEmpty else { return }
            current = stack.map { ua in
                if let hit = perUA.first(where: { $0.ua == ua }) { return hit.acc }
                let acc = Acc(ua)
                perUA.append((ua, acc))
                return acc
            }
            stack = []
        }

        for raw in (text ?? "").split(separator: "\n", omittingEmptySubsequences: false).map(String.init) {
            let stripped = raw.replacingOccurrences(of: #"[#].*$"#, with: "", options: .regularExpression)
            let line = stripped.trimmingCharacters(in: .whitespaces)
            if line.isEmpty { continue }
            guard let colon = line.firstIndex(of: ":") else { continue }
            let field = line[..<colon].trimmingCharacters(in: .whitespaces).lowercased()
            let value = String(line[line.index(after: colon)...]).trimmingCharacters(in: .whitespaces)

            switch field {
            case "user-agent":
                let ua = value.isEmpty ? "*" : value
                if !stack.contains(ua) { stack.append(ua) }
            case "disallow": flush(); current.forEach { $0.disallow.append(value) }
            case "allow": flush(); current.forEach { $0.allow.append(value) }
            case "crawl-delay": flush(); current.forEach { $0.crawlDelay = Double(value) }
            case "sitemap": flush(); sitemaps.append(value)
            default: break
            }
        }

        // Merge agents with identical rule sets into one stacked group.
        var groups: [RuleGroup] = []
        for entry in perUA {
            let acc = entry.acc
            if let i = groups.firstIndex(where: {
                $0.disallow == acc.disallow && $0.allow == acc.allow && $0.crawlDelay == acc.crawlDelay
            }) {
                groups[i].userAgents.append(entry.ua)
            } else {
                groups.append(RuleGroup(userAgents: [entry.ua], disallow: acc.disallow, allow: acc.allow, crawlDelay: acc.crawlDelay))
            }
        }
        return RobotsConfig(groups: groups, sitemaps: sitemaps)
    }
}

// Sample config -> generated body -> parse round-trip check (tool page behavior).
let config = RobotsConfig(
    groups: [
        RuleGroup(userAgents: ["*"], disallow: ["/private/", "/admin/"], allow: ["/admin/public/"]),
        RuleGroup(userAgents: ["GPTBot", "CCBot"], disallow: ["/"], crawlDelay: 10),
    ],
    sitemaps: ["https://example.com/sitemap.xml"]
)
let body = RobotsTxt.generateRobots(config)
print(body)
print("round-trip stable: \(RobotsTxt.generateRobots(RobotsTxt.parseRobots(body)) == body)")