robots.txt Generator — Kotlin source

// robots-txt-generator — Kotlin port: standards-compliant robots.txt generator + parser.
//
// Display port of the CosmoDev Robots.txt Generator tool — same contract as
// cli/robots-txt-generator/robots-txt-generator.go (the live Go twin) and
// src/lib/robotsTxt.ts (canonical TypeScript). Pure + deterministic; stdlib
// only. A group carries a LIST of user-agents (the REP spec's stacked
// User-agent lines); parsing aggregates repeated blocks per agent, then
// re-merges agents whose accumulated rules are identical into one stacked
// group, exactly like the TS lib.
import kotlin.math.floor

data class RuleGroup(
    val userAgents: List<String>,               // ["*"] or ["Googlebot", "Bingbot"]
    val disallow: List<String> = emptyList(),   // "" renders as a bare "Disallow:"
    val allow: List<String> = emptyList(),
    val crawlDelay: Double? = null,             // null = unset; non-finite never emitted
)

data class RobotsConfig(
    val groups: List<RuleGroup> = emptyList(),
    val sitemaps: List<String> = emptyList(),
)

object RobotsTxt {
    // cleanAgents in the TS lib: trim + drop blanks. A non-empty list that
    // trims away entirely degrades to ["*"]; an empty list means "no group yet".
    private fun cleanAgents(list: List<String>): List<String> {
        val trimmed = list.map { it.trim() }.filter { it.isNotEmpty() }
        return if (trimmed.isEmpty() && list.isNotEmpty()) listOf("*") else trimmed
    }

    // JS-style number rendering for Crawl-delay: 10.0 -> "10", 1.5 -> "1.5".
    private fun numStr(f: Double): String =
        if (f == floor(f)) f.toLong().toString() else f.toString()

    fun generateRobots(config: RobotsConfig): String {
        val out = mutableListOf<String>()
        for (g in config.groups) {
            val uas = cleanAgents(g.userAgents)
            if (uas.isEmpty()) continue
            uas.forEach { out += "User-agent: $it" }
            for (a in g.allow) {
                val p = a.trim()
                if (p.isNotEmpty()) out += "Allow: $p"
            }
            if (g.disallow.isEmpty()) {
                out += "Disallow:" // no entries -> the allow-all marker
            } else {
                g.disallow.forEach { out += "Disallow: $it" } // "" kept verbatim
            }
            val cd = g.crawlDelay
            if (cd != null && cd.isFinite()) out += "Crawl-delay: ${numStr(cd)}"
            out += "" // blank line separates groups
        }
        for (s in config.sitemaps) {
            val url = s.trim()
            if (url.isNotEmpty()) out += "Sitemap: $url"
        }
        return out.joinToString("\n").replace(Regex("\n{3,}"), "\n\n").trimEnd() + "\n"
    }

    private class Acc(val ua: String) {
        val disallow = mutableListOf<String>()
        val allow = mutableListOf<String>()
        var crawlDelay: Double? = null
    }

    // ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
    // (values may contain colons), unknown directives ignored. toDoubleOrNull
    // yields null for garbage where JS Number() yields NaN — both are never
    // emitted (the TS lib drops non-finite delays).
    fun parseRobots(text: String?): RobotsConfig {
        val perUA = LinkedHashMap<String, Acc>() // insertion-ordered
        val sitemaps = mutableListOf<String>()
        var stack = mutableListOf<String>()
        var current = listOf<Acc>()

        fun flush() { // a rule line applies to every agent of the pending stack
            if (stack.isEmpty()) return
            current = stack.map { ua -> perUA.getOrPut(ua) { Acc(ua) } }
            stack = mutableListOf()
        }

        for (raw in (text ?: "").split('\n')) {
            val line = raw.replace(Regex("#.*$"), "").trim()
            if (line.isEmpty()) continue
            val colon = line.indexOf(':')
            if (colon == -1) continue
            val field = line.substring(0, colon).trim().lowercase()
            val value = line.substring(colon + 1).trim()
            when (field) {
                "user-agent" -> {
                    val ua = value.ifEmpty { "*" }
                    if (ua !in stack) stack += ua
                }
                "disallow" -> { flush(); current.forEach { it.disallow += value } }
                "allow" -> { flush(); current.forEach { it.allow += value } }
                "crawl-delay" -> { flush(); val v = value.toDoubleOrNull(); current.forEach { it.crawlDelay = v } }
                "sitemap" -> { flush(); sitemaps += value }
            }
        }

        // Merge agents with identical rule sets into one stacked group.
        val groups = mutableListOf<RuleGroup>()
        for (acc in perUA.values) {
            val twin = groups.find { it.disallow == acc.disallow && it.allow == acc.allow && it.crawlDelay == acc.crawlDelay }
            if (twin != null) {
                groups[groups.indexOf(twin)] = twin.copy(userAgents = twin.userAgents + acc.ua)
            } else {
                groups += RuleGroup(listOf(acc.ua), acc.disallow.toList(), acc.allow.toList(), acc.crawlDelay)
            }
        }
        return RobotsConfig(groups, sitemaps)
    }
}

// Sample config -> generated body -> parse round-trip check (tool page behavior).
fun main() {
    val config = RobotsConfig(
        groups = listOf(
            RuleGroup(listOf("*"), disallow = listOf("/private/", "/admin/"), allow = listOf("/admin/public/")),
            RuleGroup(listOf("GPTBot", "CCBot"), disallow = listOf("/"), crawlDelay = 10.0),
        ),
        sitemaps = listOf("https://example.com/sitemap.xml"),
    )
    val body = RobotsTxt.generateRobots(config)
    print(body)
    println("round-trip stable: ${RobotsTxt.generateRobots(RobotsTxt.parseRobots(body)) == body}")
}