robots.txt Generator — Java source

// robots-txt-generator — Java port: standards-compliant robots.txt generator + parser.
//
// Display port of the CosmoDev Robots.txt Generator tool — same contract as
// cli/robots-txt-generator/robots-txt-generator.go (the live Go twin) and
// src/lib/robotsTxt.ts (canonical TypeScript). Pure + deterministic; JDK only.
// A group carries a LIST of user-agents (the REP spec's stacked User-agent
// lines); parsing aggregates repeated blocks per agent, then re-merges agents
// whose accumulated rules are identical into one stacked group, like the TS lib.
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;

final class RobotsTxtGenerator {
    // Package-private: the polyglot display filename is java.java, so the
    // top-level class cannot be public (JLS §7.6 filename rule).
    private RobotsTxtGenerator() {}

    /** One REP group — mirrors {@code RuleGroup} in the Go twin. */
    public static final class RuleGroup {
        /** {@code ["*"]} or {@code ["Googlebot", "Bingbot"]} — stacked agents. */
        public final List<String> userAgents;
        /** Paths; an empty element renders as a bare {@code "Disallow:"}. */
        public final List<String> disallow;
        /** Paths to allow. */
        public final List<String> allow;
        /** Seconds between requests; {@code null} = unset, non-finite never emitted. */
        public final Double crawlDelay;

        public RuleGroup(List<String> userAgents, List<String> disallow, List<String> allow, Double crawlDelay) {
            this.userAgents = userAgents;
            this.disallow = disallow;
            this.allow = allow;
            this.crawlDelay = crawlDelay;
        }
    }

    /** Full document: ordered rule groups + sitemap URLs. */
    public static final class RobotsConfig {
        public final List<RuleGroup> groups;
        public final List<String> sitemaps;

        public RobotsConfig(List<RuleGroup> groups, List<String> sitemaps) {
            this.groups = groups;
            this.sitemaps = sitemaps;
        }
    }

    private static final Pattern COMMENT = Pattern.compile("#.*$");
    private static final Pattern BLANK_RUN = Pattern.compile("\n{3,}");

    // cleanAgents in the TS lib: trim + drop blanks. A non-empty list that
    // trims away entirely degrades to ["*"]; an empty list means "no group yet".
    private static List<String> cleanAgents(List<String> list) {
        List<String> out = new ArrayList<>();
        for (String u : list) {
            String t = u.trim();
            if (!t.isEmpty()) out.add(t);
        }
        if (out.isEmpty() && !list.isEmpty()) out.add("*");
        return out;
    }

    // JS-style number rendering for Crawl-delay: 10.0 -> "10", 1.5 -> "1.5".
    private static String numStr(double f) {
        return f == Math.floor(f) ? String.valueOf((long) f) : String.valueOf(f);
    }

    public static String generateRobots(RobotsConfig config) {
        List<String> out = new ArrayList<>();
        for (RuleGroup g : config.groups) {
            List<String> uas = cleanAgents(g.userAgents);
            if (uas.isEmpty()) continue;
            for (String ua : uas) out.add("User-agent: " + ua);
            for (String a : g.allow) {
                String p = a.trim();
                if (!p.isEmpty()) out.add("Allow: " + p);
            }
            if (g.disallow.isEmpty()) {
                out.add("Disallow:"); // no entries -> the allow-all marker
            } else {
                for (String d : g.disallow) out.add("Disallow: " + d); // "" kept verbatim
            }
            if (g.crawlDelay != null && Double.isFinite(g.crawlDelay)) {
                out.add("Crawl-delay: " + numStr(g.crawlDelay));
            }
            out.add(""); // blank line separates groups
        }
        for (String s : config.sitemaps) {
            String url = s.trim();
            if (!url.isEmpty()) out.add("Sitemap: " + url);
        }
        return BLANK_RUN.matcher(String.join("\n", out)).replaceAll("\n\n").stripTrailing() + "\n";
    }

    private static final class Acc {
        final String ua;
        final List<String> disallow = new ArrayList<>(), allow = new ArrayList<>();
        Double crawlDelay;
        Acc(String ua) { this.ua = ua; }
    }

    // Moves the pending User-agent stack into per-agent rule buckets (Google's
    // merge semantics: a rule line applies to every agent of the immediately
    // preceding consecutive User-agent stack).
    private static List<Acc> flush(List<String> stack, LinkedHashMap<String, Acc> perUA) {
        List<Acc> current = new ArrayList<>();
        for (String ua : stack) current.add(perUA.computeIfAbsent(ua, k -> new Acc(k)));
        stack.clear();
        return current;
    }

    // ParseRobots in the Go twin: strip '#'-comments, split on the FIRST ':'
    // (values may contain colons), unknown directives ignored. Garbage
    // Crawl-delay values coerce to NaN like JS Number() — never re-emitted.
    public static RobotsConfig parseRobots(String text) {
        LinkedHashMap<String, Acc> perUA = new LinkedHashMap<>(); // insertion-ordered
        List<String> stack = new ArrayList<>(), sitemaps = new ArrayList<>();
        List<Acc> current = new ArrayList<>();

        for (String raw : (text == null ? "" : text).split("\n", -1)) {
            String line = COMMENT.matcher(raw).replaceAll("").trim();
            if (line.isEmpty()) continue;
            int colon = line.indexOf(':');
            if (colon < 0) continue;
            String field = line.substring(0, colon).trim().toLowerCase();
            String value = line.substring(colon + 1).trim();

            switch (field) {
                case "user-agent" -> {
                    String ua = value.isEmpty() ? "*" : value;
                    if (!stack.contains(ua)) stack.add(ua);
                }
                case "disallow" -> { if (!stack.isEmpty()) current = flush(stack, perUA); current.forEach(a -> a.disallow.add(value)); }
                case "allow" -> { if (!stack.isEmpty()) current = flush(stack, perUA); current.forEach(a -> a.allow.add(value)); }
                case "crawl-delay" -> {
                    if (!stack.isEmpty()) current = flush(stack, perUA);
                    double v;
                    try { v = Double.parseDouble(value); } catch (NumberFormatException e) { v = Double.NaN; }
                    final double fv = v;
                    current.forEach(a -> a.crawlDelay = fv);
                }
                case "sitemap" -> { if (!stack.isEmpty()) current = flush(stack, perUA); sitemaps.add(value); }
                default -> { }
            }
        }

        // Merge agents with identical rule sets into one stacked group.
        List<RuleGroup> groups = new ArrayList<>();
        for (Map.Entry<String, Acc> e : perUA.entrySet()) {
            Acc acc = e.getValue();
            RuleGroup twin = null;
            for (RuleGroup g : groups) {
                if (g.disallow.equals(acc.disallow) && g.allow.equals(acc.allow)
                        && Objects.equals(g.crawlDelay, acc.crawlDelay)) { twin = g; break; }
            }
            if (twin != null) twin.userAgents.add(e.getKey()); // lists are mutable for the merge
            else groups.add(new RuleGroup(new ArrayList<>(List.of(e.getKey())), new ArrayList<>(acc.disallow),
                                          new ArrayList<>(acc.allow), acc.crawlDelay));
        }
        return new RobotsConfig(groups, sitemaps);
    }

    public static void main(String[] args) {
        // Sample config -> generated body -> parse round-trip check (tool page behavior).
        RobotsConfig config = new RobotsConfig(List.of(
            new RuleGroup(new ArrayList<>(List.of("*")), new ArrayList<>(List.of("/private/", "/admin/")),
                          new ArrayList<>(List.of("/admin/public/")), null),
            new RuleGroup(new ArrayList<>(List.of("GPTBot", "CCBot")), new ArrayList<>(List.of("/")),
                          new ArrayList<>(), 10.0)),
            List.of("https://example.com/sitemap.xml"));
        String body = generateRobots(config);
        System.out.print(body);
        System.out.println("round-trip stable: " + generateRobots(parseRobots(body)).equals(body));
    }
}