robots.txt Generator — Go source

// Package robotstxt is the Go twin of CosmoDev's src/lib/robotsTxt.ts (dual
// source: the web lib is TypeScript, the CLI lib is Go — kept in lock-step).
// Pure + deterministic, never panics. The table-driven tests in
// robots-txt-generator_test.go share vectors with src/lib/robotsTxt.test.ts so
// the two implementations are held to the same contract.
//
// GenerateRobots builds standards-compliant User-agent / Allow / Disallow /
// Crawl-delay groups plus Sitemap entries; ParseRobots reads them back. Both
// mirror the TS lib's algorithm and public surface faithfully.
package robotstxt

import (
	"math"
	"regexp"
	"slices"
	"strconv"
	"strings"
)

// RuleGroup mirrors src/lib/robotsTxt.ts RuleGroup. UserAgents carries the
// REP spec's stacked User-agent lines (one group, several agents). CrawlDelay
// is a *float64 so the zero value (nil) matches the TS default
// (crawlDelay?: number | null); a non-nil non-finite value (NaN/±Inf) is
// treated like TS's Number.isFinite guard and omitted on output.
type RuleGroup struct {
	UserAgents []string // ['*'] or ['Googlebot', 'Bingbot'] - one REP group
	Disallow   []string // paths ('' means "Disallow:" → allow all); kept verbatim, NOT trimmed
	Allow      []string // whitespace-only entries dropped on output
	CrawlDelay *float64 // nil → omitted; non-finite → omitted on output
}

// RobotsConfig mirrors src/lib/robotsTxt.ts RobotsConfig.
type RobotsConfig struct {
	Groups   []RuleGroup
	Sitemaps []string
}

// multiBlank collapses runs of 3+ newlines into a single blank line, mirroring
// the TS `replace(/\n{3,}/g, '\n\n')` final normalization step.
var multiBlank = regexp.MustCompile(`\n{3,}`)

// finiteCrawlDelay reports cd's value iff it is non-nil and finite, mirroring
// the TS guard `crawlDelay !== undefined && crawlDelay !== null && Number.isFinite(crawlDelay)`.
func finiteCrawlDelay(cd *float64) (float64, bool) {
	if cd == nil {
		return 0, false
	}
	v := *cd
	if math.IsNaN(v) || math.IsInf(v, 0) {
		return 0, false
	}
	return v, true
}

// cleanAgents is the Go twin of cleanAgents() in src/lib/robotsTxt.ts: trim
// the agent list and drop empties; a non-empty list that trims away entirely
// degrades to ['*'] (a group without a User-agent line is not valid
// robots.txt); an empty list means "no group yet" and skips the group.
func cleanAgents(list []string) []string {
	trimmed := make([]string, 0, len(list))
	for _, u := range list {
		if t := strings.TrimSpace(u); t != "" {
			trimmed = append(trimmed, t)
		}
	}
	if len(trimmed) == 0 && len(list) > 0 {
		return []string{"*"}
	}
	return trimmed
}

// GenerateRobots is the Go twin of generateRobots() in src/lib/robotsTxt.ts.
// It renders a RobotsConfig into robots.txt text. Pure + deterministic.
func GenerateRobots(config RobotsConfig) string {
	var out []string
	for _, g := range config.Groups {
		uas := cleanAgents(g.UserAgents)
		if len(uas) == 0 {
			continue // skip a group whose agent list is empty
		}
		for _, ua := range uas {
			out = append(out, "User-agent: "+ua)
		}

		for _, a := range g.Allow {
			p := strings.TrimSpace(a)
			if p != "" {
				out = append(out, "Allow: "+p)
			}
		}

		if len(g.Disallow) == 0 {
			out = append(out, "Disallow:")
		} else {
			// Disallow paths are kept verbatim (NOT trimmed) — '' → "Disallow: ".
			for _, d := range g.Disallow {
				out = append(out, "Disallow: "+d)
			}
		}

		if v, ok := finiteCrawlDelay(g.CrawlDelay); ok {
			// FormatFloat with prec -1 mirrors JS `${number}`: 10 → "10", 7.5 → "7.5".
			out = append(out, "Crawl-delay: "+strconv.FormatFloat(v, 'f', -1, 64))
		}

		out = append(out, "") // blank line separating groups / trailing
	}

	for _, s := range config.Sitemaps {
		url := strings.TrimSpace(s)
		if url != "" {
			out = append(out, "Sitemap: "+url)
		}
	}

	joined := strings.Join(out, "\n")
	joined = multiBlank.ReplaceAllString(joined, "\n\n")
	// trimEnd() + '\n' — strip all trailing whitespace, then restore a single newline.
	return strings.TrimRight(joined, "\n\r\t\v\f ") + "\n"
}

// crawlDelayEq reports whether two CrawlDelay values match the TS bucket key
// `crawlDelay ?? null` — nil equals nil, and non-nil compares by value.
func crawlDelayEq(a, b *float64) bool {
	if a == nil || b == nil {
		return a == nil && b == nil
	}
	return *a == *b
}

// ParseRobots is the Go twin of parseRobots() in src/lib/robotsTxt.ts. It reads
// robots.txt text into a RobotsConfig, tolerating comments, blank lines, and
// empty input without error. Pure + deterministic.
//
// Rules attach PER AGENT (Google's merge semantics): a rule line applies to
// every agent of the immediately preceding consecutive User-agent stack. At
// the end, agents whose accumulated rule sets are identical are emitted as one
// stacked-agent group, so the common (A, B share everything) case stays
// compact while partially-overlapping blocks regroup losslessly.
func ParseRobots(text string) RobotsConfig {
	config := RobotsConfig{}

	type acc struct {
		disallow   []string
		allow      []string
		crawlDelay *float64
	}
	perUA := map[string]*acc{}
	var order []string
	var stack []string
	var current []string

	flush := func() {
		if len(stack) == 0 {
			return
		}
		for _, ua := range stack {
			if _, ok := perUA[ua]; !ok {
				perUA[ua] = &acc{}
				order = append(order, ua)
			}
		}
		current = stack
		stack = nil
	}

	for _, rawLine := range strings.Split(text, "\n") {
		// Strip comments: rawLine.replace(/#.*$/, '').
		if h := strings.IndexByte(rawLine, '#'); h >= 0 {
			rawLine = rawLine[:h]
		}
		line := strings.TrimSpace(rawLine)
		if line == "" {
			continue
		}
		// First-colon split (line.slice(0, colon) / line.slice(colon+1)).
		fieldRaw, value, found := strings.Cut(line, ":")
		if !found {
			continue
		}
		field := strings.ToLower(strings.TrimSpace(fieldRaw))
		value = strings.TrimSpace(value)

		switch field {
		case "user-agent":
			ua := value
			if ua == "" {
				ua = "*" // value || '*'
			}
			if !slices.Contains(stack, ua) {
				stack = append(stack, ua)
			}
		case "disallow":
			flush()
			for _, ua := range current {
				a := perUA[ua]
				a.disallow = append(a.disallow, value)
			}
		case "allow":
			flush()
			for _, ua := range current {
				a := perUA[ua]
				a.allow = append(a.allow, value)
			}
		case "crawl-delay":
			flush()
			cd, err := strconv.ParseFloat(value, 64)
			if err != nil {
				cd = math.NaN() // mirror Number(value) → NaN for garbage input
			}
			for _, ua := range current {
				c := cd // fresh copy per agent — each acc owns its pointer
				perUA[ua].crawlDelay = &c
			}
		case "sitemap":
			flush() // a Sitemap line is groupless; the last stack keeps receiving rules
			config.Sitemaps = append(config.Sitemaps, value)
		}
	}
	flush()

	// Regroup: agents whose accumulated rule sets are identical share one
	// stacked-agent group, buckets created in first-appearance order (mirrors
	// the TS Map<key, RuleGroup> bucket pass).
	var groups []RuleGroup
	for _, ua := range order {
		a := perUA[ua]
		merged := false
		for i := range groups {
			g := &groups[i]
			if crawlDelayEq(g.CrawlDelay, a.crawlDelay) &&
				slices.Equal(g.Disallow, a.disallow) &&
				slices.Equal(g.Allow, a.allow) {
				g.UserAgents = append(g.UserAgents, ua)
				merged = true
				break
			}
		}
		if !merged {
			groups = append(groups, RuleGroup{
				UserAgents: []string{ua},
				Disallow:   a.disallow,
				Allow:      a.allow,
				CrawlDelay: a.crawlDelay,
			})
		}
	}

	config.Groups = groups
	return config
}