# https://steeped.ai/robots.txt
#
# Posture: open to everything that can send traffic or citations back here.
# Steeped AI is a small site trying to be found, so the cost of being crawled
# is far lower than the cost of being invisible in an AI answer.

# ---------------------------------------------------------------- default --
User-agent: *
Allow: /

# Internal and utility pages. These are also marked noindex in the page head;
# the disallow just saves crawl budget.
Disallow: /test/
Disallow: /article-template/
Disallow: /smart-sheet-unsub/
Disallow: /super-sheet-portal/

# Query strings create endless duplicate URLs with no unique content.
Disallow: /*?*utm_
Disallow: /*?*fbclid
Disallow: /*?*gclid

# ------------------------------------------------------- AI answer engines --
# These are the crawlers that decide whether Steeped AI can be cited in an
# answer. They are listed explicitly so the intent survives any future edit to
# the wildcard block above.

# OpenAI: GPTBot trains, OAI-SearchBot indexes for ChatGPT Search,
# ChatGPT-User fetches a page when someone pastes or asks for the link.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI Overviews and Gemini grounding. Blocking Google-Extended removes
# the site from AI Overviews without any ranking benefit, so it stays allowed.
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft Copilot rides on Bingbot, which the wildcard already covers.
User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://steeped.ai/sitemap.xml