# Axtra — robots
#
# Public marketing pages are indexable. Authenticated app routes
# (/dashboard, /day, /todos, /inbox, /routines, /templates, /settings,
# /quick-mark, /feedback, /plans, /dock, /getting-started, /insights,
# /review) sit behind auth — they redirect to /auth for anyone without
# a session, so we explicitly disallow them to keep crawl budget on the
# pages that actually serve content.

User-agent: *
Allow: /
Allow: /auth
Allow: /privacy
Allow: /terms
Disallow: /dashboard
Disallow: /day
Disallow: /todos
Disallow: /inbox
Disallow: /routines
Disallow: /templates
Disallow: /settings
Disallow: /plans
Disallow: /dock
Disallow: /getting-started
Disallow: /insights
Disallow: /review
Disallow: /feedback
Disallow: /quick-mark

# Be explicit for the big two — same disallow list, named so future
# debugging is easier when crawler activity shows up in server logs.
User-agent: Googlebot
Allow: /
Disallow: /dashboard
Disallow: /day
Disallow: /todos
Disallow: /inbox
Disallow: /routines
Disallow: /templates
Disallow: /settings
Disallow: /plans
Disallow: /dock
Disallow: /getting-started
Disallow: /insights
Disallow: /review
Disallow: /feedback
Disallow: /quick-mark

User-agent: Bingbot
Allow: /

# Social-media link-unfurl scrapers — they need to read meta tags +
# fetch og-image.png. Allow everything; the disallow list above doesn't
# matter to them since they only crawl on demand.
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# AI crawlers — currently allowed. Flip to Disallow to opt out of
# LLM training data; Anthropic / OpenAI / Common Crawl generally honor
# robots.txt. Worth revisiting if Axtra ever has unique content
# (case studies, blog) that shouldn't be scraped wholesale.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://axtra.app/sitemap.xml