# sms-24.net — AI Worker Platform for messaging, voice, and phone automation
# https://sms-24.net
#
# We WANT to be found in AI search (Perplexity, ChatGPT browse,
# Claude with web, You.com, Phind, Kagi).  Their crawlers are
# explicitly allow-listed below.  See /llms.txt for the LLM-friendly
# product description.

User-agent: *
Allow: /
Allow: /pricing
Allow: /register
Allow: /login
Allow: /landing
Allow: /features
Allow: /use-cases
Allow: /blog
Allow: /blog/
Allow: /legal/
Allow: /docs/
Allow: /llms.txt
Allow: /.well-known/
Allow: /favicon.svg
Allow: /favicon.ico
Allow: /apple-touch-icon.png
Allow: /og-cover.png
Allow: /manifest.webmanifest

# Authenticated app surface — no indexable content behind login
Disallow: /dashboard
Disallow: /numbers
Disallow: /messages
Disallow: /analytics
Disallow: /graph
Disallow: /antifraud
Disallow: /threading
Disallow: /settings
Disallow: /users
Disallow: /parsers
Disallow: /proxies
Disallow: /fingerprints
Disallow: /accounts
Disallow: /billing
Disallow: /account/
Disallow: /admin/
Disallow: /me/

# API surfaces — never indexable
Disallow: /api/
Disallow: /v1/
Disallow: /ws/
Disallow: /graphql
Disallow: /metrics
Disallow: /health

# NOTE: /assets/ is intentionally NOT disallowed.
# Google needs to fetch the hashed JS + CSS bundles to render the SPA
# and pick up the JSON-LD / meta tags injected by the prerender. The
# Rich Results Test fails ("2 of 3 page resources blocked by robots")
# when /assets/ is forbidden. Allow it explicitly so the renderer
# works; the content-hashed filenames already discourage cache spam.
Allow: /assets/

# ── AI search engines: EXPLICITLY ALLOWED ─────────────────────
# We want our product to appear in AI-generated answers.  See
# /llms.txt for the structured product overview.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: YouBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: KagiBot
Allow: /

# ── Abusive SEO scrapers: still blocked ───────────────────────
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: BLEXBot
Disallow: /

Sitemap: https://sms-24.net/sitemap.xml