# gtm.help — crawling policy
# Search crawlers, social unfurlers and AI answer engines are all welcome.
# Grouped user-agents share the rule block that follows them.

User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /auth
Disallow: /verify
Disallow: /reset-password
Disallow: /recap/

# AI assistants and answer engines. These read pages to answer questions
# and cite sources, so they get the same access as search.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: YouBot
User-agent: Bytespider
User-agent: Omgilibot
User-agent: Omgili
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /auth
Disallow: /verify
Disallow: /reset-password
Disallow: /recap/

# MCP registries read the agent server's tool list and server card.
User-agent: Smithery
User-agent: SmitheryBot
Allow: /mcp
Allow: /mcp.json
Allow: /.well-known/

# The signed-in app is auth-gated and noindex; keep crawlers on public pages.
# /recap/ share links contain a real business's numbers — shareable, not indexable.
User-agent: *
Allow: /
Allow: /latest
Allow: /vs/
Disallow: /admin
Disallow: /api/
Disallow: /auth
Disallow: /verify
Disallow: /reset-password
Disallow: /recap/

Sitemap: https://gtm.help/sitemap.xml