Sitemap: https://www.glean.com/sitemap.xml

User-agent: *
Disallow: /internal/
Disallow: /new-pages/

# Block search engines from llm-info
User-agent: Googlebot
Disallow: /llm-info

User-agent: Bingbot
Disallow: /llm-info

User-agent: Slurp
Disallow: /llm-info

User-agent: DuckDuckBot
Disallow: /llm-info

User-agent: Baiduspider
Disallow: /llm-info

User-agent: YandexBot
Disallow: /llm-info

# Allow AI / LLM crawlers
User-agent: GPTBot
Allow: /llm-info

User-agent: ChatGPT-User
Allow: /llm-info

User-agent: ClaudeBot
Allow: /llm-info

User-agent: Claude-Web
Allow: /llm-info

User-agent: PerplexityBot
Allow: /llm-info

User-agent: Amazonbot
Allow: /llm-info

User-agent: Bytespider
Allow: /llm-info

Sitemap: https://www.glean.com/sitemap.xml