# ============================================
# ELAV8 AI - Robots Configuration
# ============================================
# Strategy: Block search engine indexing, allow social previews.
# Indexing is enabled page-by-page as content is ready.
#
# See: src/config/seo.ts for per-page indexing control
# See: docs/SEO.md for full SEO strategy documentation
# ============================================
# ──────────────────────────────────────────────
# SOCIAL BOTS - Always allowed (link previews)
# ──────────────────────────────────────────────
# These crawlers read OG/meta tags to generate link previews
# in messaging apps and social platforms. Always allow them.
User-agent: Twitterbot
Allow: /
User-agent: facebookexternalhit
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: Slackbot
Allow: /
User-agent: Discordbot
Allow: /
User-agent: TelegramBot
Allow: /
User-agent: WhatsApp
Allow: /
User-agent: Pinterest
Allow: /
# ──────────────────────────────────────────────
# AI CRAWLERS - Allowed for AI search visibility
# ──────────────────────────────────────────────
# These crawlers power AI search responses in ChatGPT, Claude,
# Perplexity, Google AI Overviews, Siri, etc. Allowing them means
# ELAV8 can appear when users ask AI about productivity assistants.
# AI-referred traffic converts 4.4x higher than organic search.
# OpenAI (ChatGPT browsing + model training)
User-agent: GPTBot
Allow: /
Disallow: /dev/
Disallow: /portal/
User-agent: ChatGPT-User
Allow: /
Disallow: /dev/
Disallow: /portal/
# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /dev/
Disallow: /portal/
# Perplexity AI Search
User-agent: PerplexityBot
Allow: /
Disallow: /dev/
Disallow: /portal/
# Google AI (Gemini training, AI Overviews)
User-agent: Google-Extended
Allow: /
Disallow: /dev/
Disallow: /portal/
User-agent: GoogleOther
Allow: /
Disallow: /dev/
Disallow: /portal/
# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /
Disallow: /dev/
Disallow: /portal/
# Common Crawl (powers many AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /dev/
Disallow: /portal/
# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /dev/
Disallow: /portal/
# Amazon Alexa AI
User-agent: Amazonbot
Allow: /
Disallow: /dev/
Disallow: /portal/
# ByteDance / TikTok AI
User-agent: Bytespider
Allow: /
Disallow: /dev/
Disallow: /portal/
# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /dev/
Disallow: /portal/
# Allen AI
User-agent: AI2Bot
Allow: /
Disallow: /dev/
Disallow: /portal/
# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /dev/
Disallow: /portal/
# ──────────────────────────────────────────────
# SEARCH ENGINES - Phased rollout
# ──────────────────────────────────────────────
# Currently blocking all search engines from broad indexing.
# Legal / compliance pages are explicitly allowed for crawling so
# that Google OAuth verification, Safe Browsing, Search Console,
# and legal-policy bots can fetch them. Page-level indexing is
# still controlled by via src/config/seo.ts.
#
# PHASE 1 (Home page only):
# User-agent: *
# Allow: /$
# Disallow: /
#
# PHASE 2 (High-value pages):
# User-agent: *
# Allow: /$
# Allow: /use-cases/
# Allow: /roadmap/
# Allow: /security/
# Disallow: /dev/
# Disallow: /portal/
# Disallow: /
#
# PHASE 3 (Broader access):
# User-agent: *
# Allow: /
# Disallow: /dev/
# Disallow: /portal/
#
# CURRENT: Phase 1.5 - Home + Voice + CONVOLENS + Blog + Honors + Partners + Use Cases + Getting Started + Brand + Contact + Legal pages
#
# IMPORTANT: /sitemap.xml is explicitly allowed in BOTH groups below. Without it,
# the Disallow: / catch-all blocks the sitemap FILE ITSELF, and Google reports
# "Sitemap could not be read" (the live URL test shows "Blocked by robots.txt").
# Google applies the longest-matching rule, so Allow: /sitemap.xml beats
# Disallow: /. Root-caused 2026-06-29 (rem-20260629-004); verified by
# scripts/check-public-assets.mjs and the Sitemap & Canonical Host panel.
#
# JUST AS IMPORTANT: rendering + brand assets (/_next/static/ JS+CSS, /fonts/,
# /images/, /videos/, the favicon, PWA icons, the manifest, and the OG images)
# are explicitly allowed in BOTH groups too. The same Disallow: / catch-all was
# blocking ALL of them from Googlebot, so Google could not render pages (JS/CSS),
# show the favicon in search results, or read OG images. Assets are NOT pages, so
# allowing them does NOT expose any not-yet-launched page; page discovery stays
# gated by the per-page Allow rules. Root-caused 2026-06-30 (rem-20260630-001);
# verified by scripts/check-public-assets.mjs and the Sitemap & Canonical Host panel.
# Explicit Googlebot rules (OAuth verification, Search Console, Safe Browsing).
# Listed before User-agent: * so Googlebot picks the most specific group.
User-agent: Googlebot
Allow: /$
Allow: /sitemap.xml
Allow: /blog/
Allow: /honors/
Allow: /convolens/
Allow: /voice/
Allow: /partners/
Allow: /use-cases/
Allow: /getting-started/
Allow: /brand/
Allow: /contact/
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /cookie-policy/
Allow: /sms-terms/
Allow: /legal/
Allow: /voice/privacy/
Allow: /voice/terms/
Allow: /voice/terms/fr/
Allow: /voice/support/
Allow: /convolens/privacy/
Allow: /convolens/terms/
Allow: /gdpr/
Allow: /security/
# Rendering + brand assets — must be crawlable so Google can render and present
# pages. Without these, Disallow: / blocks JS/CSS (rendering), the favicon, OG
# images, fonts, and media. Assets are not pages; this does not expose any page.
# /images/ covers the downloadable logo files under /images/brand//.
# /email-assets/ is the self-hosted image host for HTML email (logos, store badges, social icons).
Allow: /_next/static/
Allow: /fonts/
Allow: /images/
Allow: /email-assets/
Allow: /videos/
Allow: /favicon.ico
Allow: /apple-touch-icon.png
Allow: /icon-192.png
Allow: /icon-512.png
Allow: /icon-192-maskable.png
Allow: /icon-512-maskable.png
Allow: /site.webmanifest
Allow: /og-image.png
Allow: /og-elav8-voice.png
Allow: /og-convolens.png
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dev/
Disallow: /portal/
Disallow: /voice-alpha-7k3p9x/
Disallow: /
# Catch-all for all other search engines
User-agent: *
Allow: /$
Allow: /sitemap.xml
Allow: /blog/
Allow: /honors/
Allow: /convolens/
Allow: /voice/
Allow: /partners/
Allow: /use-cases/
Allow: /getting-started/
Allow: /brand/
Allow: /contact/
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /cookie-policy/
Allow: /sms-terms/
Allow: /legal/
Allow: /voice/privacy/
Allow: /voice/terms/
Allow: /voice/terms/fr/
Allow: /voice/support/
Allow: /convolens/privacy/
Allow: /convolens/terms/
Allow: /gdpr/
Allow: /security/
# Rendering + brand assets — must be crawlable so Google can render and present
# pages. Without these, Disallow: / blocks JS/CSS (rendering), the favicon, OG
# images, fonts, and media. Assets are not pages; this does not expose any page.
# /images/ covers the downloadable logo files under /images/brand//.
# /email-assets/ is the self-hosted image host for HTML email (logos, store badges, social icons).
Allow: /_next/static/
Allow: /fonts/
Allow: /images/
Allow: /email-assets/
Allow: /videos/
Allow: /favicon.ico
Allow: /apple-touch-icon.png
Allow: /icon-192.png
Allow: /icon-512.png
Allow: /icon-192-maskable.png
Allow: /icon-512-maskable.png
Allow: /site.webmanifest
Allow: /og-image.png
Allow: /og-elav8-voice.png
Allow: /og-convolens.png
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dev/
Disallow: /portal/
Disallow: /voice-alpha-7k3p9x/
Disallow: /
# ──────────────────────────────────────────────
# SITEMAP
# ──────────────────────────────────────────────
# Generated from src/config/seo.ts at build time.
# Only includes pages with allowIndexing: true.
Sitemap: https://elav8.ai/sitemap.xml
# FALA ships from its own host, so its pages are not in the sitemap above.
# Cross-host sitemap references are honored when the referencing site and the
# sitemap host are both covered by a verified Search Console property; our
# elav8.ai DOMAIN property covers every subdomain, this one included. This is
# a discovery aid, not a substitute for fala.elav8.ai serving its own
# robots.txt and sitemap, which it already does. Added 2026-07-28.
Sitemap: https://fala.elav8.ai/sitemap.xml