# =============================================================================
# ROBOTS.TXT - AI & LLM OPTIMIZED (2025)
# =============================================================================

# Allow all major search engines and AI crawlers
User-agent: *
Allow: /

# =============================================================================
# AI CRAWLER OPTIMIZATION (UPDATED 2025)
# =============================================================================

# ——— OPENAI ———
# Search (shows webpages as links inside ChatGPT search) - NOT used for model training
User-agent: OAI-SearchBot
Allow: /

# User-driven browsing from ChatGPT and Custom GPTs - Acts after a human click
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
Allow: /

# Model-training crawler - Allow for better representation in AI models
User-agent: GPTBot
Allow: /
Crawl-delay: 2

# ——— ANTHROPIC (Claude) ———
User-agent: anthropic-ai      # legacy but still active
User-agent: claude-web        # legacy but still active
User-agent: ClaudeBot         # current general crawler
Allow: /
Crawl-delay: 2

# ——— PERPLEXITY ———
User-agent: PerplexityBot     # index builder
Allow: /

User-agent: Perplexity-User   # human-triggered visit
Allow: /

# ——— GOOGLE (Gemini) ———
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Googlebot
Allow: /

# ——— MICROSOFT (Bing / Copilot) ———
User-agent: BingBot
Allow: /

# ——— AMAZON ———
User-agent: Amazonbot
Allow: /

# ——— APPLE ———
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# ——— META ———
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: facebookexternalhit
User-agent: meta-externalfetcher
Allow: /

# ——— LINKEDIN ———
User-agent: LinkedInBot
Allow: /

# ——— BYTEDANCE (TikTok) ———
User-agent: Bytespider
Allow: /

# ——— DUCKDUCKGO ———
User-agent: DuckAssistBot
Allow: /

# ——— COHERE ———
User-agent: cohere-ai                     # general AI crawler
User-agent: cohere-training-data-crawler  # training data collection
Allow: /
Crawl-delay: 2

# ——— MISTRAL (2025 UPDATE) ———
User-agent: MistralAI-User
Allow: /

# ——— OTHER AI/RESEARCH CRAWLERS ———
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma      # AI2 research crawler
User-agent: CCBot
User-agent: Diffbot
User-agent: omgili
User-agent: YouBot
User-agent: TimpiBot
User-agent: img2dataset       # image collection for AI
User-agent: FriendlyCrawler   # AI research crawler
Allow: /
Crawl-delay: 2

# ——— EMERGING SEARCH ENGINES ———
User-agent: PetalBot          # Huawei
Allow: /

# ——— TRADITIONAL SEARCH ENGINES ———
User-agent: Baiduspider       # Baidu
User-agent: YandexBot         # Yandex
User-agent: Slurp             # Yahoo
User-agent: SwiftBot          # Swift search
User-agent: SEMrushBot        # SEMrush
Allow: /
Crawl-delay: 1

# ——— DEFAULT FOR ALL OTHER CRAWLERS ———
User-agent: *
Allow: /
Crawl-delay: 1

# =============================================================================
# SITE CONFIGURATION
# =============================================================================

# Host
Host: https://designatives.com

# Sitemaps
Sitemap: https://designatives.com/sitemap.xml
Sitemap: https://designatives.com/sitemap-0.xml

# LLM Content Manifests
# AI-friendly content access points
# https://designatives.com/llms.txt
# https://designatives.com/llm.txt