# Robots.txt for production environment
# Allow all crawlers with specific configurations

# Default crawlers
User-agent: *
Allow: /
Disallow: /analytics/
Crawl-delay: 1
# Content signals (Cloudflare policy) — droplinked is agent-native commerce and
# WANTS AI search, AI answer-engine input, and training on its public content.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Google Bot - optimized for faster product indexing
User-agent: Googlebot
Allow: /
Disallow: /analytics/
Crawl-delay: 0.5

# Bing Bot
User-agent: Bingbot
Allow: /
Disallow: /analytics/
Crawl-delay: 0.8

# ---------------------------------------------------------------------------
# AI answer engines & LLM crawlers (GEO / AEO)
# Droplinked is agent-native commerce infrastructure and WANTS to be
# discovered, cited, and transacted with by AI shopper agents and answer
# engines, so the major LLM crawlers are explicitly allowed.
# LLM-readable site summary: https://droplinked.com/llms.txt
# ---------------------------------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini / AI Overviews grounding & training)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Common Crawl (feeds many LLM training corpora)
User-agent: CCBot
Allow: /

# ---------------------------------------------------------------------------
# Social media crawlers (link unfurls)
# ---------------------------------------------------------------------------

User-agent: facebookexternalhit
Allow: /
Disallow: /analytics/
Crawl-delay: 2

User-agent: Twitterbot
Allow: /
Disallow: /analytics/
Crawl-delay: 2

User-agent: LinkedInBot
Allow: /
Disallow: /analytics/
Crawl-delay: 2

Sitemap: https://droplinked.com/sitemap.xml

# --- Agentic-commerce surfaces (droplinked is agentic-commerce infrastructure) ---
Sitemap: https://apiv3.droplinked.com/sitemap-products.xml
Sitemap: https://apiv3.droplinked.com/marketplace-sitemap.xml
Sitemap: https://apiv3.droplinked.com/productfeed-sitemap.xml
# MCP server: https://mcp.droplinked.com/mcp  (manifest: https://mcp.droplinked.com/.well-known/mcp.json)
# Agent discovery: https://droplinked.com/.well-known/ai-catalog.json  |  /.well-known/agents.json
# Product feeds: https://apiv3.droplinked.com/feed/acp.json | /feed/acp.jsonl | /feed/gmc.xml
# LLM guide: https://droplinked.com/llms.txt  |  https://droplinked.com/llms-full.txt