# Keploy landing robots.txt
# Policy: allow selected AI search/answer engines and major training crawlers
# to improve chances of Keploy content appearing in AI-generated answers.
# Blocked: scrapers and bots with no meaningful AI product distribution
# (Meta-ExternalAgent, cohere-ai, Diffbot, Omgilibot, ImagesiftBot, Bytespider).
# Reference: Speedscale / Katalon / Testsigma split policy (2026 competitor audit)

# =============================================================================
# ALLOW — AI search / answer engines & live user agents
# =============================================================================

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Amazonbot
Allow: /

# =============================================================================
# ALLOW — Major training crawlers (may improve chances of inclusion in LLM training datasets)
# =============================================================================

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# CCBot feeds Common Crawl datasets used broadly across AI training pipelines
User-agent: CCBot
Allow: /

# =============================================================================
# DISALLOW — Scrapers with no AI product distribution value
# =============================================================================

User-agent: Meta-ExternalAgent
Disallow: /

# NOTE: FacebookBot is Facebook's link-preview / Open Graph crawler,
# not a training crawler. Blocking it breaks OG cards when keploy.io
# links are shared on Facebook, Messenger, and Instagram. Meta's
# training crawler is Meta-ExternalAgent (blocked above). FacebookBot
# is intentionally NOT listed here.

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Always-block scraper
User-agent: Bytespider
Disallow: /

# =============================================================================
# DEFAULT — all other crawlers (Googlebot, Bingbot, etc.)
# Removed 5 invalid subdomain Disallow lines (/docs.keploy.io/ etc.)
# that silently did nothing — Disallow takes paths, not hostnames.
# =============================================================================

User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /wp/
Disallow: /blog/api/
Disallow: /blog/search

# =============================================================================
# Sitemaps
# =============================================================================

Sitemap: https://keploy.io/sitemap.xml
Sitemap: https://keploy.io/blog/sitemap.xml
Sitemap: https://keploy.io/docs/sitemap.xml