# E-Waste Squad - Robots.txt
# Updated: November 2025
# Purpose: Allow all search engines and AI crawlers to index our content

# ============================================
# SEARCH ENGINE CRAWLERS - FULL ACCESS
# ============================================

# Google Search
User-agent: Googlebot
Allow: /

# Google Image Search
User-agent: Googlebot-Image
Allow: /

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Bing/Microsoft Search
User-agent: Bingbot
Allow: /

# Yahoo Search
User-agent: Slurp
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# ============================================
# SEO & ANALYTICS TOOLS
# ============================================

# SEMrush Site Audit Bot (CRITICAL - Fix for 5xx errors)
User-agent: SemrushBot
Allow: /

User-agent: SiteAuditBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

# Ahrefs
User-agent: AhrefsBot
Allow: /

# Moz
User-agent: rogerbot
Allow: /
User-agent: dotbot
Allow: /

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Allow: /

# Majestic
User-agent: MJ12bot
Allow: /

# ============================================
# AI CRAWLERS & LLM BOTS - FULL ACCESS
# ============================================

# OpenAI GPT (ChatGPT, GPT-4, etc.)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude
User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Apple AI/Applebot
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# ============================================
# SOCIAL MEDIA CRAWLERS
# ============================================

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Twitter/X
User-agent: Twitterbot
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /
User-agent: Pinterestbot
Allow: /

# Reddit
User-agent: Redditbot
Allow: /

# ============================================
# DEFAULT RULE - ALLOW ALL OTHER BOTS
# ============================================

User-agent: *
Allow: /

# ============================================
# SITEMAP LOCATION
# ============================================

Sitemap: https://ewastesquad.com/sitemap.xml
Sitemap: https://www.ewastesquad.com/sitemap.xml

# ============================================
# ADDITIONAL NOTES
# ============================================
# This robots.txt explicitly allows all major search engines,
# SEO tools, and AI crawlers to access and index all content.
# No restrictions are applied to ensure maximum visibility.
# All JavaScript, CSS, images, and static assets are accessible.