# =========================================
#   CloudTechTwitter Robots Configuration
#   Optimized for Blogger + AdSense + SEO
#   Last Updated: Nov 2025
# =========================================

# ----- Google AdSense and Display Bots -----
User-agent: Mediapartners-Google
Allow: /

User-agent: Google-Display-Ads-Bot
Allow: /

# ----- Googlebot & General Search Engines -----
User-agent: *
Disallow: /search
Allow: /

# ----- Block Known Crawler Spam & Content Scrapers -----
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: DotBot
Disallow: /

# ----- Block AI Model Training & Data Collection Bots -----
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# ----- Allow Essential Google Services -----
User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: AdsBot-Google
Allow: /

# ----- Sitemaps (Auto-Indexing Support) -----
Sitemap: https://www.cloudtechtwitter.com/sitemap.xml
Sitemap: https://www.cloudtechtwitter.com/sitemap-pages.xml

# =========================================
# Notes:
# 1. /search blocked to prevent duplicate content.
# 2. AI and scraper bots blocked to protect content.
# 3. All legitimate search & ad bots allowed.
# 4. Sitemaps ensure Google auto-discovers new posts/pages.
# 5. Verified for Blogger, AdSense & Search Console.
# =========================================