# =====================================================
# ROBOTS.TXT FOR RAFTAAR.IN
# Optimized for News, Google Discover & SEO
# =====================================================

# -----------------------------------------------------
# Default rules for all crawlers
# -----------------------------------------------------

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /search/
Disallow: /?s=
Disallow: /trackback/
Disallow: */trackback/
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /tag/*/page/
Disallow: /category/*/page/
Disallow: /author/*/page/
Disallow: /*/page/
Disallow: /*?paged=
Disallow: /comments/feed/
Disallow: */feed/comments/

Allow: /wp-admin/admin-ajax.php

# -----------------------------------------------------
# Google crawlers (primary)
# -----------------------------------------------------

User-agent: Googlebot
Crawl-delay: 1
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /search/
Disallow: /?s=
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /tag/*/page/
Disallow: /category/*/page/
Disallow: /author/*/page/
Disallow: /*/page/
Disallow: /*?paged=

User-agent: Googlebot-News
Crawl-delay: 1
Allow: /

# -----------------------------------------------------
# Sitemaps
# -----------------------------------------------------

Sitemap: https://raftaar.in/sitemap.xml
Sitemap: https://raftaar.in/sitemaps/news.xml

# -----------------------------------------------------
# Bing
# -----------------------------------------------------

User-agent: Bingbot
Crawl-delay: 5

# -----------------------------------------------------
# Apple crawler (Apple News / Siri)
# -----------------------------------------------------

User-agent: Applebot
Crawl-delay: 5

# -----------------------------------------------------
# Social media crawlers
# -----------------------------------------------------

#User-agent: facebookexternalhit
#Crawl-delay: 10

#User-agent: meta-externalagent
#Crawl-delay: 10

#User-agent: meta-webindexer
#Crawl-delay: 10

#User-agent: Twitterbot
#Crawl-delay: 5

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: meta-webindexer
Allow: /

User-agent: Twitterbot
Allow: /


# -----------------------------------------------------
# AI crawlers (allowed but slowed)
# -----------------------------------------------------

User-agent: GPTBot
Crawl-delay: 10

User-agent: ClaudeBot
Crawl-delay: 10

User-agent: PerplexityBot
Crawl-delay: 10

User-agent: OAI-SearchBot
Crawl-delay: 10

# -----------------------------------------------------
# SEO crawlers (blocked)
# -----------------------------------------------------

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

# -----------------------------------------------------
# High bandwidth crawlers
# -----------------------------------------------------

User-agent: Baiduspider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /