# --- Core: allow search engines full access to content ---
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: facebookexternalhit
Allow: /
User-agent: Facebot
Allow: /

# --- WordPress admin hygiene ---
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /?attachment_id=
Disallow: /trackback/
Disallow: /comments/feed/
Allow: /wp-admin/admin-ajax.php

# --- IMPORTANT: allow feeds (needed for News/Publisher Center & freshness) ---
Allow: /feed/
Allow: /*/feed/
Allow: /category/*/feed/

# --- Sitemaps ---
Sitemap: https://indian.community/sitemap_index.xml
Sitemap: https://indian.community/news-sitemap.xml

# --- Optional: AI bot policy (keep if you want) ---
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: Google-Extended
Allow: /
# Block specific training bots if you must
User-agent: ClaudeBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /