# robots.txt for SRKR Engineering College
# https://www.srkrec.ac.in

User-agent: *
Allow: /

# Block template/demo/backup pages from previous Astro starter
Disallow: /index-bk
Disallow: /index-copy
Disallow: /index-two
Disallow: /index-three
Disallow: /index-four
Disallow: /index-five
Disallow: /test/
Disallow: /blog/first-post/
Disallow: /blog/second-post/

# Block crawler abuse / parameterised duplicates
Disallow: /*?*ref=
Disallow: /*?*utm_

# Allow major search crawlers explicit access (these drive real traffic)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ---------------------------------------------------------------------------
# Bandwidth control
#
# The PDF library under /assets/documents/ and /files/ is ~1.75 GB. Crawlers
# repeatedly pulling it is a significant share of our Vercel egress bill and
# provides no benefit outside of search. The rules below keep Google and Bing
# fully allowed (so syllabus/newsletter PDFs still rank), while stopping
# AI-training and SEO-metrics crawlers from downloading the file library.
#
# Note: robots.txt is advisory. Well-behaved bots honour it; abusive ones do
# not. Enforcement for bad actors needs the Vercel WAF or Cloudflare — see
# docs/COST-OPTIMIZATION.md.
# ---------------------------------------------------------------------------

# AI crawlers: may read site content, but not the heavy file library.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: YouBot
Allow: /
Disallow: /assets/documents/
Disallow: /files/
Disallow: /assets/video/

# SEO-metrics and scraper bots: no search value to us, blocked entirely.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DataForSeoBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: Bytespider
User-agent: ImagesiftBot
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: Scrapy
User-agent: SiteAuditBot
User-agent: Barkrowler
User-agent: ZoominfoBot
Disallow: /

# Sitemap
Sitemap: https://www.srkrec.ac.in/sitemap-index.xml