# robots.txt — https://www.taleem360.com
# Policy: index EVERYTHING except the admin panel and the JSON API. Search engines
# AND AI answer engines are explicitly welcome — we want to be both indexed and
# cited. Nothing here blocks Googlebot/Bingbot from any public content.
#
# ⚠️ DO NOT re-add "Disallow: /gcs". The site-search page is kept out of the index
#    by its own <meta name="robots" content="noindex, follow"> (front/search/
#    site-search.blade.php), and a Disallow would stop Google from ever READING
#    that noindex — which is exactly how 10 /gcs URLs ended up as "Indexed, though
#    blocked by robots.txt" (GSC, 2026-08-01). Crawl-safe: the search query lives
#    in the URL fragment (#gsc.q=, Google's CSE widget) and PageController reads no
#    query param, so /gcs is ONE crawlable URL, not a parameter space.

# ── All crawlers (Googlebot, Bingbot, …) ──
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /thumbnail
# /open-app/* is the Android app's link: it opens the app, or 302s to the book's
# own page (Android gets an intent link first). Never useful to index — the book
# page is the canonical URL — so crawl budget stays on the real pages.
Disallow: /open-app/
# Legacy raw book files. Orphaned: NO book record points at /bookstorage/, no code
# or internal link references it, and downloads never touch it (uploads-storage books
# are PHP-streamed via /download/{slug}, BookController::download) — so this is
# crawler-only and users are unaffected. It was 30.76% of ALL Googlebot requests and
# ~all of July's 22.3 GB (one file, ebook1979.pdf, is 131 MB). Served as
# application/octet-stream, so a search hit there is a raw download with no ads and
# no site visit, cannibalising the book page that has both. (GSC crawl stats
# 2026-08-01.)
Disallow: /bookstorage/
# /download/{slug} is a per-visitor 302 to Drive/R2 — no content, nothing to index.
# It was 1.43% of all Googlebot requests (the "Redirect error" slice, GSC crawl stats
# 2026-08-01), every hit inflated books.downloads (the IP+UA dedup does not stop a
# crawler), it is doNotCacheResponse so every hit reaches the origin, and following it
# passed link equity to drive.google.com. Users are unaffected — robots.txt binds
# crawlers only, and the button still works. Nothing depends on it being crawlable:
# it appears in NO sitemap and NO structured data (the Book schema has no contentUrl
# / DownloadAction).
Disallow: /download/
# The admission twin (2026-09-13): a per-visitor 302 to the notice file, no content to index.
Disallow: /admissions/download/
# ⚠️ ONLY the pdf.js viewer SHELL — NEVER "Disallow: /vendor/". That folder holds
#    render-critical assets: vendor/lazyload/lazyload.min.js loads on EVERY front
#    page (foot.blade.php) and resolves the data-src book covers, so blocking the
#    folder would stop Googlebot rendering images. viewer.js / viewer.css / build/*
#    sit beside this file and MUST stay crawlable (prefix matching keeps them:
#    "viewer.js" does not start with "viewer.html"). The shell takes the raw PDF URL
#    as ?file= (BookController::pdf) — one crawlable URL per book, an unbounded
#    parameter space — and /pdf/{slug}, which embeds it, is noindex anyway.
Disallow: /vendor/pdfjs-dist/web/viewer.html

# ── AI assistants & answer engines — explicitly allowed (for AI citations).
#    A named bot obeys ONLY its own group and ignores "*", so the same carve-outs
#    are restated here. Google-Extended must stay allowed or we lose eligibility
#    for Google's AI Overviews / Gemini grounding. ──
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /open-app/
Disallow: /bookstorage/
Disallow: /download/
# The admission twin (2026-09-13): a per-visitor 302 to the notice file, no content to index.
Disallow: /admissions/download/
Disallow: /vendor/pdfjs-dist/web/viewer.html

Sitemap: https://www.taleem360.com/sitemap.xml