# =============================================================================
#  Intellegal — robots.txt  (CORRECTED draft, 2026-06-13)
#  Philippine AI Legal Research · https://intellegal.ai/philippines/
#
#  WHAT CHANGED vs the live file (one real fix):
#    The live file gives each AI bot its OWN group containing only "Allow: /".
#    In robots.txt a named User-agent group FULLY OVERRIDES the "*" group — so
#    those bots currently IGNORE all the Disallow rules and CAN crawl /api/,
#    /philippines/login, /admin, /payment/, etc. Fix: the AI bots share ONE group
#    (stacked User-agent lines) that MIRRORS the public Disallow policy.
#  (The live file's per-bot structure was otherwise fine — not "broken".)
#
#  Policy:
#    - Public marketing / tool / corpus pages: allow all crawlers
#    - Auth, account, admin, transactional, dev, API: Disallow (for ALL bots)
#    - AI bots: GEO opt-in (allowed), but same Disallow guardrails apply
# =============================================================================


# -------- Default rules (Googlebot, Bingbot, DuckDuckBot, etc.) --------
User-agent: *
Disallow: /philippines/login
Disallow: /philippines/register
Disallow: /philippines/google-callback
Disallow: /philippines/user-center
Disallow: /philippines/user-feedback
Disallow: /philippines/admin
Disallow: /philippines/payment/
Disallow: /philippines/case-compare-v1
Disallow: /philippines/genai
Disallow: /philippines/dev/
Disallow: /philippines/api/
Disallow: /api/
Disallow: /philippines/contract-review/review/
# Future corpus pages (Track B) live at /philippines/{case,law,topic,court}/ —
# these are public content, intentionally NOT disallowed.



# -------- Training-only crawlers - blocked (no referral benefit; anti-corpus-scraping) --------
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
Disallow: /

# -------- AI crawlers — GEO opt-in (allowed, but inherit the same guardrails) --------
# Stacked User-agent lines = one shared rule block. "Allow: /" opens everything,
# the Disallow lines still win by longest-match for the sensitive paths.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: YouBot
User-agent: Diffbot
Allow: /
Disallow: /philippines/login
Disallow: /philippines/register
Disallow: /philippines/google-callback
Disallow: /philippines/user-center
Disallow: /philippines/user-feedback
Disallow: /philippines/admin
Disallow: /philippines/payment/
Disallow: /philippines/case-compare-v1
Disallow: /philippines/genai
Disallow: /philippines/dev/
Disallow: /philippines/api/
Disallow: /api/
Disallow: /philippines/contract-review/review/


# -------- Sitemap --------
Sitemap: https://intellegal.ai/philippines/sitemap.xml


# =============================================================================
#  OPTIONAL — stricter stance (a STRATEGIC decision, not a fix; default above
#  keeps your current "allow all AI"). Ties to the data-risk of feeding the
#  corpus to training with no referral benefit. If you later want to allow only
#  the bots that DRIVE CITATIONS/TRAFFIC and block pure-training crawlers:
#    KEEP (cite / send traffic): GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot,
#      Claude-Web, anthropic-ai, PerplexityBot, Perplexity-User, Google-Extended,
#      Applebot-Extended, YouBot, Diffbot
#    BLOCK (training-mostly, little/no referral): CCBot, Bytespider, Amazonbot,
#      Meta-ExternalAgent, FacebookBot, cohere-ai
#  To block, give those a separate group:  User-agent: CCBot \n Disallow: /  (etc.)
#  Note: Google-Extended only gates Gemini training/grounding — blocking it does
#  NOT affect normal Google Search ranking (that's Googlebot, always allowed).
# =============================================================================