# Everything is crawlable by default; only the rules below change behaviour.

User-agent: *
# The JSON API is not for indexing. The /api/posts carve-out exists ONLY because
# Googlebot renders JavaScript and the /resources page re-fetches post data in
# the browser — blocking it would degrade Google's rendered view of the page.
Allow: /api/posts
Allow: /api/posts/*
Disallow: /api/
Disallow: /client-portal

# AI crawlers: read the server-rendered HTML at /resources/* and /llms.txt —
# not the JSON API. These bots do not execute JavaScript.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: PerplexityBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Googlebot-Extended
User-agent: CCBot
Disallow: /api/
Disallow: /client-portal

# XML sitemap
Sitemap: https://sarhandata.law/sitemap.xml

# LLMs.txt for AI assistants
# See: https://sarhandata.law/llms.txt