User-Agent: *
Allow: /

# GitBook's own RPC endpoints, reachable through the /docs/* proxy in
# src/proxy.ts. They answer 405 to a GET (they are POST-only), which is the
# whole of Search Console's "Blocked due to other 4xx issue" bucket — 12 URLs
# across /docs, /docs/fr, /docs/kr, /docs/zh, /docs/jp and
# /docs/morphic-help-center-fr. They are infrastructure, never content, so
# there is nothing to gain by crawling them.
#
# Scoped to these two endpoints ON PURPOSE. A broader `/*~gitbook/` rule would
# also block `~gitbook/image`, which appears 40 times in a single /docs page
# and carries every image in the documentation — blocking it would stop
# Googlebot rendering the docs properly. `~gitbook/site`, `~gitbook/static` and
# `~gitbook/visitor` are likewise live assets.
Disallow: /*~gitbook/search
Disallow: /*~gitbook/mcp

# GitBook's docs AI search renders every suggested question as a real link, so
# `?ask=<question>` is an unbounded crawlable URL space — hence a pattern rule.
# Layers with the noindex src/proxy.ts injects on those responses.
Disallow: /*?ask=
Disallow: /*&ask=

# `?message=` is unbounded like ?ask= above — leftover relative hrefs from old
# /resources CTAs left `morphic.com/chat?message=...` 404s in GSC. Studio's own
# /chat?message= is a different host, not covered here — its robots.txt still
# needs the same fix (studio #6386, docs/seo/sops/studio-crawl-control.md).
Disallow: /*?message=
Disallow: /*&message=

# HLS manifests — media transport, never pages (~33 in the GSC sample, all 404s).
# morphic.com only: external-cdn.morphic.com needs its own robots.txt plus
# `X-Robots-Tag: noindex` at the CDN, neither of which is in this repo.
Disallow: /upload/
Disallow: /*.m3u8$

Sitemap: https://morphic.com/sitemap.xml
Sitemap: https://morphic.com/docs/sitemap.xml