User-agent: *
Allow: /
Disallow: /404.html
Disallow: /tools/auto-shorts/
# Directorios internos que el deploy legacy publica pero que no deben competir
# con el contenido indexable (no estan en el sitemap).
Disallow: /projects/
Disallow: /output/
Disallow: /pins/
Disallow: /pinterest-pins/
Sitemap: https://cha0smagicklabs.com/sitemap.xml
# AI crawlers and assistants
# Machine-readable index of the site: https://llmstxt.org/
# llms.txt is the curated entry point; llms-full.txt lists all 852 articles.
# These are not listed in sitemap.xml because they are not HTML pages.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Applebot-Extended
Allow: /
# Sin Crawl-Delay: Google lo ignora y Bing/Yandex lo acatan, limitando el
# rastreo de 1001 URLs (5s entre peticiones). El servidor ya responde rapido.