# Joint Relief Institute - robots.txt # https://jointreliefinstitute.com # AI Crawler Directives # Allow reputable AI crawlers for indexing User-agent: GPTBot Allow: / Crawl-delay: 2 User-agent: ChatGPT-User Allow: / Crawl-delay: 2 User-agent: Claude-Web Allow: / Crawl-delay: 2 User-agent: ClaudeBot Allow: / Crawl-delay: 2 User-agent: Anthropic-AI Allow: / Crawl-delay: 2 User-agent: Google-Extended Allow: / Crawl-delay: 2 User-agent: CCBot Allow: / Crawl-delay: 3 User-agent: PerplexityBot Allow: / Crawl-delay: 2 # Limit aggressive crawlers User-agent: MJ12bot Crawl-delay: 10 User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 # General rules for all other crawlers User-agent: * Allow: / # Sitemap Sitemap: https://jointreliefinstitute.com/sitemap-index.xml # LLM-friendly content (llms.txt standard) # For AI assistants and language models # https://llmstxt.org/ # llms.txt: https://jointreliefinstitute.com/llms.txt # llms-full.txt: https://jointreliefinstitute.com/llms-full.txt # Crawl-delay for considerate crawling Crawl-delay: 1 # Disallow admin and private areas Disallow: /api/ Disallow: /_astro/ # Block old spam injection paths (discovered in GSC) Disallow: /gambling-online/