# START robots.txt
# Optimized for WordPress with Rank Math SEO.
# AI Crawler Policy: explicitly allow all AI crawlers, including training bots.
# Last updated: July 2026
#
# Structure: two groups sharing the same rules.
#   1. User-agent: *   -> catch-all for every unlisted crawler.
#   2. Named allowed   -> AI, search, social-preview and the SEO
#      tools we use; all share one protective ruleset.
#   Note: a crawler obeys only its own most-specific group and
#   ignores * (per Google's robots.txt spec), which is why the
#   allowed group repeats the protective Disallows.
 
# ----------------------------------------------------------------
# Group 1 - General Rules for All (unlisted) Crawlers
# ----------------------------------------------------------------
User-agent: *
 
# Block WordPress core, admin, and login areas.
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
 
# Allow files required for correct site rendering.
# Explicit, longer-path allows so plugin/theme CSS & JS are not
# caught by the directory Disallows above (Google resolves
# conflicts by longest matching path).
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/plugins/*.css$
Allow: /wp-content/plugins/*.js$
Allow: /wp-content/themes/*.css$
Allow: /wp-content/themes/*.js$
Allow: /*.css$
Allow: /*.js$
 
# ----------------------------------------------------------------
# Group 2 - Named crawlers we allow
# (AI, search engines, social-preview, and SEO tools we use)
# All bots below share the identical ruleset that follows, so they
# can crawl the site while still respecting the same admin/login/
# plugin Disallows as the catch-all group above.
# ----------------------------------------------------------------
 
# --- AI crawlers: retrieval, search, and model-training ---
# OpenAI - ChatGPT Search indexing
User-agent: OAI-SearchBot
# OpenAI - user-triggered retrieval
User-agent: ChatGPT-User
# OpenAI - model training
User-agent: GPTBot
# Anthropic - model training
User-agent: ClaudeBot
# Anthropic - user-triggered retrieval
User-agent: Claude-User
# Anthropic - search indexing
User-agent: Claude-SearchBot
# Perplexity - search discovery and citation
User-agent: PerplexityBot
# Perplexity - user-triggered retrieval
User-agent: Perplexity-User
# Google - Gemini training and grounding
User-agent: Google-Extended
# Google - Vertex AI grounding
User-agent: Google-CloudVertexBot
# Google - NotebookLM
User-agent: Google-NotebookLM
# Apple - Siri / Apple Intelligence retrieval
User-agent: Applebot
# Apple - AI training opt-in token
User-agent: Applebot-Extended
# Meta AI crawler
User-agent: Meta-ExternalAgent
# DuckDuckGo AI assistant
User-agent: DuckAssistBot
# Mistral - user-triggered retrieval
User-agent: MistralAI-User
# ByteDance AI crawler
User-agent: Bytespider
# Amazon AI crawler
User-agent: Amazonbot
 
# --- Standard search engines ---
User-agent: bingbot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: CCBot
User-agent: archive.org_bot
# Google Images - product/barrier photo discovery
User-agent: Googlebot-Image
# Huawei Petal search (APAC / Middle East visibility)
User-agent: PetalBot
 
# --- Social / link-preview crawlers (build share cards) ---
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: Slackbot
User-agent: WhatsApp
User-agent: Discordbot
 
# --- SEO & auditing tools we subscribe to ---
User-agent: AhrefsBot
User-agent: SemrushBot
 
# Shared ruleset for all named crawlers above.
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/plugins/*.css$
Allow: /wp-content/plugins/*.js$
Allow: /wp-content/themes/*.css$
Allow: /wp-content/themes/*.js$
Allow: /*.css$
Allow: /*.js$
 
# ----------------------------------------------------------------
# Sitemap
# ----------------------------------------------------------------
Sitemap: https://roxboxcontainers.com/sitemap_index.xml
 
# END robots.txt