# Default rules for all web crawlers
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/feeds/
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Disallow: /api/
Disallow: /_next/
Disallow: /*/search?*
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# OpenAI per-query retrieval / search index / ads validation crawlers + Perplexity's shopping agent, plus ChatGPT's agent-mode browsing crawler and Perplexity's shopping crawler
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: PerplexityBot
User-agent: ChatGPT-agent
User-agent: PerplexityBot-Shopper
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Allow: /api/feeds/products-acp$
Allow: /api/feeds/products-acp?
Allow: /.well-known/
Disallow: /api/
Disallow: /api/chat/
Disallow: /api/webhooks
Disallow: /api/webhooks/
Disallow: /api/auth
Disallow: /api/auth/
Disallow: /api/admin
Disallow: /api/admin/
Disallow: /api/cron/
Disallow: /api/dsar/
Disallow: /api/consent/
Disallow: /api/monitoring/
Disallow: /api/analytics
Disallow: /api/analytics/
Disallow: /api/checkout-sessions
Disallow: /api/checkout-sessions/
Disallow: /api/checkout-sessions/*
Disallow: /_next/
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /*/search
Disallow: /*/search?*
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# Training and dataset crawlers allowed (owner decision 2026-08-03): public pages open for AI training; APIs and private paths remain blocked
User-agent: GPTBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: DeepSeekBot
User-agent: MistralAI-Crawler
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: Qwen-2-Bot
User-agent: AI2Bot
User-agent: Timpibot
User-agent: ProRataBot
User-agent: Webzio-Extended
User-agent: omgili
User-agent: omgilibot
User-agent: ImagesiftBot
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Allow: /.well-known/
Disallow: /api/
Disallow: /api/chat/
Disallow: /api/webhooks
Disallow: /api/webhooks/
Disallow: /api/auth
Disallow: /api/auth/
Disallow: /api/admin
Disallow: /api/admin/
Disallow: /api/cron/
Disallow: /api/dsar/
Disallow: /api/consent/
Disallow: /api/monitoring/
Disallow: /api/analytics
Disallow: /api/analytics/
Disallow: /api/checkout-sessions
Disallow: /api/checkout-sessions/
Disallow: /api/checkout-sessions/*
Disallow: /_next/
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /*/search
Disallow: /*/search?*
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# Google-Extended allowed (owner decision 2026-08-03): enables Gemini model training and Gemini grounding; Google Search, AI Overviews, and AI Mode unaffected either way
User-agent: Google-Extended
Allow: /
# Google generic product-team and R&D fetcher allowed (owner decision 2026-08-03); public profile, APIs and private paths blocked
User-agent: GoogleOther
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Allow: /.well-known/
Disallow: /api/
Disallow: /api/chat/
Disallow: /api/webhooks
Disallow: /api/webhooks/
Disallow: /api/auth
Disallow: /api/auth/
Disallow: /api/admin
Disallow: /api/admin/
Disallow: /api/cron/
Disallow: /api/dsar/
Disallow: /api/consent/
Disallow: /api/monitoring/
Disallow: /api/analytics
Disallow: /api/analytics/
Disallow: /api/checkout-sessions
Disallow: /api/checkout-sessions/
Disallow: /api/checkout-sessions/*
Disallow: /_next/
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /*/search
Disallow: /*/search?*
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# Google Search index, Shopping/Merchant crawler, and Ads landing-page quality (Search + Shopping ads)
User-agent: Googlebot
User-agent: Storebot-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/feeds/
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Allow: /.well-known/
Disallow: /api/
Disallow: /api/chat/
Disallow: /api/webhooks
Disallow: /api/webhooks/
Disallow: /api/auth
Disallow: /api/auth/
Disallow: /api/admin
Disallow: /api/admin/
Disallow: /api/cron/
Disallow: /api/dsar/
Disallow: /api/consent/
Disallow: /api/monitoring/
Disallow: /api/analytics
Disallow: /api/analytics/
Disallow: /api/checkout-sessions
Disallow: /api/checkout-sessions/
Disallow: /api/checkout-sessions/*
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# Site-owner-requested Vertex AI Agents, Amazon/Alexa search and user retrieval, Anthropic search/user, Mistral's user-prompt answer agent (not used for training), Apple, Bing, DuckDuckGo, Yandex, Brave, You.com, Huawei, Seznam, Naver, Mojeek, and other documented answer/retrieval crawlers, plus xAI Grok, Phind, Awario/Teclis crawlers, and Perplexity's per-query retrieval (Perplexity.ai browse)
User-agent: Google-CloudVertexBot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: MistralAI-User
User-agent: Applebot
User-agent: Bingbot
User-agent: DuckAssistBot
User-agent: YandexBot
User-agent: BraveBot
User-agent: YouBot
User-agent: PetalBot
User-agent: seznambot
User-agent: Naverbot
User-agent: MojeekBot
User-agent: Grok
User-agent: PhindBot
User-agent: Awario
User-agent: Teclis
User-agent: Perplexity-User
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/mcp$
Allow: /api/ucp/discover$
Allow: /api/ucp/discover?
Allow: /api/ucp/openapi.json$
Allow: /api/ucp/products$
Allow: /api/ucp/products?
Allow: /.well-known/
Disallow: /api/
Disallow: /api/chat/
Disallow: /api/webhooks
Disallow: /api/webhooks/
Disallow: /api/auth
Disallow: /api/auth/
Disallow: /api/admin
Disallow: /api/admin/
Disallow: /api/cron/
Disallow: /api/dsar/
Disallow: /api/consent/
Disallow: /api/monitoring/
Disallow: /api/analytics
Disallow: /api/analytics/
Disallow: /api/checkout-sessions
Disallow: /api/checkout-sessions/
Disallow: /api/checkout-sessions/*
Disallow: /_next/
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /*/search
Disallow: /*/search?*
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*
# Meta link-preview, LinkedIn, Pinterest, and X (Twitter) card previews
User-agent: FacebookBot
User-agent: Meta-ExternalFetcher
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: Twitterbot
Allow: /
Allow: /.well-known/
Disallow: /api/
Disallow: /_next/
Disallow: /*/thank-you
Disallow: /privacy-center
Disallow: /*/privacy-center
Disallow: /account
Disallow: /account/
Disallow: /account/*
Disallow: /*/account
Disallow: /*/account/
Disallow: /*/account/*
Disallow: /cart
Disallow: /cart/
Disallow: /cart/*
Disallow: /*/cart
Disallow: /*/cart/
Disallow: /*/cart/*
Disallow: /checkout
Disallow: /checkout/
Disallow: /checkout/*
Disallow: /*/checkout
Disallow: /*/checkout/
Disallow: /*/checkout/*
Disallow: /admin
Disallow: /admin/
Disallow: /admin/*
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/admin/*
Disallow: /auth$
Disallow: /auth/
Disallow: /auth/*
Disallow: /*/auth$
Disallow: /*/auth/
Disallow: /*/auth/*
Disallow: /analytics
Disallow: /analytics/
Disallow: /analytics/*
Disallow: /*/analytics
Disallow: /*/analytics/
Disallow: /*/analytics/*
Disallow: /webhooks
Disallow: /webhooks/
Disallow: /webhooks/*
Disallow: /*/webhooks
Disallow: /*/webhooks/
Disallow: /*/webhooks/*

Sitemap: https://bolotstudio.com/sitemap-index.xml

# Agent instructions: https://bolotstudio.com/agents.md
# AI manifest: https://bolotstudio.com/llms.txt
# AI manifest (full): https://bolotstudio.com/llms-full.txt
# Agentic commerce (UCP): https://bolotstudio.com/.well-known/ucp
# Agentic discovery API: https://bolotstudio.com/api/ucp/discover