# robots.txt for smartg4.com
# Last updated: 2026-09-21 — full AI + global search engine coverage

# ── Standard search engines ──────────────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI crawlers — ChatGPT / OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# AI crawlers — Claude / Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# AI crawlers — Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# AI crawlers — Google Gemini
User-agent: Google-Extended
Allow: /

User-agent: Gemini
Allow: /

# AI crawlers — Meta AI
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# AI crawlers — Cohere
User-agent: cohere-ai
Allow: /

# AI crawlers — DeepSeek
User-agent: DeepSeekBot
Allow: /

# AI crawlers — ByteDance / TikTok AI
User-agent: Bytespider
Allow: /

# AI crawlers — Apple Intelligence
User-agent: Applebot-Extended
Allow: /

User-agent: Applebot
Allow: /

# AI crawlers — Baidu AI
User-agent: Baiduspider
Allow: /

# AI crawlers — Yandex AI (Alice)
User-agent: YandexBot
Allow: /

# AI crawlers — xAI / Grok
User-agent: Grok
Allow: /

# AI crawlers — Mistral AI
User-agent: MistralBot
Allow: /

# AI crawlers — Amazon Alexa / Amazonbot
User-agent: Amazonbot
Allow: /

# AI crawlers — OAI SearchBot (OpenAI search)
User-agent: OAI-SearchBot
Allow: /

# AI crawlers — Operator (OpenAI Operator agent)
User-agent: Operator
Allow: /

# AI crawlers — Diffbot (AI knowledge graph)
User-agent: Diffbot
Allow: /

# AI crawlers — Common Crawl (used by many LLMs for training)
User-agent: CCBot
Allow: /

# AI crawlers — SonixAI
User-agent: SonixAI
Allow: /

# AI crawlers — Timpibot (AI search)
User-agent: Timpibot
Allow: /

# AI crawlers — YouBot (You.com AI search)
User-agent: YouBot
Allow: /

# AI crawlers — Omgilibot (AI data)
User-agent: omgilibot
Allow: /

# AI crawlers — iaskspider (iAsk AI)
User-agent: iaskspider
Allow: /

# Standard search engines — Yahoo / Slurp
User-agent: Slurp
Allow: /

# Standard search engines — Naver (South Korea) — Yeti
User-agent: Yeti
Allow: /

User-agent: NaverBot
Allow: /

# Standard search engines — Baidu (China)
User-agent: Baiduspider-render
Allow: /

# Standard search engines — Sogou (China)
User-agent: Sogou web spider
Allow: /

# Standard search engines — 360 Spider (China)
User-agent: 360Spider
Allow: /

# Standard search engines — Shenma / UC Web (China mobile)
User-agent: Baiduspider-mobile
Allow: /

# Standard search engines — Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /

# Standard search engines — Mojeek (UK privacy search)
User-agent: MojeekBot
Allow: /

# Standard search engines — Brave Search
User-agent: BraveBot
Allow: /

# Standard search engines — Ecosia (Germany, green search)
User-agent: Ecosia
Allow: /

# Standard search engines — Qwant (France)
User-agent: Qwantify
Allow: /

# Standard search engines — Petal Search / Huawei (Arabic + Asian markets)
User-agent: PetalBot
Allow: /

# Standard search engines — Cốc Cốc (Vietnam)
User-agent: coccocbot-web
Allow: /

User-agent: coccocbot-image
Allow: /

# Standard search engines — Daum / Kakao (South Korea)
User-agent: Daumoa
Allow: /

# Standard search engines — Mail.ru / Go.Mail.ru (Russia)
User-agent: Mail.RU_Bot
Allow: /

# Standard search engines — Onet.pl (Poland)
User-agent: OnetSzukaj
Allow: /

# Standard search engines — Internet Archive / Wayback Machine
User-agent: ia_archiver
Allow: /

# Standard search engines — Googlebot News
User-agent: Googlebot-News
Allow: /

# Standard search engines — Googlebot Image
User-agent: Googlebot-Image
Allow: /

# Social media crawlers — allow for rich previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

# All other crawlers — allow public pages, block private paths
User-agent: *
Allow: /
Disallow: /api/
Disallow: /private/

Sitemap: https://smartg4.com/sitemap.xml
Sitemap: https://smartg4.com/sitemap_index.xml