# Robots.txt for GigNGo

User-agent: *
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /test/
Disallow: /facebook-groups/

# AI/LLM Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# RETRIEVAL crawlers — added 2026-09-06. These are the ones that decide whether
# we get RECOMMENDED today, and they are NOT the same agents as the training
# crawlers above. GPTBot trains; OAI-SearchBot is what fetches pages to answer a
# ChatGPT search. ClaudeBot is the current Anthropic crawler (anthropic-ai and
# Claude-Web are the older names, kept above so an old cache still matches).
# Perplexity-User is its user-initiated fetch, distinct from PerplexityBot.
# Everything is allowed by "User-agent: *" anyway; these are explicit so a future
# Disallow cannot silently take the answer engines down with it.
User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

# Baiduspider, added 2026-09-13. Baidu fetched the Flutter app bundle
# (/g/main.dart.js, cache-bypassed at Cloudflare) ~1,040 times a day: 3.1 GB/day,
# ~64% of all Firebase Hosting egress, for a US-only local marketplace. A named
# group REPLACES the "*" rules for that crawler, so those Disallows are restated.
User-agent: Baiduspider
Disallow: /g/
Disallow: /app/
Disallow: /admin/
Disallow: /test/
Disallow: /facebook-groups/

Sitemap: https://gigngo.org/sitemap.xml