# robots.txt for Big Internet Ecommerce (BIE)
# Canonical site: https://www.biginternetcommerce.com/
# Purpose: Allow search engines and AI crawlers to discover public pages while keeping non-public/internal paths out of crawl focus.

User-agent: *
Allow: /

# Important public discovery files
Allow: /llms.txt
Allow: /sitemap.xml

# Keep common internal, admin, staging, and utility URLs out of crawl focus
Disallow: /admin/
Disallow: /login/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /*?s=
Disallow: /search
Disallow: /search/
Disallow: /tag/
Disallow: /author/
Disallow: /category/
Disallow: /thank-you
Disallow: /thank-you/
Disallow: /404
Disallow: /404/
Disallow: /utility-pages/
Disallow: /draft/
Disallow: /staging/
Disallow: /test/
Disallow: /old/
Disallow: /preview/
Disallow: /*?preview=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=

# Allow major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# Allow AI and answer-engine crawlers for public content discovery
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /



Sitemap: https://biginternetcommerce.com/sitemap.xml