# =============================================================================
# Default rules for all crawlers (Googlebot, bingbot, social bots, AI
# browsing bots, etc. all inherit from this unless overridden below)
# =============================================================================

User-agent: *
Allow: /

# Block deep search pagination (allow only pages 1-2)
Allow: /s?*page=1$
Allow: /s?*page=2$
Disallow: /s?*page=

# Block search filter/geo params that create crawl traps
Disallow: /s?*boundingbox=
Disallow: /s?*filter_option_
Disallow: /s?*price_min=
Disallow: /s?*price_max=

# Block spam/garbage category slugs
Disallow: /s/pynrjqpzelyyttj
Disallow: /s/deqtrfkjotlhrkd
Disallow: /s/sgzytwlztevqjl
Disallow: /s/bjrfsnfdyl
Disallow: /s/ynogqzzokapzodkj

# Block legacy language-prefixed URLs (old site structure)
Disallow: /en/
Disallow: /fr/
Disallow: /de/
Disallow: /ja/
Disallow: /es-ES/

# Block old /listings/ URL format (replaced by /l/)
Disallow: /listings/

# Block internal/utility pages
Disallow: /listing_images/
Disallow: /invitations/
Disallow: /user_feedbacks/
Disallow: /infos/
Disallow: /listings.atom
Disallow: /people/

# Block admin, auth, and private pages
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /account
Disallow: /profile-settings
Disallow: /settings/
Disallow: /dashboard/
Disallow: /inbox
Disallow: /transactions/
Disallow: /order/
Disallow: /sale/
Disallow: /checkout/
Disallow: /payment/
Disallow: /reset-password
Disallow: /verify-email
Disallow: /confirm/
Disallow: /recover-password
Disallow: /contact-us/success

Disallow: /login
Disallow: /signup
Disallow: /authentication/

# Auth-required pages
Disallow: /favorites
Disallow: /l/new
Disallow: /listings/new

# Block test/staging/dev paths
Disallow: /test/
Disallow: /staging/
Disallow: /dev/
Disallow: /temp/
Disallow: /styleguide/

# Block session-related URLs
Disallow: /*sessionid*
Disallow: /*session_id*
Disallow: /*PHPSESSID*

# =============================================================================
# BLOCKED BOTS - Aggressive/Unwanted Crawlers
# =============================================================================

User-agent: SiteAuditBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: ImageSift
Disallow: /

User-agent: PicScout
Disallow: /

# =============================================================================
# AI CRAWLERS - Discovery, Citations, and Training (Allowed Public Content)
#
# Goal: maximize references and links in LLM products while protecting
# private/authenticated paths and crawl-trap URLs.
# =============================================================================

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap-listings.xml
Allow: /l/
Allow: /shop/wheels/
Allow: /s
Allow: /s?*
Allow: /shop/
Allow: /shops/
Allow: /for-shops
Allow: /blog/
Allow: /resources/
Allow: /help
Allow: /faq
Allow: /protection
Allow: /about
Allow: /how-it-works
Allow: /index
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /account
Disallow: /profile-settings
Disallow: /settings/
Disallow: /dashboard/
Disallow: /inbox
Disallow: /transactions/
Disallow: /order/
Disallow: /sale/
Disallow: /checkout/
Disallow: /payment/
Disallow: /reset-password
Disallow: /verify-email
Disallow: /confirm/
Disallow: /recover-password
Disallow: /authentication/
Disallow: /favorites
Disallow: /l/new
Disallow: /listings/new
Disallow: /s?*page=
Disallow: /s?*boundingbox=
Disallow: /s?*filter_option_
Disallow: /s?*price_min=
Disallow: /s?*price_max=
Disallow: /s/pynrjqpzelyyttj
Disallow: /s/deqtrfkjotlhrkd
Disallow: /s/sgzytwlztevqjl
Disallow: /s/bjrfsnfdyl
Disallow: /s/ynogqzzokapzodkj
Disallow: /test/
Disallow: /staging/
Disallow: /dev/
Disallow: /temp/
Disallow: /styleguide/
Disallow: /*sessionid*
Disallow: /*session_id*
Disallow: /*PHPSESSID*

# =============================================================================
# AI CRAWLERS - Training-Only / Scraper-Like (Still Blocked)
# =============================================================================

User-agent: Cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# =============================================================================
# AI CRAWLERS - Aggressive/Foreign (BLOCKED)
# =============================================================================

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: Nicecrawler
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

Sitemap: https://www.wheelprice.com/sitemap.xml
Sitemap: https://www.wheelprice.com/sitemap-listings.xml