# ============================================================
# Search / AI Retrieval Policy
# Search indexing and real-time AI retrieval permitted.
# AI model training / bulk data collection not permitted.
# ============================================================

User-agent: *
Disallow: /mc_db_admin/
Disallow: /staff/
Disallow: /shopify-modules/
Disallow: /serverprotect/
Disallow: /captures/




Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /


# ============================================================
# OPENAI
# ============================================================

# ChatGPT Search indexing / retrieval
User-agent: OAI-SearchBot
Allow: /

# User-requested ChatGPT page access
User-agent: ChatGPT-User
Allow: /

# OpenAI model training
User-agent: GPTBot
Disallow: /


# ============================================================
# GOOGLE
# ============================================================

# Normal Google Search
User-agent: Googlebot
Allow: /

# Google AI / model-training use
# Blocking Google-Extended does not block normal Google Search.
User-agent: Google-Extended
Disallow: /


# ============================================================
# MICROSOFT / BING
# ============================================================

User-agent: bingbot
Allow: /


# ============================================================
# ANTHROPIC
# ============================================================

# Claude Search / retrieval
User-agent: Claude-SearchBot
Allow: /

# Claude model training crawler
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /


# ============================================================
# APPLE
# ============================================================

# Apple search/indexing
User-agent: Applebot
Allow: /

# Apple AI / model-training extension
User-agent: Applebot-Extended
Disallow: /


# ============================================================
# OTHER AI / BULK DATA COLLECTION
# ============================================================

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /


# ============================================================
# SEO / SITE AUDITING
# ============================================================

User-agent: SiteAuditBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-BA
Allow: /

User-agent: SemrushBot-SI
Allow: /

User-agent: SemrushBot-SWA
Allow: /

User-agent: SemrushBot-CT
Allow: /

User-agent: SemrushBot-COUB
Allow: /


# ============================================================
# LOW-VALUE / NON-TARGET CRAWLERS
# ============================================================

User-agent: baiduspider
Disallow: /

User-agent: baiduspider-image
Disallow: /

User-agent: baiduspider-mobile
Disallow: /

User-agent: baiduspider-news
Disallow: /

User-agent: baiduspider-video
Disallow: /

User-agent: yandex
Disallow: /

User-agent: MJ12bot
Disallow: /


# ============================================================
# SITEMAP
# ============================================================

Sitemap: https://aucklandharleydavidson.co.nz/sitemap.xml