# ============================================================
# robots.txt — meubelmaatwerksite.nl
# ============================================================

# --- Googlebot ------------------------------------------
User-agent: Googlebot
Allow: /

# --- Google Ads bot -------------------------------------
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /


Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /xmlrpc.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# --- Googlebot-Image ------------------------------------
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# --- Bingbot --------------------------------------------
User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Allow: /wp-content/uploads/
Crawl-delay: 1

# --- AI training crawlers --------------------------------
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

# --- Aggressive / low-value scrapers --------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: meta-externalagent
Disallow: /
Crawl-delay: 2

# --- All other crawlers (catch-all) ----------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /xmlrpc.php
Allow: /wp-content/uploads/
Crawl-delay: 1

# --- Sitemaps -------------------------------------------
Sitemap: https://www.meubelmaatwerksite.nl/sitemap_index.xml