User-agent: *

# Block WordPress search results
Disallow: /?s=
Disallow: /search/

# Block WordPress admin/private areas
Disallow: /wp-admin/
Disallow: /wp-login.php

# Allow AJAX for WordPress plugins/themes
Allow: /wp-admin/admin-ajax.php

# Block WordPress feeds (optional - remove if you use RSS)
Disallow: /feed/
Disallow: /comments/feed/

# Block common duplicate pages
Disallow: /?author=
Disallow: /author/
Disallow: /tag/*/feed/
Disallow: /category/*/feed/

# Block trackbacks
Disallow: /trackback/

# Block temporary/cache folders
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/cache/


# --------------------------
# Block known aggressive SEO/scraper bots
# --------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: CensysInspect
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /


# --------------------------
# Keep major search engines
# --------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Google-InspectionTool
Allow: /

Sitemap: https://boldnews.al/sitemap_index.xml