# Global rules for all crawlers
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/private/
Disallow: /trackback/
Disallow: /xmlrpc.php  # Blocks access to XML-RPC functionality
Disallow: /?s=         # Blocks internal search results pages
Disallow: /author/     # Blocks author archive pages
Disallow: /tag/        # Blocks tag pages (optional, if not valuable)
Disallow: /category/   # Blocks category pages (optional)
Disallow: /comments/feed/
Disallow: /readme.html # Protects from exposing WordPress version

# Allow important public resources
Allow: /wp-content/uploads/
Allow: /wp-includes/js/

# Sitemap directive
Sitemap: https://theworkathomewife.com/sitemap_index.xml

# Crawler-specific directives
# Block known resource-heavy crawlers or scrapers
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /