User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /404.shtml

# Optional: Disallow plugin and cache folders (can reduce indexing of unneeded stuff)
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/

# Allow essential assets
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js