# public/robots.txt
# Optimized robots.txt for DataEngineer Hub

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*.json$
Disallow: /*?*debug*
Disallow: /debug

# Allow crawling of important resources
Allow: /*.css
Allow: /*.js
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.webp
Allow: /*.svg
Allow: /*.gif

# Sitemap
Sitemap: https://dataengineerhub.blog/sitemap.xml

# Block bad bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /