# Robots.txt for dreamanalyzer.net
# AI Dream Analyzer - Free Dream Interpretation & Meaning

User-agent: *
Allow: /
Allow: /favicon.ico
Allow: /favicon-*.png
Allow: /apple-touch-icon.png
Allow: /site.webmanifest
Allow: /browserconfig.xml

# Disallow private/admin areas
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /gcloud-service-account.json

# Raw article JSON consumed by the symbol pages; the pages themselves are what should be indexed
Disallow: /deep/

# Disallow referral and marketing parameters to save crawl budget
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=

# Disallow malformed URLs with multiple language codes or trailing locales
Disallow: /*/de$
Disallow: /*/fr$
Disallow: /*/es$
Disallow: /*/it$
Disallow: /*/tr$
Disallow: /*/ru$
Disallow: /*/pt$
Disallow: /*/ja$
Disallow: /*/ko$
Disallow: /*/th$
Disallow: /*/hi$
Disallow: /*/zh-CN$
Disallow: /*/zh-TW$
Disallow: /dream-meaning/
Disallow: /symbols/*/*/
Disallow: /faq/*/

# Allow search engines to crawl important canonical pages
Allow: /symbols/
Allow: /mystic-dreams/
Allow: /faq/
Allow: /feedback/
Allow: /know-about-dreams/
Allow: /about
Allow: /privacy
Allow: /terms

# Crawl delay (be respectful)
Crawl-delay: 1

# Sitemaps
Sitemap: https://dreamanalyzer.net/sitemap-index.xml
Sitemap: https://dreamanalyzer.net/sitemap.xml
Sitemap: https://dreamanalyzer.net/sitemap-bing.xml

# Additional bot instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Bing-specific configuration
User-agent: bingbot
Allow: /
Allow: /favicon.ico
Allow: /favicon-*.png
Allow: /apple-touch-icon.png
Allow: /site.webmanifest
Allow: /browserconfig.xml
Allow: /symbols/
Allow: /mystic-dreams/
Allow: /faq/
Allow: /feedback/
Allow: /know-about-dreams/
Crawl-delay: 1

# Bing image search
User-agent: Bingpreview
Allow: /favicon.ico
Allow: /favicon-*.png
Allow: /apple-touch-icon.png
Allow: /images/
Allow: /symbols/
Crawl-delay: 2

# Bing mobile search
User-agent: BingMobile
Allow: /
Crawl-delay: 1

# Baidu search engine configuration
User-agent: Baiduspider
Allow: /
Allow: /symbols/
Allow: /mystic-dreams/
Allow: /faq/
Allow: /feedback/
Allow: /know-about-dreams/
Crawl-delay: 1

# Baidu image search
User-agent: Baiduspider-image
Allow: /images/
Allow: /symbols/
Crawl-delay: 2

# Baidu mobile search
User-agent: Baiduspider-mobile
Allow: /
Crawl-delay: 1

# Baidu news search
User-agent: Baiduspider-news
Allow: /mystic-dreams/
Allow: /faq/
Allow: /know-about-dreams/
Crawl-delay: 1

User-agent: GPTBot
Allow: /symbols/
Allow: /faq/
Allow: /know-about-dreams/
Disallow: /api/
Disallow: /dashboard/

# Allow SEO analysis tools with rate limiting
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Disallow: /