# robots.txt for VentMarket.md
# This file tells search engines which pages they can and cannot crawl

# Allow all search engines to crawl the entire site
User-agent: *
Allow: /
Disallow: /sanity-debug

# Sitemap location
Sitemap: https://ventmarket.md/sitemap.xml

# Crawl delay for polite crawling (optional, helps with server load)
# Crawl-delay: 1

# Disallow specific paths if needed in the future
# Example: Admin areas, API endpoints, or private sections
# Disallow: /admin/
# Disallow: /api/
# Disallow: /private/

# Allow crawling of CSS and JS files (important for Google's rendering)
Allow: /*.css
Allow: /*.js

# Special rules for specific search engines

# Google-specific
User-agent: Googlebot
Allow: /

# Bing-specific
User-agent: Bingbot
Allow: /

# Yandex (important for Russian-speaking audience in Moldova)
User-agent: Yandex
Allow: /

# Baidu (if targeting Chinese market in the future)
User-agent: Baiduspider
Allow: /

# Block common bad bots (optional security measure)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# AI Crawlers (LLMs) - Allow to improve LLM search visibility
# OpenAI GPT crawler
User-agent: GPTBot
Allow: /

# Anthropic Claude crawler
User-agent: anthropic-ai
Allow: /

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many AI services)
User-agent: CCBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /