User-Agent: *
Disallow: /admin/  # disable crawling django admin
Disallow: /cms/  # and wagtail admin
Disallow: /app/  # disable crawling user-specific video library
Disallow: /accounts/  # disable crawling account pages
Disallow: /users/  # disable crawling user profile pages
Disallow: /dashboard/  # disable crawling user dashboard
Disallow: /hijack/  # disable crawling admin impersonation
Disallow: /stripe/  # disable crawling payment webhooks

# Allow all other content
Allow: /

# Specific allowances for AI crawlers
User-Agent: GPTBot
Allow: /
Crawl-delay: 1

User-Agent: ChatGPT-User
Allow: /

User-Agent: Claude-Web
Allow: /

User-Agent: Anthropic-AI
Allow: /

# Sitemap location
Sitemap: https://sisif.ai/robots.txt/sitemap.xml