# ============================================
# robots.txt for https://mivja.com
# Owner: Arindam Goswami
# Updated: 2025-11-01
# Purpose: Control access for search engines, social media crawlers, and AI systems
# ============================================

User-agent: *
Disallow: /back/
Disallow: /config/
Disallow: /includes/
Disallow: /uploads/
Disallow: /private/
Disallow: /tmp/
Disallow: /404error.php
Disallow: /about-me
Disallow: /admin/

Allow: /css/
Allow: /js/
Allow: /images/
Allow: /back/assets/css/
Allow: /back/assets/js/
Allow: /posts/
Allow: /posts/blogs/
Allow: /posts/web-development/
Allow: /posts/seo/

User-agent: bingbot
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: FacebookExternalHit
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: Pinterest
Disallow:

User-agent: WhatsApp
Disallow:

User-agent: facebot
Disallow:

User-agent: TelegramBot
Disallow:

User-agent: GPTBot
Disallow: /admin/
Allow: /

User-agent: Google-Extended
Disallow: /admin/
Allow: /

User-agent: ClaudeBot
Disallow: /admin/
Allow: /

User-agent: PerplexityBot
Disallow: /admin/
Allow: /

User-agent: CCBot
Allow: /

# Reference for AI systems:
# https://mivja.com/ai.txt

Sitemap: https://mivja.com/sitemap.xml