# =====================================================================
# ROBOTS.TXT — vivacioussolutions.com
# Last Updated: 2026-05-31
# =====================================================================


# =============================================================
# SECTION 1: SEARCH ENGINE CRAWLERS — FULL ACCESS
# =============================================================

# =============================================================

User-agent: Googlebot
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /contact/
Allow: /portfolio/
Allow: /faq/
Allow: /pricing/
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /offline-advertisment/
Allow: /static/
Allow: /icons/
# --- Gatsby Internal / Build Artifacts ---
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /chunk-map.json
# --- Source Maps (CRITICAL: exposed in production) ---
Disallow: /*.js.map
# --- Security-Critical Directories (found in Drive crawl) ---
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /server/
Disallow: /function/
Disallow: /include/
Disallow: /crm/
Disallow: /mangment/
Disallow: /wppanel-/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /fitlicious/
Disallow: /spn/
Disallow: /ppfj/
Disallow: /dist/
Disallow: /.trash/
# --- Exposed Archives (P0 security vulnerability) ---
Disallow: /public.zip
Disallow: /*.zip$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.env$
Disallow: /*.tar$
Disallow: /*.gz$
# --- Tracking / Analytics Parameters ---
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=
Disallow: /*?preview=

User-agent: Googlebot-Image
Allow: /static/
Allow: /icons/
Disallow: /_gatsby/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /*.js.map

User-agent: Googlebot-Video
Allow: /
Disallow: /_gatsby/
Disallow: /admin/

User-agent: Bingbot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /chunk-map.json
Disallow: /*.js.map
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /server/
Disallow: /function/
Disallow: /include/
Disallow: /crm/
Disallow: /mangment/
Disallow: /wppanel-/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /fitlicious/
Disallow: /spn/
Disallow: /ppfj/
Disallow: /dist/
Disallow: /.trash/
Disallow: /public.zip
Disallow: /*.zip$
Disallow: /*.sql$
Disallow: /*.env$
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=

User-agent: Slurp
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /*.js.map

User-agent: DuckDuckBot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /*.js.map

User-agent: Baiduspider
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/
Disallow: /*.js.map

User-agent: YandexBot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/
Disallow: /*.js.map

User-agent: Applebot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/


# =============================================================
# SECTION 2: AI RETRIEVAL AGENTS — ALLOW (RAG / CITATION BOTS)
# =============================================================

# =============================================================

# Google Gemini / AI Overviews Retrieval Agent
User-agent: Google-Extended
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /contact/
Allow: /portfolio/
Allow: /faq/
Allow: /pricing/
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /server/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /crm/
Disallow: /wppanel-/
Disallow: /fitlicious/
Disallow: /*.js.map

# OpenAI SearchGPT Retrieval Agent (cites content in search)
User-agent: OAI-SearchBot
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /portfolio/
Allow: /faq/
Allow: /pricing/
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /server/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /crm/
Disallow: /wppanel-/
Disallow: /fitlicious/
Disallow: /*.js.map

# Perplexity AI Retrieval Agent
User-agent: PerplexityBot
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /portfolio/
Allow: /faq/
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /crm/
Disallow: /wppanel-/
Disallow: /fitlicious/
Disallow: /*.js.map

# Microsoft Copilot Retrieval
User-agent: CopilotBot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/
Disallow: /*.js.map

# Anthropic Claude Web Retrieval
User-agent: ClaudeBot
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /portfolio/
Allow: /faq/
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/
Disallow: /*.js.map

# Brave Search
User-agent: BraveBot
Allow: /
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /admin/
Disallow: /backend/
Disallow: /backup/


# =============================================================
# SECTION 3: AI TRAINING SCRAPERS — BLOCK
# =============================================================

# =============================================================

# OpenAI Training Scraper (NOT the search agent)
User-agent: GPTBot
Disallow: /

# Common Crawl (bulk training data harvester)
User-agent: CCBot
Disallow: /

# ByteDance / TikTok Training Scraper
User-agent: Bytespider
Disallow: /

# Meta / Facebook Training Scraper
User-agent: FacebookBot
Disallow: /

# Cohere AI Training Scraper
User-agent: cohere-ai
Disallow: /

# Diffbot Content Scraper
User-agent: Diffbot
Disallow: /

# Anthropic Training Scraper (distinct from ClaudeBot retrieval)
User-agent: anthropic-ai
Disallow: /

# Amazon Alexa Crawler (deprecated but still active)
User-agent: ia_archiver
Disallow: /

# Omgili Data Mining Bot
User-agent: omgili
Disallow: /

# Webz.io Scraper
User-agent: Webzio-Extended
Disallow: /

# Timpi Cloud Scraper
User-agent: Timpibot
Disallow: /


# =============================================================
# SECTION 4: AGGRESSIVE SEO SCRAPERS — BLOCK
# =============================================================

# =============================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Megaindex.ru
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: Seekport Crawler
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: Rogerbot
Disallow: /


# =============================================================
# SECTION 5: DEFAULT WILDCARD — RESTRICTIVE BASELINE
# =============================================================

# =============================================================

User-agent: *
Allow: /
Allow: /about-us/
Allow: /services/
Allow: /blog/
Allow: /contact/
Allow: /portfolio/
Allow: /faq/
Allow: /pricing/
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /offline-advertisment/
Allow: /static/

# --- Gatsby Build Artifacts (NOT indexable) ---
Disallow: /_gatsby/
Disallow: /page-data/
Disallow: /chunk-map.json
Disallow: /*.js.map

# --- Security-Critical Directories (from Drive crawl) ---
# NOTE: These directories were physically found in the codebase.
# They MUST be blocked from all crawlers AND ideally removed
# from the production server entirely.
Disallow: /admin/
Disallow: /adminn/
Disallow: /admin_backup/
Disallow: /backend/
Disallow: /server/
Disallow: /function/
Disallow: /include/
Disallow: /crm/
Disallow: /mangment/
Disallow: /wppanel-/
Disallow: /backup/
Disallow: /backup-file/
Disallow: /fitlicious/
Disallow: /dist/
Disallow: /spn/
Disallow: /ppfj/
Disallow: /.trash/

# --- Exposed Archives & Sensitive Files ---
# CRITICAL: public.zip (5.8MB source code archive) is exposed
Disallow: /public.zip
Disallow: /*.zip$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.env$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.git$

# --- Tracking / Analytics Parameters ---
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=
Disallow: /*?preview=


# =============================================================
# SECTION 6: SITEMAP DECLARATIONS
# =============================================================

# =============================================================

Sitemap: https://www.vivacioussolutions.com/sitemap.xml
Sitemap: https://www.vivacioussolutions.com/sitemap-pages.xml
Sitemap: https://www.vivacioussolutions.com/sitemap-blog.xml
Sitemap: https://www.vivacioussolutions.com/sitemap-services.xml
Sitemap: https://www.vivacioussolutions.com/sitemap-images.xml

# =====================================================================