# ============================================================
# robots.txt for Servis.GE
# https://servis.ge/
# Last updated: 2026-04-29
# ============================================================

# ── Default rules for all crawlers ─────────────────────────
User-agent: *

# Allow all public content
Allow: /
Allow: /ka/
Allow: /ru/
Allow: /en/
Allow: /search/
Allow: /service/
Allow: /services
Allow: /category/
Allow: /ka/category/
Allow: /ru/category/
Allow: /en/category/

# Allow public PHP endpoints (JSON APIs that crawlers can read)
Allow: /get_services.php
Allow: /categories.php
Allow: /locations.php
Allow: /regions.php
Allow: /reviews_overview.php
Allow: /service_stats.php
Allow: /sitemap.xml
Allow: /sitemap-static.xml
Allow: /sitemap-services.xml
Allow: /sitemap-search.xml
Allow: /sitemap-combos.xml
Allow: /sitemap.php
Allow: /sitemap_static.php
Allow: /sitemap_services.php
Allow: /sitemap_search.php
Allow: /sitemap_combos.php

# Allow static assets so previews / link cards render
Allow: /uploads/
Allow: /favicon.ico
Allow: /firebase-messaging-sw.js
Allow: /sw.js
Allow: /*.css$
Allow: /*.js$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.woff
Allow: /*.woff2

# ── Block admin / internal / sensitive ─────────────────────
Disallow: /admin/
Disallow: /log/
Disallow: /api/
Disallow: /tbc/
Disallow: /uploads/tmp/
Disallow: /node_modules/
Disallow: /src/
Disallow: /dist/
Disallow: /migrations/
Disallow: /.git/
Disallow: /.claude/

# Block direct .php access (except whitelisted JSON endpoints above)
Disallow: /db.php
Disallow: /db_admin.php
Disallow: /auth.php
Disallow: /balance.php
Disallow: /chat.php
Disallow: /comments.php
Disallow: /ratings.php
Disallow: /reports.php
Disallow: /feedback.php
Disallow: /upload.php
Disallow: /upload_file.php
Disallow: /orders.php
Disallow: /packages.php
Disallow: /notifications.php
Disallow: /user_profile.php
Disallow: /user_services.php
Disallow: /set_recommended_listings.php
Disallow: /cron_check_expiry.php
Disallow: /cleanup_deleted.php
Disallow: /fcm_cron.php
Disallow: /fcm_send.php
Disallow: /fcm_token.php
Disallow: /sms.php
Disallow: /sqlrun.php
Disallow: /service_history_helper.php

# Block raw SQL / config / dev files
Disallow: /*.sql$
Disallow: /*.json$
Disallow: /*.ts$
Disallow: /*.tsx$
Disallow: /*.map$
Disallow: /*.env

# Block tracking-only query strings (saves crawl budget)
Disallow: /*?*utm_
Disallow: /*?*fbclid
Disallow: /*?*gclid
Disallow: /*?*ref=
Disallow: /*?*sessionid
Disallow: /*?*PHPSESSID

# ── Crawl-delay (default) ──────────────────────────────────
Crawl-delay: 1

# ============================================================
# Search engine specific rules
# ============================================================

# Google — full speed, no delay
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /uploads/
Allow: /*.webp$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$

User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 0

# Bing — slight delay
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yandex (important for Russian-speaking users)
User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: YandexImages
Allow: /uploads/

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Apple
User-agent: Applebot
Allow: /
Crawl-delay: 1

# Facebook / Twitter / Telegram link previews
User-agent: facebookexternalhit
Allow: /
User-agent: Twitterbot
Allow: /
User-agent: TelegramBot
Allow: /
User-agent: WhatsApp
Allow: /
User-agent: LinkedInBot
Allow: /

# ── AI / LLM training crawlers — explicit policy ───────────
# Allow indexing-style bots (so AI search products surface us);
# block bulk training scrapers that don't respect our content.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ── Block aggressive SEO-spy / scraper bots ────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: dataforseobot
Disallow: /

# ============================================================
# Sitemaps (Google honors multiple Sitemap: directives)
# ============================================================
Sitemap: https://servis.ge/sitemap.xml
Sitemap: https://servis.ge/sitemap-static.xml
Sitemap: https://servis.ge/sitemap-services.xml
Sitemap: https://servis.ge/sitemap-search.xml
Sitemap: https://servis.ge/sitemap-combos.xml

# Host preference (helps Yandex)
Host: servis.ge