# robots.txt for hotely.travel
# Canonical URLs live at /hotel/{id}. Query-string variants (?campaign=, ?custom=, ?reload=true, …)
# are handled via <link rel="canonical">, but we also block well-known tracking/duplicate
# parameters here for crawlers that don't honor canonical tags.
#
# Rule semantics (Google / RFC 9309): longest matching Allow or Disallow wins; if equal length, Allow wins.
# Sample URL matrix (path + query as seen by crawlers):
#
# | URL | User-agent: * | OAI-AdsBot / OAI-SearchBot |
# |-----|-----------------|----------------------------|
# | /hotel/12345 | allowed | allowed |
# | /hotel/12345?utm_source=chatgpt&utm_campaign=summer | allowed | allowed |
# | /hotel/12345?campaign=Hotely_GPT | blocked (campaign=) | allowed |
# | /search?city=Paris&utm_medium=cpc | allowed | allowed |
# | /checkout | blocked | blocked |
# | /booking/abc | blocked | blocked |
# | /payment-return | blocked | blocked |
# | /api/health | blocked | blocked |
# | /hotel/123/availability | blocked | blocked |
# | /hotel/verify-room-deal | blocked | blocked |
# | /hotel/123?reload=true | blocked | allowed |
# | /hotel/123?custom=foo | blocked | allowed |
# | /hotel/208625?start_date=2026-11-08&end_date=2026-11-14&party=…&campaign=Hotely_GPT&utm_* (ChatGPT ad landing) | blocked (campaign=) | allowed |
# | same + &is_first_click=true | blocked (campaign= + is_first_click=) | allowed |
# Canonical for that page remains https://hotely.travel/hotel/208625 (no query string; see HotelController CanonicalPath).

User-agent: *
Allow: /

# Block transactional / private flows — not useful for SERP and waste crawl budget.
Disallow: /checkout
Disallow: /booking
Disallow: /payment-return
Disallow: /api/
Disallow: /docs/
Disallow: /*.html$

# Block hot supplier-burning AJAX endpoints (rate verify + per-hotel availability).
# These would call booking-platform-service / search-service, which fan out to suppliers.
Disallow: /hotel/*/availability
Disallow: /hotel/verify-room-deal

# Block tracking / duplicate URL variants (safety net alongside canonical tags).
# Use ?campaign= / &campaign= (not ?*campaign=) so utm_campaign= is not blocked.
Disallow: /*?reload=true
Disallow: /*&reload=true
Disallow: /*?campaign=
Disallow: /*&campaign=
Disallow: /*?custom=
Disallow: /*&custom=
Disallow: /*?is_first_click=
Disallow: /*&is_first_click=
Disallow: /*?queryId=
Disallow: /*&queryId=
Disallow: /*?query_id=
Disallow: /*&query_id=

# AI-index crawlers that walked our search-result deeplinks under the * group anyway (Oct 2026).
# Named explicitly in case they only honor a group for their own token. A named group REPLACES *,
# so it repeats every * rule. The server also serves them no live pricing (Helpers/CrawlerGate.cs).
User-agent: meta-externalagent
User-agent: Baiduspider
Allow: /
Disallow: /checkout
Disallow: /booking
Disallow: /payment-return
Disallow: /api/
Disallow: /docs/
Disallow: /*.html$
Disallow: /hotel/*/availability
Disallow: /hotel/verify-room-deal
Disallow: /*?reload=true
Disallow: /*&reload=true
Disallow: /*?campaign=
Disallow: /*&campaign=
Disallow: /*?custom=
Disallow: /*&custom=
Disallow: /*?is_first_click=
Disallow: /*&is_first_click=
Disallow: /*?queryId=
Disallow: /*&queryId=
Disallow: /*?query_id=
Disallow: /*&query_id=

# OpenAI ad review and search crawlers: indexable pages including utm_* ad landing URLs;
# keep transactional, API, and supplier-heavy endpoints blocked (no tracking-param rules).
User-agent: OAI-AdsBot
User-agent: OAI-SearchBot
Allow: /
Disallow: /checkout
Disallow: /booking
Disallow: /payment-return
Disallow: /api/
Disallow: /docs/
Disallow: /*.html$
Disallow: /hotel/*/availability
Disallow: /hotel/verify-room-deal

# Allow Googlebot and major search engines explicit full access to indexable content.
User-agent: Googlebot
Allow: /
Disallow: /checkout
Disallow: /booking
Disallow: /payment-return
Disallow: /api/
Disallow: /docs/
Disallow: /hotel/*/availability
Disallow: /hotel/verify-room-deal
Disallow: /*?reload=true
Disallow: /*&reload=true
Disallow: /*?campaign=
Disallow: /*&campaign=
Disallow: /*?custom=
Disallow: /*&custom=
Disallow: /*?is_first_click=
Disallow: /*&is_first_click=

User-agent: Bingbot
Allow: /
Disallow: /checkout
Disallow: /booking
Disallow: /payment-return
Disallow: /api/
Disallow: /docs/
Disallow: /hotel/*/availability
Disallow: /hotel/verify-room-deal

# Google's hotel price checker (Hotel Center price accuracy). It follows robots.txt, and Hotel
# Center counts "robots.txt disallowed" as a blocked price validation. It lands on our ad
# deeplinks (?campaign=, ?custom=), loads /hotel/{id}/availability for the prices and follows the
# booking through to /checkout, so it needs all of them. (2026-10-12)
# AdsBot-Google and AdsBot-Google-Mobile ignore the * group (Google's special-case crawlers), so
# they are already unrestricted; a group for them here would only restrict them.
User-agent: Google-HotelAdsVerifier
Allow: /
Disallow: /api/
Disallow: /docs/

Sitemap: https://hotely.travel/sitemap.xml