User-agent: *
# Content Signals (https://contentsignals.org/): SelvaScore WANTS its scores
# in search indexes, in AI answers, and in trained models. Crawl volume is a
# separate matter, governed by the per-agent groups below.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# Affiliate redirects: no crawl value (they are 302s, and rel="sponsored nofollow"
# already), and a bot that follows one fires a REAL tagged click at Amazon. Keeping
# crawlers out protects the Associates click-to-conversion ratio and the click analytics.
Disallow: /go/
Allow: /
# Optimize crawl budget by ignoring duplicate product views
Disallow: /*?sort=
Disallow: /*?caps=

# --- MAJOR AI CRAWLERS: CANONICAL PAGES ONLY ---
# SelvaScore data remains open to these crawlers: every product page,
# every hub, and every lens listing is crawlable below. The filter
# permutation space (brand/size/price/feature combinations) is closed
# to keep crawl volume sane. The full catalog is in /sitemap.xml and
# the URL grammar reference at /cite/ stays open.
# (Claude-SearchBot joined this group 2026-09: it was crawling ~165K
# URLs/day, 96% of them filter permutations. Citation surfaces stay open.)

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
Disallow: /go/
Disallow: /laptop/
Allow: /laptop/$
# Country hubs
Allow: /laptop/us/$
Allow: /laptop/uk/$
Allow: /laptop/de/$
Allow: /laptop/fr/$
Allow: /laptop/it/$
Allow: /laptop/es/$
Allow: /laptop/nl/$
Allow: /laptop/ie/$
Allow: /laptop/in/$
Allow: /laptop/au/$
Allow: /laptop/ca/$
Allow: /laptop/pl/$
Allow: /laptop/sg/$
Allow: /laptop/br/$
Allow: /laptop/jp/$
Allow: /laptop/kr/$
Allow: /laptop/ae/$
Allow: /laptop/za/$
Allow: /laptop/sa/$
Allow: /laptop/tr/$
Allow: /laptop/id/$
Allow: /laptop/il/$
Allow: /laptop/my/$
Allow: /laptop/se/$
Allow: /laptop/no/$
Allow: /laptop/ch/$
Allow: /laptop/mx/$
# Lens listing bases (documented country-less US forms included)
Allow: /laptop/*/balanced/$
Allow: /laptop/*/gaming/$
Allow: /laptop/*/student/$
Allow: /laptop/*/creator/$
Allow: /laptop/*/workstation/$
Allow: /laptop/balanced/$
Allow: /laptop/gaming/$
Allow: /laptop/student/$
Allow: /laptop/creator/$
Allow: /laptop/workstation/$
# Sub-lens listing bases
Allow: /laptop/*/daily-driver/$
Allow: /laptop/*/remote-worker/$
Allow: /laptop/*/digital-nomad/$
Allow: /laptop/*/family-hub/$
Allow: /laptop/*/casual-gamer/$
Allow: /laptop/*/aaa-gamer/$
Allow: /laptop/*/esports/$
Allow: /laptop/*/streamer/$
Allow: /laptop/*/sleeper/$
Allow: /laptop/*/strategy-simulation/$
Allow: /laptop/*/stem-student/$
Allow: /laptop/*/campus-commuter/$
Allow: /laptop/*/digital-note-taker/$
Allow: /laptop/*/researcher/$
Allow: /laptop/*/multimedia/$
Allow: /laptop/*/video-editor/$
Allow: /laptop/*/3d-artist/$
Allow: /laptop/*/audio-engineer/$
Allow: /laptop/*/visualist/$
Allow: /laptop/*/social-media-creator/$
Allow: /laptop/*/cad-bim/$
Allow: /laptop/*/full-stack-dev/$
Allow: /laptop/*/financial-analyst/$
Allow: /laptop/*/field-engineer/$
Allow: /laptop/*/ai-ml/$
# Product pages (slugs end in a model year, optional variant suffix)
# (no -2000 pattern: it would also match the under-2000/over-2000 price
# brackets, unblocking filter pages. The 4 year-2000 slugs stay * -crawlable.)
Allow: /laptop/*-2014/$
Allow: /laptop/*-2014-*/$
Allow: /laptop/*-2015/$
Allow: /laptop/*-2015-*/$
Allow: /laptop/*-2016/$
Allow: /laptop/*-2016-*/$
Allow: /laptop/*-2017/$
Allow: /laptop/*-2017-*/$
Allow: /laptop/*-2018/$
Allow: /laptop/*-2018-*/$
Allow: /laptop/*-2019/$
Allow: /laptop/*-2019-*/$
Allow: /laptop/*-2020/$
Allow: /laptop/*-2020-*/$
Allow: /laptop/*-2021/$
Allow: /laptop/*-2021-*/$
Allow: /laptop/*-2022/$
Allow: /laptop/*-2022-*/$
Allow: /laptop/*-2023/$
Allow: /laptop/*-2023-*/$
Allow: /laptop/*-2024/$
Allow: /laptop/*-2024-*/$
Allow: /laptop/*-2025/$
Allow: /laptop/*-2025-*/$
Allow: /laptop/*-2026/$
Allow: /laptop/*-2026-*/$
Allow: /laptop/*-2027/$
Allow: /laptop/*-2027-*/$
Allow: /laptop/*-2028/$
Allow: /laptop/*-2028-*/$

# Google-Extended is a training opt-in read by Googlebot, not a crawler.
User-agent: Google-Extended
Disallow: /go/
Allow: /

# --- AI SEARCH BOTS ---
# Citation-based AI search — welcome.

User-agent: PerplexityBot
Disallow: /go/
Allow: /

User-agent: OAI-SearchBot
Disallow: /go/
Allow: /

# (Claude-SearchBot moved to the canonical-pages-only group above.)

# --- META LINK-PREVIEW INDEXER ---
# Allowed: powers Facebook / Instagram / WhatsApp link-preview cards.
# (Meta's AI-training agents below stay blocked — different crawler.)

User-agent: Meta-WebIndexer
Disallow: /go/
Allow: /

# --- BLOCKED: ANONYMOUS / COMMERCIAL SCRAPERS ---
# These provide no value in return.

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Cohere-ai
Disallow: /

User-agent: Omgili
Disallow: /

# AI assistant guidance (how to read & cite SelvaScore pages)
# https://selvascore.com/llms.txt

# Sitemap Location
Sitemap: https://selvascore.com/sitemap.xml