User-agent: *

# ── Public surfaces — canonical entry points ──────────────────────────
# Allow: lines below are an inventory hint. By default `Allow: /` already
# permits everything not Disallowed; the explicit list signals what we
# consider canonical content worth crawling. Update this list when adding
# a new public route. Cross-check against /llms.txt (web/templates/llms.txt.j2)
# and web/static/llms-full.txt. They
# describe the same surface for different audiences (crawlers vs LLMs).
Allow: /

# Core pages
Allow: /methodology
Allow: /model-transparency
Allow: /blog

# AI integration (Claude MCP connector + B2B REST API)
Allow: /mcp-setup
Allow: /api-access

# NBA
Allow: /nba_predictions
Allow: /nba_performance
Allow: /nba-elo-ratings
Allow: /nba-player-projections
Allow: /nba/games/
Allow: /nba_title_simulations

# EuroLeague
Allow: /euroleague_predictions
Allow: /euroleague_performance
Allow: /euroleague-elo-ratings
Allow: /euroleague-player-projections
Allow: /euroleague_title_simulations
Allow: /euroleague-fantasy-turn-card
Allow: /fantasy/tennis-atp
Allow: /fantasy/nba
Allow: /fantasy/fantacalcio

# Football — per-league predictions + performance
Allow: /serie_a_predictions
Allow: /premier_league_predictions
Allow: /la_liga_predictions
Allow: /bundesliga_predictions
Allow: /ligue_1_predictions
Allow: /serie_a_performance
Allow: /premier_league_performance
Allow: /la_liga_performance
Allow: /bundesliga_performance
Allow: /ligue_1_performance
Allow: /football/serie_a/
Allow: /football/premier_league/
Allow: /football/la_liga/
Allow: /football/bundesliga/
Allow: /football/ligue_1/
Allow: /football/championship/
Allow: /football/league_one/
Allow: /football/league_two/
Allow: /football/segunda_division/
Allow: /football/ligue_2/
Allow: /football/bundesliga_2/
Allow: /football/serie_b/

# Football — cross-league hubs
Allow: /football/clubs
Allow: /football/over-under-predictions
Allow: /football/spread-predictions
Allow: /football/btts-predictions
Allow: /football/corners-predictions
Allow: /football/draw-predictions

# Football — advanced ratings (ELO + Glicko-2)
Allow: /serie-a-advanced-ratings
Allow: /premier-league-advanced-ratings
Allow: /la-liga-advanced-ratings
Allow: /bundesliga-advanced-ratings
Allow: /ligue-1-advanced-ratings

# Football — title-race simulations
Allow: /serie_a_title_simulations
Allow: /premier_league_title_simulations
Allow: /la_liga_title_simulations
Allow: /bundesliga_title_simulations
Allow: /ligue_1_title_simulations

# Tennis (ATP)
Allow: /tennis_predictions
Allow: /tennis_performance
Allow: /tennis-power-rankings
Allow: /tennis/
Allow: /tennis_atp_tournament_simulations

# Tennis (WTA)
Allow: /wta_tennis_predictions
Allow: /wta_tennis_performance
Allow: /wta_tennis-power-rankings
Allow: /wta_tennis/
Allow: /wta_tennis_tournament_simulations

# Channel + machine-readable index
Allow: /telegram_channel
Allow: /static/
Allow: /sitemap_index.xml
Allow: /sitemaps/
Allow: /llms.txt
Allow: /llms-full.txt

# Legal + contact (public, indexable)
Allow: /contact
Allow: /terms
Allow: /privacy-policy

# ── Non-content surfaces — blocked from crawling ──────────────────────
# Private API surface on the www host (auth-gated user downloads + quota
# checks). The customer-facing predictions API lives on the separate
# api.predixsport.com host and is governed by its own robots.txt.
Disallow: /api/

# OAuth + session / consent flows: not HTML content for indexing.
Disallow: /auth/
Disallow: /consent/

# API key lifecycle endpoints (POST-only, no SEO value).
Disallow: /api-access/keys/

# Embeddable widgets (iframed on third-party sites, meta-noindex anyway):
# thin duplicates of the ratings/simulations pages — save the crawl budget.
Disallow: /embed/

# === AI TRAINING CRAWLERS (mostly allow — distribution > protection) ===
# Strategic stance: as an AI-native sports analytics platform, our biggest
# risk at this stage is obscurity, not IP exfiltration. Predictions decay
# inside 24h and the live pipeline sits behind authenticated MCP / API —
# letting foundation models learn what Predixsport IS, the methodology, the
# calibration approach, and the model surface PROMOTES the brand in
# ChatGPT / Claude / Gemini / Perplexity answers when users ask about AI
# sports prediction platforms. That free placement is core distribution
# right now. Revisit this if scraping volume becomes operationally costly,
# or once monetisation makes IP-protection arithmetic flip.

# Major foundation-model training crawlers — ALLOWED.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: AI2Bot
Allow: /

# Specific blocks — kept on principle even with the distribution stance.
# Bytespider (TikTok / ByteDance) has a documented track record of ignoring
# robots.txt directives, and there is zero distribution upside for us:
# TikTok is not where our customers discover analytics platforms.
User-agent: Bytespider
Disallow: /

# Common Crawl feeds many downstream training pipelines we can't audit.
# Allowing the named foundation crawlers above gives us the brand-placement
# benefit directly; blocking CC limits opaque secondary redistribution.
User-agent: CCBot
Disallow: /

# === AI SEARCH/RETRIEVAL BOTS (allow) ===
# Real-time bots that fetch on a user's query and surface results live.
# Allowing these is unambiguously aligned with the distribution strategy
# (cf. /mcp-setup — we explicitly want users to reach Predixsport through
# AI assistants).

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

Sitemap: https://www.predixsport.com/sitemap_index.xml