User-agent: *
Disallow:
Allow: /modules/contentbox/themes/
Allow: /modules_app/contentbox-custom/_themes/
Disallow: /bin/
Disallow: /download/
Disallow: /coldbox/
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /models/
Disallow: /modules/
Disallow: /modules_app/
Disallow: /resources/
Disallow: /views/
Disallow: /*/akibase-search/search/params/
Disallow: /index.cfm/akibase-lightbox/action/

# Honeypot: linked invisibly in the footer, requesting it bans the IP for an hour
Disallow: /de/archiv-intern/
Disallow: /en/archiv-intern/

# Block similar-image search pages (return 403, waste crawl budget)
Disallow: /*similar:

# Block model filter combo pages (duplicate IDs, no SEO value)
Disallow: /*model:

# Block facet / sort / onpage query variants of search result pages.
# These are user-preference views of the same underlying topic — already
# noindex, but we also want to save crawl budget so Google doesn't refetch
# them. See README "only first page indexed" policy (2026-04-24).
Disallow: /*?onpage=
Disallow: /*&onpage=
Disallow: /*?resultsort=
Disallow: /*&resultsort=
Disallow: /*?orientation=
Disallow: /*&orientation=
Disallow: /*?custfield1=
Disallow: /*&custfield1=
Disallow: /*?page=
Disallow: /*&page=

# SEO / scraper bots - block entirely
User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: seostar
Disallow: /

User-agent: seekport
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Sogou
Disallow: /

# AI training bots - block entirely
User-agent: GPTBot
Disallow: /

User-agent: GrokBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ExaSearchBot
Disallow: /

User-agent: ExaBot
Disallow: /

User-agent: Pumoxbot
Disallow: /

User-agent: HaloBot
Disallow: /

User-agent: KeenableBot
Disallow: /

User-agent: QlyzeBot
Disallow: /

User-agent: LinkupBot
Disallow: /

User-agent: Reflectionbot
Disallow: /

User-agent: ibou-search
Disallow: /

User-agent: IbouBot
Disallow: /

User-agent: ElevenlabsCrawler
Disallow: /

User-agent: ForestEngine
Disallow: /

# 2026-11-01 botwatch: Carpathian search + its AI-training agent, Parallel (AI web APIs), NVIDIA image fetcher, Sofya search
User-agent: CarpathianBot
Disallow: /

User-agent: Carpathian-AI
Disallow: /

User-agent: ShapBot
Disallow: /

User-agent: Shap-User
Disallow: /

User-agent: nvidia-multimodal-research-fetcher
Disallow: /

User-agent: SofyaBot
Disallow: /

User-agent: Sofya-User
Disallow: /

# 2026-11-02 botwatch: Sperid Labs research crawler (loops on robots.txt, also refused in .htaccess), Sender marketing crawler, MetaJob job crawler (hit the honeypot)
User-agent: SperidLabsResearchCrawler
Disallow: /

User-agent: SenderBot
Disallow: /

User-agent: MetaJobBot
Disallow: /

# 2026-11-03 botwatch: AI-bot names worn by a secrets scanner on GCP/AWS, and a contact harvester
User-agent: MoonshotBot
Disallow: /

User-agent: KimiBot
Disallow: /

User-agent: Kimi-SearchBot
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /

User-agent: Hunyuan
Disallow: /

User-agent: Qwenbot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Bravebot
Disallow: /

User-agent: OAI-AdsBot
Disallow: /

User-agent: b2b-contact-pipeline
Disallow: /

# 2026-11-05 botwatch: CorsairBot (GCP, never reads robots.txt), RiverBot (river.ai), tphotobot (estidraft photo collector, also refused in .htaccess)
User-agent: CorsairBot
Disallow: /

User-agent: RiverBot
Disallow: /

User-agent: tphotobot
Disallow: /

# AI search bots - allow with rate limit
User-agent: OAI-SearchBot
Crawl-delay: 10
Disallow: /de/archiv-intern/
Disallow: /en/archiv-intern/

# ChatGPT live browsing - allow (real user requests)
User-agent: ChatGPT-User
Allow: /
Disallow: /de/archiv-intern/
Disallow: /en/archiv-intern/

Sitemap: https://www.f1online.de/sitemap.xml