User-agent: *
Allow: /
Disallow: /contact/thanks
Disallow: /auth
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /_authenticated/
Disallow: /lovable/

# ============================================================
# AI search & assistant crawlers — explicitly allowed
# Keep in sync with scripts/validate-robots.mjs
# ============================================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Microsoft / Bing / Copilot
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# ByteDance (Doubao)
User-agent: Bytespider
Allow: /

# Common Crawl (training corpus most AI systems ingest)
User-agent: CCBot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Huawei Petal Search / AI
User-agent: PetalBot
Allow: /

# Kagi
User-agent: Kagibot
Allow: /

# Timpi
User-agent: Timpibot
Allow: /

# ============================================================
# Blocked: SEO snoops, scrapers, site rippers, HTTP libraries
# Legitimate search + AI bots above remain fully allowed.
# ============================================================

# SEO snoops / backlink scrapers
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SplitSignalBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: SEOkicks
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SiteAuditBot
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: seoscanners
Disallow: /
User-agent: LinkpadBot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: SerendeputyBot
Disallow: /
User-agent: SEOlyticsCrawler
Disallow: /
User-agent: MegaIndex
Disallow: /

# Site rippers / offline downloaders
User-agent: HTTrack
Disallow: /
User-agent: Wget
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: SiteSucker
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: heritrix
Disallow: /

# Generic HTTP libraries / scripted clients
User-agent: curl
Disallow: /
User-agent: Wget/
Disallow: /
User-agent: python-requests
Disallow: /
User-agent: python-urllib
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: axios
Disallow: /
User-agent: node-fetch
Disallow: /
User-agent: Go-http-client
Disallow: /
User-agent: Java
Disallow: /
User-agent: Apache-HttpClient
Disallow: /
User-agent: okhttp
Disallow: /
User-agent: libwww-perl
Disallow: /
User-agent: PhantomJS
Disallow: /
User-agent: HeadlessChrome
Disallow: /


Sitemap: https://dallasregenerativellc.com/sitemap-index.xml
Sitemap: https://dallasregenerativellc.com/sitemap.xml
Sitemap: https://dallasregenerativellc.com/sitemap-who-we-serve-images.xml
Sitemap: https://dallasregenerativellc.com/sitemap-service-images.xml