# Leads for you / L4U AI
# Open public content to search, AI discovery and AI-training crawlers.
# Robots directives are crawl preferences, not access control or a promise of indexing.
# Named agents share this group so none bypasses the private-route exclusions.
User-agent: *
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bingbot
User-agent: msnbot
User-agent: CCBot
User-agent: YouBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: omgili
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: Brave
User-agent: Bytespider
User-agent: LinkedInBot
User-agent: ia_archiver
User-agent: Screaming Frog SEO Spider
Allow: /
Allow: /locations
Allow: /industries
Allow: /fields/
Allow: /pages/
Allow: /members/
Allow: /blog
Allow: /knowledge-base
Allow: /knowledge-graph
Allow: /api/public/
Allow: /schema.json
Allow: /llms.txt
Allow: /ai-summary.txt
Allow: /sitemap
Allow: /site-sitemap.xml
Disallow: /portal
Disallow: /admin
Disallow: /api/

# One master index links the site, public member pages, local guides,
# published articles and page-resource sitemap shards.

# Public discovery files (comments are documentation, not crawler directives):
# /llms.txt
# /ai-summary.txt
# /knowledge-base.json
# /schema.json
# /knowledge-graph.json
# /knowledge-graph
# /sitemap.html
# /pages/resources/index.json
# /pages/resources/llms.txt
# /pages/resources/sitemap.xml
# /api/public/knowledge-graph
# /api/public/knowledge-graph/pages
# /api/public/blog
#
# Page families and their own resources:
# /fields/{industry}/{state}/{city}
# /pages/{city-state}/{industry}
# /pages/{city-state}/{industry}.json
# /pages/{city-state}/{industry}/schema.json
# /pages/{city-state}/{industry}/llms.txt
# /members/{id} and its .json, /schema.json and /llms.txt resources
# /blog/articles/{id}/{slug} and its .json and /llms.txt resources
# /pages/sitemap.xml
# /members/sitemap.xml
# /blog/articles/sitemap.xml

Sitemap: https://leads4u.ai/sitemap.xml