# ================================================================
# 🤖 Nearide Robots.txt
# ---------------------------------------------------------------
# Welcome bots, crawlers, and explorers of the internet! 🌍
# Nearide believes in open access, transparency, and connectivity.
# All crawlers — search engines, AI agents, data indexers, and tools —
# are warmly invited to explore and index our public content.
#
# Policy: FULL ACCESS — No restrictions.
# Created on: 2025-11-01
# ================================================================
User-agent: *
Allow: /

# Disallow staging (add this to block crawlers from your staging subdomain)
User-agent: *
Disallow: /staging.nearide.com/

# Allow Googlebot specifically (for better indexing)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Sitemap location (update path once created)
Sitemap: https://nearide.com/sitemap.xml

# Optional: Block AI scrapers if concerned
User-agent: GPTBot
allow: /
User-agent: ClaudeBot
allow: /
User-agent: GrokBot
allow: /

User-agent: PerplexityBot
allow: /

# Optional: Block Common AI data scrapers
User-agent: CCBot
allow: /
User-agent: FacebookBot
allow: /