# AllDevTool.com - Robots.txt
# 135+ Free Developer Tools - https://www.alldevtool.com
# Last updated: 2026-02-14

# Allow all crawlers to access main content
User-agent: *
Allow: /
Allow: /tools/
Allow: /category/
Allow: /discover
Allow: /sitemap-page
Allow: /privacy-policy

# Reasonable crawl rate to avoid server overload
Crawl-delay: 1

# Disallow non-content and administrative paths
Disallow: /api/
Disallow: /_next/
Disallow: /*?debug=*
Disallow: /*?print=*
Disallow: /*?preview=*

# Disallow old/removed pages (tell Google these don't exist)
Disallow: /placeprep
Disallow: /learn-roadmap
Disallow: /learn
Disallow: /roadmap
Disallow: /interview-questions

# Googlebot specific settings (Google ignores crawl-delay)
User-agent: Googlebot
Allow: /
Allow: /tools/
Allow: /category/
Disallow: /api/
Disallow: /placeprep
Disallow: /learn-roadmap
Disallow: /learn
Disallow: /roadmap

# Googlebot Image
User-agent: Googlebot-Image
Allow: /
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.ico$

# Bingbot settings
User-agent: Bingbot
Allow: /
Crawl-delay: 2
Disallow: /api/

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 3

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Facebook
User-agent: facebookexternalhit
Allow: /

# Twitter
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Main Sitemap (Auto-generated by Next.js)
Sitemap: https://www.alldevtool.com/sitemap.xml

# Host (canonical domain - www version)
Host: https://www.alldevtool.com