56 lines
1,000 B
Text
56 lines
1,000 B
Text
# robots.txt for dvorakmortonxd.dev
|
|
# https://dvorakmortonxd.dev
|
|
|
|
# General crawlers — allow main site, block self-hosted services
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /llms.txt
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Disallow: /admin/
|
|
Disallow: /darecek/
|
|
|
|
# AI training crawlers
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
# Meta bots — not welcome
|
|
User-agent: FacebookBot
|
|
Disallow: /
|
|
|
|
User-agent: meta-externalagent
|
|
Disallow: /
|
|
|
|
User-agent: meta-externalfetcher
|
|
Disallow: /
|
|
|
|
# Search engine crawlers — welcome
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: DuckDuckBot
|
|
Allow: /
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
|
|
# Sitemap and Llms
|
|
Sitemap: https://dvorakmortonxd.dev/sitemap.xml
|
|
LLMs: https://dvorakmortonxd.dev/llms.txt
|
|
|
|
# ===========================================
|
|
# 🎮 Have a great day, crawler!
|
|
# Check out my projects at dvorakmortonxd.dev
|
|
# ===========================================
|