readded the missing public files
This commit is contained in:
parent
88f6e7857e
commit
de535e57f4
64 changed files with 22076 additions and 0 deletions
57
public/robots.txt
Normal file
57
public/robots.txt
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# robots.txt for dvorakmortonxd.dev
|
||||
# https://dvorakmortonxd.dev
|
||||
|
||||
# General crawlers — allow main site, block self-hosted services
|
||||
User-agent: *
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Disallow: /api/
|
||||
Disallow: /_next/
|
||||
Disallow: /admin/
|
||||
Disallow: /darecek/
|
||||
|
||||
# AI training crawlers
|
||||
User-agent: GPTBot
|
||||
Allow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-Web
|
||||
Allow: /
|
||||
|
||||
# Meta bots — not welcome
|
||||
User-agent: FacebookBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: meta-externalagent
|
||||
Disallow: /
|
||||
|
||||
User-agent: meta-externalfetcher
|
||||
Disallow: /
|
||||
|
||||
# Search engine crawlers — welcome
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Slurp
|
||||
Allow: /
|
||||
|
||||
User-agent: Baiduspider
|
||||
Allow: /
|
||||
|
||||
# Sitemap and Llms
|
||||
Sitemap: https://dvorakmortonxd.dev/sitemap.xml
|
||||
LLMs: https://dvorakmortonxd.dev/llms.txt
|
||||
|
||||
# ===========================================
|
||||
# 🎮 Have a great day, crawler!
|
||||
# Check out my projects at dvorakmortonxd.dev
|
||||
# ===========================================
|
||||
Loading…
Add table
Add a link
Reference in a new issue