# =======================
# General User-agent Rule
# =======================

User-agent: *
Crawl-delay: 5
Disallow: /search
Disallow: /*search?

# ===========================
# Disallow Known Scraper Bots
# ===========================

User-agent: Teleport
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: ERNIE-Bot
Disallow: /

User-agent: BlowFish/1.0
Disallow: /

User-agent: grub
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Titan
Disallow: /

User-agent: Barkrowler
Disallow: /

# =======================
# Disallow Malicious Bots
# =======================

User-agent: 008
Disallow: /

User-agent: Alexibot
Disallow: /

User-agent: BackDoorBot/1.0
Disallow: /

User-agent: Bullseye/1.0
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: Kenjin Spider
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

# =================================
# Disallow Known Data Scraping Bots
# =================================

User-agent: BotALot
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: EroCrawler
Disallow: /

User-agent: Foobot
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: JennyBot
Disallow: /

User-agent: Mister PiX
Disallow: /

User-agent: moget
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: Radiation Retriever
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /

User-agent: True_Robot
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: Web Image Collector
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Zeus
Disallow: /

# =========================================
# SEO Monitoring Bots – Throttle Crawl Rate
# =========================================

User-agent: dotbot
Crawl-delay: 5

User-agent: Rogerbot
Crawl-delay: 5

User-agent: ia_archiver
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 5

User-agent: Xenu
Crawl-delay: 5

User-agent: AhrefsBot
Crawl-delay: 5

User-agent: CCBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: DataForSeoBot
Crawl-delay: 5

User-agent: XoviBot
Crawl-delay: 5

User-agent: Screaming Frog SEO Spider
Crawl-delay: 5

User-agent: Keyword Density/0.9
Crawl-delay: 5

User-agent: LinkextractorPro
Crawl-delay: 5

User-agent: LinkScan/8.1a Unix
Crawl-delay: 5

User-agent: BLEXbot
Crawl-delay: 5

User-agent: spbot
Crawl-delay: 5

# =============================
# AI Bots – Throttle Crawl Rate
# =============================

User-agent: GPTBot
Crawl-delay: 5

User-agent: ChatGPT-User
Crawl-delay: 5

User-agent: ClaudeBot
Crawl-delay: 5

User-agent: Amazonbot
Crawl-delay: 5

User-agent: omgili
Crawl-delay: 5

User-agent: PerplexityBot
Crawl-delay: 5

User-agent: YouBot
Crawl-delay: 5

User-agent: QuoraBot
Crawl-delay: 5

User-agent: facebookexternalhit
Crawl-delay: 5

User-agent: Applebot
Crawl-delay: 5

User-agent: MetaBot
Crawl-delay: 5

User-agent: Diffbot
Crawl-delay: 5

User-agent: Claude-Web
Crawl-delay: 5

User-agent: Google-Extended
Crawl-delay: 5

User-agent: PhindBot
Crawl-delay: 5

# =======
# Sitemap
# =======

Sitemap: https://www.shetland.org/sitemap.xml