2021-03-28 21:12:55 -04:00
|
|
|
# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
|
|
|
|
|
#
|
|
|
|
|
# To ban all spiders from the entire site uncomment the next two lines:
|
|
|
|
|
# User-agent: *
|
|
|
|
|
# Disallow: /
|
2021-04-06 12:05:43 -04:00
|
|
|
|
2021-04-07 13:49:25 -04:00
|
|
|
User-agent: *
|
|
|
|
|
Crawl-delay: 5
|
2021-04-06 12:05:43 -04:00
|
|
|
Disallow: /pcasts/
|
2021-04-07 13:49:25 -04:00
|
|
|
Disallow: /flux/
|
|
|
|
|
Disallow: /znc/
|
|
|
|
|
Disallow: /lounge/
|
|
|
|
|
Disallow: /books/
|
|
|
|
|
Disallow: /fs-adam/
|
|
|
|
|
Disallow: /wol/
|
|
|
|
|
Disallow: /asrv/
|
|
|
|
|
Disallow: /komga/
|
|
|
|
|
|
|
|
|
|
User-agent: PetalBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: MJ12bot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: SemrushBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: ZoominfoBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: DotBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: MauiBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: AhrefsBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: Discordbot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: BLEXBot
|
|
|
|
|
Disallow: /git/
|
|
|
|
|
|
|
|
|
|
User-agent: YandexBot
|
|
|
|
|
Disallow: /git/
|
