adam/home73k
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fixing robots.txt and adding bots

Browse source
This commit is contained in:
Adam Piontek 2021-04-07 13:49:25 -04:00
parent 0bb011b59d
commit 5db6da8b3a
1 changed files with 40 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
assets/static/robots.txt
View file

@ -4,9 +4,44 @@
# User-agent: * # User-agent: *
# Disallow: / # Disallow: /
User-agent: *
Crawl-delay: 5
Disallow: /pcasts/ Disallow: /pcasts/
Disallow: /acasts/ Disallow: /flux/
Disallow: /swm/ Disallow: /znc/
Disallow: /acasts/swm/ Disallow: /lounge/
Disallow: /mjrpt/ Disallow: /books/
Disallow: /pcasts/mjrpt/ Disallow: /fs-adam/
Disallow: /wol/
Disallow: /asrv/
Disallow: /komga/
User-agent: PetalBot
Disallow: /git/
User-agent: MJ12bot
Disallow: /git/
User-agent: SemrushBot
Disallow: /git/
User-agent: ZoominfoBot
Disallow: /git/
User-agent: DotBot
Disallow: /git/
User-agent: MauiBot
Disallow: /git/
User-agent: AhrefsBot
Disallow: /git/
User-agent: Discordbot
Disallow: /git/
User-agent: BLEXBot
Disallow: /git/
User-agent: YandexBot
Disallow: /git/