From 5db6da8b3a1a8459acd09a0d69e938d29dfae711 Mon Sep 17 00:00:00 2001 From: Adam Piontek Date: Wed, 7 Apr 2021 13:49:25 -0400 Subject: [PATCH] fixing robots.txt and adding bots --- assets/static/robots.txt | 45 +++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/assets/static/robots.txt b/assets/static/robots.txt index bc2bb9c..5624fa1 100644 --- a/assets/static/robots.txt +++ b/assets/static/robots.txt @@ -4,9 +4,44 @@ # User-agent: * # Disallow: / +User-agent: * +Crawl-delay: 5 Disallow: /pcasts/ -Disallow: /acasts/ -Disallow: /swm/ -Disallow: /acasts/swm/ -Disallow: /mjrpt/ -Disallow: /pcasts/mjrpt/ +Disallow: /flux/ +Disallow: /znc/ +Disallow: /lounge/ +Disallow: /books/ +Disallow: /fs-adam/ +Disallow: /wol/ +Disallow: /asrv/ +Disallow: /komga/ + +User-agent: PetalBot +Disallow: /git/ + +User-agent: MJ12bot +Disallow: /git/ + +User-agent: SemrushBot +Disallow: /git/ + +User-agent: ZoominfoBot +Disallow: /git/ + +User-agent: DotBot +Disallow: /git/ + +User-agent: MauiBot +Disallow: /git/ + +User-agent: AhrefsBot +Disallow: /git/ + +User-agent: Discordbot +Disallow: /git/ + +User-agent: BLEXBot +Disallow: /git/ + +User-agent: YandexBot +Disallow: /git/