diff --git a/layouts/robots.txt b/layouts/robots.txt new file mode 100644 index 0000000..206e15f --- /dev/null +++ b/layouts/robots.txt @@ -0,0 +1,61 @@ +User-agent: MJ12bot +Disallow: / + +User-agent: AhrefsBot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +# Block SISTRIX +User-agent: SISTRIX Crawler +Disallow: / +User-agent: sistrix +Disallow: / +User-agent: 007ac9 +Disallow: / +User-agent: 007ac9 Crawler +Disallow: / + +# Block Uptime robot +User-agent: UptimeRobot/2.0 +Disallow: / + +# Block Ezooms Robot +User-agent: Ezooms Robot +Disallow: / + +# Block Perl LWP +User-agent: Perl LWP +Disallow: / + +# Block netEstate NE Crawler (+http://www.website-datenbank.de/) +User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/) +Disallow: / + +# Block WiseGuys Robot +User-agent: WiseGuys Robot +Disallow: / + +# Block Turnitin Robot +User-agent: Turnitin Robot +Disallow: / + +# Block Heritrix +User-agent: Heritrix +Disallow: / + +# Block pricepi +User-agent: pimonster +Disallow: / + +User-agent: SurdotlyBot +Disallow: / + +User-agent: ZoominfoBot +Disallow: / + +User-agent: * +Allow: / + +Sitemap: {{ "/sitemap.xml" | absURL }}