host: www.segor.de
### https only
User-agent: *
Crawl-delay: 20
Disallow: /INFOBIL*
Disallow: /SUCH*
Disallow: /such*
Disallow: /sho*
#Disallow: *s.js
Allow: /Sortiment/
Allow: /INFO/
# Mozilla/5.0 (compatible; Adsbot/3.1)
# akzeptiert Delay nicht
User-agent: Adsbot
Disallow: /
# https://megaindex.com/crawler
# akzeptiert kein Crawl-delay >5 Sekunden
User-agent: MegaIndex
Disallow: /
User-agent: Seekport
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: AspiegelBot
Disallow: /
#User-agent: PetalBot
#Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: adscan
Disallow: /
User-agent: adscan/
Disallow: /
User-agent: adscanner/
Disallow: /
User-agent: seoscanner
Disallow: /
User-agent: PHPCrawl
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: G-i-g-a-b-o-t
Disallow: /
User-agent: ltx71
# - (http://ltx71.com/)
Disallow: *
User-agent: SEOkicks
Disallow: /
#
User-agent: SeznamBot
# user-language cz !!
Allow: /INFO/
Allow: /Sortiment/
User-agent: Yandex
# user-language (none)
Allow: /Sortiment/
Allow: /INFO/
User-agent: SurdotlyBot
Disallow: /
User-agent: clark-crawler2/Nutch-1.19-SNAPSHOT
Disallow: / 
User-agent: Googlebot
Allow: /INFO
Allow: /Sortiment
User-agent: W3C-checklink
Allow: /INFO
Allow: /Sortiment
Allow: /pool
User-agent: *
Disallow: /pool
#

# ##################### eof(robots.txt) ###