User-agent: *
Disallow: /private/
Allow: /public/


User-agent: psbot
Disallow: /

User-agent: CFNetwork
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: Java
Disallow: /

User-agent: HTTrack off-line browser
Disallow: /

User-agent: MBCrawler
Disallow: /

User-agent: Yandex
Crawl-delay: 10
User-agent: bingbot
Crawl-delay: 10
User-agent: AhrefsBot 
Disallow: /
User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: LinkedInBot
Disallow: /infiniterss*

User-agent: Mediapartners-Google*
Disallow: