User-Agent: *
Disallow: /api/
Disallow: /static/
Disallow: /lang/
Disallow: /my/
Disallow: /robots/
Disallow: /captcha/
User-Agent: MJ12bot
User-Agent: SapphireWebCrawler
User-Agent: ia_archiver
User-Agent: YandexSomething
User-Agent: Gaisbot
User-Agent: dotbot
User-Agent: CamontSpider
User-Agent: panscient.com
User-Agent: twiceler
Disallow: /