User-agent: *
Allow: /
Disallow: */sectors/
Disallow: */industries/
Disallow: */city/
Disallow: */noaccess/
Disallow: */ref/
Disallow: */data/
Disallow: */get/
Disallow: */api/search/jobs
Disallow: */oauth/facebook
Disallow: */oauth/google
Disallow: */job-pdf/
Disallow: */api/v2/users/
Disallow: */new-signup/start
Disallow: */login

User-agent: 008
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: gsa-crawler
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: SMTBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: heritrix
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /