User-agent: *
Disallow: /ajax
Disallow: /api
Disallow: /captcha
Disallow: /common
Disallow: /links/confirm
Disallow: /links/admin
Disallow: /shutterstock
Allow: /

User-agent: AhrefsBot
User-agent: dotbot
User-agent: exabot
User-agent: gigabot
User-agent: ia_archiver
User-agent: MJ12bot
User-agent: rogerbot
disallow: /

Allow: /