User-Agent: googlebot
User-Agent: bingbot
User-Agent: msnbot
User-Agent: Slurp
User-Agent: Twitterbot
User-Agent: DuckDuckBot
Crawl-delay: 30

Allow: /mu/
Allow: /sites/
Disallow: /*plugins
Disallow: /*wp-admin
Disallow: /*wp-includes
Disallow: /sites/cunybiz/

Sitemap: https://www1.cuny.edu/sitemap.xml

User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Dispatch/0.11.3
Disallow: /

User-agent: Dispatch
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Uptimebot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Mauibot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-Agent: dlvr.it
Disallow: /

User-agent: SemrushBot
Disallow: / 

User-agent: *
Disallow: /