#Website Sitemap
Sitemap: http://www.bestmaterials.com/sitemap.xml

# Stop Yandex from hogging bandwidth
User-agent: yandex bot 
Disallow: /

user-agent: AhrefsBot
disallow: /

# Crawlers Setup
User-agent: *
Crawl-delay: 20

Disallow: /_private/
Disallow: /aspnet_client/
Disallow: /ASPSecured/
Disallow: /bin/
Disallow: /CommonControls/
Disallow: /Controls/
Disallow: /cp/
Disallow: /db/
Disallow: /Download/
Disallow: /services/
Disallow: /sfaddons/
Disallow: /ssl/