# Crawlers Setup
User-agent: *
Sitemap: https://www.visilab.ch/sitemap.xml

# Do not crawl development files and folders (GIT/SVN directories, dump files etc.)
Disallow: /*.git$
Disallow: /*.svn$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.log$

# Do not crawl sub category pages that are sorted or filtered.
Disallow: /*?childSku*
Disallow: /*?sort*

# Do not crawl sub system directories
# Disallow: /api/

# Do not crawl filtered categories
Disallow: /*?vg_
Disallow: /*&vg_
Disallow: /*?price=
Disallow: /*&price=

# Disable agressive crawlers
User-agent: SEOkicks
Disallow: /
User-agent: SEOkicks-Robot
Disallow: / 
User-agent: MJ12bot
Disallow: /
User-agent: MJ12bot/v*
Disallow: /
User-agent: MJ12bot/v1.4.0
Disallow: /
User-agent: MJ12bot/v1.4.3
Disallow: /
User-agent: MJ12bot/v1.4.5
Disallow: /