## robots.txt for Magento Community and Enterprise

User-agent:*

Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /directory/currency/
Disallow: /sociallogin/
Disallow: /ajax/index/options/
Disallow: /wishlist/
Disallow: /newsletter/subscriber/

Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/

Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/

Disallow: /*?dir*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?limit=all
Disallow: /*?mode*
Disallow: /*?tea_region*
Disallow: /*?is_ajax*

Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/

Disallow: /tag/
Disallow: /review/

Disallow: /cdn-cgi/
Disallow: /productalert/

##DE
Disallow: /de/lib/
Disallow: /de/*.php$
Disallow: /de/pkginfo/
Disallow: /de/report/
Disallow: /de/var/
Disallow: /de/catalog/
Disallow: /de/customer/
Disallow: /de/sendfriend/
Disallow: /de/review/
Disallow: /de/*SID=
Disallow: /de/directory/currency/
Disallow: /de/sociallogin/
Disallow: /de/ajax/index/options/
Disallow: /de/wishlist/
Disallow: /de/newsletter/subscriber/

Disallow: /de/checkout/
Disallow: /de/onestepcheckout/
Disallow: /de/customer/
Disallow: /de/customer/account/
Disallow: /de/customer/account/login/

Disallow: /de/catalogsearch/
Disallow: /de/catalog/product_compare/
Disallow: /de/catalog/category/view/
Disallow: /de/catalog/product/view/

Disallow: /de/*?dir*
Disallow: /de/*?dir=desc
Disallow: /de/*?dir=asc
Disallow: /de/*?limit=all
Disallow: /de/*?mode*

Disallow: /de/app/
Disallow: /de/bin/
Disallow: /de/dev/
Disallow: /de/lib/
Disallow: /de/phpserver/

Disallow: /de/tag/
Disallow: /de/review/

Disallow: /de/cdn-cgi/
Disallow: /de/productalert/

##FR
Disallow: /fr/lib/
Disallow: /fr/*.php$
Disallow: /fr/pkginfo/
Disallow: /fr/report/
Disallow: /fr/var/
Disallow: /fr/catalog/
Disallow: /fr/customer/
Disallow: /fr/sendfriend/
Disallow: /fr/review/
Disallow: /fr/*SID=
Disallow: /fr/directory/currency/
Disallow: /fr/sociallogin/
Disallow: /fr/ajax/index/options/
Disallow: /fr/wishlist/
Disallow: /fr/newsletter/subscriber/

Disallow: /fr/checkout/
Disallow: /fr/onestepcheckout/
Disallow: /fr/customer/
Disallow: /fr/customer/account/
Disallow: /fr/customer/account/login/

Disallow: /fr/catalogsearch/
Disallow: /fr/catalog/product_compare/
Disallow: /fr/catalog/category/view/
Disallow: /fr/catalog/product/view/

Disallow: /fr/*?dir*
Disallow: /fr/*?dir=desc
Disallow: /fr/*?dir=asc
Disallow: /fr/*?limit=all
Disallow: /fr/*?mode*

Disallow: /fr/app/
Disallow: /fr/bin/
Disallow: /fr/dev/
Disallow: /fr/lib/
Disallow: /fr/phpserver/

Disallow: /fr/tag/
Disallow: /fr/review/

Disallow: /fr/cdn-cgi/
Disallow: /fr/productalert/

##SE
Disallow: /se/lib/
Disallow: /se/*.php$
Disallow: /se/pkginfo/
Disallow: /se/report/
Disallow: /se/var/
Disallow: /se/catalog/
Disallow: /se/customer/
Disallow: /se/sendfriend/
Disallow: /se/review/
Disallow: /se/*SID=
Disallow: /se/directory/currency/
Disallow: /se/sociallogin/
Disallow: /se/ajax/index/options/
Disallow: /se/wishlist/
Disallow: /se/newsletter/subscriber/

Disallow: /se/checkout/
Disallow: /se/onestepcheckout/
Disallow: /se/customer/
Disallow: /se/customer/account/
Disallow: /se/customer/account/login/

Disallow: /se/catalogsearch/
Disallow: /se/catalog/product_compare/
Disallow: /se/catalog/category/view/
Disallow: /se/catalog/product/view/

Disallow: /se/*?dir*
Disallow: /se/*?dir=desc
Disallow: /se/*?dir=asc
Disallow: /se/*?limit=all
Disallow: /se/*?mode*

Disallow: /se/app/
Disallow: /se/bin/
Disallow: /se/dev/
Disallow: /se/lib/
Disallow: /se/phpserver/

Disallow: /se/tag/
Disallow: /se/review/

Disallow: /se/cdn-cgi/
Disallow: /se/productalert/

##ES
Disallow: /es/lib/
Disallow: /es/*.php$
Disallow: /es/pkginfo/
Disallow: /es/report/
Disallow: /es/var/
Disallow: /es/catalog/
Disallow: /es/customer/
Disallow: /es/sendfriend/
Disallow: /es/review/
Disallow: /es/*SID=
Disallow: /es/directory/currency/
Disallow: /es/sociallogin/
Disallow: /es/ajax/index/options/
Disallow: /es/wishlist/
Disallow: /es/newsletter/subscriber/

Disallow: /es/checkout/
Disallow: /es/onestepcheckout/
Disallow: /es/customer/
Disallow: /es/customer/account/
Disallow: /es/customer/account/login/

Disallow: /es/catalogsearch/
Disallow: /es/catalog/product_compare/
Disallow: /es/catalog/category/view/
Disallow: /es/catalog/product/view/

Disallow: /es/*?dir*
Disallow: /es/*?dir=desc
Disallow: /es/*?dir=asc
Disallow: /es/*?limit=all
Disallow: /es/*?mode*

Disallow: /es/app/
Disallow: /es/bin/
Disallow: /es/dev/
Disallow: /es/lib/
Disallow: /es/phpserver/

Disallow: /es/tag/
Disallow: /es/review/

Disallow: /es/cdn-cgi/
Disallow: /es/productalert/

##CN
Disallow: /cn/lib/
Disallow: /cn/*.php$
Disallow: /cn/pkginfo/
Disallow: /cn/report/
Disallow: /cn/var/
Disallow: /cn/catalog/
Disallow: /cn/customer/
Disallow: /cn/sendfriend/
Disallow: /cn/review/
Disallow: /cn/*SID=
Disallow: /cn/directory/currency/
Disallow: /cn/sociallogin/
Disallow: /cn/ajax/index/options/
Disallow: /cn/wishlist/

Disallow: /cn/checkout/
Disallow: /cn/onestepcheckout/
Disallow: /cn/customer/
Disallow: /cn/customer/account/
Disallow: /cn/customer/account/login/

Disallow: /cn/catalogsearch/
Disallow: /cn/catalog/product_compare/
Disallow: /cn/catalog/category/view/
Disallow: /cn/catalog/product/view/

Disallow: /cn/*?dir*
Disallow: /cn/*?dir=desc
Disallow: /cn/*?dir=asc
Disallow: /cn/*?limit=all
Disallow: /cn/*?mode*

Disallow: /cn/app/
Disallow: /cn/bin/
Disallow: /cn/dev/
Disallow: /cn/lib/
Disallow: /cn/phpserver/

Disallow: /cn/tag/
Disallow: /cn/review/

Disallow: /cn/cdn-cgi/
Disallow: /cn/productalert/

## disallow crawlers to avoid high bandwidth use
User-agent: dotbot
Disallow: /

User-agent: ClaudeBot
User-agent: CCbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: PiplBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: fast
Disallow: /

User-agent: wget
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: Sogou web spider/4.0
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: MJ12bot
Disallow: /

## delay user agents for future when low on bandwidth
User-agent: bingbot
Crawl-delay: 1