########################################################################################
##################### Creare's robots.txt file for Magento websites ####################
########################################################################################
# robots.txt for Magento 1.8 Shop
# Sitemap declaration
Sitemap: https://www.beadsandtreasures.ch/sitemap/sitemap.xml
###  Remarks ChZ:
## NOINDEX,NOFOLLOW => captured / blocked on Meta level in app/design/frontend/ultimo/new/layout/local.xml file
#######################################
##### MAGENTO DIRECTORIES & FILES #####
#######################################
##### Directories ##### 
# General directives for all robots
User-agent: *
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /index.php/
Disallow: /magento/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /rss/
Disallow: /scripts/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/
Disallow: /captcha/
Disallow: /*?SID=
Disallow: /*?___SID=
Disallow: /*.php$
# Layered Navigation / Filter & Sort Parameter - added to reduce session/DB load from crawlers
Disallow: /*?p=
Disallow: /*?dir=
Disallow: /*?order=
Disallow: /*?mode=
Disallow: /*?limit=
##### Files #####
# Disallowed paths for enhanced security and to avoid indexing of irrelevant pages
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
# Directives for search engine bots to enhance crawling efficiency
# Googlebot specific crawl setup
User-agent: Googlebot
Disallow: /*?
Disallow: /catalogsearch/
# Google Image Crawler Setup for optimized image indexing
User-agent: Googlebot-Image
Disallow:
# Preventing Yandex from indexing the site
User-agent: YandexBot 
Disallow: /
#######################################
########### SCREAMING FROG ############
#######################################
# Screaming Frog SEO Spider Setup to allow full site crawling but limit image, CSS, and JS file indexing
User-agent: Screaming Frog SEO Spider
Allow: /
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.png$
Disallow: /*.bmp$
Disallow: /*.xml$
Disallow: /*.css$
Disallow: /*.js$
# Additional setup can be considered based on custom paths, parameters, or specific files that should not be indexed.