Difference between revisions of "X-Cart:Setting up robots.txt"

From X-Cart 4 Classic
Jump to: navigation, search
(Created page with '== About robots.txt == Search engines allow webmasters to define what kind of web pages should be included into the database and become available within search results. Some pag…')
 
(Updated disallow rules)
Line 17: Line 17:
 
== Dynamic version (without HTML catalog) ==
 
== Dynamic version (without HTML catalog) ==
  
If you're not using HTML catalog, only the following pages should be included into the index:  
+
If you're not using HTML catalog, only the following pages should be included into the index:
  
* home and category pages (home.php)  
+
* home and category pages (home.php)
* product pages (product.php)  
+
* product pages (product.php)
* static pages (pages.php)  
+
* static pages (pages.php)
* manufacturers (manufacturers.php)  
+
* manufacturers (manufacturers.php)
  
All other pages are recommended for exclusion. Therefore, we suggest using this configuration:  
+
All other pages are recommended for exclusion. Therefore, we suggest using this configuration:
  
 
<pre>
 
<pre>
Line 33: Line 33:
  
 
User-agent: *
 
User-agent: *
 +
Disallow: /admin/
 
Disallow: /catalog/
 
Disallow: /catalog/
Disallow: /cart.php
+
Disallow: /customer/
Disallow: /config.local.php
 
Disallow: /config.php
 
Disallow: /admin/
 
 
Disallow: /files/
 
Disallow: /files/
Disallow: /register.php
+
Disallow: /images/
 +
Disallow: /include/
 +
Disallow: /mail/
 +
Disallow: /modules/
 +
Disallow: /partner/
 +
Disallow: /payment/
 +
Disallow: /provider/
 +
Disallow: /schemes/
 +
Disallow: /shipping/
 +
Disallow: /skin/
 +
Disallow: /skin_backup/
 +
Disallow: /sql/
 +
Disallow: /upgrade/
 +
Disallow: /var/
 +
Disallow: /404.php
 +
Disallow: /CHANGELOG
 
Disallow: /COPYRIGHT
 
Disallow: /COPYRIGHT
Disallow: /COPYRIGHT_var_DE
 
Disallow: /CVS
 
 
Disallow: /INSTALL
 
Disallow: /INSTALL
Disallow: /INSTALL.X-AFFILIATE
+
Disallow: /NEW-4.4.0
Disallow: /INSTALL.X-AFFILIATE_var_DE
 
Disallow: /INSTALL.X-AOM
 
Disallow: /INSTALL.X-AOM_var_DE
 
Disallow: /INSTALL.X-FANCYCAT
 
Disallow: /INSTALL.X-FANCYCAT_var_DE
 
Disallow: /INSTALL.X-FCOMP
 
Disallow: /INSTALL.X-FCOMP_var_DE
 
Disallow: /INSTALL.X-GIFTREG
 
Disallow: /INSTALL.X-GIFTREG_var_DE
 
Disallow: /INSTALL.X-MAGNIFIER
 
Disallow: /INSTALL.X-MAGNIFIER_var_DE
 
Disallow: /INSTALL.X-OFFERS
 
Disallow: /INSTALL.X-OFFERS_var_DE
 
Disallow: /INSTALL.X-PCONF
 
Disallow: /INSTALL.X-PCONF_var_DE
 
Disallow: /INSTALL.X-RMA
 
Disallow: /INSTALL.X-RMA_var_DE
 
Disallow: /INSTALL.X-SURVEY
 
Disallow: /INSTALL_var_DE
 
Disallow: /Makefile
 
Disallow: /NEW-4.1.0
 
 
Disallow: /README
 
Disallow: /README
Disallow: /README_var_DE
 
Disallow: /Smarty-2.6.12
 
Disallow: /UPGRADE.readme
 
Disallow: /UPGRADE.readme_var_DE
 
 
Disallow: /VERSION
 
Disallow: /VERSION
 
Disallow: /adaptive.php
 
Disallow: /adaptive.php
Disallow: /admin_banner.php
+
Disallow: /address_book.php
 
Disallow: /adv_counter.php
 
Disallow: /adv_counter.php
 
Disallow: /antibot_image.php
 
Disallow: /antibot_image.php
 
Disallow: /auth.php
 
Disallow: /auth.php
 
Disallow: /banner.php
 
Disallow: /banner.php
Disallow: /banner_element.php
 
 
Disallow: /bonuses.php
 
Disallow: /bonuses.php
Disallow: /catalog_de/
+
Disallow: /cart.php
 
Disallow: /change_password.php
 
Disallow: /change_password.php
 
Disallow: /check_requirements.php
 
Disallow: /check_requirements.php
Line 88: Line 73:
 
Disallow: /comparison.php
 
Disallow: /comparison.php
 
Disallow: /comparison_list.php
 
Disallow: /comparison_list.php
Disallow: /customer/
+
Disallow: /config.php
 +
Disallow: /cron.php
 
Disallow: /default_icon.gif
 
Disallow: /default_icon.gif
 
Disallow: /default_image.gif
 
Disallow: /default_image.gif
Disallow: /default_image.gif_var_DE
 
 
Disallow: /default_logo.gif
 
Disallow: /default_logo.gif
Disallow: /doc/
+
Disallow: /dispatcher.php
 
Disallow: /download.php
 
Disallow: /download.php
 
Disallow: /error_message.php
 
Disallow: /error_message.php
Disallow: /fcategory.php
 
 
Disallow: /featured_products.php
 
Disallow: /featured_products.php
 +
Disallow: /get_block.php
 +
Disallow: /get_info.php
 
Disallow: /giftcert.php
 
Disallow: /giftcert.php
 
Disallow: /giftreg_manage.php
 
Disallow: /giftreg_manage.php
Line 104: Line 90:
 
Disallow: /https.php
 
Disallow: /https.php
 
Disallow: /image.php
 
Disallow: /image.php
Disallow: /images/
 
Disallow: /include/
 
Disallow: /index.html
 
Disallow: /index.html_var_DE
 
 
Disallow: /index.php
 
Disallow: /index.php
 
Disallow: /init.php
 
Disallow: /init.php
Disallow: /install-xaff.php
 
Disallow: /install-xaom.php
 
Disallow: /install-xbench.php
 
Disallow: /install-xfancycat.php
 
Disallow: /install-xfcomp.php
 
Disallow: /install-xgiftreg.php
 
Disallow: /install-xmagnifier.php
 
Disallow: /install-xoffers.php
 
Disallow: /install-xpconf.php
 
Disallow: /install-xrma.php
 
Disallow: /install-xsurvey.php
 
 
Disallow: /install.php
 
Disallow: /install.php
 +
Disallow: /login.php
 
Disallow: /magnifier_xml.php
 
Disallow: /magnifier_xml.php
Disallow: /mail/
+
Disallow: /message.html
 
Disallow: /minicart.php
 
Disallow: /minicart.php
Disallow: /modules/
 
 
Disallow: /news.php
 
Disallow: /news.php
 
Disallow: /offers.php
 
Disallow: /offers.php
 
Disallow: /order.php
 
Disallow: /order.php
 
Disallow: /orders.php
 
Disallow: /orders.php
Disallow: /page_sample1.html
+
Disallow: /pages.php
Disallow: /partner/
 
Disallow: /payment/
 
 
Disallow: /pconf.php
 
Disallow: /pconf.php
Disallow: /php.ini
+
Disallow: /permission_denied.html
 +
Disallow: /popup_address.php
 +
Disallow: /popup_ask.php
 
Disallow: /popup_edit_label.php
 
Disallow: /popup_edit_label.php
 +
Disallow: /popup_estimate_shipping.php
 
Disallow: /popup_fc_products.php
 
Disallow: /popup_fc_products.php
 
Disallow: /popup_image.php
 
Disallow: /popup_image.php
Line 141: Line 113:
 
Disallow: /popup_magnifier.php
 
Disallow: /popup_magnifier.php
 
Disallow: /popup_poptions.php
 
Disallow: /popup_poptions.php
 +
Disallow: /postauth.php
 +
Disallow: /preauth.php
 
Disallow: /prepare.php
 
Disallow: /prepare.php
 
Disallow: /probe.php
 
Disallow: /probe.php
 +
Disallow: /preview_image.gif
 
Disallow: /process_order.php
 
Disallow: /process_order.php
Disallow: /prodconf.gif
 
 
Disallow: /products.php
 
Disallow: /products.php
Disallow: /provider/
+
Disallow: /products_map.php
 
Disallow: /recommends.php
 
Disallow: /recommends.php
 
Disallow: /referer.php
 
Disallow: /referer.php
 +
Disallow: /register.php
 
Disallow: /returns.php
 
Disallow: /returns.php
Disallow: /schemes/
 
 
Disallow: /search.php
 
Disallow: /search.php
Disallow: /secure_login.php
 
 
Disallow: /send_to_friend.php
 
Disallow: /send_to_friend.php
Disallow: /shipping/
 
 
Disallow: /shop_closed.html
 
Disallow: /shop_closed.html
Disallow: /shop_closed.html_var_DE
+
Disallow: /shop_registration.php
Disallow: /skin1/
+
Disallow: /sitemap.php
Disallow: /skin1_images/
 
Disallow: /skins_dev/
 
 
Disallow: /slabel.php
 
Disallow: /slabel.php
 
Disallow: /smarty.php
 
Disallow: /smarty.php
Disallow: /sql/
 
 
Disallow: /survey.php
 
Disallow: /survey.php
Disallow: /swedish/
 
Disallow: /tests/
 
 
Disallow: /top.inc.php
 
Disallow: /top.inc.php
Disallow: /under_update.html
 
Disallow: /under_update.html_var_DE
 
Disallow: /upgrade/
 
Disallow: /var/
 
Disallow: /vote.php
 
 
Disallow: /wysiwyg.php
 
Disallow: /wysiwyg.php
 
</pre>
 
</pre>

Revision as of 21:44, 15 March 2011

About robots.txt

Search engines allow webmasters to define what kind of web pages should be included into the database and become available within search results. Some pages may also be undesired for inclusion since they do not contain any information relevant for potential customers. Besides, the capacity of listings available for individual websites might be limited, therefore, it's quite reasonable to include only content-rich part of your site into the index.

Another important action highly recommended for successful search engine indexing is the exclusion of so-called duplicate content from database. For example, we may speak of printable versions and non-JavaScript versions as main sources of the same content with different presentations.

If we speak of online stores based on X-Cart, their main traffic sources are pages containing product and industry-related materials. These pages include: home page, category listings, product details, static pages and manufacturers sections.

Below we describe how to setup X-Cart-based online store for proper search engine indexing.

Note: the description below is based on default X-Cart files and folders location. For customized stores, the configuration may slightly vary.

This is done by placing a special text file called robots.txt into the root folder of your domain (please note that it cannot be located within a subfolder, for example, /store). Such file can be created using any plain-text editor (like NotePad) and should be uploaded using an FTP client or online file manager of your hosting control panel.

If we speak of a content, for robots.txt it can easily be varied in accordance with your needs. In our case, main differences take place for two cases: dynamic store or the store based on HTML catalog. Working examples for each case are presented below:

Dynamic version (without HTML catalog)

If you're not using HTML catalog, only the following pages should be included into the index:

  • home and category pages (home.php)
  • product pages (product.php)
  • static pages (pages.php)
  • manufacturers (manufacturers.php)

All other pages are recommended for exclusion. Therefore, we suggest using this configuration:

User-agent: Googlebot
Disallow: /*printable=Y*
Disallow: /*js=*
Disallow: /*print_cat=*

User-agent: *
Disallow: /admin/
Disallow: /catalog/
Disallow: /customer/
Disallow: /files/
Disallow: /images/
Disallow: /include/
Disallow: /mail/
Disallow: /modules/
Disallow: /partner/
Disallow: /payment/
Disallow: /provider/
Disallow: /schemes/
Disallow: /shipping/
Disallow: /skin/
Disallow: /skin_backup/
Disallow: /sql/
Disallow: /upgrade/
Disallow: /var/
Disallow: /404.php
Disallow: /CHANGELOG
Disallow: /COPYRIGHT
Disallow: /INSTALL
Disallow: /NEW-4.4.0
Disallow: /README
Disallow: /VERSION
Disallow: /adaptive.php
Disallow: /address_book.php
Disallow: /adv_counter.php
Disallow: /antibot_image.php
Disallow: /auth.php
Disallow: /banner.php
Disallow: /bonuses.php
Disallow: /cart.php
Disallow: /change_password.php
Disallow: /check_requirements.php
Disallow: /choosing.php
Disallow: /cleanup.php
Disallow: /cmpi_popup.php
Disallow: /comparison.php
Disallow: /comparison_list.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /default_icon.gif
Disallow: /default_image.gif
Disallow: /default_logo.gif
Disallow: /dispatcher.php
Disallow: /download.php
Disallow: /error_message.php
Disallow: /featured_products.php
Disallow: /get_block.php
Disallow: /get_info.php
Disallow: /giftcert.php
Disallow: /giftreg_manage.php
Disallow: /giftregs.php
Disallow: /help.php
Disallow: /https.php
Disallow: /image.php
Disallow: /index.php
Disallow: /init.php
Disallow: /install.php
Disallow: /login.php
Disallow: /magnifier_xml.php
Disallow: /message.html
Disallow: /minicart.php
Disallow: /news.php
Disallow: /offers.php
Disallow: /order.php
Disallow: /orders.php
Disallow: /pages.php
Disallow: /pconf.php
Disallow: /permission_denied.html
Disallow: /popup_address.php
Disallow: /popup_ask.php
Disallow: /popup_edit_label.php
Disallow: /popup_estimate_shipping.php
Disallow: /popup_fc_products.php
Disallow: /popup_image.php
Disallow: /popup_info.php
Disallow: /popup_magnifier.php
Disallow: /popup_poptions.php
Disallow: /postauth.php
Disallow: /preauth.php
Disallow: /prepare.php
Disallow: /probe.php
Disallow: /preview_image.gif
Disallow: /process_order.php
Disallow: /products.php
Disallow: /products_map.php
Disallow: /recommends.php
Disallow: /referer.php
Disallow: /register.php
Disallow: /returns.php
Disallow: /search.php
Disallow: /send_to_friend.php
Disallow: /shop_closed.html
Disallow: /shop_registration.php
Disallow: /sitemap.php
Disallow: /slabel.php
Disallow: /smarty.php
Disallow: /survey.php
Disallow: /top.inc.php
Disallow: /wysiwyg.php

Static version (using HTML catalog)

For static version, it is suggested that only corresponding HTML part is included into index. Therefore, all of dynamic (PHP-based) part can be closed from search engines while all HTML pages will be kept for inclusion:

User-agent: Googlebot
Disallow: /*.php$
Disallow: /*?*
Disallow: /*printable=Y*
Disallow: /*js=*
Disallow: /*print_cat=*

User-agent: *
Disallow: /product.php
Disallow: /manufacturers.php
Disallow: /home.php
Disallow: /pages.php
Disallow: /config.local.php
Disallow: /config.php
Disallow: /cart.php
Disallow: /register.php
Disallow: /admin/
Disallow: /files/
Disallow: /COPYRIGHT
Disallow: /COPYRIGHT_var_DE
Disallow: /CVS
Disallow: /INSTALL
Disallow: /INSTALL.X-AFFILIATE
Disallow: /INSTALL.X-AFFILIATE_var_DE
Disallow: /INSTALL.X-AOM
Disallow: /INSTALL.X-AOM_var_DE
Disallow: /INSTALL.X-FANCYCAT
Disallow: /INSTALL.X-FANCYCAT_var_DE
Disallow: /INSTALL.X-FCOMP
Disallow: /INSTALL.X-FCOMP_var_DE
Disallow: /INSTALL.X-GIFTREG
Disallow: /INSTALL.X-GIFTREG_var_DE
Disallow: /INSTALL.X-MAGNIFIER
Disallow: /INSTALL.X-MAGNIFIER_var_DE
Disallow: /INSTALL.X-OFFERS
Disallow: /INSTALL.X-OFFERS_var_DE
Disallow: /INSTALL.X-PCONF
Disallow: /INSTALL.X-PCONF_var_DE
Disallow: /INSTALL.X-RMA
Disallow: /INSTALL.X-RMA_var_DE
Disallow: /INSTALL.X-SURVEY
Disallow: /INSTALL_var_DE
Disallow: /Makefile
Disallow: /NEW-4.1.0
Disallow: /README
Disallow: /README_var_DE
Disallow: /Smarty-2.6.12
Disallow: /UPGRADE.readme
Disallow: /UPGRADE.readme_var_DE
Disallow: /VERSION
Disallow: /adaptive.php
Disallow: /admin_banner.php
Disallow: /adv_counter.php
Disallow: /antibot_image.php
Disallow: /auth.php
Disallow: /banner.php
Disallow: /banner_element.php
Disallow: /bonuses.php
Disallow: /change_password.php
Disallow: /check_requirements.php
Disallow: /choosing.php
Disallow: /cleanup.php
Disallow: /cmpi_popup.php
Disallow: /comparison.php
Disallow: /comparison_list.php
Disallow: /customer/
Disallow: /default_icon.gif
Disallow: /default_image.gif
Disallow: /default_image.gif_var_DE
Disallow: /default_logo.gif
Disallow: /doc/
Disallow: /download.php
Disallow: /error_message.php
Disallow: /fcategory.php
Disallow: /featured_products.php
Disallow: /giftcert.php
Disallow: /giftreg_manage.php
Disallow: /giftregs.php
Disallow: /help.php
Disallow: /https.php
Disallow: /image.php
Disallow: /images/
Disallow: /include/
Disallow: /index.html
Disallow: /index.html_var_DE
Disallow: /index.php
Disallow: /init.php
Disallow: /install-xaff.php
Disallow: /install-xaom.php
Disallow: /install-xbench.php
Disallow: /install-xfancycat.php
Disallow: /install-xfcomp.php
Disallow: /install-xgiftreg.php
Disallow: /install-xmagnifier.php
Disallow: /install-xoffers.php
Disallow: /install-xpconf.php
Disallow: /install-xrma.php
Disallow: /install-xsurvey.php
Disallow: /install.php
Disallow: /magnifier_xml.php
Disallow: /mail/
Disallow: /minicart.php
Disallow: /modules/
Disallow: /news.php
Disallow: /offers.php
Disallow: /order.php
Disallow: /orders.php
Disallow: /page_sample1.html
Disallow: /partner/
Disallow: /payment/
Disallow: /pconf.php
Disallow: /php.ini
Disallow: /popup_edit_label.php
Disallow: /popup_fc_products.php
Disallow: /popup_image.php
Disallow: /popup_info.php
Disallow: /popup_magnifier.php
Disallow: /popup_poptions.php
Disallow: /prepare.php
Disallow: /probe.php
Disallow: /process_order.php
Disallow: /prodconf.gif
Disallow: /products.php
Disallow: /provider/
Disallow: /recommends.php
Disallow: /referer.php
Disallow: /returns.php
Disallow: /schemes/
Disallow: /search.php
Disallow: /secure_login.php
Disallow: /send_to_friend.php
Disallow: /shipping/
Disallow: /shop_closed.html
Disallow: /shop_closed.html_var_DE
Disallow: /skin1/
Disallow: /skin1_images/
Disallow: /skins_dev/
Disallow: /slabel.php
Disallow: /smarty.php
Disallow: /sql/
Disallow: /survey.php
Disallow: /swedish/
Disallow: /tests/
Disallow: /top.inc.php
Disallow: /under_update.html
Disallow: /under_update.html_var_DE
Disallow: /upgrade/
Disallow: /var/
Disallow: /vote.php
Disallow: /wysiwyg.php

Blocking of the secure part

Most of e-commerce websites can be reached via both HTTP and HTTPs protocols since checkout is usually done securely. Such situation can be treated by search robots as a duplicate content issue as all HTTP pages can be accessed via HTTPs as well. However, if your web server allows using mod_rewrite, all of secure pages can easily be blocked.

First of all, you will need to create a file called .htaccess containing the following lines (or add them into the existing one):

.htaccess

RewriteEngine on
RewriteCond %{HTTPS} on
RewriteRule ^robots\.txt$ robots-https.txt 

Next, create a file which will be used instead of robots.txt via the secure protocol and name it robots-https.txt:

robots-https.txt

User-agent: *
Disallow: / 

Finally, you need to upload both files along with the actual version of robots.txt that you intend to use.

If you ever require our assistance with the robots.txt setup for your online store, please, do not hesitate to contact our technical support for assistance. Our specialists will examine your store and perform the configuration which will be the most suitable from the search engines point of view.

See also