#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
#
# For syntax checking, see:
# http://www.frobee.com/robots-txt-check

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /emergo/

# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: */users/*
Disallow: */user/*
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /logout/
Disallow: /node/add*
Disallow: /node/*
Disallow: /de/node/*
Disallow: /fr/node/*
Disallow: /jp/node/*
Disallow: /es/node/*
Disallow: /pt/node/*
Disallow: /nl/node/*
Disallow: /it/node/*
Disallow: /ko/node/*
Disallow: /ru/node/*
Disallow: /cn/node/*
Disallow: /tr/node/*
Disallow: /node*
Disallow: /de/node*
Disallow: /fr/node*
Disallow: /jp/node*
Disallow: /es/node*
Disallow: /pt/node*
Disallow: /nl/node*
Disallow: /it/node*
Disallow: /ko/node*
Disallow: /ru/node*
Disallow: /cn/node*
Disallow: /tr/node*
Disallow: /contact-sent
Disallow: /*/contact-sent*
Disallow: /*/contact-sent
Disallow: /*/contact-sent*
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /?q=logout/
Disallow: /eform*
Disallow: /eform/*
Disallow: /content/*
# At the recommendation of: http://tips.webdesign10.com/robots-txt-and-drupal
Disallow: /node$
Disallow: /user$
Disallow: /search$
Disallow: /*/feed$
Disallow: /*/track$

# Other
Disallow: /aggregator/*
Disallow: /aggregator/sources/*
Disallow: /de/aggregator/*
Disallow: /de/aggregator/sources/*
Disallow: /nl/aggregator/*
Disallow: /nl/aggregator/sources/*
Disallow: /pt/aggregator/*
Disallow: /pt/aggregator/sources/*
Disallow: /es/aggregator/*
Disallow: /es/aggregator/sources/*
Disallow: /fr/aggregator/*
Disallow: /fr/aggregator/sources/*
Disallow: /it/aggregator/*
Disallow: /it/aggregator/sources/*
Disallow: /ko/aggregator/*
Disallow: /ko/aggregator/sources/*
Disallow: /ru/aggregator/*
Disallow: /ru/aggregator/sources/*
Disallow: /cn/aggregator/*
Disallow: /cn/aggregator/sources/*
Disallow: /tr/aggregator/*
Disallow: /tr/aggregator/sources/*
Disallow: /feed/
Disallow: /feed/*
Disallow: /aggregator/
Disallow: /aggregator/*

Disallow: /blog-full/rss
Disallow: /en/blog-full/rss
Disallow: /ko/blog-full/rss
Disallow: /jp/blog-full/rss
Disallow: /de/blog-full/rss
Disallow: /fr/blog-full/rss
Disallow: /nl/blog-full/rss
Disallow: /es/blog-full/rss
Disallow: /pt/blog-full/rss

Disallow: /views/resources/*
Disallow: /de/views/resources/*
Disallow: /fr/views/resources/*
Disallow: /es/views/resources/*
Disallow: /pt/views/resources/*
Disallow: /jp/views/resources/*
Disallow: /nl/views/resources/*
Disallow: /it/views/resources/*
Disallow: /ko/views/resources/*
Disallow: /ru/views/resources/*
Disallow: /cn/views/resources/*
Disallow: /tr/views/resources/*

Disallow: /views/*
Disallow: /de/views/*
Disallow: /fr/views/*
Disallow: /es/views/*
Disallow: /pt/views/*
Disallow: /jp/views/*
Disallow: /nl/views/*
Disallow: /it/views/*
Disallow: /ko/views/*
Disallow: /ru/views/*
Disallow: /cn/views/*
Disallow: /tr/views/*
Disallow: /views*
Disallow: /de/views*
Disallow: /fr/views*
Disallow: /es/views*
Disallow: /pt/views*
Disallow: /jp/views*
Disallow: /nl/views*
Disallow: /it/views*
Disallow: /ko/views*
Disallow: /ru/views*
Disallow: /cn/views*
Disallow: /tr/views*

Disallow: /taxonomy*
Disallow: /de/taxonomy*
Disallow: /fr/taxonomy*
Disallow: /nl/taxonomy*
Disallow: /jp/taxonomy*
Disallow: /ko/taxonomy*
Disallow: /es/taxonomy*
Disallow: /pt/taxonomy*


Disallow: /archive*
Disallow: /de/archive*
Disallow: /fr/archive*
Disallow: /nl/archive*
Disallow: /jp/archive*
Disallow: /ko/archive*
Disallow: /es/archive*
Disallow: /pt/archive*


Disallow: /resources/rss*
Disallow: /de/resources/rss*
Disallow: /fr/resources/rss*
Disallow: /es/resources/rss*
Disallow: /pt/resources/rss*
Disallow: /jp/resources/rss*
Disallow: /nl/resources/rss*
Disallow: /it/resources/rss*
Disallow: /ko/resources/rss*
Disallow: /ru/resources/rss*
Disallow: /cn/resources/rss*
Disallow: /tr/resources/rss*
Disallow: /es/blog*
Disallow: /pt/blog/2014*
Disallow: /jp/blog*
Disallow: /it/blog*
Disallow: /ko/blog*
Disallow: /ru/blog*
Disallow: /cn/blog*
Disallow: /tr/blog*
#disallow blog term and page archives for better SEO
Disallow: /blog/term*
Disallow: /blog/term/*
Disallow: /blog/author*
Disallow: /blog/author/*
Disallow: /de/blog/term*
Disallow: /de/blog/term/*
Disallow: /de/blog/author*
Disallow: /de/blog/author/*
Disallow: /nl/blog/term*
Disallow: /nl/blog/term/*
Disallow: /nl/blog/author*
Disallow: /nl/blog/author/*
#disallow admin pages
Disallow: /admin/*
Disallow: /admin*
#disallow country pages until we launch them
Disallow: /cn*
Disallow: /it*
Disallow: /ru*
Disallow: /tr/*
Disallow: */payment-received*
Disallow: */pay*
Disallow: */pay-step-2*

#reference Wistia Video Sitemap
Sitemap: http://app.wistia.com/sitemaps/41911.xml
Sitemap: http://www.emergobyul.com/sitemap.xml
Sitemap: http://store.emergogroup.com/sitemap_products_1.xml
Sitemap: http://emergo-group.myshopify.com/sitemap.xml
Sitemap: https://www.emergobyul.com/sitemap.xml