# Validate robots.txt in http://tool.motoricerca.info/robots-checker.phtml
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

User-agent: Googlebot
# disallow all files ending with these extensions
Disallow: /*.rb$
Disallow: /*.rake$
Disallow: /*.js$
Disallow: /*.sass$
Disallow: /*.css$
Disallow: /*.gz$

User-agent: *
Crawl-Delay: 30
Disallow: /images/
Disallow: /javascripts/
Disallow: /stylesheets/

# Allow: /posts/
# Disallow: /posts/*/comment/*
# Logs with word comment (paginado de comentarios). Estos casos pertenecen a la idea de no indexar contenido duplicado.

Sitemap: http://aitorfernandez.com/sitemap.xml.gz