static/robots.txt

   1 User-agent: *
   2 Disallow: /tools/
   3 Disallow: /gbrowse/
   4 Disallow: /jbrowse/
   5 Disallow: /documents/tempfiles/
   6 Disallow: /cview/
   7 Disallow: /phenome/qtl_analysis.pl
   8 Disallow: /phenome/qtl.pl
   9 Disallow: /phenome/population.pl
  10 Disallow: /qtl/download/
  11 Disallow: /solgs/
  12 Disallow: /*genotypes
  13 Crawl-delay: 30