# small robots.txt # more information about this file can be found at # http://www.robotstxt.org/ # lines beginning with the pund ("#") sign are comments and can be deleted. # if case your drupal site is in a directory # lower than your docroot (e.g. /drupal) # please add this before the /-es below # to stop a polite robot indexing an example dir # add a line like (delete the #'s) # user-agent: polite-bot # Disallow: /example-dir/ User-agent: * Crawl-Delay: 10 Disallow: /aggregator Disallow: /book/print Disallow: /comment/reply Disallow: /database Disallow: /images Disallow: /misc Disallow: /node/add Disallow: /pathauto Disallow: /page_category Disallow: /search Disallow: /site Disallow: /SysVol Disallow: /sysvol Disallow: /taxonomy Disallow: /tracker Disallow: /user