User-agent: * # Crawl-delay is not in the robots.txt standard, but an extension by # Yahoo. Such references are explicitly allowed as per the standard. # References: # Crawl-delay: 10 Disallow: /external.php