User-agent: * # directed to all robots Disallow: /gp Disallow: /cgi-bin Disallow: /download Crawl-delay: 3