# All robots will spider the domain User-agent: * Disallow: # Disallow directory /csr/ User-agent: * Disallow: /abc123/ # Disallow directory /images/ User-agent: * Disallow: /images/ # Disallow directory /affadmin/ User-agent: * Disallow: /affadmin/ # Disallow directory /afflib/ User-agent: * Disallow: /afflib/ # Disallow directory /cgi-bin/ User-agent: * Disallow: /cgi-bin/ # Disallow directory /fpdb/ User-agent: * Disallow: /fpdb/ # Disallow directory /gstftp/ User-agent: * Disallow: /gstftp/ # Disallow directory /guestadmin/ User-agent: * Disallow: /guestadmin/ # Disallow directory /shop/admin/ User-agent: * Disallow: /shop/admin/ # Disallow directory /shop/cache/ User-agent: * Disallow: /shop/cache/ # Disallow directory /shop/docs/ User-agent: * Disallow: /shop/docs/ # Disallow directory /shop/email/ User-agent: * Disallow: /shop/email/ # Disallow directory /shop/extras/ User-agent: * Disallow: /shop/extras/ # Disallow directory /shop/media/ User-agent: * Disallow: /shop/media/ # Disallow directory /shop/readme_files/ User-agent: * Disallow: /shop/readme_files/ # Disallow directory /shop/temp/ User-agent: * Disallow: /shop/temp/