# Disallow all crawlers access. User-agent: * Disallow: / Disallow: /nhsverification/terms-of-use.asp # Disallow crawlers access site. User-agent: ia_archiver-web.archive.org Disallow: / User-agent: ia_archiver Disallow: / User-agent: ia_archiver-web.archive.org Disallow: /robots.txt User-agent: ia_archiver-web.archive.org Disallow: robots.txt User-agent: ia_archiver Disallow: /robots.txt User-agent: ia_archiver Disallow: robots.txt User-agent: AboutUsBot Disallow: /