# KB Airport Consulting Robots.txt updated 9/2/2005 # # If you decide to allow all bots use the below # User-agent: * # Disallow: ################### # GOOD BOTS - ALLOW ################### #Google User-agent: Googlebot Disallow: #MSN User-agent: MSNBot Disallow: #AOL, MSN, etc. User-agent: Slurp Disallow: #Ask Jeeves User-agent: Teoma Disallow: #dMoz User-agent: Robozilla Disallow: #AlltheWeb.com, Yahoo, Lycos User-agent: fast Disallow: #Looksmart User-agent: looksmart Disallow: #AltaVista User-agent: scooter Disallow: #Mozilla User-agent: mozilla Disallow: #Alexa User-agent: ia_archiver Disallow: #Gigablast User-agent: Gigabot Disallow: #scrubtheweb.com User-agent: Scrubby Disallow: ################### # BAD BOTS - DON'T ALLOW ################### User-agent: Black Hole Disallow: / User-agent: WebStripper Disallow: / User-agent: Titan Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: WebBandit Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: CherryPicker Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: SiteSnagger Disallow: / User-agent: ProWebWalker Disallow: / User-agent: CheeseBot Disallow: /