# Popular bots known to visit
User-agent: googlebot               #google.com
User-agent: adsbot-google           #google.com
User-agent: googlebot-image         #google.com
User-agent: googlebot-mobile        #google.com
User-agent: msnbot                  #msn.com
User-agent: msnbot-products         #msn.com
User-agent: msnbot-media            #msn.com
User-agent: librabot                #msn.com
User-agent: Ask                     #ask.com
User-agent: Teoma                   #ask.com
User-agent: Jeeves                  #ask.com
User-agent: Slurp                   #yahoo.com
User-agent: YahooSeeker             #yahoo.com
Crawl-delay: 10                     # 10 seconds between requests
Disallow: /cgi-bin/
Disallow: /online/              
Request-rate: 1/60m 1300-1659		# 8:00 am to noon EST
Request-rate: 1/60m 1700-0459		# noon to 11:59 pm EST
Request-rate: 6/1m  0500-1259		# midnight to 7:59 am EST

# Limited bots known to visit
User-agent: babaloospider
User-agent: BecomeBot 
User-agent: cazoodlebot
User-agent: ccBot
User-Agent: Charlotte
User-Agent: dibot
User-Agent: digext
User-Agent: docomo
User-agent: dotbot
User-agent: envolk
User-agent: exabot
User-agent: gigabot
User-agent: ips-agent
User-agent: fedora
User-agent: gurujibot
User-agent: holmes
User-agent: jyxobo
User-agent: konqueror
User-agent: RedCarpet
User-agent: mj12bot
User-agent: msnptc
User-agent: naverbot
User-agent: python-urllib
User-agent: shopwiki
User-agent: snapbot
User-agent: Scooter
User-agent: scoutjet
User-agent: seznambot
User-agent: speedy spider
User-agent: Speedy
User-agent: surveybot
User-agent: twiceler
User-agent: voilabot
User-agent: Voyager 
User-agent: woriobot
User-agent: yacybot
User-agent: yebolbot
User-agent: youdaobot
Disallow: /cgi-bin/
Disallow: /online/
Crawl-delay: 60                     # 60 seconds between requests
Request-rate: 1/60m 1300-1659		# 8:00 am to noon EST
Request-rate: 1/60m 1700-0459		# noon to 11:59 pm EST
Request-rate: 1/1m  0500-1259		# midnight to 7:59 am EST

# Excluded bots known to visit
User-agent: amznkassocbot
User-agent: baiduspider
User-agent: bdfetch
User-agent: busiversebot
User-agent: discobot
User-agent: fatbot
User-agent: Gaisbot
User-agent: grub-client
User-agent: grub
User-agent: heritrix
User-agent: ia_archiver
User-agent: linkalarm
User-agent: mlbot
User-agent: mutant
User-agent: netseer
User-agent: Nutch
User-agent: obot
User-agent: OOZBOT
User-agent: psbot
User-agent: robotgenius
User-agent: trident
User-agent: turnitinbot
User-agent: wget
User-agent: worldsearch
User-agent: wwwster
User-agent: yandex
User-agent: yanga
User-agent: yeti
Disallow: /

# Limited bot might visit
User-agent: Atomz
User-agent: bbot
User-agent: calif
User-agent: christcrawler
User-agent: Internet Cruiser Robot
User-agent: cyberspyder
User-agent: cydralspider
User-agent: digger
User-agent: grabber
User-agent: dwcp
User-agent: elfinbot
User-agent: esther
User-agent: fastcrawler
User-agent: ESI
User-agent: fido
User-agent: KIT-Fireball
User-agent: Freecrawl
User-agent: gulliver
User-agent: gulper
User-agent: hotwired
User-agent: INGRID
User-agent: Informant
User-agent: InfoSpiders
User-agent: irobot
User-agent: Iron33
User-agent: Linkidator
User-agent: linkwalker
User-agent: marvin
User-agent: MindCrawler
User-agent: MuscatFerret
User-agent: PortalBSpider
User-agent: Raven
User-agent: roadrunner
User-agent: Robbie
User-agent: robi
User-agent: RoboCrawl
User-agent: robofox
User-agent: SapphireWebCrawler
User-agent: Search-AU
User-agent: skymob
User-agent: spanwebbot
User-agent: ESI
User-agent: spider_monkey
User-agent: SpiderBot
User-agent: spiderline
User-agent: vacuumweb
User-agent: verticrawl
User-agent: VWbot_K
User-agent: wapspider
User-agent: WebBandit
User-agent: T-H-U-N-D-E-R-S-T-O-N-E
User-agent: webquest
User-agent: webs
User-agent: webspider
User-agent: WebWalker
User-agent: winona
User-agent: whowhere
User-agent: WWWC
Disallow: /cgi-bin/
Disallow: /online/
Crawl-delay: 60                     # 60 seconds between requests
Request-rate: 1/60m 1300-1659		# 8:00 am to noon EST
Request-rate: 1/60m 1700-0459		# noon to 11:59 pm EST
Request-rate: 1/1m  0500-1259		# midnight to 7:59 am EST

# Excluded bots that might visit:
User-agent: IsraBot
User-agent: UbiCrawler
User-agent: DOC
User-agent: Zao
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: k2spider
User-agent: NPBot
User-agent: WebReaper
User-agent: anthill
User-agent: appie
User-agent: arale
User-agent: BotLink
User-agent: ahoy
User-agent: AlkalineBOT 
User-agent: araneo
User-agent: AraybOt
User-agent: ariadne
User-agent: arks
User-agent: ATN_Worldwide
User-agent: Bjaaland
User-agent: Ukonline
User-agent: borg-bot
User-agent: boxseabot
User-agent: bspider
User-agent: CMC
User-agent: combine
User-agent: confuzzledbot
User-agent: CoolBot
User-agent: cosmos
User-agent: cusco
User-agent: desertrealm
User-agent: DIIbot
User-agent: downloadexpress
User-agent: DragonBot
User-agent: ecollector
User-agent: ebiness
User-agent: esculapio
User-agent: FDSE
User-agent: FELIX IDE
User-agent: Hämähäkki
User-agent: fouineur
User-agent: gammaSpider
User-agent: gazz
User-agent: gcreep
User-agent: golem
User-agent: griffon
User-agent: Gromit
User-agent: hambot
User-agent: havIndex
User-agent: htdig
User-agent: iajabot
User-agent: inspectorwww
User-agent: JBot
User-agent: jcrawler
User-agent: jobo
User-agent: image.kapsi.net
User-agent: KDD-Explorer
User-agent: ko_yappo_robot
User-agent: label-grabber
User-agent: larbin
User-agent: legs
User-agent: Lockon
User-agent: User-agent: logo_gif_crawler
User-agent: mattie
User-agent: mediafox
User-agent: MerzScope
User-agent: NEC-MeshExplorer
User-agent: UdmSearch
User-agent: moget
User-agent: User-agent: Motor
User-agent: muncher
User-agent: muninn
User-agent: MwdSearch
User-agent: sharp-info-agent
User-agent: WebMechanic
User-agent: NetScoop
User-agent: newscan-online
User-agent: ObjectsSearch
User-agent: Occam
User-agent: Orbsearch
User-agent: User-agent: packrat
User-agent: pageboy
User-agent: ParaSite
User-agent: patric
User-agent: pegasus
User-agent: perlcrawler
User-agent: phpdig
User-agent: piltdownman
User-agent: Pimptrain
User-agent: User-agent: pjspider
User-agent: PlumtreeWebAccessor
User-agent: Getterrobo-Plus
User-agent: RHCS
User-agent: RixBot
User-agent: searchprocess
User-agent: Senrigan
User-agent: Shagseeker
User-agent: sift
User-agent: SimBot
User-agent: Site Valet
User-agent: SLCrawler
User-agent: snooper
User-agent: solbot
User-agent: suke
User-agent: tach_bw
User-agent: Tarantula
User-agent: TechBOT
User-agent: templeton
User-agent: titin
User-agent: topiclink
User-agent: UdmSearch
User-agent: uptimebot
User-agent: urlck
User-agent: Victoria
User-agent: void-bot
User-agent: crawlpaper
User-agent: webcatcher
User-agent: WebMoose
User-agent: webreaper
User-agent: wlm
User-agent: WOLP
User-agent: XGET
User-agent: Nederland.zoek
User-agent: YoBot 
Disallow: /

# Unknown bots that might visit
User-agent: *
Disallow: /cgi-bin/
Disallow: /online/
Crawl-delay: 10                     # 10 seconds between requests
Request-rate: 1/60m 1300-1659		# 8:00 am to noon EST
Request-rate: 1/60m 1700-0459		# noon to 11:59 pm EST
Request-rate: 6/1m  0500-1259		# midnight to 7:59 am EST

# GGH 02/18/09 Generated TMS sitemap.xml
sitemap: http://www.cholesterol-monitor-tester.com/sitemap.xls