Hello,
I inserted an actual list of 631 more spiders. After some time its good to remove all not needed spiders.
//631 more bots...
array('VoilaBot', 'Voila'),
array('searchdnabot', 'SearchDNA'),
array('kalooga/KaloogaBot', 'KaloogaBot'),
array('iajaBot', 'IajaBot'),
array('Girafabot', 'Girafa'),
array('Gigabot', 'Giga Blast'),
array('ExaBot', 'ExaLead Beta'),
array('Exabot', 'Exabot'),
array('EnaBot', 'EnaBall'),
array('DNAbot/1.0', 'DNAbot'),
array('facebookexternalhit', 'Facebook'),
array('Googlebot-Image', 'Google Images'),
array('ichiro', 'Ichiro Mobile'),
array('008/0.83', '80legs'),
array('urlfan-bot', '://URLFAN'),
array('abot', 'A Bot'),
array('ABACHOBot', 'ABACHOBot'),
array('ABCdatos', 'ABCdatos BotLink'),
array('Aboundex', 'Aboundex'),
array('AboutUsBot', 'AboutUs:Bot'),
array('Accelatech RSSCrawler', 'Accelatech RSS'),
array('Accoona-AI-Agent', 'Accoona'),
array('aconon Index', 'Aconon Index'),
array('AcoonBot', 'Acoon'),
array('AddThis', 'AddThis'),
array('Ahoy!', 'Ahoy!'),
array('AhrefsBot', 'AhrefsBot'),
array('AideRSS', 'AideRSS (PostRank.com)'),
array('ia_archiver', 'Alexa'),
array('bitlybot', 'Alexa Bitlybot'),
array('AlkalineBOT', 'Alkaline'),
array('scooter', 'AltaVista'),
array('crawler/3.0.0', 'Amazon AWS Cloud Based'),
array('JS-Kit URL Resolver', 'Amazon AWS Cloud Based'),
array('EMC Spider', 'Ananzi'),
array('Anthill', 'Anthill'),
array('Synapse', 'Apache Synapse ESB'),
array('Robot/v1.34', 'Apnoti Search Robot'),
array('Aport', 'Aport'),
array('AppleSyndication', 'Apple RSS'),
array('Arachnophilia', 'Arachnophilia'),
array('Araneo', 'Araneo'),
array('ArchitextSpider', 'Architext'),
array('arks/1.0', 'Arks'),
array('T312461', 'Artabus'),
array('Ask Jeeves', 'Ask.com'),
array('ASpider', 'ASpider'),
array('ATN_Worldwide', 'ATN Worldwide'),
array('Atomz', 'Atomz'),
array('atraxbot', 'Atrax Solutions'),
array('Attentio', 'Attentio'),
array('attributor/1.13.2', 'Attributor'),
array('AURESYS', 'Auresys'),
array('bbot', 'B Bot'),
array('BabalooSpider', 'Babaloo'),
array('BackRub', 'BackRub'),
array('BaiduMobaider', 'Baidu'),
array('BaiduImagespider', 'Baidu'),
array('Baiduspider', 'Baidu'),
array('BecomeBot', 'Become'),
array('Begun', 'Begun Robot Crawler'),
array('BejiBot', 'BejiBot'),
array('Big Brother', 'Big Brother'),
array('BigBoardsUpdater', 'Big-Boards.com'),
array('BigmirSpider', 'Bigmir'),
array('bingbot', 'Bing'),
array('Birubot', 'Birubot'),
array('Bitacle bot', 'Bitacle'),
array('Biz360 Spider', 'Biz 360'),
array('Bjaaland', 'Bjaaland'),
array('BlackWidow', 'Black Widow'),
array('BlogCrawler by Xango', 'BlogCrawler'),
array('blogdb', 'BlogDB RSS'),
array('blog search engine by BlogFan.ORG', 'BlogFan'),
array('Bloglines', 'Bloglines RSS'),
array('BlogPulse (ISSpider-3.0)', 'BlogPulse RSS'),
array('BlogSearch', 'BlogSearch RSS'),
array('BlogsNowBot', 'BlogsNow'),
array('BlogStreetBot', 'BlogStreet RSS'),
array('BoardTracker', 'Board Tracker'),
array('BoardPulse', 'BoardPulse'),
array('BoardReader', 'BoardReader'),
array('BoardViewer', 'BoardViewer'),
array('boitho.com-robot', 'Boitho'),
array('boitho.com-dc', 'Boitho Web Crawler'),
array('borg-bot', 'Borg'),
array('BotOnParade', 'BotOnParade'),
array('BOTW Spider', 'BotW'),
array('BDFetch', 'BrandProtect'),
array('BSpider', 'BSpider'),
array('Bulkfeeds', 'Bulkfeeds'),
array('Butterfly', 'Butterfly Topsy Crawler'),
array('CACTVS Chemistry Spider', 'CACTVS Chemistry'),
array('Calif', 'Calif'),
array('CaRP/3.6Evolution', 'CaRP RSS'),
array('CatchBot', 'Catch'),
array('mxbot', 'Chainn'),
array('ChangeDetection', 'ChangeDetection'),
array('Charlotte', 'Charlotte'),
array('Checkbot', 'Checkbot'),
array('ChitikaBot', 'ChitikaBot'),
array('ChristCrawler.com', 'Christ Crawler'),
array('www.cienciaficcion.net', 'cIeNcIaFiCcIoN'),
array('CipinetBot', 'Cipinet'),
array('CJNetworkQuality', 'CJ'),
array('YesupBot', 'Clicksor.com'),
array('CligooRobot', 'Cligoo'),
array('CLX.ru Bot', 'CLX.ru'),
array('CMC/0.01', 'CMC/0.01'),
array('ColdFusion', 'ColdFusion'),
array('combine', 'Combine System'),
array('Crawler (cometsearch@cometsystems.com)', 'Comet Systems'),
array('CCBot', 'CommonCrawl'),
array('COMODOspider', 'COMODO'),
array('ComputingSite Robi', 'ComputingSite Robi'),
array('conceptbot', 'ConceptBot'),
array('Cooby.de Crawler', 'Cooby'),
array('CoolBot', 'CoolBot'),
array('Covario', 'Covario'),
array('twiceler.', 'Cuil'),
array('Twiceler-0.9', 'Cuil'),
array('Cusco', 'Cusco'),
array('CyberSpyder', 'CyberSpyder'),
array('daumoa', 'Daum'),
array('daypopbot', 'Daypop RSS'),
array('DeadLinkCheck', 'Dead Link Check'),
array('Deepnet Explorer', 'Deepnet Explorer'),
array('DesertRealm.com', 'Desert Realm'),
array('Deweb', 'Deweb'),
array('Die Blinde Kuh', 'Die Blinde Kuh'),
array('dienstspider', 'Dienst'),
array('Diffbot', 'Diffbot'),
array('Digger/1.0 JDK/1.3.0', 'Digger'),
array('Digimarc WebReader', 'Digimarc MarcSpider'),
array('Digimarc CGIReader', 'Digimarc Marcspider/CGI'),
array('DIIbot', 'Digital Integrity Robot'),
array('grabber', 'Direct Hit Grabber'),
array('discobot', 'Discobot'),
array('DLE_Spider', 'DLE'),
array('Dolphin', 'Dolphin'),
array('Domnutch-Bot', 'Domnutch-Bot'),
array('DotBot', 'DotBot'),
array('DotBot/1.1', 'dotnetdotcom.org'),
array('DotTK SiteCheck', 'DotTK SiteCheck'),
array('DragonBot/1.0 libwww/5.0', 'DragonBot'),
array('Drupal', 'Drupal'),
array('DWCP/2.0', '<![CDATA[DWCP (Dridus Web Cataloging Project)]]>'),
array('e-SocietyRobot', 'e-Society'),
array('EbiNess/0.01a', 'EbiNess'),
array('dragonfly(ebingbong@playstarmusic.com)', 'eBingBong'),
array('edgeio-retriever', 'Edgeio'),
array('EIT-Link-Verifier-Robot/0.2', 'EIT Link Verifier Robot'),
array('elfinbot', 'Elfin Bot'),
array('abby', 'Ellerdale Project'),
array('Emacs-w3/v[0-9\.]+', 'Emacs-w3 Search Engine'),
array('Embedly', 'Embedly'),
array('envolk', 'envolk'),
array('ESISmartSpider', 'ESI Smart'),
array('esther', 'Esther'),
array('eSyndiCat Bot', 'eSyndiCat Bot'),
array('EuripBot', 'EuripBot'),
array('Eurobot/1.1', 'Eurobot'),
array('EventGuruBot', 'EventGuruBot'),
array('Evliya Celebi', 'Evliya Celebi'),
array('exactseek-pagereaper', 'Exact Seek'),
array('ExactSeek_Spider', 'Exact Seek'),
array('NG/2.0', 'ExaLead'),
array('Ezooms', 'Ezooms'),
array('Facebook', 'Facebook share follower'),
array('factbot', 'FactBites'),
array('fast-webcrawler', 'FAST / AlltheWeb'),
array('FastCrawler', 'FastCrawler'),
array('Feed24.com', 'Feed 24 RSS'),
array('FeedBlitz', 'Feed Blitz'),
array('UniversalFeedParser', 'Feed Parser'),
array('FeedBurner', 'FeedBurner RSS'),
array('FeedHub MetaDataFetcher', 'FeedHub'),
array('Feedster Crawler', 'Feedster Inc.'),
array('Feedtrace-bot', 'Feedtrace'),
array('FeedValidator/', 'FeedValidator'),
array('FEHLSTART Superspider', 'Fehlstart'),
array('FelixIDE', 'Felix IDE'),
array('ESIRover', 'FetchRover'),
array('fido', 'Fido'),
array('findlinks', 'FindLinks'),
array('FindoryBot', 'Findroy'),
array('Firebat', 'Firebat'),
array('Fish-Search-Robot', 'Fish Search'),
array('Mozilla/4.0 (compatible: FDSE robot)', 'Fluid Dynamics'),
array('FollowSite Bot', 'FollowSite Bot'),
array('libwww-perl/5.810', 'Forexitalia'),
array('fouineur.9bit.qc.ca', 'Fouineur'),
array('Freecrawl', 'Freecrawl'),
array('FreeWebMonitoring', 'FreeWebMonitoring'),
array('FunnelWeb', 'FunnelWeb'),
array('GaisBot', 'Gais'),
array('gamekitbot', 'Gamekit'),
array('gamma', 'gammaSpider'),
array('GarlikCrawler', 'Garlik Crawler'),
array('gazz', 'Gazz'),
array('gcreep', 'GCreep'),
array('genieBot', 'Genie Bot'),
array('GeoHasher', 'GeoHasher'),
array('geourl', 'GeoURL'),
array('GetterroboPlus', 'GetterroboPlus Puu'),
array('GetURL.rexx', 'GetURL'),
array('GingerCrawler', 'GingerCrawler'),
array('UnwindFetchor', 'GNIP'),
array('Goku', 'Goku'),
array('Golem', 'Golem'),
array('gonzo', 'Gonzo'),
array('gooblogsearch', 'Goo Blog Search'),
array('Googlebot', 'Google'),
array('Mediapartners-Google', 'Google AdSense'),
array('Adsbot-Google', 'Google Adwords'),
array('AppEngine-Google', 'Google AppEngine'),
array('FeedFetcher-Google', 'Google FeedFetcher'),
array('kw-lp-suggest', 'Google LP Keyword Checker Bot'),
array('Googlebot-Mobile', 'Google Mobile'),
array('PageFetcher-Google-CoOp', 'Google PageFetcher CoOp'),
array('Google-Sitemaps/1.0', 'Google Sitemaps'),
array('Googlebot-Video', 'Google Video'),
array('Google Web Preview', 'Google Web Preview'),
array('Google Wireless Transcoder', 'Google Wireless Transcoder'),
array('GosoSpider', 'Goso'),
array('Gpostbot', 'Gpost'),
array('griffon', 'Griffon'),
array('Gromit', 'Gromit'),
array('http://grub.org', 'Grub Client'),
array('Gulper Web Bot', 'Gulper'),
array('GurujiBot', 'Guruji'),
array('H?m?h?kki', 'H?m?h?kki'),
array('havIndex', 'HavIndex'),
array('HeinrichderMiragoRobot', 'Heinrichder Mirago'),
array('HenryTheMiragoRobot', 'Henry The Mirago Robot'),
array('archive.org_bot', 'Heritrix'),
array('HKU WWW Robot', 'HKU WWW Octopus'),
array('HolyCowDude', 'HolyCowDude RSS'),
array('Hometown', 'Hometown'),
array('HostTracker.com/1.0', 'HostTracker'),
array('htdig', 'ht://Dig'),
array('HTMLgobble', 'HTML Gobble'),
array('AITCSRobot', 'HTML Index'),
array('HuaweiSymantecSpider', 'Huawei Symantec'),
array('I Robot', 'I, Robot'),
array('http://www.almaden.ibm.com/cs/crawler', 'IBM Almaden'),
array('IBM_Planetwide', 'IBM Planetwide'),
array('+http://www.icerocket.com/', 'IceRocket'),
array('ichiro', 'Ichiro'),
array('igde', 'igdeSpyder'),
array('IlTrovatore-Setaccio', 'IlTrovatore Setaccio'),
array('Mozilla 3.01 PBWF (Win95)', 'Imagelock'),
array('IncyWincy', 'IncyWincy'),
array('Indy Library', 'Indy Library'),
array('Informant', 'Informant'),
array('InfoSeek Robot', 'InfoSeek Robot 1.0'),
array('Infoseek Sidewinder', 'Infoseek Sidewinder'),
array('infoSpider', 'infoSpider'),
array('INGRID', 'Ingrid'),
array('slurp@inktomi', 'Inktomi'),
array('Insitor', 'Insitor'),
array('inspectorwww', 'Inspector Web'),
array('IAGENT', 'IntelliAgent'),
array('Intelliseek', 'Intelliseek'),
array('Internet Cruiser Robot', 'Internet Cruiser'),
array('SCHOOLCARE; SV1; InfoPath.1', 'Internet for learning'),
array('InternetLinkAgent', 'Internet Link Agent'),
array('3GSE bot', 'Internet Research Institute UK'),
array('internetseer', 'Internet Seer'),
array('sharp-info-agent', 'Internet Shinchakubin'),
array('Pogodak', 'Interseek'),
array('Iron33', 'Iron33'),
array('IsraeliSearch', 'Israeli Search'),
array('itchBot', 'Itch'),
array('JavaBee', 'JavaBee'),
array('JBot', 'JBot'),
array('JCrawler', 'JCrawler'),
array('JetBot', 'JetEye'),
array('JoBo', 'JoBo'),
array('Jobot', 'Jobot'),
array('JoeBot', 'JoeBot'),
array('JSpider', 'JSpider'),
array('jumpstation', 'JumpStation'),
array('Jyxobot', 'Jyxo'),
array('image.kapsi.net', 'Kapsi Images'),
array('Katipo', 'Katipo'),
array('KDD-Explorer', 'KDD Explorer'),
array('KIT-Fireball', 'KIT Fireball'),
array('kmbot', 'knowmore'),
array('KO_Yappo_Robot', 'KO Yappo'),
array('LabelGrab', 'LabelGrabber'),
array('larbin', 'Larbin'),
array('LeapTag', 'LeapTag News Reader'),
array('LexxeBot', 'Lexxe'),
array('lwp-trivial', 'libwww-perl'),
array('linkalarm', 'Link Alarm'),
array('Linkidator', 'Link Validator'),
array('LinkedInBot', 'LinkedIn'),
array('LinkScan Server', 'LinkScan'),
array('LinkWalker', 'LinkWalker'),
array('livedoorCheckers/', 'Livedoor Checkers'),
array('Lockon', 'Lockon'),
array('logo.gif crawler', 'logo.gif'),
array('LuminateBot', 'LuminateBot'),
array('Lycos', 'Lycos'),
array('Apple-PubSub/59', 'Mac OS X RSS'),
array('Magpie', 'Magpie'),
array('magpie-crawler', 'Magpie Crawler'),
array('Mail.Ru', 'Mail.Ru'),
array('MJ12bot', 'Majestics MJ12bot'),
array('Mammoth', 'Mammoth'),
array('Marvin', 'Marvin'),
array('marvin/infoseek', 'marvin/infoseek'),
array('M/3.8', 'Mattie'),
array('MediaFox', 'MediaFox'),
array('mercator', 'Mercator'),
array('MerzScope', 'MerzScope'),
array('METASpider', 'Meta'),
array('Metaeuro', 'MetaEuro'),
array('MetaGer-LinkChecker', 'MetaGer'),
array('MetaURI', 'MetaURI'),
array('MSR-ISRCCrawler', 'Microsoft Research'),
array('MindCrawler', 'MindCrawler'),
array('Miva', 'Miva'),
array('MLBot', 'MLBot'),
array('UdmSearch', 'MNO GoSearch'),
array('mnoGoSearch', 'mnoGoSearch'),
array('moget', 'Moget'),
array('MOMspider', 'MOM'),
array('Monster', 'Monster'),
array('Moreoverbot', 'Moreover'),
array('Mp3Bot', 'Mp3Realm'),
array('msnbot', 'MSNBot'),
array('msnbot-media', 'MSNBot (Media Search)'),
array('msnbot-mobile', 'MSNBot (Mobile)'),
array('msnbot-newsblogs', 'MSNBot (News Search)'),
array('msnbot-products', 'MSNBot (Product Search)'),
array('MSRBOT', 'MSRBot'),
array('MuscatFerret', 'Muscat Ferret'),
array('MwdSearch', 'Mwd.Search'),
array('Najdi.si', 'Najdi.si'),
array('NPBot', 'NameProtect'),
array('NaverBot', 'NaverBot'),
array('NEC-MeshExplorer', 'NEC MeshExplorer'),
array('Nederland.zoek', 'Nederland.zoek'),
array('NerdByNature.Bot', 'NerdByNature'),
array('NetCarta CyberPilot Pro', 'NetCarta WebMap'),
array('Netcraft', 'Netcraft Web Server Survey'),
array('NetMechanic', 'NetMechanic'),
array('NetNewsWire', 'NetNewsWire RSS'),
array('NetScoop', 'NetScoop'),
array('NIF', 'News is Free RSS'),
array('newscan-online', 'Newscan Online'),
array('(X11; compatible; crawler@newstin.com; HTTPClient 3.1', 'Newstin'),
array('NextGenSearchBot 1', 'NextGen Search Bot'),
array('NHSEWalker', 'NHSE Web Forager'),
array('NimbleCrawler', 'NimbleCrawler'),
array('NjuiceBot', 'NjuiceBot'),
array('Nomad', 'Nomad'),
array('Norbert the Spider', 'Norbert'),
array('Gulliver', 'Northern Light'),
array('Nutch', 'Nutch'),
array('explorersearch', 'NZ Explorer'),
array('Occam', 'Occam'),
array('Ocelli', 'Ocelli'),
array('omgilibot', 'Omgili'),
array('omgilibot/0.3 +http://www.omgili.com/Crawler.html', 'omgilibot'),
array('OneRiot', 'OneRiot'),
array('Me.dium', 'OneRiot.com'),
array('LargeSmall', 'OneSpot'),
array('Online24-Bot', 'Online24-Bot'),
array('OOZBOT', 'OOZBOT'),
array('Openbot', 'Openfind'),
array('Openfind', 'Openfind data gatherer'),
array('OpenISearch', 'OpenISearch'),
array('Orbsearch', 'Orb Search'),
array('OWPBot', 'OWPBot'),
array('PackRat', 'Pack Rat'),
array('PageBoy', 'PageBoy'),
array('panscient.com', 'Panscient'),
array('ParaSite', 'ParaSite'),
array('ParchBot', 'ParchmentHill'),
array('Patric', 'Patric'),
array('PEGASUS', 'Pegasus'),
array('PerlCrawler/1.0 Xavatoria/2.0', 'PerlCrawler 1.0'),
array('PGP-KA', 'PGP Key Agent'),
array('Duppies', 'Phantom'),
array('phpdig', 'PhpDig'),
array('psbot/0.1 (+http://www.picsearch.com/bot.html) (51dc65875976ac434c09274f7e46dec6)', 'Picsearch'),
array('PiltdownMan', 'Piltdown Man'),
array('Pimptrains robot', 'Pimptrain'),
array('pingalink', 'Ping A Link'),
array('pingdom.com_bot', 'Pingdom.com Bot'),
array('Pioneer', 'Pioneer'),
array('PluckFeedCrawler', 'Pluck'),
array('Plukkie', 'Plukkie'),
array('PlumtreeWebAccessor', 'Plumtree Web Accessor'),
array('PodNova', 'Pod Nova'),
array('Pompos', 'Pompos'),
array('Poppi', 'Poppi'),
array('gestaltIconoclast', 'Popular Iconoclast'),
array('PortalBSpider', 'Portal B'),
array('PortalJuice.com', 'Portal Juice'),
array('PostRank', 'PostRank'),
array('ProCogBot', 'ProCog Bot'),
array('psbot', 'PSBot'),
array('PycURL', 'PycURL'),
array('Qango.com Web Directory', 'Qango'),
array('R6_FeedFetcher', 'Radian6'),
array('R6_CommentReader', 'Radian6'),
array('R6_CommentReader', 'Radian6 Comment Reader'),
array('R6_FeedFetcher', 'Radian6 FeedFetcher'),
array('R6_FeedFetcher', 'Radian6 FeedFetcher'),
array('StackRambler', 'Rambler'),
array('Raven', 'Raven Search'),
array('RixBot', 'REBOL IndeXer'),
array('rdfbot', 'Rediff'),
array('Resume Robot', 'Resume Robot'),
array('Road Runner: ImageScape Robot', 'Road Runner: ImageScape Robot'),
array('RHCS', 'RoadHouse Crawling System'),
array('Robbie', 'Robbie'),
array('RoboCrawl', 'RoboCrawl'),
array('Robofox', 'RoboFox'),
array('Robot du CRIM 1.0a', 'Robot Francoroute'),
array('Robozilla', 'Robozilla'),
array('Roverbot', 'Roverbot'),
array('RSS-SPIDER', 'RSS Feed Seeker'),
array('RuLeS', 'RuLeS'),
array('SafetyNet Robot', 'SafetyNet'),
array('SBIder', 'SBIder RSS'),
array('Scarlett', 'Scarlett'),
array('Scharia', 'Scharia'),
array('Science-Index', 'Science Index'),
array('ScooperBot', 'ScooperBot'),
array('ScoutJet', 'ScoutJet'),
array('Scrubby/3.0', 'Scrubby'),
array('SearchNZ', 'Search NZ'),
array('search17', 'Search17'),
array('searchprocess', 'SearchProcess'),
array('SBSearch', 'Secret Search Engine Labs'),
array('Seekbot', 'Seekbot'),
array('SemrushBot', 'SemrushBot'),
array('SemtoBot', 'SemtoBot'),
array('Senrigan', 'Senrigan'),
array('Sensis Web Crawler', 'Sensis'),
array('spbot', 'SEOprofiler'),
array('ServiceUptime.robot', 'ServiceUptime'),
array('SeznamBot', 'Seznam Fulltext Blog'),
array('SG-Scout', 'SG Scout'),
array('Shagseeker', 'ShagSeeker'),
array('ShaiHulud', '<![CDATA[ShaiHulud]]>'),
array('SheenBot', 'SheenBot'),
array('ShopWiki', '<![CDATA[Shopwiki [Bot]]]>'),
array('SimilarPages/Nutch', '<![CDATA[SimilarPages/Nutch [Crawler]]]>'),
array('SimBot/1.0', 'Simmany Robot Ver 1.0'),
array('ssearcher100', 'Site Searcher'),
array('Site Valet', 'Site Valet'),
array('http://www.site-list.net', 'Site-List RSS'),
array('SiteBot', 'SiteBot'),
array('SiteTech-Rover', 'SiteTech-Rover'),
array('SiteUptime.com', 'SiteUptime'),
array('SiteVibeBot', 'SiteVibeBot'),
array('+SitiDi.net/SitiDiBot/', 'SitiDi'),
array('SkimBot', 'SkimBot'),
array('aWapClient', 'Skymob'),
array('SLCrawler', 'SLCrawler'),
array('Sleek Spider', 'Sleek'),
array('Snapbot', 'Snap Shots'),
array('Snapbot/1.0', 'Snapbot'),
array('SnapPreviewBot', 'SnapPreviewBot'),
array('Snooper', 'Snooper'),
array('socbot', 'SocBot'),
array('Sogou web spider', 'Sogou'),
array('sohu-search', 'Sohu Search'),
array('Solbot', 'Solbot'),
array('Sosospider', 'Soso'),
array('www.entireweb.com/speedy.html', 'Speedy'),
array('Speedy', 'Speedy'),
array('Sphere Scout', 'Sphere'),
array('Sphider2', 'Sphider'),
array('mouse.house', 'Spider Monkey'),
array('SpiderBot', 'SpiderBot'),
array('spiderline', 'Spiderline Crawler'),
array('SpiderMan', 'SpiderMan'),
array('SpiderPig', 'SpiderPig'),
array('SpiderView', 'SpiderView'),
array('Spinn3r', 'Spinn3r'),
array('squadbot', 'SQuADbot'),
array('suke', 'Suke'),
array('suntek', 'Suntek Search Engine'),
array('superbot.com', 'Super.info Search Bot'),
array('Superfeedr', 'Superfeedr'),
array('Synthesio', 'Synthesio'),
array('Szukacz', 'Szukacz'),
array('Black Widow', 'TACH Black Widow'),
array('Tagoobot', 'Tagoo.ru'),
array('tailsweepblogcrawler', 'Tailsweep'),
array('Tarantula', 'Tarantula'),
array('tarspider', 'TarSpider'),
array('dlw3robot', 'Tcl W3 Robot'),
array('TechBOT', 'TechBOT'),
array('Technoratibot', 'Technorati'),
array('Templeton', 'Templeton'),
array('teoma', 'Teoma/Ask Jeeves'),
array('JubiiRobot', 'The Jubii'),
array('NorthStar', 'The NorthStar Robot'),
array('w3index', 'The NWI Robot'),
array('Peregrinator-Mathematics', 'The Peregrinator'),
array('thumbshots-de-Bot', 'Thumbshots'),
array('T-H-U-N-D-E-R-S-T-O-N-E', 'Thunderstone'),
array('TinEye', 'TinEye'),
array('TITAN', 'Titan'),
array('TitIn', 'TitIn'),
array('TLSpider', 'TLSpider'),
array('turnitinbot', 'Turn it in'),
array('slysearch', 'Turn it in slysearch'),
array('TurtleScanner', 'Turtle'),
array('Tweetmeme', 'Tweetmeme.com'),
array('Twiceler', 'Twiceler (Cuill.com)'),
array('Twingbot', 'Twingbot'),
array('Twingly', 'Twingly'),
array('Twitterbot', 'Twitterbot'),
array('Twitturls', 'Twitturls.com'),
array('Python-urllib', 'Twitturls.com (Python-urllib)'),
array('UCSD-Crawler', 'UCSD Crawl'),
array('UMBC-memeta-Bot', 'UMBC RSS'),
array('Unpartisan', 'Unpartisan RSS'),
array('urlck', 'URL Check'),
array('URL Spider Pro', 'URL Spider Pro'),
array('Valkyrie', 'Valkyrie'),
array('ClickSense', 'ValueClick LM'),
array('Mozilla/4.0 (vBSEO; http://www.vbseo.com)', 'vBSEO'),
array('Verticrawl', 'Verticrawl'),
array('Victoria', 'Victoria'),
array('vision-search', 'Vision Search'),
array('Visions Search', 'Visions'),
array('voyager/1.0', 'Voyager'),
array('VWbot_K', 'VWbot'),
array('W3C-checklink', 'W3C'),
array('W3C_CSS_Validator', 'W3C CSS Validator'),
array('W3C_Validator', 'W3C Validator'),
array('Unicorn', '<![CDATA[W3Cs United Validator]]>'),
array('W3M2', 'W3M2'),
array('w3mir', 'W3mir'),
array('w@pspider', 'w@p'),
array('appie', 'Walhello Appie'),
array('CrawlPaper', 'WallPaper'),
array('root', 'Web Core / Roots'),
array('WebMoose', 'Web Moose'),
array('WebAlta', 'WebAlta'),
array('WebAlta Crawler', 'WebAlta'),
array('WebBandit', 'WebBandit'),
array('WebCatcher', 'WebCatcher'),
array('Webclipping', 'Webclipping'),
array('WebCopy', 'WebCopy'),
array('WebFetcher', 'WebFetcher'),
array('weblayers', 'WebLayers'),
array('WebLinker', 'WebLinker'),
array('wlm', 'Weblog Monitor'),
array('WebQuest', 'WebQuest'),
array('WebReaper', 'WebReaper'),
array('webs@recruit.co.jp', 'Webs'),
array('websearchbench', 'WebSearchBench'),
array('WOLP', 'WebStolperer'),
array('webvac', 'WebVac'),
array('webwalk', 'WebWalk'),
array('WebWalker', 'WebWalker'),
array('WebWatch', 'WebWatch'),
array('www.WebWombat.com.au', 'WebWombat'),
array('Wget', 'Wget'),
array('whatUseek_winona', 'What U Seek Winona'),
array('Whitevector Crawler', 'Whitevector Crawler'),
array('www.whoisde.de', 'Whois DE'),
array('SurveyBot', 'Whois Source'),
array('wikiwix', 'Wikiwix'),
array('Hazels Ferret Web hopper', 'Wild Ferret Web Hopper'),
array('Willow Internet Crawler by Twotrees', 'Willow'),
array('Windows-Live-Social-Object-Extractor-Engine', 'Windows Live SOEE'),
array('Windows-RSS-Platform/1.0', 'Windows RSS Platform 1.0'),
array('Windows-RSS-Platform/2.0', 'Windows RSS Platform 2.0'),
array('WinHTTP', 'WinHTTP'),
array('wired-digital-newsbot', 'Wired Digital'),
array('Bilbo', 'Wise-Guys'),
array('Vagabondo', 'Wise-Guys'),
array('zyborg', 'WiseNut'),
array('WordPress', 'WordPress'),
array('woriobot', 'Worio'),
array('OmniExplorer_Bot', 'WorldIndexer'),
array('Project Kolinka Forum Search', 'www.kolinka.com'),
array('WWWC', 'WWWC'),
array('WWWeasel Robot', 'WWWeasel'),
array('wwwster', 'WWWSter'),
array('WWWWanderer', 'WWWWanderer'),
array('TECOMAC-Crawler', 'X-Crawler'),
array('Xenu', 'Xenu Link Sleuth'),
array('XGET', 'XGET'),
array('cosmos', 'XYLEME Robot'),
array('yacybot', 'YaCy'),
array('YahooYSMcm', 'Yahoo Publisher Network'),
array('Yahoo-Blogs', 'Yahoo! Blogs'),
array('YahooFeedSeeker', 'Yahoo! FeedSeeker'),
array('Yahoo-MMCrawler', 'Yahoo! Image Search'),
array('YahooSeeker/M1A1-R2D2', 'Yahoo! Mobile'),
array('Yahoo! Slurp', 'Yahoo! Slurp'),
array('Yahoo-VerticalCrawler', 'Yahoo! Vertical Crawler'),
array('YandexAntivirus', 'Yandex Antivirus'),
array('YandexBlog', 'Yandex Blog'),
array('YandexBot', 'Yandex Bot'),
array('YandexCatalog', 'Yandex Catalog'),
array('YandexDirect', 'Yandex Direct'),
array('YandexFavicon', 'Yandex Favicon'),
array('YandexImageResizer', 'Yandex ImageResizer'),
array('YandexImages', 'Yandex Images'),
array('YandexMedia', 'Yandex Media'),
array('YandexMetrika', 'Yandex Metrika'),
array('YandexNews', 'Yandex News'),
array('YandexPagechecker', 'Yandex Pagechecker'),
array('YandexVideo', 'Yandex Video'),
array('YandexWebmaster', 'Yandex Webmaster'),
array('YandexZakladki', 'Yandex Zakladki'),
array('Yanga WorldSearch Bot', 'Yanga'),
array('Yanga WorldSearch Bot', 'Yanga WorldSearch Bot'),
array('YebolBot', 'YebolBot'),
array('yeti', 'Yeti'),
array('Yeti', 'Yeti'),
array('Yeti/1.0', 'Yeti/1.0'),
array('YodaoBot', 'Yodao'),
array('YoudaoBot', 'Youdao'),
array('YRSpider', 'YunRang'),
array('zeus', 'Zeus Internet Marketing'),
array('http://www.zorkk.com', 'Zork RSS'),
