Line 58 | Line 58 |
---|
*/ protected $bot_list = array( 'AdsBot [Google]' => array('AdsBot-Google', ''),
|
*/ protected $bot_list = array( 'AdsBot [Google]' => array('AdsBot-Google', ''),
|
| 'Ahrefs [Bot]' => array('AhrefsBot/', ''),
|
'Alexa [Bot]' => array('ia_archiver', ''), 'Alta Vista [Bot]' => array('Scooter/', ''),
|
'Alexa [Bot]' => array('ia_archiver', ''), 'Alta Vista [Bot]' => array('Scooter/', ''),
|
| 'Amazon [Bot]' => array('Amazonbot/', ''),
|
'Ask Jeeves [Bot]' => array('Ask Jeeves', ''), 'Baidu [Spider]' => array('Baiduspider', ''), 'Bing [Bot]' => array('bingbot/', ''),
|
'Ask Jeeves [Bot]' => array('Ask Jeeves', ''), 'Baidu [Spider]' => array('Baiduspider', ''), 'Bing [Bot]' => array('bingbot/', ''),
|
'Exabot [Bot]' => array('Exabot', ''),
| 'DuckDuckGo [Bot]' => array('DuckDuckBot/', ''), 'Exabot [Bot]' => array('Exabot/', ''),
|
'FAST Enterprise [Crawler]' => array('FAST Enterprise Crawler', ''), 'FAST WebCrawler [Crawler]' => array('FAST-WebCrawler/', ''), 'Francis [Bot]' => array('http://www.neomo.de/', ''),
| 'FAST Enterprise [Crawler]' => array('FAST Enterprise Crawler', ''), 'FAST WebCrawler [Crawler]' => array('FAST-WebCrawler/', ''), 'Francis [Bot]' => array('http://www.neomo.de/', ''),
|
Line 82 | Line 85 |
---|
'MSN NewsBlogs' => array('msnbot-NewsBlogs/', ''), 'MSN [Bot]' => array('msnbot/', ''), 'MSNbot Media' => array('msnbot-media/', ''),
|
'MSN NewsBlogs' => array('msnbot-NewsBlogs/', ''), 'MSN [Bot]' => array('msnbot/', ''), 'MSNbot Media' => array('msnbot-media/', ''),
|
| 'NG-Search [Bot]' => array('NG-Search/', ''),
|
'Nutch [Bot]' => array('http://lucene.apache.org/nutch/', ''),
|
'Nutch [Bot]' => array('http://lucene.apache.org/nutch/', ''),
|
| 'Nutch/CVS [Bot]' => array('NutchCVS/', ''), 'OmniExplorer [Bot]' => array('OmniExplorer_Bot/', ''),
|
'Online link [Validator]' => array('online link validator', ''), 'psbot [Picsearch]' => array('psbot/0', ''),
|
'Online link [Validator]' => array('online link validator', ''), 'psbot [Picsearch]' => array('psbot/0', ''),
|
| 'Seekport [Bot]' => array('Seekbot/', ''), 'Semrush [Bot]' => array('SemrushBot/', ''),
|
'Sensis [Crawler]' => array('Sensis Web Crawler', ''), 'SEO Crawler' => array('SEO search Crawler/', ''), 'Seoma [Crawler]' => array('Seoma [SEO Crawler]', ''), 'SEOSearch [Crawler]' => array('SEOsearch/', ''), 'Snappy [Bot]' => array('Snappy/1.1 ( http://www.urltrends.com/ )', ''), 'Steeler [Crawler]' => array('http://www.tkl.iis.u-tokyo.ac.jp/~crawler/', ''),
|
'Sensis [Crawler]' => array('Sensis Web Crawler', ''), 'SEO Crawler' => array('SEO search Crawler/', ''), 'Seoma [Crawler]' => array('Seoma [SEO Crawler]', ''), 'SEOSearch [Crawler]' => array('SEOsearch/', ''), 'Snappy [Bot]' => array('Snappy/1.1 ( http://www.urltrends.com/ )', ''), 'Steeler [Crawler]' => array('http://www.tkl.iis.u-tokyo.ac.jp/~crawler/', ''),
|
| 'Synoo [Bot]' => array('SynooBot/', ''),
|
'Telekom [Bot]' => array('[email protected]', ''), 'TurnitinBot [Bot]' => array('TurnitinBot/', ''), 'Voyager [Bot]' => array('voyager/', ''), 'W3 [Sitesearch]' => array('W3 SiteSearch Crawler', ''), 'W3C [Linkcheck]' => array('W3C-checklink/', ''),
|
'Telekom [Bot]' => array('[email protected]', ''), 'TurnitinBot [Bot]' => array('TurnitinBot/', ''), 'Voyager [Bot]' => array('voyager/', ''), 'W3 [Sitesearch]' => array('W3 SiteSearch Crawler', ''), 'W3C [Linkcheck]' => array('W3C-checklink/', ''),
|
'W3C [Validator]' => array('W3C_Validator', ''),
| 'W3C [Validator]' => array('W3C_*Validator', ''), 'WiseNut [Bot]' => array('http://www.WISEnutbot.com', ''),
|
'YaCy [Bot]' => array('yacybot', ''), 'Yahoo MMCrawler [Bot]' => array('Yahoo-MMCrawler/', ''), 'Yahoo Slurp [Bot]' => array('Yahoo! DE Slurp', ''),
| 'YaCy [Bot]' => array('yacybot', ''), 'Yahoo MMCrawler [Bot]' => array('Yahoo-MMCrawler/', ''), 'Yahoo Slurp [Bot]' => array('Yahoo! DE Slurp', ''),
|