diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ed242b6..448c6f52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Add GarlikCrawler, ImplisenseBot and WikiDo bots. - Add Mastodon URL expander bot. +- Add eZ Publish Link Validator, GermCrawler, Pu_iN Crawler, ZoomBot, and ZoominfoBot bots. ## v2.5.3 diff --git a/bots.yml b/bots.yml index 6ffcb7ee..f263525f 100644 --- a/bots.yml +++ b/bots.yml @@ -66,6 +66,7 @@ everyonesocialbot: "EveryoneSocial" evrinid: "Evri bot" exabot: "Exalead's bot" exaleadcloudview: "ExaleadCloudView" +ez publish: "eZ Publish Link Validator" facebookexternalhit: "Facebook Bot" facebot: "Facebook Bot" feedburner: "RSS bot" @@ -75,6 +76,7 @@ flipboardproxy: "FlipboardProxy" friendfeedbot: "FriendFeed" garlik: "GarlikCrawler" genieo: "Genieo Web filter bot" +germcrawler: 'GermCrawler' getprismatic.com: "getprismatic.com" gigabot: "Gigabot spider" gimme60bot: "Gimme60 (gimme60.com)" @@ -175,12 +177,14 @@ plukkie: "botje.com/plukkie.htm" privacyawarebot: "PrivacyAwareBot" proximic: "Proximic Spider" psbot-page: "Picsearch" +pu_in: 'Pu_iN Crawler' publiclibraryarchive.org: "publiclibraryarchive.org" pycurl: "Python http library" python-httplib2: "Python-httplib2" python-requests: "Python http library" python-urllib: "Python http library" queryseeker: "QuerySeekerSpider" +quick-crawler: "Quick-Crawler" quicklook: "QuickLook" re-animator: "Domain Re-Animator Bot" readability: "Readability" @@ -268,4 +272,6 @@ yourls: "YOURLS" zelist.ro: "feed parser" zibb: "ZIBB spider" zitebot: "Zite" +zoombot: 'ZoomBot' +zoominfobot: 'ZoominfoBot' zyborg: "Zyborg" diff --git a/search_engines.yml b/search_engines.yml index ce216419..98669e2f 100644 --- a/search_engines.yml +++ b/search_engines.yml @@ -1,6 +1,6 @@ +ask jeeves: "Ask Jeeves" baidu: "Chinese search engine" bingbot: "Microsoft bing bot" +duckduckbot: "Duck Duck Go" googlebot: "Google spider" slurp: "Yahoo spider" -duckduckbot: "Duck Duck Go" -ask jeeves: "Ask Jeeves" diff --git a/test/ua_bots.yml b/test/ua_bots.yml index 1418260b..141eb09e 100644 --- a/test/ua_bots.yml +++ b/test/ua_bots.yml @@ -19,8 +19,10 @@ DAUMOA: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) DOMAINAREANIMATOR: 'Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com' DOT_BOT: 'Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)' DUCKDUCKGO: 'DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)' +EZPUBLISH: 'eZ Publish Link Validator' FACEBOOK_BOT: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)' GARLIK: 'GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)' +GERMCRAWLER: 'GermCrawler' GOOGLE_BOT: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' GOOGLE_PAGE_SPEED_INSIGHTS: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Chrome/22.0.1229 Safari/537.4' GOOGLE_SITE_VERIFICATION: Mozilla/5.0 (compatible; Google-Site-Verification/1.0) @@ -28,9 +30,9 @@ GOOGLE_STACKDRIVER_UPTIME_CHECKS: 'GoogleStackdriverMonitoring-UptimeChecks' GOOGLE_STRUCTURED_DATA_TESTING_TOOL2: 'Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)' GOOGLE_STRUCTURED_DATA_TESTING_TOOL: 'Mozilla/5.0 (compatible; X11; Linux x86_64; Google-StructuredDataTestingTool; +http://www.google.com/webmasters/tools/richsnippets)' GRAPESHOT: 'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)' +IMPLISENSEBOT: 'ImplisenseBot 1.0' JOBSEEKER: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) JobBot/5.0 (compatible; +http://www.jobseeker.com.au/bot.html) Safari/538.1' LINKDEXBOT: 'Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)' -IMPLISENSEBOT: 'ImplisenseBot 1.0' LOAD_TIME_BOT: 'Mozilla/5.0 (compatible; LoadTimeBot/0.9; +http://www.loadtime.net/bot.html)' LTX71: 'ltx71 - (http://ltx71.com/)' MAIL_RU: 'Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)' @@ -46,7 +48,9 @@ NEWRELICPINGER: NewRelicPinger/1.0 (12345) PAESSLER: Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com); Windows) PRIVACYAWAREBOT: 'Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)' PROXIMIC: 'Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)' +PUINCRAWLER: 'Pu_iN Crawler (+http://semanticjuice.com/)' QUERYSEEKER: 'QuerySeekerSpider ( http://queryseeker.com/bot.html )' +QUICKCRAWLER: "Quick-Crawler (+https://www.scrapinghub.com/)" SCRAPY: 'Scrapy/0.18.4 (+http://scrapy.org)' SEMANTICBOT: 'Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)' SEO_AUDIT: 'Mozilla/5.0 (compatible; seo-audit-check-bot/1.0)' @@ -70,3 +74,5 @@ YAHOO_SLURP: 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/ YANDEX_DIRECT: 'Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)' YANDEX_METRIKA: 'Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)' YANGA: 'Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)' +ZOOMBOT: 'ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)' +ZOOMINFOBOT: 'ZoominfoBot (zoominfobot at zoominfo dot com)'