Skip to content

Commit 73c51f1

Browse files
mitchellkrogzamitchellkrogza
committed
Referers / Bots (+/-)
1 parent 547f0ba commit 73c51f1

File tree

10 files changed

+43
-9
lines changed

10 files changed

+43
-9
lines changed

‎Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
### - https://github.com/oohnoitz/nginx-blacklist
1818

1919
### Last Updated
20-
### Updated: Thu May 25 10:02:53 SAST 2017
20+
### Updated: Thu May 25 10:45:40 SAST 2017
2121
### End Last Updated
2222

2323
### Generated in
24-
### 0.380031347275 seconds
24+
### 0.314699649811 seconds
2525
### End Generated in
2626

2727
### Tested on: nginx/1.10.0 (Ubuntu 16.04)
@@ -192,6 +192,7 @@ map $http_user_agent $bad_bot {
192192
"~Alexa" 2;
193193
"~archive.org" 2;
194194
"~Baidu" 2;
195+
"~BUbiNG" 2;
195196
"~FlipboardProxy" 2;
196197
"~ia_archiver" 2;
197198
"~Mozilla/4.0" 2;
@@ -220,6 +221,7 @@ map $http_user_agent $bad_bot {
220221
"~*Abonti" 3;
221222
"~*Aboundex" 3;
222223
"~*Acunetix" 3;
224+
"~*ADmantX" 3;
223225
"~*AhrefsBot" 3;
224226
"~*AIBOT" 3;
225227
"~*AiHitBot" 3;
@@ -295,6 +297,7 @@ map $http_user_agent $bad_bot {
295297
"~*Discoverybot" 3;
296298
"~*DittoSpyder" 3;
297299
"~*DomainAppender" 3;
300+
"~*DomainCrawler" 3;
298301
"~*DomainSigmaCrawler" 3;
299302
"~*Dotbot" 3;
300303
"~*Download\ Demon" 3;
@@ -330,6 +333,7 @@ map $http_user_agent $bad_bot {
330333
"~*Foobot" 3;
331334
"~*Freeuploader" 3;
332335
"~*FrontPage" 3;
336+
"~*Fyrebot" 3;
333337
"~*GalaxyBot" 3;
334338
"~*Genieo" 3;
335339
"~*Getintent" 3;
@@ -517,6 +521,7 @@ map $http_user_agent $bad_bot {
517521
"~*Ripper" 3;
518522
"~*RocketCrawler" 3;
519523
"~*Rogerbot" 3;
524+
"~*SalesIntelligent" 3;
520525
"~*SBIder" 3;
521526
"~*ScanAlert" 3;
522527
"~*Scanbot" 3;
@@ -670,6 +675,7 @@ map $http_user_agent $bad_bot {
670675
"~*WWW-Mechanize" 3;
671676
"~*WWW::Mechanize" 3;
672677
"~*WWWOFFLE" 3;
678+
"~*x22Mozilla" 3;
673679
"~*Xaldon_WebSpider" 3;
674680
"~*Xaldon\ WebSpider" 3;
675681
"~*Xenu" 3;
@@ -1892,6 +1898,7 @@ map $http_referer $bad_referer {
18921898
"~*doktoronline.no" 1;
18931899
"~*dokumentalkino.net" 1;
18941900
"~*domain2008.com" 1;
1901+
"~*domaincrawler.com" 1;
18951902
"~*domainsatcost.com" 1;
18961903
"~*domain-submit.info" 1;
18971904
"~*domain-tracker.com" 1;
@@ -4013,6 +4020,7 @@ map $http_referer $bad_referer {
40134020
"~*school-diplomat.ru" 1;
40144021
"~*scoopquest.com" 1;
40154022
"~*score-ads.men" 1;
4023+
"~*scrapy.org" 1;
40164024
"~*screentoolkit.com" 1;
40174025
"~*scripted.com" 1;
40184026
"~*sc-specialhost.com" 1;

‎_generator_lists/bad-referrers.list

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,6 +1059,7 @@ dokumentalkino.net
10591059
domain-submit.info
10601060
domain-tracker.com
10611061
domain2008.com
1062+
domaincrawler.com
10621063
domainsatcost.com
10631064
domcran.net
10641065
domik-derevne.ru
@@ -3179,6 +3180,7 @@ scenicmissouri.us
31793180
school-diplomat.ru
31803181
scoopquest.com
31813182
score-ads.men
3183+
scrapy.org
31823184
screentoolkit.com
31833185
scripted.com
31843186
sdelai-prosto.ru

‎_generator_lists/bad-user-agents.list

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
Abonti
44
Aboundex
55
Acunetix
6+
ADmantX
67
AhrefsBot
78
AIBOT
89
AiHitBot
@@ -78,6 +79,7 @@ Discobot
7879
Discoverybot
7980
DittoSpyder
8081
DomainAppender
82+
DomainCrawler
8183
DomainSigmaCrawler
8284
Dotbot
8385
Download\ Demon
@@ -113,6 +115,7 @@ Flunky
113115
Foobot
114116
Freeuploader
115117
FrontPage
118+
Fyrebot
116119
GalaxyBot
117120
Genieo
118121
Getintent
@@ -300,6 +303,7 @@ RepoMonkey
300303
Ripper
301304
RocketCrawler
302305
Rogerbot
306+
SalesIntelligent
303307
SBIder
304308
ScanAlert
305309
Scanbot
@@ -453,6 +457,7 @@ WWW-Collector-E
453457
WWW-Mechanize
454458
WWW::Mechanize
455459
WWWOFFLE
460+
x22Mozilla
456461
Xaldon_WebSpider
457462
Xaldon\ WebSpider
458463
Xenu

‎_generator_lists/limited-user-agents.list

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
Alexa
22
archive.org
33
Baidu
4+
BUbiNG
45
FlipboardProxy
56
ia_archiver
67
Mozilla/4.0

‎conf.d/globalblacklist.conf

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
### - https://github.com/oohnoitz/nginx-blacklist
1818

1919
### Last Updated
20-
### Updated: Thu May 25 10:02:53 SAST 2017
20+
### Updated: Thu May 25 10:45:40 SAST 2017
2121
### End Last Updated
2222

2323
### Generated in
24-
### 0.380031347275 seconds
24+
### 0.314699649811 seconds
2525
### End Generated in
2626

2727
### Tested on: nginx/1.10.0 (Ubuntu 16.04)
@@ -192,6 +192,7 @@ map $http_user_agent $bad_bot {
192192
"~Alexa" 2;
193193
"~archive.org" 2;
194194
"~Baidu" 2;
195+
"~BUbiNG" 2;
195196
"~FlipboardProxy" 2;
196197
"~ia_archiver" 2;
197198
"~Mozilla/4.0" 2;
@@ -220,6 +221,7 @@ map $http_user_agent $bad_bot {
220221
"~*Abonti" 3;
221222
"~*Aboundex" 3;
222223
"~*Acunetix" 3;
224+
"~*ADmantX" 3;
223225
"~*AhrefsBot" 3;
224226
"~*AIBOT" 3;
225227
"~*AiHitBot" 3;
@@ -295,6 +297,7 @@ map $http_user_agent $bad_bot {
295297
"~*Discoverybot" 3;
296298
"~*DittoSpyder" 3;
297299
"~*DomainAppender" 3;
300+
"~*DomainCrawler" 3;
298301
"~*DomainSigmaCrawler" 3;
299302
"~*Dotbot" 3;
300303
"~*Download\ Demon" 3;
@@ -330,6 +333,7 @@ map $http_user_agent $bad_bot {
330333
"~*Foobot" 3;
331334
"~*Freeuploader" 3;
332335
"~*FrontPage" 3;
336+
"~*Fyrebot" 3;
333337
"~*GalaxyBot" 3;
334338
"~*Genieo" 3;
335339
"~*Getintent" 3;
@@ -517,6 +521,7 @@ map $http_user_agent $bad_bot {
517521
"~*Ripper" 3;
518522
"~*RocketCrawler" 3;
519523
"~*Rogerbot" 3;
524+
"~*SalesIntelligent" 3;
520525
"~*SBIder" 3;
521526
"~*ScanAlert" 3;
522527
"~*Scanbot" 3;
@@ -670,6 +675,7 @@ map $http_user_agent $bad_bot {
670675
"~*WWW-Mechanize" 3;
671676
"~*WWW::Mechanize" 3;
672677
"~*WWWOFFLE" 3;
678+
"~*x22Mozilla" 3;
673679
"~*Xaldon_WebSpider" 3;
674680
"~*Xaldon\ WebSpider" 3;
675681
"~*Xenu" 3;
@@ -1892,6 +1898,7 @@ map $http_referer $bad_referer {
18921898
"~*doktoronline.no" 1;
18931899
"~*dokumentalkino.net" 1;
18941900
"~*domain2008.com" 1;
1901+
"~*domaincrawler.com" 1;
18951902
"~*domainsatcost.com" 1;
18961903
"~*domain-submit.info" 1;
18971904
"~*domain-tracker.com" 1;
@@ -4013,6 +4020,7 @@ map $http_referer $bad_referer {
40134020
"~*school-diplomat.ru" 1;
40144021
"~*scoopquest.com" 1;
40154022
"~*score-ads.men" 1;
4023+
"~*scrapy.org" 1;
40164024
"~*screentoolkit.com" 1;
40174025
"~*scripted.com" 1;
40184026
"~*sc-specialhost.com" 1;

‎google-disavow.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,6 +1057,7 @@ domain:dokfilms.net
10571057
domain:doktoronline.no
10581058
domain:dokumentalkino.net
10591059
domain:domain2008.com
1060+
domain:domaincrawler.com
10601061
domain:domainsatcost.com
10611062
domain:domain-submit.info
10621063
domain:domain-tracker.com
@@ -3178,6 +3179,7 @@ domain:scenicmissouri.us
31783179
domain:school-diplomat.ru
31793180
domain:scoopquest.com
31803181
domain:score-ads.men
3182+
domain:scrapy.org
31813183
domain:screentoolkit.com
31823184
domain:scripted.com
31833185
domain:sc-specialhost.com

‎google-exclude-01.txt

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎google-exclude-02.txt

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎google-exclude-03.txt

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎travisCI/globalblacklist.tpl.conf

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
### - https://github.com/oohnoitz/nginx-blacklist
1818

1919
### Last Updated
20-
### Updated: Thu May 25 10:02:53 SAST 2017
20+
### Updated: Thu May 25 10:45:40 SAST 2017
2121
### End Last Updated
2222

2323
### Generated in
24-
### 0.380031347275 seconds
24+
### 0.314699649811 seconds
2525
### End Generated in
2626

2727
### Tested on: nginx/1.10.0 (Ubuntu 16.04)
@@ -194,6 +194,7 @@ map $http_user_agent $bad_bot {
194194
"~Alexa" 2;
195195
"~archive.org" 2;
196196
"~Baidu" 2;
197+
"~BUbiNG" 2;
197198
"~FlipboardProxy" 2;
198199
"~ia_archiver" 2;
199200
"~Mozilla/4.0" 2;
@@ -222,6 +223,7 @@ map $http_user_agent $bad_bot {
222223
"~*Abonti" 3;
223224
"~*Aboundex" 3;
224225
"~*Acunetix" 3;
226+
"~*ADmantX" 3;
225227
"~*AhrefsBot" 3;
226228
"~*AIBOT" 3;
227229
"~*AiHitBot" 3;
@@ -297,6 +299,7 @@ map $http_user_agent $bad_bot {
297299
"~*Discoverybot" 3;
298300
"~*DittoSpyder" 3;
299301
"~*DomainAppender" 3;
302+
"~*DomainCrawler" 3;
300303
"~*DomainSigmaCrawler" 3;
301304
"~*Dotbot" 3;
302305
"~*Download\ Demon" 3;
@@ -332,6 +335,7 @@ map $http_user_agent $bad_bot {
332335
"~*Foobot" 3;
333336
"~*Freeuploader" 3;
334337
"~*FrontPage" 3;
338+
"~*Fyrebot" 3;
335339
"~*GalaxyBot" 3;
336340
"~*Genieo" 3;
337341
"~*Getintent" 3;
@@ -519,6 +523,7 @@ map $http_user_agent $bad_bot {
519523
"~*Ripper" 3;
520524
"~*RocketCrawler" 3;
521525
"~*Rogerbot" 3;
526+
"~*SalesIntelligent" 3;
522527
"~*SBIder" 3;
523528
"~*ScanAlert" 3;
524529
"~*Scanbot" 3;
@@ -672,6 +677,7 @@ map $http_user_agent $bad_bot {
672677
"~*WWW-Mechanize" 3;
673678
"~*WWW::Mechanize" 3;
674679
"~*WWWOFFLE" 3;
680+
"~*x22Mozilla" 3;
675681
"~*Xaldon_WebSpider" 3;
676682
"~*Xaldon\ WebSpider" 3;
677683
"~*Xenu" 3;
@@ -1900,6 +1906,7 @@ map $http_referer $bad_referer {
19001906
"~*doktoronline.no" 1;
19011907
"~*dokumentalkino.net" 1;
19021908
"~*domain2008.com" 1;
1909+
"~*domaincrawler.com" 1;
19031910
"~*domainsatcost.com" 1;
19041911
"~*domain-submit.info" 1;
19051912
"~*domain-tracker.com" 1;
@@ -4021,6 +4028,7 @@ map $http_referer $bad_referer {
40214028
"~*school-diplomat.ru" 1;
40224029
"~*scoopquest.com" 1;
40234030
"~*score-ads.men" 1;
4031+
"~*scrapy.org" 1;
40244032
"~*screentoolkit.com" 1;
40254033
"~*scripted.com" 1;
40264034
"~*sc-specialhost.com" 1;

0 commit comments

Comments
 (0)