Commit 89b5c740 authored by Elias Ojala's avatar Elias Ojala

Update

parent bf6b03c4
......@@ -53,9 +53,16 @@ nmap.org
wwranks
DomainStatsBot
evc-batch
pdrlabs
turnitinbot
woorank
Zune
# If you use these technologies for legitimate purposes, please send issue report
Bytespider
bytedance
# If you use these technologies for legitimate purposes, please change your UA to application-specific
HeadlessChrome
PhantomJS
\ No newline at end of file
PhantomJS
Scrapy
......@@ -134,9 +134,11 @@ Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
Cloud mapping experiment. Contact research@pdrlabs.net
# Fake crawler:
Googlebot (gocrawl v0.4)
# The IP of the domain has cert for dailygieselmann.com / "e.ventures"
Mozilla/5.0 (compatible; evc-batch/2.0)
\ No newline at end of file
Mozilla/5.0 (compatible; evc-batch/2.0)
# http://rosslawley.co.uk/archive/old/2010/01/04/nginx-how-to-multiple-if-statements/
if ($http_user_agent ~* "ltx71|Barkrowler|ahrefs|infergy|SEMrush|Tsunami|Solstice|Hentai|BLEXbot|webmeup|CCBot|commoncrawl|Exabot|yoozBot|dotBot|opensiteexplorer|checkmarknetwork|yak|megaindex|domainsigma|elefent|extlinksbot|daum|uptimebot|iabtechlab|dataprovider|ips-agent|mauibot|bnf.fr_bot|spbot|MJ12bot|filterdb.iss.net|YisouSpider|archive|Wayback Machine|PiplBot|ipip.net|majestic12|zgrab|uptime.com|netEstate|CipaCrawler|domaincrawler|Wappalyzer|Nimbostratus|aihitdata|aiHitBot|Datanyze|LightspeedSystemsCrawler|PrivacyScore|Bytespider|bytedance.com" ) {
if ($http_user_agent ~* "ltx71|Barkrowler|ahrefs|infergy|SEMrush|Tsunami|Solstice|Hentai|BLEXbot|webmeup|CCBot|commoncrawl|Exabot|yoozBot|dotBot|opensiteexplorer|checkmarknetwork|yak|megaindex|domainsigma|elefent|extlinksbot|daum|uptimebot|iabtechlab|dataprovider|ips-agent|mauibot|bnf.fr_bot|spbot|MJ12bot|filterdb.iss.net|YisouSpider|archive|Wayback Machine|PiplBot|ipip.net|majestic12|zgrab|uptime.com|netEstate|CipaCrawler|domaincrawler|Wappalyzer|Nimbostratus|aihitdata|aiHitBot|Datanyze|LightspeedSystemsCrawler|PrivacyScore|NetcraftSurveyAgent|nmap.org|wwranks|DomainStatsBot|evc-batch|pdrlabs|turnitinbot|woorank|Zune|Bytespider|bytedance|# If you use these technologies for legitimate purposes, please change your UA to application-specific|HeadlessChrome|PhantomJS|Scrapy" ) {
set $robot Y;
# return 403 "User-agent blocked";
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment