Nginx反爬虫模块

添加配置文件

vim /etc/nginx/conf.d/agent_deny.conf 

内容

# 禁止Scrapy|curl等工具的抓取
if ($http_user_agent ~* (Scrapy|Curl|HttpClient))
{
    return 403;
}

# 禁止指定UA及UA为空的访问
if ($http_user_agent ~ "Bytespider|FeedDemon|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|Feedly|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|YisouSpider|HttpClient|MJ12bot|heritrix|EasouSpider|Ezooms|^$" )
{
    return 403;
}
# 禁止非GET|HEAD|POST方式的抓取
if ($request_method !~ ^(GET|HEAD|POST)$)
{
    return 403;
}
#屏蔽单个IP的命令是
#deny 123.45.6.7
#封整个段即从123.0.0.1到123.255.255.254的命令
#deny 123.0.0.0/8
#封IP段即从123.45.0.1到123.45.255.254的命令
#deny 124.45.0.0/16
#封IP段即从123.45.6.1到123.45.6.254的命令是
#deny 123.45.6.0/24
#以下IP皆为流氓
#deny 58.95.66.0/24;

重启

nginx -s reload

server块导入配置

测试

#curl -X GET -I -A 'YYSpider' www.test.com
HTTP/1.1 403 Forbidden
Server: nginx/1.16.1
Date: Fri, 15 May 2020 09:12:55 GMT
Content-Type: text/html
Content-Length: 153
Connection: keep-alive

#curl -I -A ' ' www.test.com
HTTP/1.1 403 Forbidden
Server: nginx/1.16.1
Date: Fri, 15 May 2020 09:13:36 GMT
Content-Type: text/html
Content-Length: 153
Connection: keep-alive