nginx 屏蔽采集的 $http_user_agent

block_cralwer

# save at /etc/nginx/block_cralwer
# then use it `include block_cralwer` at `server` directive

set $fbd 0;

if ($http_user_agent ~* "yandex|Ahref|MJ12bot|XoviBot|SemrushBot|AhrefsBot|Twitterbot|Claritybot|Crawler|Python") {
    set $fbd 1;
}

location ~* \/(plus|data|trust|include|shtml|bbs|rank|rxcq|tager) {
    set $fbd 1;
}

location ~ ^/(wp-admin|wp-login\.php) {
    set $fbd 1;
}

if ($fbd = 1 ) {
    return 403;
}