1.查看HA状态
yarn rmadmin -getServiceState rm1
hdfs haadmin -getServiceState nn1
hadoop dfs -ls -R -lrt /tmp/mc/'`default`'.db/stg_mh_lpdyrmyy_radinfo_pacsimages_mc_temp_mc_temp/
700 hadoop dfs -ls -lrt /tmp/mc/'`default`'.db/stg_mh_lpdyrmyy_radinfo_pacsimages_mc_temp_mc_temp/
701 hadoop fs -ls /tmp/mc/'`default`'.db/stg_mh_lpdyrmyy_radinfo_pacsimages_mc_temp_mc_temp/ | sort -k6,7
702 hadoop fs -ls /user/iteblog | sort -r -k6,7
hadoop fs -ls /tmp/mc/'`default`'.db/stg_mh_lpdyrmyy_radinfo_pacsimages_mc_temp_mc_temp/ | sort -r -k6,7
./hive --orcfiledump -d /user/hive/warehouse/stg_bq_lpdwrmyy_mz_chufang1/dt=20221125/000000_0 > 25_0.txt
sed -n '/2022-11-25T10:06:*/,/2022-11-25T10:10:*/p' hadoop-fri-namenode-sdw1.log
sudo du -sh /opt/module/* | sort
tail -f postgresql-2023-06-01_000000.log
cat postgresql-2023-06-05_000000.log | grep hzjbxx2.hadoop页面文件的大小
hdfs dfs -du -s -h /tmp/mc
1.1 T 3.2 T /tmp/mc
hadoop fs -du -h /tmp/hive/* | grep G
36.1 G 108.2 G /tmp/hive/fri/6300c87c-91ed-4a11-b3f5-ad0146f1d6eb
hadoop fs -count -h /tmp/mc
703.1 K 703.2 K 1.1 T /tmp/mc参考国外论坛的脚本，写了如下脚本，定时清理或者手动执行均可。
#!/bin/bash
#cleanup hive session dir on hdfs named with /tmp/hive/hdfs
usage="Usage: cleanup.sh [days]"
if [ ! "$1" ]
then
echo $usage
exit 1
fi
now=$(date +%s)
hadoop fs -ls /tmp/hive/hdfs | grep "^d" | while read f; do
dir_date=`echo $f | awk '{print $6}'`
difference=$(( ( $now - $(date -d "$dir_date" +%s) ) / (24 * 60 * 60 ) ))
if [ $difference -gt $1 ]; then
echo $f
hadoop fs -rm -r -skipTrash `echo $f| awk '{ print $8 }'`; //此处直接对上边的过滤结果进行删除，建议先改成 hadoop fs -ls -R 命令先进行测试，一切正常在改回删除。
fi
done[bque@sdw7 ~]$ hdfs dfs -find /tmp/hive/ -name "*" |grep 2024
/tmp/hive/anonymous/4792e1e4-1c9b-4283-b20b-085b84b22024
/tmp/hive/anonymous/4792e1e4-1c9b-4283-b20b-085b84b22024/_tmp_space.db
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269/-mr-10003
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269/-mr-10004
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269/-mr-10004/49e82cad-46ec-4118-85d6-30b945310269
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269/-mr-10004/49e82cad-46ec-4118-85d6-30b945310269/map.xml
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_191_8247420852406730837-6269/-mr-10004/49e82cad-46ec-4118-85d6-30b945310269/reduce.xml
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_348_3147415929326670410-2543
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_348_3147415929326670410-2543/-mr-10000
/tmp/hive/anonymous/9d969536-c1b3-41c7-91db-92430c757fc7/hive_2024-05-21_17-22-56_348_3147415929326670410-2543/-mr-10000/.hive-staging_hive_2024-05-21_17-22-56_348_3147415929326670410-2543[bque@sdw7 ~]$ hdfs dfs -ls /tmp/hive/* |grep 2024
drwx------ - bque supergroup 0 2024-05-21 23:59 /tmp/hive/_resultscache_/results-8e9b7b05-7775-4cab-a74a-271fad78c153
drwx------ - bque supergroup 0 2024-05-21 17:29 /tmp/hive/_resultscache_/results-9d093e9a-843a-49b8-8b22-a01b1bd38e0c
drwx------ - anonymous supergroup 0 2024-05-21 17:40 /tmp/hive/anonymous/00404f9f-4b94-4173-9f63-fabcc93d56fc
drwx------ - anonymous supergroup 0 2024-05-21 13:20 /tmp/hive/anonymous/00d09ced-efd7-423c-8b9f-5b6bcda1d712
drwx------ - anonymous supergroup 0 2024-05-21 23:54 /tmp/hive/anonymous/01573cd2-d364-4102-a5dd-10b253ef135b
drwx------ - anonymous supergroup 0 2024-05-21 18:01 /tmp/hive/anonymous/01be451d-3e86-4c57-b899-910d06654f8d
drwx------ - anonymous supergroup 0 2024-05-21 17:45 /tmp/hive/anonymous/01ea3fc1-00c7-4629-bcf5-c8fe7f7ae992
drwx------ - anonymous supergroup 0 2024-05-21 17:46 /tmp/hive/anonymous/027d7ddf-6a9e-45a8-ac36-64a1be40802f
drwx------ - anonymous supergroup 0 2024-05-21 18:14 /tmp/hive/anonymous/030fb51e-05a0-4753-9800-779ff3881290
drwx------ - anonymous supergroup 0 2024-05-21 20:11 /tmp/hive/anonymous/0348c550-2de0-415a-9488-a30fb6ec26a7
drwx------ - anonymous supergroup 0 2024-05-21 17:41 /tmp/hive/anonymous/034c0ebf-2305-4816-96d3-e9cb8f30f4aa
drwx------ - anonymous supergroup 0 2024-05-21 17:15 /tmp/hive/anonymous/03bbcd41-962f-4b68-9dc0-fead9e581d96
drwx------ - anonymous supergroup 0 2024-05-21 17:15 /tmp/hive/anonymous/0473e831-a191-4d30-9b3b-1376794cd74c
drwx------ - anonymous supergroup 0 2024-05-21 21:00 /tmp/hive/anonymous/04bc1656-3c1b-4645-97eb-114c1c037c06
drwx------ - anonymous supergroup 0 2024-05-21 17:40 /tmp/hive/anonymous/057245c9-1c59-4ea0-9d94-6f9ee6092b76
drwx------ - anonymous supergroup 0 2024-05-21 17:20 /tmp/hive/anonymous/0601e50a-eac5-41be-a471-15e13718d44d[bque@sdw7 ~]$ hdfs dfs -ls /tmp/hive/* |grep 2024-05-21 | awk '{print $8}' | xargs hdfs dfs -rm -r
Deleted /tmp/hive/_resultscache_/results-8e9b7b05-7775-4cab-a74a-271fad78c153
Deleted /tmp/hive/_resultscache_/results-9d093e9a-843a-49b8-8b22-a01b1bd38e0c
Deleted /tmp/hive/anonymous/00404f9f-4b94-4173-9f63-fabcc93d56fc
Deleted /tmp/hive/anonymous/00d09ced-efd7-423c-8b9f-5b6bcda1d712
Deleted /tmp/hive/anonymous/01573cd2-d364-4102-a5dd-10b253ef135b
Deleted /tmp/hive/anonymous/01be451d-3e86-4c57-b899-910d06654f8d
Deleted /tmp/hive/anonymous/01ea3fc1-00c7-4629-bcf5-c8fe7f7ae992
Deleted /tmp/hive/anonymous/027d7ddf-6a9e-45a8-ac36-64a1be40802f
Deleted /tmp/hive/anonymous/030fb51e-05a0-4753-9800-779ff38812903.清理日志
/opt/module/zookeeper/bin/zkCleanup.sh /opt/module/zookeeper/zkData -n 5
留下最近五条日志，其他的清除du -h /opt/module* | grep G
pwd cd 进入到日志层 rm -rf
df -h
4.TCP连接数
[fri@sdw7 ~]$
netstat -an |grep 181:10000 |wc -l
140
You have mail in /var/spool/mail/fri
[fri@sdw7 ~]$
netstat -an |grep 10000
tcp6 0 0 :::10000 :::* LISTEN
tcp6 0 0 192.26.244.181:10000 192.26.244.161:34286 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.218:44038 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.147:51928 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.149:54344 ESTABLISHED
tcp6 0 0 192.26.244.181:41290 192.26.244.181:10000 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.164:61912 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.208:39974 ESTABLISHED
tcp6 0 0 192.26.244.181:41152 192.26.244.181:10000 ESTABLISHED
tcp6 0 0 192.26.244.181:10000 192.26.244.149:27986 ESTABLISHED其他端口是dolphinscheduler发起的，来连接hiveserver2的10000端口
5.找到端口被什么进程发出
[fri@sdw7 ~]$
sudo lsof -i:41276
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
java 14417 fri 2450u IPv6 618861604 0t0 TCP sdw7:ndmp->sdw7:41276 (ESTABLISHED)
java 18506 fri 568u IPv6 618845901 0t0 TCP sdw7:41276->sdw7:ndmp (ESTABLISHED)
[fri@sdw7 ~]$
netstat -nltp | grep 14417
(Not all processes could be identified, non-owned process info
will not be shown, you would have to be root to see it all.)
tcp6 0 0 :::10000 :::* LISTEN 14417/java
tcp6 0 0 :::10002 :::* LISTEN 14417/java
[fri@sdw7 ~]$
netstat -nltp | grep 18506
(Not all processes could be identified, non-owned process info
will not be shown, you would have to be root to see it all.)
tcp6 0 0 :::1234 :::* LISTEN 18506/java
[fri@sdw7 ~]$
ps -ef18506
PID TTY STAT TIME COMMAND
18506 ? Sl 6:31 /opt/module/jdk/bin/java -Dlogging.config=classpath:logback-worker.xml -Xms2g -Xmx2g -Xmn1g -server -XX:MetaspaceSize=128m -XX:MaxMetasp
[fri@sdw7 ~]$
jps
227507 LoggerServer
14417 RunJar
67920 Jps
163920 DataNode
211557 JournalNode
18506 WorkerServer
2138
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
