大数据常用命令

hadoop常用命令

##查看集群状态
hdfs dfsadmin -report
 
##查看目录下的文件
hadoop dfs  -ls  /user/hive/
 
## hdfs删除快照 
hdfs dfs -deleteSnapshot /user/hive/hive-snapshot-20210509
 
下载数据块
hadoop fs -get /user/hive/warehouse/ods.db/table_name/part-00021-d20b1d30-dce1-4a38-a0c1-2b4ef68b7da8-c000.snappy.orc  ./
 
#下载Yarn日志
yarn logs -applicationId application_1606201254014_1003     > application_1606201254014_1003.txt
 
#查看正在写入文件
hdfs fsck /user/events -openforwrite
 
#恢复文件租约
hdfs debug recoverLease -path /user/hive/warehouse/app.db/test_order_sample1/order_sample -retries 3
 
## 将edtis目录下二进制文件改为xml,可以伪装相关命令处理
hdfs oev -i edits_0000000001177152591-0000000001177162768 -o edits_0000000001177152591-0000000001177162768.xml
 
## 将xml改为 二进制文件,可以伪装相关命令处理
hdfs oev -i edits_0000000001177152591-0000000001177162768.xml -o edits_0000000001177152591-0000000001177162768 -p binary
 
##检查hdfs坏的数据块
hadoop fsck /user -files -blocks -locations
 
##删除坏的block
hadoop fsck -rm /user/spark/sparkstreamingcheckpoint/checkpoint-1517487380000
 
 
##统计文件下数据块
hdfs dfs -count /user/hive/warehouse/dws.db/*/

kafka常用命令

##518400000 为毫秒,转化一下即为天数
kafka-configs --zookeeper zkhost:2181 --alter --entity-name topic_name --entity-type topics --add-config retention.ms=518400000
 
##设置某一个kafka 对应的topic 永久存储
kafka-configs --zookeeper zkhost:2181 --alter --entity-name topic_name --entity-t

线上问题常用命令

jsatck 处理线上问题
 
##查找进程里面cpu占用最高的线程
top -Hp  pid
##top 线程
printf "%x\n" 21742
## 建议使用root权限                          
jstack  pid | grep 54ee

##查找大于800m的文件
find . -type f -size +800M
 
##查找大文件夹
du -hm --max-depth=2 | sort -nr | head -12
 
##查看nginx 请求最多ip
cat /***/nginx/logs/access.log|awk '{print $1,$2}'sort|uniq -c|sort -rn -k1|head -3
 
## nohup 启动线上任务
nohup java -jar publish_online.jar  --server.port=8106 &>/opt/data/xxx_8106.log &
 
## 查找/opt/shell目录下以及子目录包含redis关键词
grep  -ril "redis" /opt/shell/*

原文链接https://blog.csdn.net/tangpengtao/article/details/118752347?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522170131194816800180625785%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fall.%2522%257D&request_id=170131194816800180625785&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2_allfirst_rank_ecpm_v1~rank_v31_ecpm-4-118752347-null-null.142^v96pc_search_result_base3&utm_term=Giving%20up%20on%20recoverLease%20for%20%2Fwarehouse%2Fgame%2Fods%2Fods_betRecord%2Fdt%3D2023-11-26%2Fevents-.1700966897623.gz%20after%201%20try.&spm=1018.2226.3001.4187