some tips

最新推荐文章于 2023-08-13 11:00:17 发布

bpmbp

最新推荐文章于 2023-08-13 11:00:17 发布

阅读量127

点赞数

文章标签：其他

本文链接：https://blog.csdn.net/enjoystu/article/details/120529807

版权

cd /tpdata/clientkey
source /tpdata/client/bigdata_env
kinit ods2sqoop -kt user.keytab
beeline --hiveconf hive.server2.logging.operation.level=NONE

spark-sql --master yarn-client --driver-memory 5G --executor-memory 15G --executor-cores 6 --num-executors 10 --queue sqoop

sudo su -
su - omm
. source_env

#hdfs文件查看数据量
hdfs dfs -cat /user/sqoop/tpprod/VSLOSE/part-m-00000 |wc -l
hdfs dfs -du /user/sqoop/tpprod |wc -l
hdfs dfs -du /user/sqoop/tpprod | cat > /tmp/linh/ttt.txt
hdfs dfs -du /user/sqoop/test |grep GSAPPLYINFOMAIN
chmod 777 ttt.txt

select oid from pg_class where relname='可能锁表了的表'
select pid from pg_locks where relation='上面查出的oid'
--如果查询到了结果，表示该表被锁则需要释放锁定
select pg_cancel_backend(上面查到的pid)

grant select on tpi_rpt.t_report_list_111053 to tpi_finebi;
--查看任务状态===============================================================
select * from pg_stat_activity where state='active' and query like '%111053%';
select * from pg_stat_activity where state='active' and query like '%100291%';
select * from pg_stat_activity where state='active' and query_id = '291889550882008369';
select * from pgxc_thread_wait_STATUS where wait_status !='wait cmd' and wait_status !='none' and node_name like 'cn%' and thread_name='PostgreSQL JDBC Driver';

--杀任务====================================================================
ALTER SYSTEM KILL SESSION '140136638949120, 0' IMMEDIATE ;

--数据库中查找表================================================================
SELECT table_name FROM information_schema.tables WHERE table_schema = 'tpi_rpt' and table_name like '%100300%';

--
select * from dlk_data_check

--表分析====================================================================
vacuum analyze T_100094_history;

--赋权给 tpi_rpt=============================================================
grant select on tpods.gppolicytransinfo to tpi_rpt;

--赋权给 tpi_finebi==============================================================
grant select on tpi_rpt.t_report_list_102042_view to tpi_finebi;

SELECT * FROM pgxc_get_table_skewness ORDER BY totalsize DESC;

--==========================================
select * from pgxc_stat_activity where state='active' and query like '%100094%' and usename='tpods';
select * from pgxc_stat_activity where state='active' and query like '%100398%';
select * from pgxc_stat_activity where state='active' and query_id = '149744687618141449';
ALTER SYSTEM KILL SESSION '140068790404864, 0' IMMEDIATE ;
SELECT a.count,b.node_name FROM (SELECT count(*) AS count,xc_node_id FROM tpi_rpt.T_103075_v503_tmp1 GROUP BY xc_node_id) a, pgxc_node b WHERE a.xc_node_id=b.node_id ORDER BY a.count desc;
SELECT a.count,b.node_name FROM (SELECT count(*) AS count,xc_node_id FROM tpods.v_605_tmp GROUP BY xc_node_id) a, pgxc_node b WHERE a.xc_node_id=b.node_id ORDER BY a.count desc;

grant select on tpods.T_100094_history to tpi_finebi;
grant select on tpods.gupolicykindlimit to tpi_rpt;

select * from pg_tables where tablename ='test20210318'
select * from test20210318_1 limit 10
select * from tpi_rpt.t_116030_feegroup2_test2 limit 10
select * from t_100094_history limit 10
SELECT table_name FROM information_schema.tables WHERE table_schema = 'tpods' and table_name like '%100094%';
SELECT table_name FROM information_schema.tables WHERE table_schema = 'tpi_rpt' and table_name like '%2021%';

select relname, pg_size_pretty(pg_relation_size(relid)) from pg_stat_user_tables where schemaname='tpods' order by pg_relation_size(relid) desc;
select pg_size_pretty(pg_relation_size('tpi_rpt.t_report_list_100051'));
select pg_size_pretty(pg_relation_size('tpods.t_100094_history_20210317'));

select * from gupolicyrisk limit 1

select query_start,pid, state_change, waiting, enqueue, state, a.query_id, substr(replace(query, chr(10), ' '), 0, 10),
node_name,thread_name,tid,lwtid,ptid,tlevel,smpid,wait_status,wait_event from pgxc_stat_activity a, pgxc_thread_wait_status b
where state = 'active' and a.query_id = b.query_id and a.query_id <> 0 ;
select * from pgxc_stat_activity where state='active' and query_id = '149744687618166031';

select oid from pg_class where relname='t_100094_history'
select pid from pg_locks where relation ='98758289'
select pg_cancel_backend('140048529880832')

spark-sql --driver-memory 5G --executor-memory 10G --executor-cores 4 --num-executors 8
spark-sql --name test --driver-memory 5G --executor-memory 10G --executor-cores 4 --total-executor-cores 40
SET spark.app.name=I17_SD_TREATY_RI;
set hive.execution.engine=spark;

/ETL/sqoop_tool/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop eval \
--connect jdbc:oracle:thin:@10.28.133.42:1521/TPISFXTDEV \
--username tpi_uat_query \
--password TPI#uatquery%2021 \
--query "select count(1) from dual"

38服务器对应的赋权操作
source /home/sqoop/hadoopclient/bigdata_env
kinit -kt /home/sqoop/dataplatform/user.keytab su_dataplatform

git更新
git pull origin master

git branch dev
git checkout dev
git pull origin 开发分支\(develop\)