全域数据处理

最新推荐文章于 2024-09-09 00:00:00 发布

郭新宇

最新推荐文章于 2024-09-09 00:00:00 发布

阅读量124

点赞数 3

文章标签： python

本文链接：https://blog.csdn.net/u010924845/article/details/140296585

版权

#!/usr/bin/env python
# coding: utf-8

# In[6]:


# encoding=utf-8
# encoding=utf-8
import pandas as pd
import numpy as np
import re
import time
import subprocess,sys
#import logging
from operator import itemgetter
import os
import logging
'''
定义一个类，用于下文进行加载，从而多线程进行数据连接hive
'''
class Get_data:      
    def run_cmd(self,cmdstr,encoding = 'utf-8'):
        try:
            res = subprocess.Popen(cmdstr,shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        except Exception as e:
            raise e
        print('===========res============%s'%res)
        results = []
        while True:
            line = res.stdout.readline().decode(encoding)
            if  line == '' and res.poll() is not None:
                break
            else:
                results.append(line.strip())
                if line.strip() != "":
                    print(line)
        if res.returncode != 0:
            print("执行cmd失败,cmd语句为：%s" % cmdstr)
            sys.exit(res.returncode)
        return [res.returncode, '\n'.join(results)]
'''
第二步骤：获取数据
'''
def load_data(yesterday):
    executor=Get_data()
    sql_jd=f"select user_log_account,union_id,master_time,master_source_type,master_account_type,slave_time,slave_source_type,slave_account_type from app.app_user_source_jdwx_join "
    cmdstr_jd=f'hive -e " set hive.cli.print.header=true;{sql_jd}" > ./data/data.csv'
    executor.run_cmd(cmdstr_jd)
def get_data(filename):
    df_data = pd.read_csv(filename,usecols=['user_log_account','union_id','master_time','master_source_type','master_account_type','slave_time','slave_source_type','slave_account_type'], sep='\t',encoding="utf-8",low_memory=False)
    data_list = df_data.values.tolist()
    return data_list
'''
第三步：处理数据
'''
def process_data(data_list):
    l=[]
    for i in range(len(data_list)):
         l.append(data_list[i][0:2])
    return l  
def process_data_nw(data_list):
    import networkx as nx
    l = process_data(data_list)
    G = nx.Graph()
    #将节点添加到Graph    
    G.add_nodes_from(sum(l, []))
    #从节点列表创建边
    q = [[(s[i],s[i+1]) for i in range(len(s)-1)] for s in l]
    #print(q)
    for i in q:
        #向Graph添加边
        G.add_edges_from(i)
    #查找每个组件的图形和列表节点中的所有连接组件
    result=[list(i) for i in nx.connected_components(G)]
    return result
def p_data(data_list,result):
    r={}
    for i in range(len(data_list)):
        for j in range(len(result)):
            if data_list[i][0] in result[j] and data_list[i][1] in result[j]:
                r.setdefault(j,[]).append(data_list[i])
                break
    return r
def write_csv(result_csv,r,yesterday):
    h=[]
    for k in r:
        for i in range(len(r[k])):
            a=r[k][i]
            a.append(k)
            a.append(yesterday)
            h.append(a)
    h = np.array(h)
    h = pd.DataFrame(h)
    pd.DataFrame(h).to_csv(result_csv,index=False ,header=None,sep='\t')
    return result_csv
'''
第四步：数据加载回大数据平台
'''
def Upload_Data(yesterday):
    executor=Get_data()
    '''
    推送到大数据平台
    '''
    sql2=f"load data local inpath './data/result.csv' overwrite into table app.xcx_app_land_data_result partition (dt='{yesterday}')"
    cmdstr2=f'hive -e "{sql2}" '
    executor.run_cmd(cmdstr2)
def main(yesterday):
    filename='./data/data.csv'
    result_csv='./data/result.csv'
    load_data(yesterday)
    data_list=get_data(filename)
    result=process_data_nw(data_list)
    r=p_data(data_list,result)
    write_csv(result_csv,r,yesterday)
    Upload_Data(yesterday)
    
if __name__ == '__main__':
    yesterday=str(sys.argv[1])
    main(yesterday)



jd_45a74c8a80fae oCwKwuA–nUcBf-AjibsvsIDqeFk
jd_TyTPaUnfTyuy oCwKwuH_fqvHNPESGLUzWUkchluk
rib6lg oCwKwuHk1udYCo82sUtEmxSVX6_0
jd_UKsFoffVBvtB oCwKwuHk1udYCo82sUtEmxSVX6_0
rib6lg oCwKwuMEkxKZIxFpAXZwei-2axMY
jd_RlVQwU8z2b6Lqbz oCwKwuMEkxKZIxFpAXZwei-2axMY
jd_TyTPaUnfTyuy oCwKwuMEkxKZIxFpAXZwei-2axMY
cm870321 oCwKwuMEkxKZIxFpAXZwei-2axMY
jd_UKsFoffVBvtB oCwKwuMEkxKZIxFpAXZwei-2axMY
rib6lg oCwKwuOPk1ldt-yFjqlI7qpziyGw
wyl166246 oCwKwuA–yFZ6imDGIBuNx0iU6ZU
jd_aLDqFGhChTWE oCwKwuA–yFZ6imDGIBuNx0iU6ZU
jd_UKsFoffVBvtB oCwKwuOPk1ldt-yFjqlI7qpziyGw
jd_ukDdpAAgqHcs oCwKwuA–kXG85Khc5cJQv4j0f2o