大数据处理过程中知识汇总

背景：接到临时需求，要处理3000w的数据，分为3000个txt文件，而每个文件1万域名。

问题：大量数据无法直接跑，太耗费时间，需首先清洗处理数据；

思路：筛选出有ip的网站进一步看剩下多少域名

1、服务器上32线程分别跑这3000个txt,生成3000个csv文件，里边保存四列数据

2、python pandas 进行数据分析

一、多线程编程知识

思路：

编程实现32线程跑2600个txt，先分list，每个list里边为32个str的数字，作为读取txt的路径以及生成的csv的文件名

#代码参考---多线程多个文件
import sys
sys.path.append(r'/usr/local/lib/python3.7/site-packages')
sys.path.append('/root/con_log')
from qqwry import QQwry
from IPy import IP
import socket
import threading
import csv
import time
from con_log import logger_rule
import eventlet
eventlet.monkey_patch()
q = QQwry()
q.load_file('qqwry.dat')
​
def batch_query_and_print(path):
    """
    参数为一个str的数字，表示文件名
    输出为同名csv文件
    :param batch_list:
    :return:
    """
    with open(path+".csv", "a") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["country", "serve", "url", "ip"])
        with open(path+".txt") as f:
            url_list = f.read().splitlines()
            for url in url_list:
                ip =''
                address =[]
                try:
                    with eventlet.Timeout(1, False):
                        ip = socket.gethostbyname(url)
                        address = list(q.lookup(ip))
                    address.append(url)
                    address.append(ip)
                    writer.writerow(address)
                    logger_rule.info('{}  ip success'.format(url))
                except Exception as e:
                    #logger_rule.error()
                    #print('{}  ip fail,{}'.format(url, e))
                    pass
​
                #logger_rule.info()
​
def batch_query(batch_list):
    """
    参数为一个list，32个线程
    输出为32线程的结果
    :param batch_list:
    :return:
    """
    threads = []
    for batch in batch_list:
        logger_rule.info('********************')
        logger_rule.info('{} pian '.format(batch))
        t = threading.Thread(target=batch_query_and_print, args=(batch,))
        threads.append(t)
    for t in threads:
        t.setDaemon(True)
        t.start()
    for t in threads:
        t.join()
​
def threads_test(thread_num):
    per_num = int(128/thread_num)
    for i in range(per_num):
        batch_list = []
        for j in range(thread_num):
            batch_list.append(str(32*i+(j+1)))
        a = time.time()