python 模块使用

模块

Elasticsearch

from elasticsearch import Elasticsearch
from elasticsearch import helpers
es = Elasticsearch(["192.168.1.2:9200"])
def insertDate(word,num,my_index,my_doc,one_bulk):
    id_key_num = 0
    body_count = 0
    body = []
    pbar = tqdm(total=len(word))
    print("\n此次需要插入%d条数据"%len(word))
    for i in range(num):
        id = word[i][0]
        p_name = word[i][1]
        p_price = word[i][2]
        p_shop = word[i][3]
        data= {
        "id":id,
        "p_name":p_name,
        "p_price":p_price,
        "p_shop":p_shop
        }
        # helps.bulk 函数body
        every_body = \
            {
                "_index": my_index,
                "_type": my_doc,
                "_source": data
            }
        # 判断小于一次插入的最大数据  并给helps请求 添加body
        if body_count<one_bulk:
            body.append(every_body)
            body_count+=1
        else:
            # 插入数据****************************************************helps
            helpers.bulk(es, body) #还是要用bulk啊，不然太慢了
            # 更新进度条
            pbar.update(one_bulk)
            # 重新计数插入数据
            body_count = 0
            body = []
            body.append(every_body)
            body_count+=1

    if len(body) > 0:
        # 如果body里面还有，则再插入一次（最后非整块的）
        helpers.bulk(es, body)
        pbar.update(len(body))
        print('done2')
        id_key_num += 1
        #*********************************************************** 插入
        es.index(index=my_index,doc_type=my_doc,id=id_key_num,body=data)
    pbar.close()

tqdm–进度条

import time
from tqdm import tqdm
# word = [1,2,32,4,2,42,4,2,42,4,2,]
# pbar = tqdm(total =(len(word)) )
# print("共需要插入%d条..."%len(word))
# for i in range(len(word)):
#     time.sleep(1)
#     pbar.update(1)
#



with tqdm(total=100) as pbar:
    for i in range(10):
        pbar.update(10)

sqlite3 -SQLite

import sqlite3
cx = sqlite3.connect("jd.db")
c = cx.cursor()

def getData(num):
    sql = "select * from product limit"
    c.execute(sql+str(num))
    # 提取全部 fetchall 
    res = c.fetchall()