Python大文件CSV导入数据库

最新推荐文章于 2024-05-02 15:04:24 发布

繁梦溪

最新推荐文章于 2024-05-02 15:04:24 发布

阅读量277

点赞数

文章标签： python 数据库开发语言

本文链接：https://blog.csdn.net/fg24151110876/article/details/132898580

版权

import pandas as pd
from sqlalchemy import create_engine


postgres_engine = create_engine('postgresql+psycopg2://postgres:postgres@127.0.0.1/SOURCEDATA')
mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format('root','admin','127.0.0.1','3306','sourcedata'))


file="D:\SubSystem\Downloads\pp-complete.csv"


# 测试
df=pd.read_csv(file, header=None,nrows=2)
headers = ['UUID_STRING','PRICE_STRING','TIME','POSTCODE','A','B','C','ADDR1','ADDR2','STREET','LOCALITY','TOWN','DISTRICT','COUNTY','D','E']
df.columns = headers
df

df.to_sql('pp-complete',postgres_engine,if_exists ='append',index = False)

df.to_sql('pp-complete',mysql_engine,if_exists ='append',index = False)



# 执行
from tqdm import tqdm # 进度条
import warnings # 忽略警告信息
warnings.filterwarnings('ignore')
for chunk in tqdm(pd.read_csv(file, chunksize=10000,header=None)):
    chunk.columns = headers
    chunk.to_sql('UK_PRICE_PAID',postgres_engine,if_exists ='append',index = False)
    chunk.to_sql('UK_PRICE_PAID',mysql_engine,if_exists ='append',index = False)