python读取csv大文件存入数据库_Python读取CSV并将值存入MySQL数据库

我认为这不是一个更好的选择“标记”CSV文件是为了保存文件，你是否存储了你处理的最后一行的编号。

因此，如果文件不存在（一个是存储最后处理的行的编号），则会处理整个CSV文件。如果此文件存在，则仅处理此行后的记录。

终极密码在工作系统：

#!/usr/bin/python

import csv

import MySQLdb

import os

mydb = MySQLdb.connect(host='localhost',

user='root',

passwd='*******',

db='kestrel_keep')

cursor = mydb.cursor()

csv_data = csv.reader(file('data_csv.log'))

start_row = 0

def getSize(fileobject):

fileobject.seek(0,2) # move the cursor to the end of the file

size = fileobject.tell()

return size

file = open('data_csv.log', 'rb')

curr_file_size = getSize(file)

# Get the last file Size

if os.path.exists("file_size"):

with open("file_size") as f:

saved_file_size = int(f.read())

# Get the last processed line

if os.path.exists("lastline"):

with open("lastline") as f:

start_row = int(f.read())

if curr_file_size < saved_file_size: start_row = 0

cur_row = 0

for row in csv_data:

if cur_row >= start_row:

cursor.execute('INSERT INTO `heating` VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ,%s)', row)

# Other processing if necessary

cur_row += 1

mydb.commit()

cursor.close()

# Store the last processed line

with open("lastline", 'w') as f:

start_line = f.write(str(cur_row + 1)) # you want to start at the **next** line

# next time

# Store Current File Size To Find File Flush

with open("file_size", 'w') as f:

start_line = f.write(str(curr_file_size))

# not necessary but good for debug

print (str(cur_row))

print "Done"

编辑：终极密码由ZeroG提供Submited现在工作在系统上！谢谢你也是太Xion345帮助