#-*- conding:utf-8 -*-
import requests,re,pymysql
from pyquery import PyQuery as pq
url_index = ['http://finance.eastmoney.com/news/cgnjj_{}.html'.format(pages) for pages in range(1,2)]
def index_content(url_index,title=0,time=0):
#print('起始页:' + url_index)
index_req = requests.get(url_index,timeout = 10)
index_req.encoding = 'utf-8'
index_content = index_req.text
content_url = re.findall('
title = re.search('
.*?(.*?)',index_content,re.S).group(1)
time = re.search('
(.*?)
',index_content,re.S).group(1).replace("年", "-").replace("月", "-").replace("日", "")#time = time_content
#re.sub(".", "", title)
#print('标题:' + title)
#print('时间:' + time)
return content_url
def content(content_url):
content_req = requests.get(content_url,timeout = 10)
content_req.encoding = 'utf-8'
content = content_req.text
title = re.search('
(.*?)
',content,re.S).group(1)time = re.search('
sources = re.search('
content_content = re.search('(.*?)',content,re.S).group(1)
pic = re.findall('''.*?src="https://z1.dfcfw.com(.*?)"''',content_content,re.S)#.sub('.*?same.*?','',content_content)
content_body = {'title:':title,
'time:':time,
'source:':sources,
'content:':content_content,
'pic:':pic
}
# print('网址:' + content_url)
# print ('标题:' + title)
# print ('时间:' + time)
# print ('来源:' + source)
# print ('内容:' + content_content )
# print (pic)
print(type(title))
return content_body
def save_sql(content_body):
try:
db = pymysql.connect("localhost","root","root","test" )
cursor = db.cursor()
#sql = """INSERT INTO test(title,source_time, source, content_body, pageurl,pic)VALUES (title,time,source,content_content,content_url,pic)"""
# 执行sql语句
sql = """INSERT INTO test(title,source_time, sources, content_body, pageurl,pic)VALUES (%s,%s,%s,%s,%s,%s)"""
# 执行 sql 语句
cursor.execute(sql,(title,time,sources,content_content,content_url,pic))
#cursor.execute(sql)
# 提交到数据库执行
db.commit()
except:
# 如果发生错误则回滚
db.rollback()
# 关闭数据库连接
db.close()
def save_csv(con):
with open('C:\\Users\\Administrator\\Desktop\\finance.csv','wb+') as f: #wb为以二进制方式打开
con = str.encode('utf-8')
f.write(con)
f.close()
for url_list in url_index:
content_url = index_content(url_list,title=0,time=0)
for con_url in content_url:
cons = content(con_url)
print(cons)
save_sql(cons)
# for con in cons:
# #print(con)
# save_csv(con)
#save_sql(con)
运行没报错,也能正常打印输出,但是就是保存不到数据库...而且保存到文件的时候,老是提示类型不对,字典、元组、字符串都不行,帮忙看下..