利用MySql数据库接受维基百科信息

最新推荐文章于 2022-07-24 17:07:50 发布

sharecord

最新推荐文章于 2022-07-24 17:07:50 发布

阅读量728

点赞数

分类专栏： python 文章标签： linux python

本文链接：https://blog.csdn.net/lzh_86/article/details/79547541

版权

python 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

from bs4 import  BeautifulSoup
from urllib  import  urlopen
from bs4 import  BeautifulSoup
import re
import datetime
import  random
import  pymysql

conn  = pymysql.connect(host='localhost',user='root',passwd='001',db='mysql',charset='utf8' )

cur = conn.cursor()
cur.execute('USE scraping')
random.seed(datetime.datetime.now())

def store(title,content):
	cur.execute("INSERT INTO pages (title,content) VALUES  (\"%s\",\"%s\")",(title,content))
	cur.connection.commit()
	
def getLinks(articleUrl):
	html=urlopen("http://en.wikipedia.org"+articleUrl)
	bsObj = BeautifulSoup(html)
	title = bsObj.find("h1").get_text()
	content = bsObj.find("div",{"id":"mw-content-text"}).find("p").get_text()
	store(title,content)
	return bsObj.find("div",{"id":"bodyContent"}).findAll("a",href = re.compile("^(/wiki/)((?!:).)*$"))
	
links = getLinks("/wiki/Kevin_Bacon") 
try:
	while len(links)>0:
		newArticle = links[random.randint(0,len(links)-1)].attrs["href"]
		print(newArticle)
		links = getLinks(newArticle)
finally:
	cur.close()
	conn.close()