Python抓取网页信息并存储到excel中

目的：抓取网页上的人物和相关简介信息
代码分享：
import urllib.request
import ssl
ssl._create_default_https_context=ssl._create_unverified_context
def getHtml(url):
page = urllib.request.urlopen(url)
html = page.read()
html = html.decode(‘utf-8’)
return html

if name == “main”:
# from requests_html import HTMLSession
# import requests
import pandas as pd
import bs4
import re
# testurl1 = “http://dtcc.it168.com/2010/”
url_typ1_list = [“http://dtcc.it168.com/2010/",“http://dtcc.it168.com/2011/”,“http://dtcc.it168.com/2012/”,“http://dtcc.it168.com/2013/”,“http://dtcc.it168.com/2014/”,“http://dtcc.it168.com/2015/”,“http://dtcc.it168.com/2016/”,"http://dtcc.it168.com/2017/”]
result_list = []
for url in url_typ1_list:
try:
html_info = getHtml(url)
soup = bs4.BeautifulSoup(html_info, ‘html.parser’)
info_list = soup.select(".yjjb")
for info in info_list:
tmp = info.contents[-1]
if len(tmp) < 2:
continue
# try:
# tmp = str(tmp)
# pat = re.compile(’>(.*?)<’)
# tmp = ‘’.join(pat.findall(tmp))
# except:
# tmp = tmp
try:
info = tmp.strip()
except:
info = tmp
try:
info = info.split("：")
if len(info) == 2:
result_list.append(info)
except:
continue
except Exception as e:
print("grab web info Error, url=%s, Exception=%s. " % (url, e))
print(result_list)
print(len(result_list))
df = pd.DataFrame(result_list)
df.columns = [‘名字’,‘简介’]
df.to_csv(‘网页人物信息抓取类型1.csv’, encoding=‘gbk’, index=False)