python extract函数乱码_爬虫爬下来的怎么都是乱码...

’’’

import requests

import re

import os

from lxml.etree import HTML

def request_link(link):

"""链接请求"""

headers_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36

(KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'}

request = requests.get(link, headers=headers_dict, allow_redirects=True, verify=True) # 提交网址

if 200 <= request.status_code <= 206: # 判断响应状态码

htmlData = request.text.encode('utf-8').decode('ISO-8859-1')

return htmlData

else:

return None

def extract_fields(pattern, datas, flags, m=1):

"""提取字段"""

findField = re.search(pattern, datas, flags=flags)

if findField:

findField = findField.group(1)

if m:

if '

findField = re.sub('', '', findField, flags=re.S)

return findField.replace(' ', '')

else:

return findField

else:

return 'null'

def download(fileName, url, path):

"""下载附件"""

try:

print('下载中，请耐心等待...')

r = requests.get(url)

with open(path + fileName, "wb") as File:

File.write(r.content)

print(fileName, '下载Ok')

except:

print(fileName, '下载失败')

def main():

for i in range(98): # 循环迭代列表翻页

if not i: # 判断第一页

url = 'http://www.liyang.gov.cn/default.php?mod=article&fid=163250&s99679207_start=0'

else: # 其它列表页

url = 'http://www.liyang.gov.cn/default.php?mod=article&fid=163250&s99679207_start=+i'

print('\n正在提取列表第%s页item的详情链接...\n' % str(i + 1))

data = request_link(url) # 列表页面的链接请求

if data:

# print(len(data))

tree = HTML(data) # 列表页面

urlList = tree.xpath('//span[@class="link_14"]/a') # 匹配列表页面的item的详情链接

if urlList:

print('列表第%s页共计条目:' % str(i + 1), len(urlList))

v = 1

for url in urlList:

url_i = 'http://www.liyang.gov.cn/'+ url.get('href')

print(url_i)

# try:

print(v, '正在提取链接:', url_i, '的字段数据...')

datas = request_link(url_i) # 详情页面的链接请求

print(len(datas))

if datas:

trees = HTML(datas) # 详情页面

title = trees.xpath('//font[@class="link_19"]/text()') # 匹配标题

if title:

title = title[0]

# print(title)

date = trees.xpath('//font[@class="link_19"]/text()')[0].split()[0] # 发布日期

print(date)

path = './data/%s/' %title

try:

os.mkdir(path) # 创建储存数据和附件的目录

except:

pass

content = extract_fields(r'(.*?)', datas,re.S) # 公示文本纯内容

content = content.replace(' ', '').replace(' ', '').replace('“', '')

# print(content)

contents = extract_fields(r'

(.*?)', datas,re.S, 0) # 公示文本div

# print(contents)

if '

imgUrls = re.findall(r'src="(.*?)"', contents, 0)

for imgUrl in imgUrls:

imgUrl = 'http://www.liyang.gov.cn' + imgUrl

print(imgUrl)

download(imgUrl.split('/')[-1], imgUrl, path) # 下载图片

if '

fileUrls = re.findall(r'href="(.*?)"', contents, 0)

for fileUrl in fileUrls:

fileUrl = 'http://www.liyang.gov.cn' + fileUrl

print(fileUrl)

download(fileUrl.split('/')[-1], fileUrl, path) # 下载附件

with open(path + 'data.txt', 'w', encoding='utf-8') as f: # 储存页面源代码及数据内容

f.write(title + '\n\n' + date + ' ' + '溧阳环保' + '\n\n' + content)

# except:

# print('响应或解析异常！！')

# pass

v += 1

if name == 'main':

main()

’’’

这里是源码

这个是目标网站的详情页‘http://www.liyang.gov.cn/default.php?mod=article&do=detail&tid=146309700&sid=’

求助求助