python爬取网页正文_深度剖析使用python抓取网页正文的源码

#coding:utf-8

import re

def remove_js_css (content):

""" remove the the javascript and the stylesheet and the comment content ( and ) """

r = re.compile(r'''''',re.I|re.M|re.S)

s = r.sub ('',content)

r = re.compile(r'''''',re.I|re.M|re.S)

s = r.sub ('', s)

r = re.compile(r'''''', re.I|re.M|re.S)

s = r.sub('',s)

r = re.compile(r'''''', re.I|re.M|re.S)

s = r.sub('',s)

r = re.compile(r'''''', re.I|re.M|re.S)

s = r.sub('',s)

return s

def remove_empty_line (content):

"""remove multi space """

r = re.compile(r'''^\s+$''', re.M|re.S)

s = r.sub ('', content)

r = re.compile(r'''\n+''',re.M|re.S)

s = r.sub('\n',s)

return s

def remove_any_tag (s):

s = re.sub(r'''<[^>]+>''','',s)

return s.strip()

def remove_any_tag_but_a (s):

text = re.findall (r''']*>(.*?)''',s,re.I|re.S|re.S)

text_b = remove_any_tag (s)

return len(''.join(text)),len(text_b)

def remove_image (s,n=50):

image = 'a' * n

r = re.compile (r'''''',re.I|re.M|re.S)

s = r.sub(image,s)

return s

def remove_video (s,n=1000):

video = 'a' * n

r = re.compile (r'''''',re.I|re.M|re.S)

s = r.sub(video,s)

return s

def sum_max (values):

cur_max = values[0]

glo_max = -999999

left,right = 0,0

for index,value in enumerate (values):

cur_max += value

if (cur_max > glo_max) :

glo_max = cur_max

right = index

elif (cur_max < 0):

cur_max = 0

for i in range(right, -1, -1):

glo_max -= values[i]

if abs(glo_max < 0.00001):

left = i

break

return left,right+1

def method_1 (content, k=1):

if not content:

return None,None,None,None

tmp = content.split('\n')

group_value = []

for i in range(0,len(tmp),k):

group = '\n'.join(tmp[i:i+k])

group = remove_image (group)

group = remove_video (group)

text_a,text_b= remove_any_tag_but_a (group)

temp = (text_b - text_a) - 8

group_value.append (temp)

left,right = sum_max (group_value)

return left,right, len('\n'.join(tmp[:left])), len ('\n'.join(tmp[:right]))

def extract (content):

content = remove_empty_line(remove_js_css(content))

left,right,x,y = method_1 (content)

return '\n'.join(content.split('\n')[left:right])