编写了一个python爬虫,可以抓取目标网页,并进行简单的删选。如下
# -*- coding: UTF-8 -*-
import urllib
import re
#定义获取目标网页函数
def getHtml(url):
page = urllib.urlopen(url)
html = page.read()
return html
#定义获取url的函数
def getUrl(html):
reg = r'href="(.+?)"'
urlre = re.compile(reg)
urllist = re.findall(urlre,html)
return urllist
if __name__=='__main__':
s = []
html = getHtml("http://www.scyongdong.com")
gurl = getUrl(html)
#gurl得到的是list
s = gurl
i = 0
fw = ''
for i in range(0,len(s)) :
fp = open('test.txt','w')
#判断url是否含有http....头部,没有的加上
if "http" in s[i]:
fw += (s[i]+'\n')
fp.write(fw)
else:
fw += ("http://www.scyongdong.com" +s[i] + '\n' )