最近组里某老师做研究需求一些站点的水文数据,刚好最近在学习python,了解到python中有很多方便的库用于网页抓取和网页分析,于是就着手写了一个。勉强可以用吧,哈哈。
实现上,简单用到了线程中队列的使用,线程池的使用,网页分析。主要参考了:
1.http://www.ibm.com/developerworks/aix/library/au-threadingpython/
2.http://www.pythoner.cn/home/blog/parallelism-in-one-line/
3.http://www.crummy.com/software/BeautifulSoup/bs4/doc/index.zh.html
上代码,随后补上简要说明
代码块
# -*- coding: utf-8 -*-
import urllib
import urllib2
from BeautifulSoup import BeautifulSoup
from datetime import *
from dateutil.relativedelta import *
import Queue
import threading
queue = Queue.Queue()
out_queue = Queue.Queue()
#日期迭代器,迭代从start到end中间的每一天
def loopDay(start, end):
while(start < end):
yield start
start = start + relativedelta(days = 1)
#向指定网站(湖南省水文查询系统)请求所需的页面
def post(url, data):
req = urllib2.Request(url)
data = urllib.urlencode(data)
#enable cookie
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
response = opener.open(req, data)
return response.read()
def getHtml(time):
posturl = 'http://61.187.56.156/wap/hnsq_BB2.asp'
data = {}
data['nian'] = str(time.year)
data['yue'] = str(time.month)
data['ri'] = str(time.day)
data['shi'] = '08:00'
html = post(posturl, data)
return html
#查找我们需要的关键字在Tag列表中的位置
def findKeyword(keyword, tagList):
location = -1
for tag in tagList:
location += 1
if (keyword.encode('GBK') in unicode(tag.string).encode("windows-1252")):
break
return location
#用于抓取某一天的对应的页面
class ThreadUrl(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue, out_queue):
threading.Thread.__init__(self)
self.queue = queue
self.out_queue = out_queue
def run(self):
while True:
oneDay = self.queue.get()
print oneDay
#grabs webpage
webpage = getHtml(oneDay)
#place page into out queue
self.out_queue.put((oneDay, webpage))
#抓取获取到的指定日期对应的网页中我们需要的数据
class DatamineThread(threading.Thread):
"""Threaded Url Parsing"""
def __init__(self, out_queue):
threading.Thread.__init__(self)
self.out_queue = out_queue
def run(self):
while True:
(oneDay, page) = self.out_queue.get()
#parse the page
soup = BeautifulSoup(page, fromEncoding = "gb2312")
tagList = soup.findAll('td')
locations = []
for keyword in keywords:
locations.append(findKeyword(keyword, tagList))
for i in range(5):
fileName = keywords[i] + ".txt"
f = file(fileName, "a+")
waterLevel = unicode(tagList[locations[i] + 2].string).strip().encode("windows-1252")
waterStorage = unicode(tagList[locations[i] + 5].string).strip().encode("windows-1252")
f.write(str(oneDay) +"\t" + waterLevel + "\t" + waterStorage + "\n")
f.close()
if __name__ == '__main__':
keywords = [u"寸滩", u"万县", u"巫山", u"清溪场", u"忠县", u"武隆"]
startDay = date(2006, 1, 1)
endDay = date(2007, 1, 1)
days = loopDay(startDay, endDay)
t = ThreadUrl(queue, out_queue)
t.setDaemon(True)
t.start()
for oneDay in days:
queue.put(oneDay)
dt = DatamineThread(out_queue)
dt.setDaemon(True)
dt.start()
queue.join()
out_queue.join()
注册好久了,第一次动手写。网速不好,后面再改吧。