# encoding:utf-8
__author__ = 'Sun'
import re
import urllib.request
import urllib
import queue
import threading
import os
queue = queue.Queue()
visited = set()
cnt = 0
class CsdnBlogSpider(threading.Thread):
def __init__(self, queue, opener, blog_name):
threading.Thread.__init__(self)
self.queue = queue
self.opener = opener
self.blog_name = blog_name
self.lock = threading.Lock()
def htmltotxt(self, data, fout):
html = data.read().decode("utf-8")
html.encode("utf-8")
content=re.findall('<div style=".*?">(.*?)</div>',html,re.S)
reg=re.compile('<[^>]*>')
fout.write(fout)
def save_data(self, data, filename):
if not os.path.exists('blog'):
blog_path = os.path.join(os.path.abspath('.'),'blog')
os.mkdir(blog_path)
try:
fout = open('./blog/' + 'lalal' + '.txt', 'wb')
htmltotxt(data,fout)
#fout.write(data);
except IOError as e:
print(e)
# finally:
# fout.close()
def find_title(self,data):
data = data.decode('utf-8')
begin = data.find(r'<title') + 7
end = data.find('\r\n',begin)
title = data[begin:end]
return title
def run(self):
global cnt
global visited
while True:
url = self.queue.get()
self.lock.acquire()
cnt += 1
print('已经抓取:' + str(cnt-1) + '正在抓取---->' + url)
self.lock.release()
try:
res = self.opener.open(url, timeout=1000)
except Exception as e:
if hasattr(e, 'reason'):
print('reason:', e.reason)
elif hasattr(e, 'code'):
print('error code:', e.code)
cnt -= 1
self.queue.task_done()
continue
else:
data = res.read()
title = self.find_title(data)
self.save_data(data,title)
data = data.decode('utf-8')
blog_urls = re.compile('/' + self.blog_name + '/article/details/' + '\d*')
for url in blog_urls.findall(data):
url = 'http://blog.csdn.net' + url
if url not in visited:
self.queue.put(url)
visited |= {url}
# print('加入队列---》' + url)
#当调用一次get函数之后就调用此函数,让线程阻塞或退出
self.queue.task_done()
def init(name, number=10):
#抓取的url个数
global cnt
global visited
# blog_name = input('输入博客名称:')
# thread_num = input('输入启动线程数:')
blog_name = name.lower()
th_num = int(number)
url = 'http://blog.csdn.net/' + blog_name + '/'
#模仿浏览器访问CSDN
opener = urllib.request.build_opener(urllib.request.HTTPHandler)
headers = [
('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko')
]
urllib.request.install_opener(opener)
opener.addheaders = headers
queue.put(url)
visited |= {url}
cnt = 0
for i in range(th_num):
t = CsdnBlogSpider(queue,opener,blog_name)
t.setDaemon(True)
t.start()
queue.join()
print('--------end!!!-----')
print('共抓取:' + str(cnt))
if __name__ == '__main__':
init("xiaoxiaoCYG")