# _*_ conding: utf-8 _*_
import urllib.request
import re
import pandas as pd
class itcast_teacher:
'''
传智播客老师信息爬取类
'''
def load_page(self):
url = "http://www.itcast.cn/channel/teacher.shtml#aandroid"
user_agent = "Mozilla"
headers = {"User-Agent": user_agent}
request = urllib.request.Request(url, headers=headers)
html = urllib.request.urlopen(request).read()
new_html = html.decode('utf-8')
# pattern = re.compile(r'<div.*?class="li_txt".*?>(.*?)</div>', re.S)
pattern = re.compile(r'<div.*?class="li_txt".*?>.*?<h3>(.*?)</h3>.*?<h4>(.*?)</h4>.*?<p>(.*?)</p>.*?</div>', re.S)
item_list = pattern.findall(new_html)
'''
pattern2 = re.compile(r'<h3>(.*?)</h3>', re.S)
item_list2 = pattern2.findall(str(item_list1))
pattern3 = re.compile(r'<h4>(.*?)</h4>', re.S)
item_list3 = pattern3.findall(str(item_list1))
pattern = re.compile(r'<p>(.*?)</p>', re.S)
item_list = pattern.findall(str(item_list1))
ah = pd.DataFrame(item_list2)
ah.columns = ['作者']
ad = pd.DataFrame(item_list3)
ad.columns = ["职务"]
ak = pd.DataFrame(item_list)
ak.columns = ["个人信息"]
z = ah.join(ad)
k = z.join(ak)
'''
return item_list
def do_one_page(self, item_list):
for item in item_list:
'''
item = item.replace("<h3>", "").replace("</h3>", "").replace("<h4>", "").replace("</h4>", "").\
replace("<p>", "").replace("</p>", "")
print(item.strip())
'''
self.write_to_file(item)
def write_to_file(self, txt):
file = open("F:/anaconda/work/itcast_teacher2.txt", "a")
file.write(txt + '\n')
file.close()
def do_work(self):
a = self.load_page()
self.do_one_page(a)
if __name__ == "__main__":
'''
main主函数
'''
myspider = itcast_teacher()
myspider.do_work()