传智博客老师信息爬虫

# _*_ conding: utf-8 _*_
import urllib.request
import re
import pandas as pd

class itcast_teacher:
    '''
        传智播客老师信息爬取类
    '''
    def load_page(self):
        url = "http://www.itcast.cn/channel/teacher.shtml#aandroid"
        user_agent = "Mozilla"
        headers = {"User-Agent": user_agent}
        request = urllib.request.Request(url, headers=headers)
        html = urllib.request.urlopen(request).read()
        new_html = html.decode('utf-8')

        # pattern = re.compile(r'<div.*?class="li_txt".*?>(.*?)</div>', re.S)

        pattern = re.compile(r'<div.*?class="li_txt".*?>.*?<h3>(.*?)</h3>.*?<h4>(.*?)</h4>.*?<p>(.*?)</p>.*?</div>', re.S)

        item_list = pattern.findall(new_html)

        '''
        pattern2 = re.compile(r'<h3>(.*?)</h3>', re.S)
        item_list2 = pattern2.findall(str(item_list1))

        pattern3 = re.compile(r'<h4>(.*?)</h4>', re.S)
        item_list3 = pattern3.findall(str(item_list1))

        pattern = re.compile(r'<p>(.*?)</p>', re.S)
        item_list = pattern.findall(str(item_list1))

        ah = pd.DataFrame(item_list2)
        ah.columns = ['作者']

        ad = pd.DataFrame(item_list3)
        ad.columns = ["职务"]

        ak = pd.DataFrame(item_list)
        ak.columns = ["个人信息"]

        z = ah.join(ad)
        k = z.join(ak)
        '''
        return item_list

    def do_one_page(self, item_list):
        for item in item_list:
            '''
            item = item.replace("<h3>", "").replace("</h3>", "").replace("<h4>", "").replace("</h4>", "").\
                replace("<p>", "").replace("</p>", "")


            print(item.strip())
            '''
            self.write_to_file(item)

    def write_to_file(self, txt):
        file = open("F:/anaconda/work/itcast_teacher2.txt", "a")
        file.write(txt + '\n')
        file.close()



    def do_work(self):
        a = self.load_page()
        self.do_one_page(a)

if __name__ == "__main__":
    '''
        main主函数
    '''
    myspider = itcast_teacher()
    myspider.do_work()