爬虫——海大起点论坛房屋租赁信息

爬虫——海大起点论坛房屋租赁信息

多协程
CSV存储

引用包

import gevent
from gevent import monkey
monkey.patch_all(select=False)
from gevent.queue import Queue
import csv
import re
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

爬取流程

代码

class CZW_Spider(object):

    def __init__(self):
        # 初始化数据
        try:
            self.fake_user = UserAgent()
        except Exception as ex:
            print(ex)
            print('>>> 正在修复异常功能模块')
            get_funcModule('fake_useragent', Update=True)
        self.user = user
        self.password = password
        self.out_path = out_fp

    def layer_01(self):
        """
        DEMO 采集层， 完成目标网站的试探性采集
        :return:
        """

        def get_user_id(trace=None, user_name=None):
            """
            依据 用户昵称及页面元素确定用户id，当用户匿名时id=0
            :param trace: 暂时弃用，正则语法
            :param user_name: 用户名
            :return: host_id
            """

            if user_name == '匿名':
                user_id = '0'
                return user_id
            else:
                try:
                    user_id = tag.find('cite').a['href']
                    user_id = user_id.split('=')[-1]
                    return user_id
                except TypeError:
                    print(user_name, '该用户ID获取进程发生未知错误')

        # HTML下载器,默认页面为房屋租赁首页(page=1)
        def Handle(target=None):
            """
            数据请求句柄
            :return: Response 对象
            """

            # 当任务队列为空时，默认访问qdRent
            if target is None:
                target = qdRent
            # 生成随机请求头
            try:
                headers = {'user-agent': self.fake_user.random}
            except AttributeError:
                headers = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/' \
                          '537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'

            try:
                Response = requests.get(target, headers=headers)
                Response.raise_for_status()
                # Response.encoding = 'utf-8'
                return Response.text
            except Exception as e:
                raise e

        res = Handle()
        soup = BeautifulSoup(res, 'html.parser')
        # 定位表单
        tags = soup.find('div', id='threadlist').find_all('tbody')
        Out_flow = []
        for tag in tags[2:]:
            # 获取项目名
            item = tag.find(attrs={'class': 's xst'})
            # 获取作者名
            host_name = tag.find(attrs={'class': 'by'}).text.strip().split('\n')[0]
            # 获取作者ID
            host_id = get_user_id(user_name=host_name)
            # 获取项目链接
            href = qdHome + tag.find_all('a')[-1]['href']

            Out_flow.append([item.text.strip(), host_id, host_name, href])
            # print(href)
            # print('id:{}'.format(host_id))
            # print('作者:{}\n'.format(host_name))

        return Out_flow

    def layer_02(self):
        # 所有房屋租赁列表URL采集

        def QUEUE(url_list):
            work = Queue()
            for url in url_list:
                work.put_nowait(url)
            return work

        def get_HTML(target=None):
            """
            数据请求句柄
            :return: Response 对象
            """
            # 当任务队列为空时，默认访问qdRent
            if target is None:
                target = qdRent
            # 生成随机请求头
            try:
                headers = {'User-Agent': self.fake_user.random}
            except AttributeError:
                headers = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' \
                                  'Chrome/84.0.4147.105 Safari/537.36 '}

            try:
                Response = requests.get(target, headers=headers)
                Response.raise_for_status()
                # Response.encoding = 'utf-8'
                return Response.text
            except Exception as e:
                raise e

        def crwaler__01():

            def Parser__01(response):
                bs = BeautifulSoup(response, 'html.parser')
                tags = bs.find_all('tbody', id=re.compile('^normalthread_[0-9]{6}'))
                URL_lists = []
                for tag in tags:
                    URL_Program = qdHome + tag.find('a')['href']
                    URL_lists.append(URL_Program)
                return URL_lists


            while not work_0.empty():
                url = work_0.get_nowait()
                html = get_HTML(url)
                URL_LIST = Parser__01(html)
                out_flow_01.append(URL_LIST)

        work_0 = Queue()
        qdurl = 'http://www.ihain.cn/forum.php?mod=forumdisplay&fid=175&typeid=' \
                '24&typeid=24&filter=typeid&page={page1}'
        out_flow_01 = []
        for x in range(1, 200):
            url = qdurl.format(page1=x)
            work_0.put_nowait(url)
        self.Multi_program(crwaler__01)
        work = Queue()
        for list in out_flow_01:
            for url in list:
                work.put_nowait(url)

        def crwaler_02():

            def get_url(tag):
                href = tag.find('td', class_='plc ptm pbn vwthd').find('span', class_='xg1').a['href'].strip()
                url = qdHome + href
                return url

            def get_item(tag):
                # 项目名
                try:
                    item = tag.find('h1', class_='ts').find('span', id='thread_subject').text
                    print(item)
                    return item
                except AttributeError as e:
                    print('项目名获取进程错误')

            def get_date(tag):
                element = tag.find('div', class_='pti').find('div', class_='authi').find('em')
                if element.find('span') is None:
                    date = element.text.replace('发表于 ', '')
                    print(date)
                    return date
                else:
                    date = element.find('span')['title']
                    print(date)
                    return date

            def get_user_name_id(tag, href):
                element_1 = tag.find('td', class_='pls', rowspan='2').find('div', class_='pi')
                try:
                    element_2 = element_1.find('div')
                except TypeError:
                    print(href + '网页在ID处出现意外元素')
                if element_2 is None:
                    host_name = element_1.text
                    print('匿名')
                    host_id = None
                else:
                    host_name = element_2.text.replace('\n', '')
                    id = element_2.a['href']
                    host_id = id.split('=')[-1]
                return host_name, host_id

            def get_item_intro(tag, href):
                element = tag.find('div', class_='pcb').td
                if element.find('div', align='left') is not None:
                    try:
                        # 不存在图片
                        intro = element.find('div', align='left').text.replace(' ', '')
                        intro = intro.replace('\n', '')
                    except TypeError:
                        print(href + '网页出现意外元素')
                else:
                    intro = tag.text.strip()
                return intro

            def get_weixin(text):
                text.replace(' ', '')
                weixin = re.search('(微信[:： ]?|weixin[:： ]?|VX[:： ]?)?([a-zA-Z0-9\-\_]{6,16})', text, re.I)
                if weixin is not None:
                    weixin = weixin.group(2)
                    return weixin
                else:
                    return None

            def get_QQ(text):
                text.replace(' ', '')
                qq = re.search('(QQ[:： ]?)([0-9]{6,10})', text, re.I)
                if qq is not None:
                    qq = qq.group(2)
                    return qq
                else:
                    return None

            def get_cellphone(text):
                text.replace(' ', '')
                phone = re.search('(电话[:： ]?|联系.*[:： ]?|手机.[:： ]?|cellphone[:： ]?|mobilephone[:： ]?)?(1[0-9]{10})',
                                  text, re.I)
                if phone is not None:
                    phone = phone.group(2)
                    return phone
                else:
                    return None

            def get_item_attribute(title):
                if re.search('女.*', title) is not None:
                    attribute_1 = '仅限女生居住'
                    if re.search('研究生.*', title) is not None:
                        attribute = attribute_1 + '研究生可咨询'
                    else:
                        attribute = attribute_1
                    return attribute
                else:
                    if re.search('研究生.*', title) is not None:
                        attribute = '研究生可咨询'
                        return attribute
                    else:
                        return None

            def get_ortherpost(tags, host_name):
                data = {}
                list1 = []
                list2 = []
                if len(tags) <= 1:
                    print(host_name + "无帖回复")
                    return None
                else:
                    for x in range(1, len(tags)):
                        Ans_name = tags[x].find('td', class_='pls', rowspan='2').find('div', class_='pi').text
                        try:
                            quetion = tags[x].find('div', class_='t_fsz').find('td', class_='t_f').get_text()
                        except Exception as e:
                            print('获取回复贴进程错误')
                            quetion = tags[x].find('div', class_='t_fsz').text
                        if host_name == Ans_name:
                            list1.append(quetion)
                        # 其他人询问或答复
                        else:
                            list2.append(quetion)
                    data[host_name] = list1
                    data[Ans_name] = list2
                    return data

            def Parser__02(response):
                bs = BeautifulSoup(response, 'html.parser')
                # 定位所有楼层列表的元素
                postlist = bs.find_all('div', id=re.compile('^post_[0-9]+'))
                # 项目名
                item = get_item(bs)
                # 项目链接
                href = get_url(bs)
                # 日期
                date = get_date(postlist[0])
                # 发布者名和ID
                host_name, host_id = get_user_name_id(postlist[0], href)
                # 项目简介
                item_intro = get_item_intro(postlist[0], href)
                # 联系方式
                Wechat = get_weixin(item_intro)
                qq = get_QQ(item_intro)
                phone = get_cellphone(item_intro)
                contact = {'微信': Wechat, 'QQ': qq, '电话': phone}
                print(contact)
                # 项目属性
                attribute = get_item_attribute(item)
                print(attribute)
                # 项目回复（其他楼层）：
                dic_post = get_ortherpost(postlist, host_name)
                Out_flow = [item, host_id, host_name, href, date, item_intro, contact, attribute, dic_post]
                return Out_flow

            while not work.empty():
                URL = work.get_nowait()
                HTML = get_HTML(URL)
                results = Parser__02(HTML)
                writer.writerow(results)

        writer = self.outFlow()
        self.Multi_program(crwaler_02)

    def Multi_program(self, crawler):

        def __tasks__(crawler1):
            tasks_list = []
            for x in range(10):
                task = gevent.spawn(crawler1)
                tasks_list.append(task)
            return tasks_list

        task_list = __tasks__(crawler)
        gevent.joinall(task_list)

    def outFlow(self):
        """
        弹出数据
        :param dataFlow: list(list)
        :return:
        """
        f = open(self.out_path, 'w', encoding='utf-8', newline='')
        writer = csv.writer(f)
        writer.writerow(['项目名', '用户ID', "用户昵称", '项目链接', '发布日期', '项目简介', '联系方式', '项目属性', '项目回复'])
        return writer

    def run(self):
        """爬虫入口"""
        # 连接数据库，建立数据库中的表结构

        # 运行demo
        dataFlow = self.layer_02()


if __name__ == '__main__':
    czw = CZW_Spider()
    czw.run()

问题