python爬虫

已于 2023-09-19 15:36:56 修改
阅读量94 收藏
点赞数
分类专栏： python基础与实践 文章标签： python 爬虫 开发语言
于 2023-09-18 20:53:48 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/weixin_38716567/article/details/132996271
版权 

import pandas as pd
import requests
from bs4 import BeautifulSoup
#
import requests
from lxml import etree
import re


class Job_info:
    def __init__(self):
        self.company = ''
        self.job_name = ''
        self.job_money = ''
        self.job_year = ''
        self.job_city = ''
        self.job_people_number = ''
        self.url = ''


class Info_python:
    def __init__(self, company, company_url, job_url):
        self.url = r'https://www.kanzhun.com'
        self.company_url = company_url
        self.job_url = job_url
        self.company = company
        self.headers = {'Accept': 'text/html, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, sdch',
                        'Accept-Language': 'zh-CN,zh;q=0.8',
                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'}
        self.cookies = {
            'Cookie': 'aliyungf_tc=AQAAAD2MPjQrcwQAp4x2Dgdwc71am5e9; __c=1491732911; W_CITY_S_V=57; __g=-; isHasPushRecommentMessage=true; thirtyMinutes=true; isShowDownload=false; thirtyMinutesCount=2; pageType=2; ac="544397501@qq.com"; __t=ZPp3Vr6QMt1cLNx; __l=r=&l=%2Fgsr194222.html%3Fka%3Dpercent-review-list; __a=29429174.1491732911..1491732911.7.1.7.7; t=ZPp3Vr6QMt1cLNx; AB_T=abvb'}

    def get_company_url(self):
        current_url = r'https://www.kanzhun.com/search?pageNum=1&query=' + self.company + '&type=1'
        print(current_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'lxml')
        print(html)
        name = soup.find_all('div', class_='middle')
        print(name)

    def loop_diff_page(self, page_num, url_str):
        replacedStr = re.sub("/p\d+", "/p" + str(page_num), url_str)
        return replacedStr

    def loop_diff_page_1(self, page_num, url_str):
        if page_num == 1:
            replacedStr = re.sub(".html", "/p" + str(page_num) + ".html", url_str)
        else:
            replacedStr = re.sub("/p\d+.html", "/p" + str(page_num) + ".html", url_str)
        return replacedStr

    def get_all_pages_salary(self):  # 公司页面下所有岗位页的遍历
        page_num = 1
        current_url = self.loop_diff_page(page_num, self.company_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        name = soup.find_all('ul', class_='rc-pagination')
        a_list = name[0].find_all('li')
        page_limit = int(a_list[-3].text)
        all_pages_df = pd.DataFrame()
        # a = soup.find_all('div', class_='kz-empty-description')[0].text
        a = soup.find_all('div', class_='kz-empty-description')
        # while a != " 没有人发布过这家公司的工资信息":  # 遍历直到没有信息
        while len(a) == 0:
            one_page_df = self.get_one_page_salary(response)
            if all_pages_df.shape[0] == 0:
                all_pages_df = one_page_df
            else:
                all_pages_df = pd.concat([all_pages_df, one_page_df], axis=0)
            page_num += 1
            current_url = self.loop_diff_page(page_num, self.company_url)
            response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
            # a = soup.find_all('div', class_='kz-empty-description')[0].text
            html = response.text
            soup = BeautifulSoup(html, 'html.parser')
            a = soup.find_all('div', class_='kz-empty-description')
        # print(all_pages_df)
        all_pages_df.insert(loc=0, column='公司全称', value=self.company)
        return all_pages_df

    def check_status(self, job):
        dic = {}
        dic['code'] = 1
        if job not in ['硬件产品经理', '硬件测试工程师', '硬件测试', '硬件工程师', '硬件产品', '测试工程师', '嵌入式', '嵌入式软件工程师', '嵌入式工程师', '工业设计', '工业设计工程师', '机械结构', '机械结构工程师', '质量测试', '质量管理', '质量管理/测试', '质量管理工程师']:
        # if job != '工业工程师':
            dic['code'] = -1
        return dic

    def get_one_page_salary(self, response):  # 岗位页下每个岗位的遍历
        # current_url = r'https://www.kanzhun.com/firm/wage/0nJ92tS5EQ~~/p1.html'
        # current_url = self.loop_diff_page(page_num)
        # response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        # job_list = soup.find_all('div', class_='salary-list_1vfxU')
        job_list = soup.find_all('a', class_='salary-item')
        one_page_df = pd.DataFrame()
        for job1 in job_list:  #
            job = Job_info()
            try:
                job.job_name = job1.find('div', class_='name').text.strip()
                a = job1.find('div', class_='extra').text.strip()
                job.job_people_number = re.findall(r'\d+', a)[0]
                job.job_money = job1.find('div', class_='number').text.strip()
                job.url = job1['href']
                job_dic = job.__dict__
                detail_url = self.url + job.url
                print(detail_url)
                dic = self.check_status(job.job_name)
                print(dic['code'])
                if dic['code'] == 1:
                    one_position_df = self.get_all_pages_detail(detail_url)  # 已拿到公司-岗位url，准备进到公司-岗位，拿个人单挑信息
                    one_position_df['job_name'] = job.job_name
                    one_position_df['月均值'] = job.job_money
                    one_position_df['贡献人数'] = job.job_people_number
                    if one_page_df.shape[0] == 0:
                        one_page_df = one_position_df
                    else:
                        one_page_df = pd.concat([one_page_df, one_position_df], axis=0)
                    print(one_position_df)
                else:
                    continue
            except Exception as e:
                print(str(e))
            # if one_page_df.shape[0] == 0:
            #     one_page_df = one_position_df
            # else:
            #     one_page_df = pd.concat([one_page_df, one_position_df], axis=0)
        return one_page_df

    def get_all_pages_detail(self, url):
        page_num = 1
        current_url = self.loop_diff_page_1(page_num, url)
        # print(current_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        # print(html)
        soup = BeautifulSoup(html, 'html.parser')
        page_box = soup.find_all('div', class_='page-box')
        # print(page_box)
        # name = soup.find('ul', class_='rc-pagination')
        # print(name)
        a_list = page_box[0].find_all('li')
        # print(a_list)
        # print(a_list)
        a_list_proce = [a for a in a_list if a['class'] != ['rc-pagination-options']]
        # print(a_list_proce)
        # if not a_list[-1].text:
        #     page_limit = int(a_list[-3].text)
        # else:
        #     page_limit = int(a_list[-2].text)
        page_limit = int(a_list_proce[-2].text)

        print(page_limit)
        all_pages_person_df = pd.DataFrame()
        # a = soup.find_all('div', class_='kz-empty-description')[0].text
        while page_num <= page_limit:
            print(page_num)
            one_page_person_df = self.get_detail_salary(response)
            # print(current_url)
            # print(one_page_person_df)
            if all_pages_person_df.shape[0] == 0:
                all_pages_person_df = one_page_person_df
            else:
                all_pages_person_df = pd.concat([all_pages_person_df, one_page_person_df], axis=0)
            page_num += 1
            current_url = self.loop_diff_page_1(page_num, current_url)
            response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
            # a = soup.find_all('div', class_='kz-empty-description')[0].text

            # print(all_pages_person_df)
        return all_pages_person_df

    def get_detail_salary(self, response):  # 个人单条数据遍历
        # print(url)
        # response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        job_list = soup.find_all('a', class_='salary-item second')
        one_page_person_df = pd.DataFrame()
        for job1 in job_list:
            url_2nd = self.url + job1['href']
            one_person_df = self.get_detail_salary_2nd(url_2nd)  # 拿到公司-岗位-个人url
            if one_page_person_df.shape[0] == 0:
                one_page_person_df = one_person_df
            else:
                one_page_person_df = pd.concat([one_page_person_df, one_person_df], axis=0)

        return one_page_person_df

    def get_detail_salary_2nd(self, url):  # 个人详细数据
        # print(url)
        response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text  # 拿到公司-岗位-个人 detail
        soup = BeautifulSoup(html, 'html.parser')
        info_list = soup.find_all('div', class_='card-container')

        detail_info = {}
        for infos in info_list:  # 遍历每个blocker
            for info in infos:  # 遍历每行
                a = info.find('span', class_='title')
                if a is None:
                    continue
                else:
                    title_list = a.text.strip()
                    content_list = info.find('span', class_='content').text.strip()
                    detail_info[title_list] = content_list
                    # print(detail_info)
        one_person_df = pd.DataFrame([detail_info])
        return one_person_df


#  岗位循环可为空
#  个人循环需判断是否为重复


if __name__ == '__main__':
    # replacedStr = re.sub("/p\d+", "/p2", 'https://www.kanzhun.com/firm/wage/0nZz3tk~/p1.html')
    # print(replacedStr)
    company_url_dict = {
        # "科大讯飞": {
        #     "company": "https://www.kanzhun.com/firm/wage/0nZz3tk~/p1.html",
        #     "job": ""
        # },
        "百度": {
            "company": "https://www.kanzhun.com/firm/wage/1nV_2tk~/p1.html",
            "job": ""
        },
        "作业帮": {
            "company": "https://www.kanzhun.com/firm/wage/0nJ73961GQ~~/p1.html",
            "job": ""
        },
        "网易有道": {
            "company": "https://www.kanzhun.com/firm/wage/1Xd60tg~/p1.html",
            "job": ""
        }
    }

    # info_python = Info_python('江苏彤明', 'https://www.kanzhun.com/firm/wage/1nF729i5GQ~~/p1.html', '')
    # df = info_python.get_all_pages_salary()
    # print(df)
    # df.to_csv(r'D:\chatgpt/' + '江苏彤明.csv', encoding='utf-8')
    for key in company_url_dict:
        info_python = Info_python(key, company_url_dict[key]['company'], company_url_dict[key]['job'])
        df = info_python.get_all_pages_salary()
        df.to_csv(r'D:\chatgpt/' + key + '1.csv', index=False, encoding='utf-8')


import pandas as pd
import requests
from bs4 import BeautifulSoup
#
import requests
from lxml import etree
import re


class Job_info:
    def __init__(self):
        self.company = ''
        self.job_name = ''
        self.job_money = ''
        self.job_year = ''
        self.job_city = ''
        self.job_people_number = ''
        self.url = ''


class Info_python:
    def __init__(self, company, company_url, job_url):
        self.url = r'https://www.kanzhun.com'
        self.company_url = company_url
        self.job_url = job_url
        self.company = company
        self.headers = {'Accept': 'text/html, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, sdch',
                   'Accept-Language': 'zh-CN,zh;q=0.8',
                   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'}
        self.cookies = {
            'Cookie': 'aliyungf_tc=AQAAAD2MPjQrcwQAp4x2Dgdwc71am5e9; __c=1491732911; W_CITY_S_V=57; __g=-; isHasPushRecommentMessage=true; thirtyMinutes=true; isShowDownload=false; thirtyMinutesCount=2; pageType=2; ac="544397501@qq.com"; __t=ZPp3Vr6QMt1cLNx; __l=r=&l=%2Fgsr194222.html%3Fka%3Dpercent-review-list; __a=29429174.1491732911..1491732911.7.1.7.7; t=ZPp3Vr6QMt1cLNx; AB_T=abvb'}

    def get_company_url(self):
        current_url = r'https://www.kanzhun.com/search?pageNum=1&query=' + self.company + '&type=1'
        print(current_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'lxml')
        print(html)
        name = soup.find_all('div', class_='middle')
        print(name)

    def loop_diff_page(self, page_num, url_str):
        replacedStr = re.sub("/p\d+", "/p" + str(page_num), url_str)
        return replacedStr

    def loop_diff_page_1(self, page_num, url_str):
        if page_num == 1:
            replacedStr = re.sub(".html", "/p" + str(page_num) + ".html", url_str)
        else:
            replacedStr = re.sub("/p\d+.html", "/p" + str(page_num) + ".html", url_str)
        return replacedStr

    def get_all_pages_salary(self):  # 公司页面下所有岗位页的遍历
        page_num = 1
        current_url = self.loop_diff_page(page_num, self.company_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        name = soup.find_all('ul', class_='rc-pagination')
        a_list = name[0].find_all('li')
        page_limit = int(a_list[-3].text)
        all_pages_df = pd.DataFrame()
        # a = soup.find_all('div', class_='kz-empty-description')[0].text
        a = soup.find_all('div', class_='kz-empty-description')
        # while a != " 没有人发布过这家公司的工资信息":  # 遍历直到没有信息
        while len(a) == 0:
            one_page_df = self.get_one_page_salary(response)
            if all_pages_df.shape[0] == 0:
                all_pages_df = one_page_df
            else:
                all_pages_df = pd.concat([all_pages_df, one_page_df], axis=0)
            page_num += 1
            current_url = self.loop_diff_page(page_num, self.company_url)
            response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
            # a = soup.find_all('div', class_='kz-empty-description')[0].text
            a = soup.find_all('div', class_='kz-empty-description')

        all_pages_df.insert(loc=0, column='公司全称', value=self.company)
        return all_pages_df
      

    def get_one_page_salary(self, response):   # 岗位页下每个岗位的遍历
        # current_url = r'https://www.kanzhun.com/firm/wage/0nJ92tS5EQ~~/p1.html'
        # current_url = self.loop_diff_page(page_num)
        # response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        # job_list = soup.find_all('div', class_='salary-list_1vfxU')
        job_list = soup.find_all('a', class_='salary-item')
        one_page_df = pd.DataFrame()
        for job1 in job_list:  # 
            job = Job_info()
            try:
                job.job_name = job1.find('div', class_='name').text.strip()
                a = job1.find('div', class_='extra').text.strip()
                job.job_people_number = re.findall(r'\d+', a)
                job.job_money = job1.find('div', class_='number').text.strip()
                job.url = job1['href']
                job_dic = job.__dict__
                detail_url = self.url + job.url
                print(detail_url)
                one_position_df = self.get_all_pages_detail(detail_url)  # 已拿到公司-岗位url，准备进到公司-岗位，拿个人单挑信息
                one_position_df['job_name'] = job.job_name
                one_position_df['月均值'] = job.job_money
                one_position_df['贡献人数'] = job.job_people_number
                if one_page_df.shape[0] == 0:
                    one_page_df = one_position_df
                else:
                    one_page_df = pd.concat([one_page_df, one_position_df], axis=0)
                print(one_position_df)
            except Exception as e:
                print(str(e))
            # if one_page_df.shape[0] == 0:
            #     one_page_df = one_position_df
            # else:
            #     one_page_df = pd.concat([one_page_df, one_position_df], axis=0)
        return one_page_df
    
    def get_all_pages_detail(self, url):
        page_num = 1
        current_url = self.loop_diff_page_1(page_num, url)
        # print(current_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        # print(html)
        soup = BeautifulSoup(html, 'html.parser')
        name = soup.find_all('ul', class_='rc-pagination')
        # print(name)
        a_list = name[0].find_all('li')
        # print(a_list)
        page_limit = int(a_list[-3].text)
        # print(page_limit)
        all_pages_person_df = pd.DataFrame()
        # a = soup.find_all('div', class_='kz-empty-description')[0].text
        while page_num <= page_limit:
            one_page_person_df = self.get_detail_salary(response)
            # print(current_url)
            # print(one_page_person_df)
            if all_pages_person_df.shape[0] == 0:
                all_pages_person_df = one_page_person_df
            else:
                all_pages_person_df = pd.concat([all_pages_person_df, one_page_person_df], axis=0)
            page_num += 1
            current_url = self.loop_diff_page_1(page_num, current_url)
            response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
            # a = soup.find_all('div', class_='kz-empty-description')[0].text
        return all_pages_person_df

    def get_detail_salary(self, response):  # 个人单条数据遍历
        # print(url)
        # response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        job_list = soup.find_all('a', class_='salary-item second')
        one_page_person_df = pd.DataFrame()
        for job1 in job_list:
            url_2nd = self.url + job1['href']
            one_person_df = self.get_detail_salary_2nd(url_2nd)  # 拿到公司-岗位-个人url
            if one_page_person_df.shape[0] == 0:
                one_page_person_df = one_person_df
            else:
                one_page_person_df = pd.concat([one_page_person_df, one_person_df], axis=0)

        return one_page_person_df

    def get_detail_salary_2nd(self, url):  # 个人详细数据
        # print(url)
        response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text  # 拿到公司-岗位-个人 detail
        soup = BeautifulSoup(html, 'html.parser')
        info_list = soup.find_all('div', class_='card-container')

        detail_info = {}
        for infos in info_list:  # 遍历每个blocker
            for info in infos:  # 遍历每行
                a = info.find('span', class_='title')
                if a is None:
                    continue
                else:
                    title_list = a.text.strip()
                    content_list = info.find('span', class_='content').text.strip()
                    detail_info[title_list] = content_list
                    # print(detail_info)
        one_person_df = pd.DataFrame([detail_info])
        return one_person_df

#  岗位循环可为空
#  个人循环需判断是否为重复


if __name__ == '__main__':
    # replacedStr = re.sub("/p\d+", "/p2", 'https://www.kanzhun.com/firm/wage/0nZz3tk~/p1.html')
    # print(replacedStr)
    company_url_dict = {
        "科大讯飞": {
            "company": "https://www.kanzhun.com/firm/wage/0nZz3tk~/p1.html",
            "job": ""
            },
        "百度": {
            "company": "httzzps://www.kanzhun.com/firm/wage/1nV_2tk~/p2.html",
            "job": ""
            },
        "作业帮": {
            "company": "https://www.kanzhun.com/firm/wage/0nJ73961GQ~~/p1.html",
            "job": ""
            },
        "网易有道": {
            "company": "https://www.kanzhun.com/firm/wage/1Xd60tg~/p1.html",
            "job": ""
            }
    }
    # for key, value in company_url_dict:
    #     print(key, value)
    info_python = Info_python('科大讯飞', 'https://www.kanzhun.com/firm/wage/0nZz3tk~/p1.html', '')
    df = info_python.get_all_pages_salary()
    df.to_csv(r'' + key + '1.csv', encoding='gbk')

    


import pandas as pd
import requests
from bs4 import BeautifulSoup
#
import requests
from lxml import etree
import re


class Job_info:
    def __init__(self):
        self.company = ''
        self.job_name = ''
        self.job_money = ''
        self.job_year = ''
        self.job_city = ''
        self.job_people_number = ''
        self.url = ''


class Info_python:
    def __init__(self, company):
        self.url = r'https://www.kanzhun.com'
        self.company = company
        self.headers = {'Accept': 'text/html, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, sdch',
                   'Accept-Language': 'zh-CN,zh;q=0.8',
                   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'}
        self.cookies = {
            'Cookie': 'aliyungf_tc=AQAAAD2MPjQrcwQAp4x2Dgdwc71am5e9; __c=1491732911; W_CITY_S_V=57; __g=-; isHasPushRecommentMessage=true; thirtyMinutes=true; isShowDownload=false; thirtyMinutesCount=2; pageType=2; ac="544397501@qq.com"; __t=ZPp3Vr6QMt1cLNx; __l=r=&l=%2Fgsr194222.html%3Fka%3Dpercent-review-list; __a=29429174.1491732911..1491732911.7.1.7.7; t=ZPp3Vr6QMt1cLNx; AB_T=abvb'}

    def get_company_url(self):
        current_url = r'https://www.kanzhun.com/search?pageNum=1&query=' + self.company + '&type=1'
        print(current_url)
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'lxml')
        print(html)
        name = soup.find_all('div', class_='middle')
        print(name)

    def get_total_salary(self):
        current_url = r'https://www.kanzhun.com/firm/wage/0nJ92tS5EQ~~/p1.html'
        response = requests.get(current_url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        # job_list = soup.find_all('div', class_='salary-list_1vfxU')
        job_list = soup.find_all('a', class_='salary-item')
        for job1 in job_list:
            job = Job_info()
            try:
                job.job_name = job1.find('div', class_='name').text.strip()
                a = job1.find('div', class_='extra').text.strip()
                job.job_people_number = re.findall(r'\d+', a)
                job.job_money = job1.find('div', class_='number').text.strip()
                job.url = job1['href']
                job_dic = job.__dict__
                detail_url = self.url + job.url
                position_info_df = self.get_detail_salary(detail_url)  # 拿到公司-岗位url，准备进到公司-岗位
                position_info_df['job_name'] = job.job_name
                position_info_df['月均值'] = job.job_money
                position_info_df['贡献人数'] = job.job_people_number

            except Exception as e:
                print(str(e))

    def get_detail_salary(self, url):
        print(url)
        response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        job_list = soup.find_all('a', class_='salary-item second')
        position_info_df = pd.DataFrame()
        for job1 in job_list:
            url_2nd = self.url + job1['href']
            detail_info_df = self.get_detail_salary_2nd(url_2nd)  # 拿到公司-岗位-个人url
            if position_info_df.shape[0] == 0:
                position_info_df = detail_info_df
            else:
                position_info_df = pd.concat([position_info_df, detail_info_df], axis=0)

        return position_info_df

    def get_detail_salary_2nd(self, url):
        print(url)
        response = requests.get(url, headers=self.headers, cookies=self.cookies)
        html = response.text  # 拿到公司-岗位-个人 detail
        soup = BeautifulSoup(html, 'html.parser')
        info_list = soup.find_all('div', class_='card-container')

        detail_info = {}
        for infos in info_list:  # 遍历每个blocker
            for info in infos:  # 遍历每行
                a = info.find('span', class_='title')
                if a is None:
                    continue
                else:
                    title_list = a.text.strip()
                    content_list = info.find('span', class_='content').text.strip()
                    detail_info[title_list] = content_list
                    # print(detail_info)
        detail_info_df = pd.DataFrame([detail_info])
        return detail_info_df

#  岗位循环可为空
#  个人循环需判断是否为重复


if __name__ == '__main__':
    a = Info_python('宁德时代')
    a.get_company_url()


# # 需要的公司
# needed_companys = ['科大讯飞', '百度', '作业帮', '网易有道']
# # 需要的岗位
# needed_position = ['硬件产品经理', '硬件测试工程师', '硬件工程师', '嵌入式软件工程师']




# headers = {'Accept': 'text/html, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, sdch',
#            'Accept-Language': 'zh-CN,zh;q=0.8',
#            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'}
# cookies = {
#     'Cookie': 'aliyungf_tc=AQAAAD2MPjQrcwQAp4x2Dgdwc71am5e9; __c=1491732911; W_CITY_S_V=57; __g=-; isHasPushRecommentMessage=true; thirtyMinutes=true; isShowDownload=false; thirtyMinutesCount=2; pageType=2; ac="544397501@qq.com"; __t=ZPp3Vr6QMt1cLNx; __l=r=&l=%2Fgsr194222.html%3Fka%3Dpercent-review-list; __a=29429174.1491732911..1491732911.7.1.7.7; t=ZPp3Vr6QMt1cLNx; AB_T=abvb'}
# url1 = 'https://www.kanzhun.com/search?pageNum=1&query=%E5%AE%81%E5%BE%B7%E6%97%B6%E4%BB%A3&type=4'
# response = requests.get(url1, headers=headers, cookies=cookies)
# soup = BeautifulSoup(response.text, 'lxml')
# print(soup)
# name = soup.find_all('div', class_='company-name-left')
# print(name)
#
# url2 = '&cityCode=&sortMethod=1&employeeStatus=0'
# name2 = []  # 合并name字段各列表内容
# score2 = []  # 合并score字段各列表内容
# content2 = []  # 合并content字段各列表内容
# question2 = []
# for i in range(1, 8):
#     url = url1 + str(i) + url2
#     response = requests.get(url, headers=headers, cookies=cookies)
#     soup = BeautifulSoup(response.text, 'lxml')
#     name = soup.find_all('p', class_='f_14 grey_99 dd_bot')
#     for n in name:
#         name1 = n.get_text()
#         name2.append(name1)
#     score = soup.find_all('span', class_='grade')
#     for s in score:
#         score1 = s.get_text()
#         score2.append(score1)
#     content = soup.find_all('h3', class_='question_title')
#     for c in content:
#         content1 = c.get_text()
#         content11 = content1.replace('\n', '')
#         content2.append(content11)
#     question = soup.find_all('p', class_='question_content')
#     for q in question:
#         question1 = q.get_text()
#         question1 = question1.replace('\n', '')
#         question2.append(question1)
#         print(len(question1))
#
# table = pd.DataFrame({'name': name2, 'score': score2, 'content': content2})
# print(table)
# from selenium import webdriver
#
# url = "https://www.kanzhun.com/firm/wage/0nJ92tS5EQ~~.html"
# driver = webdriver.Chrome()
# driver.get(url)

# search_input = driver.find_element_by_name("wd")
# search_input.send_keys("Python")
# search_input.submit()

# html = driver.page_source
# soup = BeautifulSoup(html, 'lxml')
# # print(soup)
# name = soup.find_all('a')
# print(html)
#
# driver.quit()
# import scrapy
#
# class MySpider(scrapy.Spider):
#     name = "myspider"
#     start_urls = [
#         "http://www.example.com",
#     ]
#
#     def parse(self, response):
#         results = response.css("#element_id").extract_first()
#         print(results)
xzen
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
Python爬虫
LegendaryChen的博客
01-11 3651
选择一个主题，用Python语言编写一个网络爬虫程序，将文字和图像等信息抓取到MySQL中保存，（如果有图片数据，图片数据可以只在数据库存放路径，图片资源存储到文件夹）。
Python爬虫100例教程导航帖（已完结）
梦想橡皮擦，专栏100例写作模式先行者，现象级专栏 《Python 爬虫 100 例》作者、《滚雪球学 Python 专栏》原创者
08-14 20万+
Python爬虫入门教程导航，目标100篇。
带你玩转Python爬虫（爬取电影资源篇）
热门推荐
阿玥的小博客
03-16 76万+
跟着我，python你也可以！
Python爬虫实战：利用代理IP爬取百度翻译
努力让自己发光，对的人才能迎着光而来
08-06 25万+
Python爬虫实战：利用代理IP爬取百度翻译
Python爬虫详解：原理、常用库与实战案例
Why_does_it_work的博客
04-01 7万+
通过本文的讲解，相信读者已经对Python爬虫有了较为全面的认识。爬虫技能在数据分析、自然语言处理等领域具有广泛的应用，希望读者能够动手实践，不断提高自己的技能水平。同时，请注意合法合规地进行爬虫，遵守相关法律法规。祝您学习愉快！
Python爬虫实战（五）：根据关键字爬取某度图片批量下载到本地（附上完整源码）
努力让自己发光，对的人才能迎着光而来
08-21 14万+
Python爬虫实战（五）：根据关键字爬取某度图片批量下载到本地（附上完整源码）
Python爬虫详解（一看就懂）
我的博客
06-21 9万+
爬虫简单的来说就是用程序获取网络上数据这个过程的一种名称。如果要获取网络上数据，我们要给爬虫一个网址（程序中通常叫URL），爬虫发送一个HTTP请求给目标网页的服务器，服务器返回数据给客户端（也就是我们的爬虫），爬虫再进行数据解析、保存等一系列操作。爬虫可以节省我们的时间，比如我要获取豆瓣电影 Top250 榜单，如果不用爬虫，我们要先在浏览器上输入豆瓣电影的 URL ，客户端（浏览器）通过解析查到豆瓣电影网页的服务器的 IP 地址，然后与它建立连接，浏览器再创造一个 HTTP 请求发送给豆瓣电影的服务器，
Python爬虫之入门保姆级教程，学不会我去你家刷厕所
小袁同学的博客
05-26 8万+
注重版权，转载请注明原作者和原文链接 作者：Bald programmer 今天这个教程采用最简单的爬虫方法，适合小白新手入门，代码不复杂 爬虫的介绍以及原理等等七七八八的东西我就不多bb了，咋们直接上教程 本案例我就以 彼岸图网 这个网站做教程，原网址下方链接 https://pic.netbian.com/ 首先打开咋们的网站 可以看到有很多好看的图片，一页总共21张图片 我们右键选择检查或者直接按F12来到控制台 点击左上角的箭头或者快捷键ctrl+shift+c，然后随便点在一张图片上面 ..
Python爬虫实战（七）——批量下载4K高清小姐姐图片（附上完整源码）
努力让自己发光，对的人才能迎着光而来
12-18 29万+
Python爬虫实战（七）——批量下载4K高清小姐姐图片（附上完整源码）
Python爬虫系列（一）——手把手教你写Python爬虫
cun的博客
10-23 4万+
适合初学者快速上手入门，以爬取CSDN和百度图片为例。
Python爬虫框架Scrapy教程 完整版PDF
04-06
《Python爬虫框架Scrapy教程》主要是针对学习python爬虫的课程，又基础的python爬虫框架scrapy开始，一步步学习到最后完整的爬虫完成，现在python爬虫应用的非常广泛，本文档详细介绍了scrapy爬虫和其他爬虫技术的...
python爬虫课件+代码.zip
07-25
Python爬虫技术是一种用于自动化网络数据获取的编程技术，它在大数据分析、网站维护、市场研究等领域具有广泛应用。本课程由“路飞学城樵夫”老师指导，通过实际操作帮助学习者掌握Python爬虫的基本原理和实战技巧。...
python爬虫从入门到精通（模块）
03-17
这份文档旨在帮助想要学习Python爬虫的初学者，从入门到精通逐步提升自己的技能。以下是我们将要涵盖的主题： ## 入门篇 1. Python爬虫简介 2. Requests库的使用 3. Beautiful Soup库的使用 4. 爬虫实战：爬取百度...
Python爬虫教学视频-最全的Python爬虫视频教程全集
09-12
本Python爬虫教学视频，全集共51天课程，整套课程以Python语言为核心，通过各种经典案例的讲解，很好的演示了python爬虫如何抓取数据的全过程，非常值得Python爬虫工程师和想掌握python爬虫技术的同学借鉴学习。...
81个Python爬虫源代码
12-13
81个Python爬虫源代码，内容包含新闻、视频、中介、招聘、图片资源等网站的爬虫资源
【python】—— Python爬虫实战：爬取珠海市2011-2023年天气数据并保存为CSV文件
weixin_63106307的博客
09-05 1114
本文将介绍如何使用Python编写一个简单的爬虫程序，以爬取珠海市2011年至2023年的天气数据，并将这些数据保存为CSV文件。
【FFMPEG】Install FFmpeg CUDA gltransition in Ubuntu
RockWang的博客
09-05 483
FFMPEG gltranlations CUDA
pip-tools：打造可重复、可控的 Python 开发环境，解决依赖关系，让代码更稳定
weixin_53707653的博客
09-09 774
是一个强大且易用的工具，可以帮助开发者轻松管理 Python 项目的依赖关系，确保代码的可重复性和稳定性。是一组命令行工具，旨在简化 Python 依赖关系的管理，确保项目环境的稳定性和可重复性。可以确保每次构建环境时都使用相同的依赖项版本，从而避免由于依赖项版本不一致导致的错误，提高代码可重复性和稳定性。自动化了依赖关系管理过程，节省了开发者的时间和精力，可以将更多时间投入到实际的开发工作中。文件中的信息，更新虚拟环境，安装、升级或卸载所需的软件包，确保虚拟环境与。命令可以从你的项目配置文件中生成。
Python实现多线程、多进程及协程
最新发布
qq_42568323的博客
09-09 1008
本文详细介绍了 Python 中多线程、多进程和协程的并发模型及其实现方式，并通过具体场景演示了如何使用面向对象思想实现这些模型。在实际应用中，应根据任务的类型和需求选择合适的并发模型，从而优化程序的性能和资源利用率。本文将详细介绍 Python 中的多线程、多进程和协程的概念及其实现方式，并通过具体场景展示如何在 Python 中使用面向对象的思想实现这些并发模型。接下来，我们通过一个计算密集型任务的示例来演示多进程的实现：计算一系列大数字的阶乘。主程序中创建并启动了多个计算进程，并使用。
Python爬虫入门指南
"Python爬虫入门指南" Python爬虫是一个用于自动化网络数据抓取的技术，它允许程序员从网页中提取大量信息。本指南将带你逐步走进Python爬虫的世界。 首先，你需要具备Python编程基础。Python是一种易学易用的编程...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值