爬虫案例

最新推荐文章于 2020-10-27 08:49:20 发布
最新推荐文章于 2020-10-27 08:49:20 发布
阅读量350 收藏 2
点赞数
原文链接：http://www.cnblogs.com/changwoo/p/9930610.html
版权
一、爬取猫眼电影TOP100
二、爬取魔法花园所有花种
三、爬取魔法花园魔法屋
四、爬取pixabay图片
五、爬取饿了么单个地区商家信息
六、爬取淘女郎
七、爬取FIND ICONES
八、爬取斗图网图片
九、未完待续..
 
一、爬取猫眼电影TOP100
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
import re
import os
from random import choice
 
 
def get_page_url():
    pass
 
 
def parse_page_url(url):
    print('Downloading:',url)
    headers = [
        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
        'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
        'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0'
        ]
    header = {'user-agent': choice(headers)}
 
    try:
        r = requests.get(url,headers=header)
        if r.status_code is 200:
            print('访问成功')
            return r.text
        else:
            print('返回状态码不为200,请修改代码')
            return None
    except (ConnectionError):
        print('Error occurred')
        return None
    except Exception as result:
        print('捕获到其他异常')
        print(result)
        return None
 
 
def get_text(text):
    soup = BeautifulSoup(text, 'lxml')
    pattern = re.compile(
        '<dd>.*?board-index.*?>(.*?)</i>' + '.*?data-val="{movieId:(.*?)}" href='
        '.*?src="(.*?)"/>\n<img.*?name.*?>(.*?)</a></p>.*?class="star">(.*?)</p>' +
        '.*?"releasetime">(.*?)</p>.*?</dd>', re.S
        )
    
    items = re.findall(pattern, str(soup))
    for item in items:
        yield {
            'name': item[3].split('>')[1],
            'id':item[1],
            'action': item[4].split('：')[1].split(' ')[0],
            'time': item[5].split('：')[1],
            #'img':'http:' + item[2]
            }
 
 
def write_text(content):
    print('正在保存:%s,请稍等...'%content['name'])
    with open('c://moayanmove.txt', 'a', encoding='utf-8') as f:
        f.write(str(content) + '\n')
        print('保存成功')
        f.close()
 
 
def main(Groups):
    urls = 'http://maoyan.com/board/4?offset=' + str(Groups)
    texts = parse_page_url(urls)
    for item in get_text(texts):
        write_text(item)
 
 
if __name__ == '__main__':
    pool = Pool()
    Groups = ([i*10 for i in range(10)])
    pool.map(main, Groups)
    pool.close()
    pool.join()
 
 
二、爬取魔法花园所有花种
import requests
import re
from bs4 import BeautifulSoup
import time
import random
import xlwt
url_half = 'http://h5.pinpinhu.com/gd-graph/index.action?uid=3640469&sid=7ddb5d624f398d043b74d1fa21af3c8f4ec83ad1&pageNo={}'
url_list = [url_half.format(i) for i in range(24)]
 
 
# print(url_list)
def get_html(url):
    res = requests.get(url, headers=headers)
    r_text = res.text.replace(u'\u266d', u'').replace(u'\xa9', u'')
    pattern = r'<img src="(.*?)"/>\n<a href=\'(.*?)\'>(.*?)</a>'
 
    t = re.findall(pattern, r_text, re.S)
    flower_list = []
    for item in t:
        time.sleep(random.random())
        flower_dict = {}
        flower_img = 'http://h5.pinpinhu.com' + item[0]
        flower_id = item[1].rsplit(';', maxsplit=1)[1].rsplit('=', maxsplit=1)[1]
        flower_name = item[2]
        _s = item[1].split(';')
        valid_url = 'http://h5.pinpinhu.com' + _s[0] + _s[1].split('amp')[0] + _s[2].split('amp')[0] + _s[3]
        pattern = '(.*?)HOME_SESSION.*?jvm1(.*)'
        url_half = re.findall(pattern, valid_url, re.S)[0]
        flower_url = url_half[0] + url_half[1]
        flower_dict['flower_name'] = flower_name
        flower_dict['flower_id'] = flower_id
        flower_dict['flower_url'] = flower_url
 
        res_2 = requests.get(flower_url, headers=headers)
        a_2 = res_2.text.replace(u'\u266d', u'').replace(u'\xa9', u'')
        li_2 = re.findall(r'花之图谱(.*?)返回花之图谱', a_2, re.S)[0]
        pattern_level = r'.*花种等级:(.*?)<br/>.*'
        pattern_price = r'.*种子价格:(.*?)家币<br/>.*'
        pattern_max = r'.*[预计成花|成花数量]:(.*?)<br/>.*保底数量.*'
        pattern_min = r'.*保底数量:(.*?)<br/>.*'
 
        pattern_time = r'.*预计时间:(.*?)<br/>.*'
        pattern_word = r'.*[鲜花花语|鲜花话语]:(.*?)<br/>.*'
        try:
            s_level = re.findall(pattern_level, li_2, re.S)[0]
        except:
            s_level = ''
        try:
            s_price = re.findall(pattern_price, li_2, re.S)[0]
        except:
            s_price = ''
        try:
            s_max = re.findall(pattern_max, li_2, re.S)[0]
        except:
            s_max = re.findall(pattern_max, li_2, re.S)
        try:
            s_min = re.findall(pattern_min, li_2, re.S)[0]
        except:
            s_min = re.findall(pattern_min, li_2, re.S)
        try:
            s_time = re.findall(pattern_time, li_2, re.S)[0]
        except:
            s_time = re.findall(pattern_time, li_2, re.S)
        try:
            s_word = re.findall(pattern_word, li_2, re.S)[0]
        except:
            s_word = re.findall(pattern_word, li_2, re.S)
        if s_price:
            pattern_other = '商城购买'
        else:
            pattern_other = r'.*[鲜花花语|鲜花话语]:.*?<br/>(.*?)<br/>'
        try:
            s_other = re.findall(pattern_other, li_2, re.S)[0]
        except:
            s_other = re.findall(pattern_other, li_2, re.S)
        
        flower_dict['level'] = s_level
        flower_dict['price'] = s_price
        flower_dict['max'] = s_max
        flower_dict['min'] = s_min
        flower_dict['time'] = s_time
        flower_dict['word'] = s_word
        flower_dict['other'] = s_other
        flower_dict['flower_img'] = flower_img
        flower_list.append(flower_dict)
    return flower_list
 
 
headers = {
'user-agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
 
}
flower_list = []
for i in range(1,24):
    # 普通：http://h5.pinpinhu.com/gd-graph/index.action?uid=3640469&sid=7ddb5d624f398d043b74d1fa21af3c8f4ec83ad1&pageNo=23
    # 独特：http://h5.pinpinhu.com/gd-graph/index.action?uid=3640469&sid=7ddb5d624f398d043b74d1fa21af3c8f4ec83ad1&type=1&pageNo=6
    # 珍稀：http://h5.pinpinhu.com/gd-graph/index.action?uid=3640469&sid=7ddb5d624f398d043b74d1fa21af3c8f4ec83ad1&type=2&pageNo=6
    # 极品：http://h5.pinpinhu.com/gd-graph/index.action?uid=3640469&sid=7ddb5d624f398d043b74d1fa21af3c8f4ec83ad1&type=3&pageNo=7
 
    url = 'http://h5.pinpinhu.com/gd-graph/index.action?uid=3981003&sid=3dd8cff09524f7af218b2af16ef6d26a5b69c6f2&type=0&pageNo={}'.format(i)
    print('------------第{}页--------共8页--------'.format(i))
    a = get_html(url)
    flower_list += a
    time.sleep(random.randint(1,5))
 
print(flower_list)
time.sleep(1)
workbook = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = workbook.add_sheet('普通花朵', cell_overwrite_ok=True)
x = 1
for item in flower_list:
        time.sleep(0.1)
 
        sheet.write(x, 0, item['flower_name'])
        sheet.write(x, 1, item['flower_id'])
        sheet.write(x, 2, item['flower_url'])
        sheet.write(x, 3, item['level'])
        sheet.write(x, 4, item['price'])
        sheet.write(x, 5, item['max'])
        sheet.write(x, 6, item['min'])
        sheet.write(x, 7, item['time'])
        sheet.write(x, 8, item['word'])
        sheet.write(x, 9, item['other'])
        sheet.write(x, 10, item['flower_img'])
        x += 1
 
workbook.save(r'普通.xls')
 
 
三、爬取魔法花园魔法屋
import requests
import re
import time
import random
 
 
def get_html(url):
    headers = {
    'user-agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
    }
    res = requests.get(url=url, headers=headers)
    s = res.text.replace(u'\u266d', u'').replace(u'\xa9', u'')
    pattern = r'.*极品(.*?)返回花园首页.*'
    a = re.findall(pattern, s, re.S)[0]
    pattern = r'.*?<img src="(.*?)"/><a href=\'(.*?)\'>(.*?)</a>.*?'
    b = re.findall(pattern, a, re.S)
    flower_list = []
    for item in b:
        time.sleep(random.random())
        flower_dict = {}
        flower_img = 'http://h5.pinpinhu.com' + item[0]
        flower_id = item[1].rsplit('id=', maxsplit=1)[1].split('&')[0]
        flower_name = item[2]
        _s = item[1].split(';')
        valid_url = 'http://h5.pinpinhu.com' + _s[0] + _s[1].split('amp')[0] + _s[2].split('amp')[0] + _s[3]
        pattern = '(.*?)HOME_SESSION.*?jvm1(.*)'
        url_half = re.findall(pattern, valid_url, re.S)[0]
        flower_url = url_half[0] + url_half[1]
 
        res_2 = requests.get(flower_url, headers=headers)
        a_2 = res_2.text.replace(u'\u266d', u'').replace(u'\xa9', u'')
        li_2 = re.findall(r'魔法屋(.*?)返回花园首页', a_2, re.S)[0]
        pattern_level = r'.*种子等级:(.*?)<br/>.*'
        pattern_compound = r'.*合成需消耗:(.*?)所需花朵不足.*'
        level = re.findall(pattern_level, li_2, re.S)[0]
        compound = re.findall(pattern_compound, li_2, re.S)[0]
        flower_dict['name'] = flower_name
        flower_dict['level'] = level
        flower_dict['flower_id'] = flower_id
 
        _a = compound.replace(r'<br/>', '').replace('\n', '')
        li_half = [i.rsplit('(', maxsplit=1) for i in _a.replace('/0)', '|').split('|')]
        detail_list = [i for i in li_half if not i==['']]
        flower_dict['detail_list'] = detail_list
        flower_dict['flower_img'] = flower_img
        flower_dict['flower_url'] = flower_url
        flower_list.append(flower_dict)
    return flower_list
 
flower_list = []
for i in range(1,2):
    url = 'http://h5.pinpinhu.com/gd-synthesis/index.action?uid=3981003&sid=0a05853e3cc8ea83eeb896806280c894be5aa59b&type=0&subType=0'
    print('---------%s-------'%i)
    a = get_html(url)
    flower_list += a
    time.sleep(3)
 
import xlwt
time.sleep(1)
workbook = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = workbook.add_sheet('普通花朵合成', cell_overwrite_ok=True)
x = 1
for item in flower_list:
    time.sleep(0.1)
    sheet.write(x, 0, item['name'])
    sheet.write(x, 1, int(item['flower_id']))
    sheet.write(x, 2, str(item['level']))
    sheet.write(x, 3, str(item['detail_list']))
    sheet.write(x, 4, item['flower_img'])
    sheet.write(x, 5, item['flower_url'])
    x += 1
 
workbook.save(r'1.xls')
 
 
四、爬取pixabay图片
#pixabay图片下载
#原网址  https://pixabay.com/
import requests
from bs4 import BeautifulSoup
import os
root = 'F:\\图片'
url_s = []
pixabay_day = []
base_url = 'https://pixabay.com/zh/photos/?order=ec&pagi='
for i in range(1,2):
    urls = base_url + str(i)
    url_s.append(urls)
for urls in url_s:
    request = requests.get(urls)
    content = request.content
    soup = BeautifulSoup(content,'lxml')
    img_list = soup('img')
    for img in img_list:
        url =img['src']
        url = url.split('/static')[0]
        pixabay_day.append(url)
        path = root + url.split('/')[-1]
        try:
            if not os.path.exists(root):
                os.mkdir(root)
            if not os.path.exists(path):
                r = requests.get(url)
                with open(path,'wb') as f:
                    f.write(r.content)
                    f.close()
                    print('文件保存成功')
            else:
                print('文件已存在')
        except:
            print('爬取失败')
 
五、爬取饿了么单个地区商家信息
import urllib.request
import os
import time
import json
from  openpyxl  import Workbook
from  openpyxl  import load_workbook
 
keywordExcel="keyword.xlsx"  #关键字检索外卖地点保存路径
keywords=["郑州","商丘"]  #关键字集合
targetDir ="test"  #文件保存路径
 
 
def reqsetting():#首先构造请求头headers，url目前暂时保存根路径
    weburl = "https://mainsite-restapi.ele.me/v2/pois?"
    #extra1="extras%5B%5D=count&geohash=wx4g0bmjetr7&keyword=%E6%9C%9D%E9%98%B3&limit=20&type=nearby"
    webheaders={
    "Accept":"application/json, text/plain, */*",
    "Accept-Language":"zh-CN,zh;q=0.8",
    "Connection":"keep-alive",
    "Cookie":"ubt_ssid=plds7ye19rj2rghg3oaar8hkt89yy7f1_2017-02-07; _utrace=ac9073c509bedb74b28a1482bd95a9d8_2017-02-07",
    "Host":"mainsite-restapi.ele.me",
    "Origin":"https://www.ele.me",
    "Referer":"https://www.ele.me/place/wx4g4h5shqf",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36"
        }
    req=urllib.request.Request(url=weburl,headers=webheaders)
    return req
 
 
def write2Excel(jsondata,title):#根据不同的关键字将数据写入到excel中
    fileName=keywordExcel
    if(os.path.exists(fileName)):
        wb=load_workbook(fileName)
    else:
        wb=Workbook()
    ws=wb.create_sheet(title)    
    ws.column_dimensions["A"].width =10.0
    ws.append(["ID","城市","geohash","名称","地址","商家总数","经度","纬度","request_id","short_address"])
    ws.column_dimensions["A"].width =30.0
    ws.column_dimensions["B"].width =10.0
    ws.column_dimensions["C"].width =18.0
    ws.column_dimensions["D"].width =20.0
    ws.column_dimensions["E"].width =50.0
    ws.column_dimensions["F"].width =10.0
    ws.column_dimensions["G"].width =10.0
    ws.column_dimensions["H"].width =10.0
    ws.column_dimensions["I"].width =25.0
    ws.column_dimensions["J"].width =40.0
    for i in range(len(jsondata)):
        row=jsondata[i]
        ws.append([row["id"],row["city"],row["geohash"],row["name"],row["address"],row["count"],
                   row["longitude"],row["latitude"],row["request_id"],row["short_address"]])
    wb.save(fileName)
    
 
def excelName():#根据日期生成文件
    if not os.path.isdir(targetDir):  
        os.mkdir(targetDir)
    excelName=str(time.strftime ("%Y-%m-%d")+".xlsx")
    completePath=targetDir+"\\"+excelName
    return completePath
 
 
def reqsetting():#初始化url请求，需要实时替换的是extral  和  header里的referer
    weburl = "https://mainsite-restapi.ele.me/shopping/restaurants?"
    extra1="extras%5B%5D=activities&geohash=wx4g56v1d2m&latitude=39.91771&limit=24&longitude=116.51698&offset=0&terminal=web"
    webheaders={
    "Accept":"application/json, text/plain, */*",
    "Accept-Language":"zh-CN,zh;q=0.8",
    "Connection":"keep-alive",
    "Cookie":"ubt_ssid=plds7ye19rj2rghg3oaar8hkt89yy7f1_2017-02-07; _utrace=ac9073c509bedb74b28a1482bd95a9d8_2017-02-07",
    "Host":"mainsite-restapi.ele.me",
    "Origin":"https://www.ele.me",
    #"Referer":"https://www.ele.me/place/wx4g56v1d2m",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36"
        }
    req=urllib.request.Request(url=weburl,headers=webheaders)
    return req
 
 
def write2Excel(jsondata,title):#根据不同的商圈地点写入数据，每个商圈地点占用excel 的一个sheet
    fileName=excelName()
    isexit="false"
    if(os.path.exists(fileName)):
        wb=load_workbook(fileName)
        isexit="true"
    else :
       wb=Workbook()
    if(wb.__contains__(title)):
        ws=wb[title]
        ws.append([])
    else:
        ws=wb.create_sheet(title)
        ws.column_dimensions["A"].width =10.0
        ws.column_dimensions["B"].width =40.0
        ws.column_dimensions["C"].width =60.0
        ws.column_dimensions["D"].width =10.0
        ws.column_dimensions["E"].width =18.0
        ws.column_dimensions["F"].width =10.0
        ws.append(["ID","店名","地址","距离","人均消费","月销售额"])
    
    for i  in range(len(jsondata)):
        row=jsondata[i]
        #print(type(row))
        if("average_cost" not  in row.keys()):
            row["average_cost"]="无人均消费数据"
        ws.append([row["id"],row["name"],row["address"],row["distance"],row["average_cost"],row["recent_order_num"]])
    wb.save(fileName)
    
 
def readKeyWordFromExcel():#从上一节生成的keywordExcel 中加载商圈地点
    fileName=keywordExcel
    if(os.path.exists(fileName)):
        wb=load_workbook(fileName)
    else:
        return
    for title in wb.sheetnames:
        ws=wb[title]
        for i in range(2,ws.max_row):
            infos={}#商圈地点数据，为生成请求参数做准备
            infos["name"]=ws.cell(row=i,column=4).value
            print("正在爬取 %s 附近外卖商家的数据..." % infos["name"])
            infos["ID"]=ws.cell(row=i,column=1).value
            infos["geohash"]=ws.cell(row=i,column=3).value
            infos["longitude"]=ws.cell(row=i,column=7).value
            infos["latitude"]=ws.cell(row=i,column=8).value
            if(infos["geohash"]):
                req=reqsetting()
                req.add_header("Refer","https://www.ele.me/place/%s" % infos["geohash"])#修改请求头的refer
                newUrl=req.get_full_url()
                offset=0
                contentBytes=""
                while(contentBytes!="[]"):#构造请求参数，基本上只修改offset 偏移量数据
                    params={
                    "extras[]":"activities",
                    "geohash":"%s" % infos["geohash"],
                    "latitude":"%s" % infos["latitude"],
                    "longitude":"%s" % infos["longitude"],
                    "terminal":"web",
                    "limit":24,
                    "offset":offset
                       }
                    params=urllib.parse.urlencode(params)#请求参数编码
                    req.full_url=newUrl+params   #重新生成url请求
                    webpage=urllib.request.urlopen(req)
                    contentBytes = webpage.read().decode("utf-8")
                    if(contentBytes!="[]"):
                        jsondata=json.loads(contentBytes)        
                        write2Excel(jsondata,infos["name"])#将请求数据写入excel中
                        offset+=24 #便宜
                    else :
                        break
 
 
if __name__ == '__main__':  #程序运行入口
    if(os.path.exists(keywordExcel)):
        os.remove(keywordExcel)
    req=reqsetting()
    newUrl=req.get_full_url()
    for keyword in keywords:#遍历关键字集合，构造不同的请求参数，附加到URL 请求上
        params={
        "extras[]":"count",
        "geohash":"wx4g0bmjetr7",
        "keyword":"%s" % keyword,
        "limit":"20",
        "type":"nearby"
            }
        params=urllib.parse.urlencode(params)#将请求参数进行编码
        req.full_url=newUrl+params#重新构造请求参数
       
        webpage=urllib.request.urlopen(req)#获取数据
        contentBytes = webpage.read().decode("utf-8")
        jsondata=json.loads(contentBytes)#将数据解析成json格式
        write2Excel(jsondata,keyword)#将数据写入excel 中
    time.sleep(10)
    offset=0;
    readKeyWordFromExcel()
 
 
六、爬取淘女郎
#!-*- coding:utf-8 -*-
 
import urllib.request
import re
def getUrlList(url):
    html = urllib.request.urlopen(url)
    response = str(html.read())
    return response
def getImageList(html):
    pattern = re.compile('.*?"avatarUrl":"(.*?)","cardUrl":"(.*?)","city":"(.*?)","height":"(.*?)",.*?"realName":"(.*?)",.*?"totalFavorNum":(.*?),"userId":(.*?),.*?"weight":"(.*?)".*?',re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'avatarUrl': 'http:' + str(items[0]).split('\'')[1],
            'cardUrl': 'http:' + str(item[1]),
            'city': item[2],
            'height': item[3],
            'realName': item[4],
            'totalFavorNum': item[5],
            'userId': item[6],
            'weight': item[7],
        }
 
def main():
    url = 'https://mm.taobao.com/tstar/search/tstar_model.do?_input_charset=utf-8'
    html = getUrlList(url)
    for item in getImageList(html):
        print(item)
 
if __name__ == '__main__':
    main()
 
七、爬取FIND ICONES
#FIND ICONES图标集
#源网址 https://findicons.com/
 
import requests
import os
from bs4 import BeautifulSoup
 
root = 'D:/图片/'
base_urls = 'https://findicons.com/pack/2787/beautiful_flat_icons/'
for i in range(1,8):
    base_url = base_urls + str(i)
    r = requests.get(base_url)
    content = r.content
    soup = BeautifulSoup(content,'lxml')
    img_list = soup('img')
    for img in img_list:
        url = img['src']
        path = root + url.split('/')[-1]
        try:
            if not os.path.exists(root):
                os.mkdir(root)
            if not os.path.exists(path):
                r = requests.get(url)
                with open(path,'wb') as f:
                    f.write(r.content)
                    f.close()
                    print('文件保存成功')
            else:
                print('文件已存在')
        except:
            print('爬取失败')
 
 
八、爬取斗图网图片
#斗图网图片下载
 
from tkinter import *
import os
import requests
from bs4 import BeautifulSoup
 
def do():
    urls = 'https://www.doutula.com/photo/list/?page=3'
    root = 'E:/图片/'
    request = requests.get(urls)
    content = request.content
    soup = BeautifulSoup(content,'lxml')
    img_list = soup.find_all('img',attrs = {'class':'img-responsive lazy image_dta'})
    for img in img_list:
        url_a = 'http:' + img['data-original']
        url = url_a.split('!')[0]
        path = root + url.split('/')[-1]
        try:
            if not os.path.exists(root):
                os.mkdir(root)
            if not os.path.exists(path):
                r = requests.get(url)
                with open(path,'wb') as f:
                    f.write(r.content)
                    f.close()
                    print('文件保存成功')
            else:
                print('文件已存在')
        except:
            print('爬取失败')
app = Tk()
Button(text = 'click',command = do).pack()
 
app.mainloop()
 
九、未完待续..
 
 

转载于:https://www.cnblogs.com/changwoo/p/9930610.html

weixin_30526593
关注
爬虫简单案例
老菜鸟
09-16 493
1.原生jdk发送get请求 //演示: 原生jdk发送get请求 public class JDKRequest { public static void main(String[] args) throws Exception { //1.确定url String indexUrl = "http://www.baidu.com"; ...
【自用】Python爬虫学习（一）：爬虫基础与四个简单案例
最新发布
Lucky_云佳的博客
08-13 1167
可以看到，已经获取到预览中看到的所有数据，但略显杂乱，后续只需要对该部分内容进行感兴趣提取就行，显然这是python基础，与爬虫无关了，毕竟已经获取到了数据。如下所示，页面中含有“美丽人生”，但右键查看页面源代码，使用Ctrl+F搜索却没有该文本，说明该网页应该就属于第2种类型，即客户端渲染。依次点击左侧红色方框中的条目内容，查看右侧预览信息，发现第二个就应该是我们需要的内容，其中就有“美丽人生”的文本。对于一个网页，浏览器右键可以查看页面源代码，但是这与使用开发者工具的检查看到的结果不一定相同。
网络爬虫案例解析
08-31
网络爬虫案例解析 网络爬虫是一种按照一定的规则，自动抓取万维网信息的程序或者脚本，已被广泛应用于互联网领域。搜索引擎使用网络爬虫抓取Web网页、文档甚至图片、音频、视频等资源，通过相应的索引技术组织这些...
拉钩scrapy爬虫案例
03-29
在这个“拉钩scrapy爬虫案例”中，我们将探讨如何利用Scrapy来爬取拉钩网（一个知名的招聘网站）上的职位信息，并将这些数据保存到CSV文件中，进一步上传到MySQL数据库。 首先，我们需要了解Scrapy的基本架构。...
python爬虫案例简介及实际案例及案例详解.rar
06-29
python爬虫案例python爬虫案例简介及实际案例及案例详解.rarpython爬虫案例简介及实际案例及案例详解.rarpython爬虫案例简介及实际案例及案例详解.rarpython爬虫案例简介及实际案例及案例详解.rarpython爬虫案例简介...
7个经典的python爬虫案例附源码分享
09-14
在这个案例中，我们将深入探讨7个经典Python爬虫小案例，这些案例涵盖了正则表达式（re）、XPath、BeautifulSoup和Selenium等关键技术。 首先，让我们从第一个案例开始，它涉及到在百度贴吧中抓取NBA吧的一个帖子的...
python爬虫案例获取歌曲.zip
03-24
python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲python爬虫案例获取歌曲...
一个简单的爬虫例子
05-30
一个简单的爬虫，亲测可用
爬虫实例
weixin_30387663的博客
08-04 281
实例1：中国大学排名定向爬虫 功能描述 输入：大学排名URL链接 输出：大学排名信息的屏幕输出（排名，大学名称，总分） 技术路线：requests-bs4 定向爬虫：仅对输入URL进行爬取，不扩展爬取 程序的结构设计 步骤1：从网络上获取大学排名网页内容getHTMLText() 步骤2：提取网页内容中信息到合适的数据结构fillUnivList() 步骤3：利用数据结...
爬虫小案例
若晨
07-15 543
爬取“http://bjjqe.com”网站的内容，解析数据，输出该网站中的图书信息 爬取网站内容，查看到图书信息都是子啊’div’的标签中，并包含“class=“bk_show_info”属性，因此调用BeautifulSoup对象的find_all()方法进行查找时，可用语句”bsObj.find_all("div",{'class':'bk_show_info'})“实现，然后再通过遍历查找找到的内容列表，取出有用的信息进行输出。 from bs4 import BeautifulSoup #
爬虫案例（一）
weixin_44797793的博客
10-27 437
1.百度产品 ## 需求：将百度全部产品页面保存到本地 import requests response = requests.get(url='https://www.baidu.com/more/') # print(response.text) # print(response.content.decode() # 保存图片 img_url = requests.get( 'https://timgsa.baidu.com/timg?image&quality=80&size=b99
pycharm爬虫案例
07-03
PyCharm是一款强大的集成开发环境（IDE），特别适合Python开发者使用，包括但不限于编写爬虫。以下是一个简单的PyCharm爬虫案例： 1. **设置项目**： 首先，在PyCharm中创建一个新的Python Web Scrapper项目，选择"File" > "New" > "Project"，然后选择"Python"，勾选"Supports Django/Flask/Web App Development"。 2. **安装依赖**： 使用`pip install`安装必要的库，如`requests`用于发送HTTP请求，`beautifulsoup4`或`lxml`解析HTML，`pandas`用于数据处理等。 ```python pip install requests beautifulsoup4 pandas ``` 3. **编写爬虫代码**： ```python import requests from bs4 import BeautifulSoup url = 'https://www.example.com' # 替换为你要爬取的网站地址 response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') # 例如抓取页面标题 title = soup.title.string print("Title:", title) # 如果有链接需要提取 links = soup.find_all('a', href=True) for link in links: print(link['href']) ``` 4. **调试与运行**： 在PyCharm中，你可以使用其丰富的调试工具设置断点、单步执行和查看变量值，检查爬虫是否按预期工作。 **相关问题--:** 1. PyCharm如何支持爬虫项目的调试？ 2. 如何在PyCharm中管理爬虫项目的依赖包？ 3. 用PyCharm编写爬虫时，如何处理网页动态加载的内容？
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值