简单网站爬取：糗事百科

简单网站爬取：糗事百科

总结遇到的问题：
①xpah在响应页面定位时：
首先分组，分组后的子元素定位，是在当前定位内进行搜寻，因此点非常重要，他表示在当前目录中进行定位，不然可能出现，分组内爬取的内容一样，循环无效

-----------------************************=-------------------------

错误代码示例：

item["u-user-name"]=li.xpath("//div[@class='u-txt']/a/text()")[0] 

正确代码示例

item["u-user-name"]=li.xpath(".//div[@class='u-txt']/a/text()")[0]

全部代码

# -*- coding: utf-8 -*-
"""
Created on Sun Mar  1 15:09:54 2020

@author: Administrator
"""
import requests
from lxml import etree
import json
import os
os.chdir(r"H:\实操\学习\01\xiaohua")

class Xiaohua:
    def __init__(self):
        self.url_temp = "http://www.budejie.com/"
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"}
        self.part_url = "http://www.budejie.com/"

    def parse_url(self,url):
        response = requests.get(url,headers = self.headers)
        return response.content.decode(encoding = "utf-8")
        
    def get_url_content(self,html_str):
        html = etree.HTML(html_str)
        li_list = html.xpath("//div[@class ='j-r-list']/ul/li")
        print(li_list)
        content_list = []
        for li in li_list:
            item = {}
            item["u-logo"] = li.xpath("./div[@class='j-list-user']//img/@src")[0] if len(li.xpath("./div[@class='j-list-user']//img/@src"))>0 else None
            
            item["u-user-name"]=li.xpath(".//div[@class='u-txt']/a/text()")[0] if len(li.xpath(".//div[@class='u-txt']/a/text()"))>0 else None
           
            item["u-text"] = li.xpath(".//div[@class='j-r-list-c-desc']/a/text()")[0] if len(li.xpath(".//div[@class='j-r-list-c-desc']/a/text()"))>0 else None
            item["u-text-img"] = li.xpath(".//div[@class='j-r-list-c-img']//img/@src")[0] if len(li.xpath(".//div[@class='j-r-list-c-img']//img/@src"))>0 else None
            content_list.append(item)
        next_url = self.part_url+html.xpath("//a[text()='下一页']/@href")[0] if len(html.xpath("//a[text()='下一页']/@href"))>0 else None
        return content_list,next_url
        
    def save_content(self,content_list,num):
        filepath  = str(num) +'.txt'
        with open(filepath,"a",encoding = "utf-8") as f:
            for content in content_list:
                f.write(json.dumps(content,ensure_ascii=False,indent = 2))
                f.write("\n")
        print("保存成功")
                
        
            
    def run(self):
        num = 1
        next_url = self.url_temp
        while next_url is not None:
            #1.创建首页地址，
            
            #2.发出请求，获取响应
            html_str = self.parse_url(next_url)
            #3.提取数据：
            content_list,next_url = self.get_url_content(html_str)
              #头像，网名。时间，内容，内容图片
            #4.保存数据
            self.save_content(content_list,num)
            num+=1
        #5.获取下一页地址
            print(next_url)
        #6.重复2-5操作
        
        
if __name__=="__main__": 
    xiaohua = Xiaohua()
    xiaohua.run()         

-------------视频学习 小酥仙 2020.3.1------------------------------------------------------