python爬虫----爬取糗事百科

一个练手的小爬虫，这个是写在图片爬取的那个前面

熟悉了下requests，BeautifulSoup，正则表达式的操作

本来不想放上来，因为感觉写的不好，放上来方便自己以后修改下

import requests
from bs4 import BeautifulSoup
import re


def gethtml(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return 'wrong!!!'



def save_way(url,fpath):
    html = gethtml(url)
    soup = BeautifulSoup(html,'html.parser')
    ID = []
    level = []
    content = []
    haoxiao = []
    pinglun = []
    Fun_pinglun = []
    data = ['ID：','等级：','内容：','好笑：','评论：']

    ids = soup.find_all('h2')
    for k in ids:
        ID.append(k.text)


    Levels = soup.find_all('div',attrs={'class':'author clearfix'})
    for k in Levels:
        l = re.findall(r'<div class="articleGender .+">(\d+)</div>',str(k))[0]
        
        level.append(l)


    Contents = soup.find_all('div',attrs={'class':"content"})
    for k in Contents:
        c = re.sub(r'<br/>','\n',str(k))
        
        cend = re.findall(r'<span>(.+)</span>',c,flags = re.S)[0]

        content.append(cend)


    num_haoxiao = soup.find_all('span',attrs={'class':"stats-vote"})
    for k in num_haoxiao:
        haoxiao.append(k.i.text)

    share = soup.find_all('a',attrs={'class':'qiushi_comments'})
    for k in share:
        try:
            
            p = k.find('i',attrs={'class':'number'})
            
            pinglun.append(p.text)
        except:
            continue


    count = 0


    with open(fpath,'w',encoding = 'utf-8') as f:
        print(len(ID),len(level),len(content),len(haoxiao),len(pinglun))
        for i in range(len(ID)):
            count = count + 1
            print('\r正在爬取第%d个段子......'%count)
            f.write(data[0] + ID[i] +'\n')
            f.write(data[1] + level[i] +'\n')
            f.write(data[2] + content[i]+'\n')
            f.write(data[3] + haoxiao[i]+'\n')
            f.write(data[4] + pinglun[i]+'\n\n\n')
           


def main():
    url = 'https://www.qiushibaike.com/'
    fpath = 'E:qiushibaike.txt'
    save_way(url,fpath)


main()