爬取周公解梦数据(二)

最新推荐文章于 2021-02-27 05:29:37 发布

xiaominuo

最新推荐文章于 2021-02-27 05:29:37 发布

阅读量220

点赞数

分类专栏： python 文章标签： mysql python

本文链接：https://blog.csdn.net/qq_40923390/article/details/107613580

版权

python 专栏收录该内容

10 篇文章 0 订阅

订阅专栏

爬取周公解梦更多数据

具体解析参考爬取周公解梦数据(一)

效果

在这里插入图片描述

贴所有代码

import json
import requests
import time
from bs4 import BeautifulSoup
import pymysql
def add_data(url,title,type_name):
    headers = {
        "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }
    # json_data = requests.get(url, params=form_data, headers=headers).json()
    try:
        json_data = requests.get(url)
        html=json_data.content
        
        html_doc = str(html,'utf-8')
        soup = BeautifulSoup(html_doc,"lxml")
        content = soup.find(class_='article-content')

        title = soup.find(class_='article-title')

        cur.execute("insert into dream(type,title,content,index_type) VALUES ('"+type_name+"','"+str(title.string)+"','"+str(content.text)+"',0)")
    except UnicodeDecodeError as err:
        print(title,"Unicodeerror")
        pass
    
def down_meng():
    url = 'https://www.zgjm.net'
    arr={0:'/b/renwu/',1:'/b/dongwu/',2:'/b/zhiwu/',3:'/b/wupin/',4:'/b/huodong/',5:'/b/qinggan/',6:'/b/shenghuo/',7:'/b/guishen/',8:'/b/ziran/',9:'/b/jianzhu/',10:'/b/qita/'}
    headers = {
        "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }
    arr_type = {0:"人物",1:"动物",2:"植物",3:"物品",4:"活动",5:"情感",6:"生活",7:"鬼神",8:"自然",9:"建筑",10:"其他"}
    for key in arr:
        for i in range(1,50):
            target_url = url+str(arr[key])+"index_"+str(i)+".html"
            headers = {
                "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
            }
            try:
                json_data = requests.get(target_url)
                html=json_data.content
                html_doc = str(html,'utf-8')
                soup = BeautifulSoup(html_doc,"lxml")
                li_list = soup.find(class_='postitemot')
                for lilist in li_list:
                    add_data(url+lilist.a['href'],lilist.a.string,arr_type[key])
                    print(lilist.a.string,"完成")
            except UnicodeDecodeError as err:
                print(target_url,"Unicodeerror")
                pass
            except TypeError as err:
                print(target_url,"Unicodeerror")
                pass
        print(arr_type[key],"完成")
        conn.commit()
        time.sleep(3)

if __name__ == '__main__':
    conn = pymysql.connect(host=ip,port=port,user='root',passwd='',db='',charset='utf8mb4')
    # print(conn)
    cur = conn.cursor()
    down_meng()
    # arr = {0:"人物",1:"动物",2:"植物",3:"物品",4:"活动",5:"情感",6:"生活",7:"鬼神",8:"自然",9:"建筑",10:"其他"}
    # add_data('https://www.zgjm.net/b/4053/','aa',arr[0])

    cur.close()
    conn.close()